sys/vfs/hammer2/hammer2_vfsops.c

   1 /*
   2  * Copyright (c) 2011-2015 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  * by Daniel Flores (GSOC 2013 - mentored by Matthew Dillon, compression)
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in
  16  *    the documentation and/or other materials provided with the
  17  *    distribution.
  18  * 3. Neither the name of The DragonFly Project nor the names of its
  19  *    contributors may be used to endorse or promote products derived
  20  *    from this software without specific, prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  26  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  27  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  28  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  29  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  30  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  31  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  32  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  33  * SUCH DAMAGE.
  34  */
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/kernel.h>
  38 #include <sys/nlookup.h>
  39 #include <sys/vnode.h>
  40 #include <sys/mount.h>
  41 #include <sys/fcntl.h>
  42 #include <sys/buf.h>
  43 #include <sys/uuid.h>
  44 #include <sys/vfsops.h>
  45 #include <sys/sysctl.h>
  46 #include <sys/socket.h>
  47 #include <sys/objcache.h>
  48
  49 #include <sys/proc.h>
  50 #include <sys/namei.h>
  51 #include <sys/mountctl.h>
  52 #include <sys/dirent.h>
  53 #include <sys/uio.h>
  54
  55 #include <sys/mutex.h>
  56 #include <sys/mutex2.h>
  57
  58 #include "hammer2.h"
  59 #include "hammer2_disk.h"
  60 #include "hammer2_mount.h"
  61 #include "hammer2_lz4.h"
  62
  63 #include "zlib/hammer2_zlib.h"
  64
  65 #define REPORT_REFS_ERRORS 1    /* XXX remove me */
  66
  67 MALLOC_DEFINE(M_OBJCACHE, "objcache", "Object Cache");
  68
  69 struct hammer2_sync_info {
  70         int error;
  71         int waitfor;
  72 };
  73
  74 TAILQ_HEAD(hammer2_mntlist, hammer2_dev);
  75 TAILQ_HEAD(hammer2_pfslist, hammer2_pfs);
  76 static struct hammer2_mntlist hammer2_mntlist;
  77 static struct hammer2_pfslist hammer2_pfslist;
  78 static struct lock hammer2_mntlk;
  79
  80 int hammer2_debug;
  81 int hammer2_cluster_enable = 1;
  82 int hammer2_hardlink_enable = 1;
  83 int hammer2_flush_pipe = 100;
  84 int hammer2_synchronous_flush = 1;
  85 int hammer2_dio_count;
  86 long hammer2_limit_dirty_chains;
  87 long hammer2_count_modified_chains;
  88 long hammer2_iod_file_read;
  89 long hammer2_iod_meta_read;
  90 long hammer2_iod_indr_read;
  91 long hammer2_iod_fmap_read;
  92 long hammer2_iod_volu_read;
  93 long hammer2_iod_file_write;
  94 long hammer2_iod_file_wembed;
  95 long hammer2_iod_file_wzero;
  96 long hammer2_iod_file_wdedup;
  97 long hammer2_iod_meta_write;
  98 long hammer2_iod_indr_write;
  99 long hammer2_iod_fmap_write;
 100 long hammer2_iod_volu_write;
 101 long hammer2_ioa_file_read;
 102 long hammer2_ioa_meta_read;
 103 long hammer2_ioa_indr_read;
 104 long hammer2_ioa_fmap_read;
 105 long hammer2_ioa_volu_read;
 106 long hammer2_ioa_fmap_write;
 107 long hammer2_ioa_file_write;
 108 long hammer2_ioa_meta_write;
 109 long hammer2_ioa_indr_write;
 110 long hammer2_ioa_volu_write;
 111
 112 MALLOC_DECLARE(M_HAMMER2_CBUFFER);
 113 MALLOC_DEFINE(M_HAMMER2_CBUFFER, "HAMMER2-compbuffer",
 114                 "Buffer used for compression.");
 115
 116 MALLOC_DECLARE(M_HAMMER2_DEBUFFER);
 117 MALLOC_DEFINE(M_HAMMER2_DEBUFFER, "HAMMER2-decompbuffer",
 118                 "Buffer used for decompression.");
 119
 120 SYSCTL_NODE(_vfs, OID_AUTO, hammer2, CTLFLAG_RW, 0, "HAMMER2 filesystem");
 121
 122 SYSCTL_INT(_vfs_hammer2, OID_AUTO, debug, CTLFLAG_RW,
 123            &hammer2_debug, 0, "");
 124 SYSCTL_INT(_vfs_hammer2, OID_AUTO, cluster_enable, CTLFLAG_RW,
 125            &hammer2_cluster_enable, 0, "");
 126 SYSCTL_INT(_vfs_hammer2, OID_AUTO, hardlink_enable, CTLFLAG_RW,
 127            &hammer2_hardlink_enable, 0, "");
 128 SYSCTL_INT(_vfs_hammer2, OID_AUTO, flush_pipe, CTLFLAG_RW,
 129            &hammer2_flush_pipe, 0, "");
 130 SYSCTL_INT(_vfs_hammer2, OID_AUTO, synchronous_flush, CTLFLAG_RW,
 131            &hammer2_synchronous_flush, 0, "");
 132 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, limit_dirty_chains, CTLFLAG_RW,
 133            &hammer2_limit_dirty_chains, 0, "");
 134 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, count_modified_chains, CTLFLAG_RW,
 135            &hammer2_count_modified_chains, 0, "");
 136 SYSCTL_INT(_vfs_hammer2, OID_AUTO, dio_count, CTLFLAG_RD,
 137            &hammer2_dio_count, 0, "");
 138
 139 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_read, CTLFLAG_RW,
 140            &hammer2_iod_file_read, 0, "");
 141 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_meta_read, CTLFLAG_RW,
 142            &hammer2_iod_meta_read, 0, "");
 143 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_indr_read, CTLFLAG_RW,
 144            &hammer2_iod_indr_read, 0, "");
 145 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_fmap_read, CTLFLAG_RW,
 146            &hammer2_iod_fmap_read, 0, "");
 147 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_volu_read, CTLFLAG_RW,
 148            &hammer2_iod_volu_read, 0, "");
 149
 150 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_write, CTLFLAG_RW,
 151            &hammer2_iod_file_write, 0, "");
 152 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_wembed, CTLFLAG_RW,
 153            &hammer2_iod_file_wembed, 0, "");
 154 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_wzero, CTLFLAG_RW,
 155            &hammer2_iod_file_wzero, 0, "");
 156 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_wdedup, CTLFLAG_RW,
 157            &hammer2_iod_file_wdedup, 0, "");
 158 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_meta_write, CTLFLAG_RW,
 159            &hammer2_iod_meta_write, 0, "");
 160 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_indr_write, CTLFLAG_RW,
 161            &hammer2_iod_indr_write, 0, "");
 162 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_fmap_write, CTLFLAG_RW,
 163            &hammer2_iod_fmap_write, 0, "");
 164 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_volu_write, CTLFLAG_RW,
 165            &hammer2_iod_volu_write, 0, "");
 166
 167 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, ioa_file_read, CTLFLAG_RW,
 168            &hammer2_ioa_file_read, 0, "");
 169 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, ioa_meta_read, CTLFLAG_RW,
 170            &hammer2_ioa_meta_read, 0, "");
 171 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, ioa_indr_read, CTLFLAG_RW,
 172            &hammer2_ioa_indr_read, 0, "");
 173 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, ioa_fmap_read, CTLFLAG_RW,
 174            &hammer2_ioa_fmap_read, 0, "");
 175 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, ioa_volu_read, CTLFLAG_RW,
 176            &hammer2_ioa_volu_read, 0, "");
 177
 178 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, ioa_file_write, CTLFLAG_RW,
 179            &hammer2_ioa_file_write, 0, "");
 180 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, ioa_meta_write, CTLFLAG_RW,
 181            &hammer2_ioa_meta_write, 0, "");
 182 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, ioa_indr_write, CTLFLAG_RW,
 183            &hammer2_ioa_indr_write, 0, "");
 184 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, ioa_fmap_write, CTLFLAG_RW,
 185            &hammer2_ioa_fmap_write, 0, "");
 186 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, ioa_volu_write, CTLFLAG_RW,
 187            &hammer2_ioa_volu_write, 0, "");
 188
 189 static int hammer2_vfs_init(struct vfsconf *conf);
 190 static int hammer2_vfs_uninit(struct vfsconf *vfsp);
 191 static int hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 192                                 struct ucred *cred);
 193 static int hammer2_remount(hammer2_dev_t *, struct mount *, char *,
 194                                 struct vnode *, struct ucred *);
 195 static int hammer2_recovery(hammer2_dev_t *hmp);
 196 static int hammer2_vfs_unmount(struct mount *mp, int mntflags);
 197 static int hammer2_vfs_root(struct mount *mp, struct vnode **vpp);
 198 static int hammer2_vfs_statfs(struct mount *mp, struct statfs *sbp,
 199                                 struct ucred *cred);
 200 static int hammer2_vfs_statvfs(struct mount *mp, struct statvfs *sbp,
 201                                 struct ucred *cred);
 202 static int hammer2_vfs_vget(struct mount *mp, struct vnode *dvp,
 203                                 ino_t ino, struct vnode **vpp);
 204 static int hammer2_vfs_fhtovp(struct mount *mp, struct vnode *rootvp,
 205                                 struct fid *fhp, struct vnode **vpp);
 206 static int hammer2_vfs_vptofh(struct vnode *vp, struct fid *fhp);
 207 static int hammer2_vfs_checkexp(struct mount *mp, struct sockaddr *nam,
 208                                 int *exflagsp, struct ucred **credanonp);
 209
 210 static int hammer2_install_volume_header(hammer2_dev_t *hmp);
 211 static int hammer2_sync_scan2(struct mount *mp, struct vnode *vp, void *data);
 212
 213 static void hammer2_update_pmps(hammer2_dev_t *hmp);
 214
 215 static void hammer2_mount_helper(struct mount *mp, hammer2_pfs_t *pmp);
 216 static void hammer2_unmount_helper(struct mount *mp, hammer2_pfs_t *pmp,
 217                                 hammer2_dev_t *hmp);
 218
 219 /*
 220  * HAMMER2 vfs operations.
 221  */
 222 static struct vfsops hammer2_vfsops = {
 223         .vfs_init       = hammer2_vfs_init,
 224         .vfs_uninit     = hammer2_vfs_uninit,
 225         .vfs_sync       = hammer2_vfs_sync,
 226         .vfs_mount      = hammer2_vfs_mount,
 227         .vfs_unmount    = hammer2_vfs_unmount,
 228         .vfs_root       = hammer2_vfs_root,
 229         .vfs_statfs     = hammer2_vfs_statfs,
 230         .vfs_statvfs    = hammer2_vfs_statvfs,
 231         .vfs_vget       = hammer2_vfs_vget,
 232         .vfs_vptofh     = hammer2_vfs_vptofh,
 233         .vfs_fhtovp     = hammer2_vfs_fhtovp,
 234         .vfs_checkexp   = hammer2_vfs_checkexp
 235 };
 236
 237 MALLOC_DEFINE(M_HAMMER2, "HAMMER2-mount", "");
 238
 239 VFS_SET(hammer2_vfsops, hammer2, 0);
 240 MODULE_VERSION(hammer2, 1);
 241
 242 static
 243 int
 244 hammer2_vfs_init(struct vfsconf *conf)
 245 {
 246         static struct objcache_malloc_args margs_read;
 247         static struct objcache_malloc_args margs_write;
 248         static struct objcache_malloc_args margs_vop;
 249
 250         int error;
 251
 252         error = 0;
 253
 254         if (HAMMER2_BLOCKREF_BYTES != sizeof(struct hammer2_blockref))
 255                 error = EINVAL;
 256         if (HAMMER2_INODE_BYTES != sizeof(struct hammer2_inode_data))
 257                 error = EINVAL;
 258         if (HAMMER2_VOLUME_BYTES != sizeof(struct hammer2_volume_data))
 259                 error = EINVAL;
 260
 261         if (error)
 262                 kprintf("HAMMER2 structure size mismatch; cannot continue.\n");
 263
 264         margs_read.objsize = 65536;
 265         margs_read.mtype = M_HAMMER2_DEBUFFER;
 266
 267         margs_write.objsize = 32768;
 268         margs_write.mtype = M_HAMMER2_CBUFFER;
 269
 270         margs_vop.objsize = sizeof(hammer2_xop_t);
 271         margs_vop.mtype = M_HAMMER2;
 272
 273         /*
 274          * Note thaht for the XOPS cache we want backing store allocations
 275          * to use M_ZERO.  This is not allowed in objcache_get() (to avoid
 276          * confusion), so use the backing store function that does it.  This
 277          * means that initial XOPS objects are zerod but REUSED objects are
 278          * not.  So we are responsible for cleaning the object up sufficiently
 279          * for our needs before objcache_put()ing it back (typically just the
 280          * FIFO indices).
 281          */
 282         cache_buffer_read = objcache_create(margs_read.mtype->ks_shortdesc,
 283                                 0, 1, NULL, NULL, NULL,
 284                                 objcache_malloc_alloc,
 285                                 objcache_malloc_free,
 286                                 &margs_read);
 287         cache_buffer_write = objcache_create(margs_write.mtype->ks_shortdesc,
 288                                 0, 1, NULL, NULL, NULL,
 289                                 objcache_malloc_alloc,
 290                                 objcache_malloc_free,
 291                                 &margs_write);
 292         cache_xops = objcache_create(margs_vop.mtype->ks_shortdesc,
 293                                 0, 1, NULL, NULL, NULL,
 294                                 objcache_malloc_alloc_zero,
 295                                 objcache_malloc_free,
 296                                 &margs_vop);
 297
 298
 299         lockinit(&hammer2_mntlk, "mntlk", 0, 0);
 300         TAILQ_INIT(&hammer2_mntlist);
 301         TAILQ_INIT(&hammer2_pfslist);
 302
 303         hammer2_limit_dirty_chains = desiredvnodes / 10;
 304
 305         return (error);
 306 }
 307
 308 static
 309 int
 310 hammer2_vfs_uninit(struct vfsconf *vfsp __unused)
 311 {
 312         objcache_destroy(cache_buffer_read);
 313         objcache_destroy(cache_buffer_write);
 314         objcache_destroy(cache_xops);
 315         return 0;
 316 }
 317
 318 /*
 319  * Core PFS allocator.  Used to allocate the pmp structure for PFS cluster
 320  * mounts and the spmp structure for media (hmp) structures.
 321  *
 322  * pmp->modify_tid tracks new modify_tid transaction ids for front-end
 323  * transactions.  Note that synchronization does not use this field.
 324  * (typically frontend operations and synchronization cannot run on the
 325  * same PFS node at the same time).
 326  *
 327  * XXX check locking
 328  */
 329 hammer2_pfs_t *
 330 hammer2_pfsalloc(hammer2_chain_t *chain, const hammer2_inode_data_t *ripdata,
 331                  hammer2_tid_t modify_tid)
 332 {
 333         hammer2_inode_t *iroot;
 334         hammer2_pfs_t *pmp;
 335         int count;
 336         int i;
 337         int j;
 338
 339         /*
 340          * Locate or create the PFS based on the cluster id.  If ripdata
 341          * is NULL this is a spmp which is unique and is always allocated.
 342          */
 343         if (ripdata) {
 344                 TAILQ_FOREACH(pmp, &hammer2_pfslist, mntentry) {
 345                         if (bcmp(&pmp->pfs_clid, &ripdata->meta.pfs_clid,
 346                                  sizeof(pmp->pfs_clid)) == 0) {
 347                                         break;
 348                         }
 349                 }
 350         } else {
 351                 pmp = NULL;
 352         }
 353
 354         if (pmp == NULL) {
 355                 pmp = kmalloc(sizeof(*pmp), M_HAMMER2, M_WAITOK | M_ZERO);
 356                 hammer2_trans_manage_init(pmp);
 357                 kmalloc_create(&pmp->minode, "HAMMER2-inodes");
 358                 kmalloc_create(&pmp->mmsg, "HAMMER2-pfsmsg");
 359                 lockinit(&pmp->lock, "pfslk", 0, 0);
 360                 lockinit(&pmp->lock_nlink, "h2nlink", 0, 0);
 361                 spin_init(&pmp->inum_spin, "hm2pfsalloc_inum");
 362                 spin_init(&pmp->xop_spin, "h2xop");
 363                 RB_INIT(&pmp->inum_tree);
 364                 TAILQ_INIT(&pmp->sideq);
 365                 spin_init(&pmp->list_spin, "hm2pfsalloc_list");
 366
 367                 /*
 368                  * Distribute backend operations to threads
 369                  */
 370                 for (j = 0; j < HAMMER2_MAXCLUSTER; ++j)
 371                         TAILQ_INIT(&pmp->xopq[j]);
 372                 for (j = 0; j < HAMMER2_XOPGROUPS; ++j)
 373                         hammer2_xop_group_init(pmp, &pmp->xop_groups[j]);
 374
 375                 /*
 376                  * Save the last media transaction id for the flusher.  Set
 377                  * initial
 378                  */
 379                 if (ripdata)
 380                         pmp->pfs_clid = ripdata->meta.pfs_clid;
 381                 TAILQ_INSERT_TAIL(&hammer2_pfslist, pmp, mntentry);
 382
 383                 /*
 384                  * The synchronization thread may start too early, make
 385                  * sure it stays frozen until we are ready to let it go.
 386                  * XXX
 387                  */
 388                 /*
 389                 pmp->primary_thr.flags = HAMMER2_THREAD_FROZEN |
 390                                          HAMMER2_THREAD_REMASTER;
 391                 */
 392         }
 393
 394         /*
 395          * Create the PFS's root inode.
 396          */
 397         if ((iroot = pmp->iroot) == NULL) {
 398                 iroot = hammer2_inode_get(pmp, NULL, NULL, -1);
 399                 pmp->iroot = iroot;
 400                 hammer2_inode_ref(iroot);
 401                 hammer2_inode_unlock(iroot);
 402         }
 403
 404         /*
 405          * Stop here if no chain is passed in.
 406          */
 407         if (chain == NULL)
 408                 goto done;
 409
 410         /*
 411          * When a chain is passed in we must add it to the PFS's root
 412          * inode, update pmp->pfs_types[], and update the syncronization
 413          * threads.
 414          *
 415          * At the moment empty spots can develop due to removals or failures.
 416          * Ultimately we want to re-fill these spots but doing so might
 417          * confused running code. XXX
 418          */
 419         hammer2_inode_ref(iroot);
 420         hammer2_mtx_ex(&iroot->lock);
 421         j = iroot->cluster.nchains;
 422
 423         kprintf("add PFS to pmp %p[%d]\n", pmp, j);
 424
 425         if (j == HAMMER2_MAXCLUSTER) {
 426                 kprintf("hammer2_mount: cluster full!\n");
 427                 /* XXX fatal error? */
 428         } else {
 429                 KKASSERT(chain->pmp == NULL);
 430                 chain->pmp = pmp;
 431                 hammer2_chain_ref(chain);
 432                 iroot->cluster.array[j].chain = chain;
 433                 pmp->pfs_types[j] = ripdata->meta.pfs_type;
 434                 pmp->pfs_names[j] = kstrdup(ripdata->filename, M_HAMMER2);
 435                 pmp->pfs_hmps[j] = chain->hmp;
 436
 437                 /*
 438                  * If the PFS is already mounted we must account
 439                  * for the mount_count here.
 440                  */
 441                 if (pmp->mp)
 442                         ++chain->hmp->mount_count;
 443
 444                 /*
 445                  * May have to fixup dirty chain tracking.  Previous
 446                  * pmp was NULL so nothing to undo.
 447                  */
 448                 if (chain->flags & HAMMER2_CHAIN_MODIFIED)
 449                         hammer2_pfs_memory_inc(pmp);
 450                 ++j;
 451         }
 452         iroot->cluster.nchains = j;
 453
 454         /*
 455          * Update nmasters from any PFS inode which is part of the cluster.
 456          * It is possible that this will result in a value which is too
 457          * high.  MASTER PFSs are authoritative for pfs_nmasters and will
 458          * override this value later on.
 459          *
 460          * (This informs us of masters that might not currently be
 461          *  discoverable by this mount).
 462          */
 463         if (ripdata && pmp->pfs_nmasters < ripdata->meta.pfs_nmasters) {
 464                 pmp->pfs_nmasters = ripdata->meta.pfs_nmasters;
 465         }
 466
 467         /*
 468          * Count visible masters.  Masters are usually added with
 469          * ripdata->meta.pfs_nmasters set to 1.  This detects when there
 470          * are more (XXX and must update the master inodes).
 471          */
 472         count = 0;
 473         for (i = 0; i < iroot->cluster.nchains; ++i) {
 474                 if (pmp->pfs_types[i] == HAMMER2_PFSTYPE_MASTER)
 475                         ++count;
 476         }
 477         if (pmp->pfs_nmasters < count)
 478                 pmp->pfs_nmasters = count;
 479
 480         /*
 481          * Create missing synchronization and support threads.
 482          *
 483          * Single-node masters (including snapshots) have nothing to
 484          * synchronize and do not require this thread.
 485          *
 486          * Multi-node masters or any number of soft masters, slaves, copy,
 487          * or other PFS types need the thread.
 488          *
 489          * Each thread is responsible for its particular cluster index.
 490          * We use independent threads so stalls or mismatches related to
 491          * any given target do not affect other targets.
 492          */
 493         for (i = 0; i < iroot->cluster.nchains; ++i) {
 494                 /*
 495                  * Single-node masters (including snapshots) have nothing
 496                  * to synchronize and will make direct xops support calls,
 497                  * thus they do not require this thread.
 498                  *
 499                  * Note that there can be thousands of snapshots.  We do not
 500                  * want to create thousands of threads.
 501                  */
 502                 if (pmp->pfs_nmasters <= 1 &&
 503                     pmp->pfs_types[i] == HAMMER2_PFSTYPE_MASTER) {
 504                         continue;
 505                 }
 506
 507                 /*
 508                  * Sync support thread
 509                  */
 510                 if (pmp->sync_thrs[i].td == NULL) {
 511                         hammer2_thr_create(&pmp->sync_thrs[i], pmp,
 512                                            "h2nod", i, -1,
 513                                            hammer2_primary_sync_thread);
 514                 }
 515         }
 516
 517         /*
 518          * Create missing Xop threads
 519          */
 520         if (pmp->mp)
 521                 hammer2_xop_helper_create(pmp);
 522
 523         hammer2_mtx_unlock(&iroot->lock);
 524         hammer2_inode_drop(iroot);
 525 done:
 526         return pmp;
 527 }
 528
 529 /*
 530  * Destroy a PFS, typically only occurs after the last mount on a device
 531  * has gone away.
 532  */
 533 static void
 534 hammer2_pfsfree(hammer2_pfs_t *pmp)
 535 {
 536         hammer2_inode_t *iroot;
 537         int i;
 538         int j;
 539
 540         /*
 541          * Cleanup our reference on iroot.  iroot is (should) not be needed
 542          * by the flush code.
 543          */
 544         TAILQ_REMOVE(&hammer2_pfslist, pmp, mntentry);
 545
 546         iroot = pmp->iroot;
 547         if (iroot) {
 548                 for (i = 0; i < iroot->cluster.nchains; ++i) {
 549                         hammer2_thr_delete(&pmp->sync_thrs[i]);
 550                         for (j = 0; j < HAMMER2_XOPGROUPS; ++j)
 551                                 hammer2_thr_delete(&pmp->xop_groups[j].thrs[i]);
 552                 }
 553 #if REPORT_REFS_ERRORS
 554                 if (pmp->iroot->refs != 1)
 555                         kprintf("PMP->IROOT %p REFS WRONG %d\n",
 556                                 pmp->iroot, pmp->iroot->refs);
 557 #else
 558                 KKASSERT(pmp->iroot->refs == 1);
 559 #endif
 560                 /* ref for pmp->iroot */
 561                 hammer2_inode_drop(pmp->iroot);
 562                 pmp->iroot = NULL;
 563         }
 564
 565         kmalloc_destroy(&pmp->mmsg);
 566         kmalloc_destroy(&pmp->minode);
 567
 568         kfree(pmp, M_HAMMER2);
 569 }
 570
 571 /*
 572  * Remove all references to hmp from the pfs list.  Any PFS which becomes
 573  * empty is terminated and freed.
 574  *
 575  * XXX inefficient.
 576  */
 577 static void
 578 hammer2_pfsfree_scan(hammer2_dev_t *hmp)
 579 {
 580         hammer2_pfs_t *pmp;
 581         hammer2_inode_t *iroot;
 582         hammer2_chain_t *rchain;
 583         int didfreeze;
 584         int i;
 585         int j;
 586
 587 again:
 588         TAILQ_FOREACH(pmp, &hammer2_pfslist, mntentry) {
 589                 if ((iroot = pmp->iroot) == NULL)
 590                         continue;
 591                 if (hmp->spmp == pmp) {
 592                         kprintf("unmount hmp %p remove spmp %p\n",
 593                                 hmp, pmp);
 594                         hmp->spmp = NULL;
 595                 }
 596
 597                 /*
 598                  * Determine if this PFS is affected.  If it is we must
 599                  * freeze all management threads and lock its iroot.
 600                  *
 601                  * Freezing a management thread forces it idle, operations
 602                  * in-progress will be aborted and it will have to start
 603                  * over again when unfrozen, or exit if told to exit.
 604                  */
 605                 for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 606                         if (pmp->pfs_hmps[i] == hmp)
 607                                 break;
 608                 }
 609                 if (i != HAMMER2_MAXCLUSTER) {
 610                         /*
 611                          * Make sure all synchronization threads are locked
 612                          * down.
 613                          */
 614                         for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 615                                 if (pmp->pfs_hmps[i] == NULL)
 616                                         continue;
 617                                 hammer2_thr_freeze_async(&pmp->sync_thrs[i]);
 618                                 for (j = 0; j < HAMMER2_XOPGROUPS; ++j) {
 619                                         hammer2_thr_freeze_async(
 620                                                 &pmp->xop_groups[j].thrs[i]);
 621                                 }
 622                         }
 623                         for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 624                                 if (pmp->pfs_hmps[i] == NULL)
 625                                         continue;
 626                                 hammer2_thr_freeze(&pmp->sync_thrs[i]);
 627                                 for (j = 0; j < HAMMER2_XOPGROUPS; ++j) {
 628                                         hammer2_thr_freeze(
 629                                                 &pmp->xop_groups[j].thrs[i]);
 630                                 }
 631                         }
 632
 633                         /*
 634                          * Lock the inode and clean out matching chains.
 635                          * Note that we cannot use hammer2_inode_lock_*()
 636                          * here because that would attempt to validate the
 637                          * cluster that we are in the middle of ripping
 638                          * apart.
 639                          *
 640                          * WARNING! We are working directly on the inodes
 641                          *          embedded cluster.
 642                          */
 643                         hammer2_mtx_ex(&iroot->lock);
 644
 645                         /*
 646                          * Remove the chain from matching elements of the PFS.
 647                          */
 648                         for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 649                                 if (pmp->pfs_hmps[i] != hmp)
 650                                         continue;
 651                                 hammer2_thr_delete(&pmp->sync_thrs[i]);
 652                                 for (j = 0; j < HAMMER2_XOPGROUPS; ++j) {
 653                                         hammer2_thr_delete(
 654                                                 &pmp->xop_groups[j].thrs[i]);
 655                                 }
 656                                 rchain = iroot->cluster.array[i].chain;
 657                                 iroot->cluster.array[i].chain = NULL;
 658                                 pmp->pfs_types[i] = 0;
 659                                 if (pmp->pfs_names[i]) {
 660                                         kfree(pmp->pfs_names[i], M_HAMMER2);
 661                                         pmp->pfs_names[i] = NULL;
 662                                 }
 663                                 if (rchain) {
 664                                         hammer2_chain_drop(rchain);
 665                                         /* focus hint */
 666                                         if (iroot->cluster.focus == rchain)
 667                                                 iroot->cluster.focus = NULL;
 668                                 }
 669                                 pmp->pfs_hmps[i] = NULL;
 670                         }
 671                         hammer2_mtx_unlock(&iroot->lock);
 672                         didfreeze = 1;  /* remaster, unfreeze down below */
 673                 } else {
 674                         didfreeze = 0;
 675                 }
 676
 677                 /*
 678                  * Cleanup trailing chains.  Gaps may remain.
 679                  */
 680                 for (i = HAMMER2_MAXCLUSTER - 1; i >= 0; --i) {
 681                         if (pmp->pfs_hmps[i])
 682                                 break;
 683                 }
 684                 iroot->cluster.nchains = i + 1;
 685
 686                 /*
 687                  * If the PMP has no elements remaining we can destroy it.
 688                  * (this will transition management threads from frozen->exit).
 689                  */
 690                 if (iroot->cluster.nchains == 0) {
 691                         kprintf("unmount hmp %p last ref to PMP=%p\n",
 692                                 hmp, pmp);
 693                         hammer2_pfsfree(pmp);
 694                         goto again;
 695                 }
 696
 697                 /*
 698                  * If elements still remain we need to set the REMASTER
 699                  * flag and unfreeze it.
 700                  */
 701                 if (didfreeze) {
 702                         for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 703                                 if (pmp->pfs_hmps[i] == NULL)
 704                                         continue;
 705                                 hammer2_thr_remaster(&pmp->sync_thrs[i]);
 706                                 hammer2_thr_unfreeze(&pmp->sync_thrs[i]);
 707                                 for (j = 0; j < HAMMER2_XOPGROUPS; ++j) {
 708                                         hammer2_thr_remaster(
 709                                                 &pmp->xop_groups[j].thrs[i]);
 710                                         hammer2_thr_unfreeze(
 711                                                 &pmp->xop_groups[j].thrs[i]);
 712                                 }
 713                         }
 714                 }
 715         }
 716 }
 717
 718 /*
 719  * Mount or remount HAMMER2 fileystem from physical media
 720  *
 721  *      mountroot
 722  *              mp              mount point structure
 723  *              path            NULL
 724  *              data            <unused>
 725  *              cred            <unused>
 726  *
 727  *      mount
 728  *              mp              mount point structure
 729  *              path            path to mount point
 730  *              data            pointer to argument structure in user space
 731  *                      volume  volume path (device@LABEL form)
 732  *                      hflags  user mount flags
 733  *              cred            user credentials
 734  *
 735  * RETURNS:     0       Success
 736  *              !0      error number
 737  */
 738 static
 739 int
 740 hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 741                   struct ucred *cred)
 742 {
 743         struct hammer2_mount_info info;
 744         hammer2_pfs_t *pmp;
 745         hammer2_pfs_t *spmp;
 746         hammer2_dev_t *hmp;
 747         hammer2_key_t key_next;
 748         hammer2_key_t key_dummy;
 749         hammer2_key_t lhc;
 750         struct vnode *devvp;
 751         struct nlookupdata nd;
 752         hammer2_chain_t *parent;
 753         hammer2_chain_t *chain;
 754         hammer2_cluster_t *cluster;
 755         const hammer2_inode_data_t *ripdata;
 756         hammer2_blockref_t bref;
 757         struct file *fp;
 758         char devstr[MNAMELEN];
 759         size_t size;
 760         size_t done;
 761         char *dev;
 762         char *label;
 763         int ronly = 1;
 764         int error;
 765         int cache_index;
 766         int i;
 767
 768         hmp = NULL;
 769         pmp = NULL;
 770         dev = NULL;
 771         label = NULL;
 772         devvp = NULL;
 773         cache_index = -1;
 774
 775         kprintf("hammer2_mount\n");
 776
 777         if (path == NULL) {
 778                 /*
 779                  * Root mount
 780                  */
 781                 bzero(&info, sizeof(info));
 782                 info.cluster_fd = -1;
 783                 ksnprintf(devstr, sizeof(devstr), "%s",
 784                           mp->mnt_stat.f_mntfromname);
 785                 kprintf("hammer2_mount: root '%s'\n", devstr);
 786         } else {
 787                 /*
 788                  * Non-root mount or updating a mount
 789                  */
 790                 error = copyin(data, &info, sizeof(info));
 791                 if (error)
 792                         return (error);
 793
 794                 error = copyinstr(info.volume, devstr, MNAMELEN - 1, &done);
 795                 if (error)
 796                         return (error);
 797         }
 798
 799         /* Extract device and label */
 800         dev = devstr;
 801         label = strchr(devstr, '@');
 802         if (label == NULL ||
 803             ((label + 1) - dev) > done) {
 804                 return (EINVAL);
 805         }
 806         *label = '\0';
 807         label++;
 808         if (*label == '\0')
 809                 return (EINVAL);
 810
 811         if (mp->mnt_flag & MNT_UPDATE) {
 812                 /*
 813                  * Update mount.  Note that pmp->iroot->cluster is
 814                  * an inode-embedded cluster and thus cannot be
 815                  * directly locked.
 816                  *
 817                  * XXX HAMMER2 needs to implement NFS export via
 818                  *     mountctl.
 819                  */
 820                 pmp = MPTOPMP(mp);
 821                 cluster = &pmp->iroot->cluster;
 822                 for (i = 0; i < cluster->nchains; ++i) {
 823                         if (cluster->array[i].chain == NULL)
 824                                 continue;
 825                         hmp = cluster->array[i].chain->hmp;
 826                         devvp = hmp->devvp;
 827                         error = hammer2_remount(hmp, mp, path,
 828                                                 devvp, cred);
 829                         if (error)
 830                                 break;
 831                 }
 832
 833                 return error;
 834         }
 835
 836         /*
 837          * HMP device mount
 838          *
 839          * Lookup name and verify it refers to a block device.
 840          */
 841         if (path) {
 842                 error = nlookup_init(&nd, dev, UIO_SYSSPACE, NLC_FOLLOW);
 843                 if (error == 0)
 844                         error = nlookup(&nd);
 845                 if (error == 0)
 846                         error = cache_vref(&nd.nl_nch, nd.nl_cred, &devvp);
 847                 nlookup_done(&nd);
 848         } else {
 849                 /* root mount */
 850                 cdev_t cdev = kgetdiskbyname(dev);
 851                 error = bdevvp(cdev, &devvp);
 852                 if (error)
 853                         kprintf("hammer2: cannot find '%s'\n", dev);
 854         }
 855
 856         if (error == 0) {
 857                 if (vn_isdisk(devvp, &error))
 858                         error = vfs_mountedon(devvp);
 859         }
 860
 861         /*
 862          * Determine if the device has already been mounted.  After this
 863          * check hmp will be non-NULL if we are doing the second or more
 864          * hammer2 mounts from the same device.
 865          */
 866         lockmgr(&hammer2_mntlk, LK_EXCLUSIVE);
 867         TAILQ_FOREACH(hmp, &hammer2_mntlist, mntentry) {
 868                 if (hmp->devvp == devvp)
 869                         break;
 870         }
 871
 872         /*
 873          * Open the device if this isn't a secondary mount and construct
 874          * the H2 device mount (hmp).
 875          */
 876         if (hmp == NULL) {
 877                 hammer2_chain_t *schain;
 878                 hammer2_xid_t xid;
 879
 880                 if (error == 0 && vcount(devvp) > 0)
 881                         error = EBUSY;
 882
 883                 /*
 884                  * Now open the device
 885                  */
 886                 if (error == 0) {
 887                         ronly = ((mp->mnt_flag & MNT_RDONLY) != 0);
 888                         vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
 889                         error = vinvalbuf(devvp, V_SAVE, 0, 0);
 890                         if (error == 0) {
 891                                 error = VOP_OPEN(devvp,
 892                                                  ronly ? FREAD : FREAD | FWRITE,
 893                                                  FSCRED, NULL);
 894                         }
 895                         vn_unlock(devvp);
 896                 }
 897                 if (error && devvp) {
 898                         vrele(devvp);
 899                         devvp = NULL;
 900                 }
 901                 if (error) {
 902                         lockmgr(&hammer2_mntlk, LK_RELEASE);
 903                         return error;
 904                 }
 905                 hmp = kmalloc(sizeof(*hmp), M_HAMMER2, M_WAITOK | M_ZERO);
 906                 ksnprintf(hmp->devrepname, sizeof(hmp->devrepname), "%s", dev);
 907                 hmp->ronly = ronly;
 908                 hmp->devvp = devvp;
 909                 kmalloc_create(&hmp->mchain, "HAMMER2-chains");
 910                 TAILQ_INSERT_TAIL(&hammer2_mntlist, hmp, mntentry);
 911                 RB_INIT(&hmp->iotree);
 912                 spin_init(&hmp->io_spin, "hm2mount_io");
 913                 spin_init(&hmp->list_spin, "hm2mount_list");
 914                 TAILQ_INIT(&hmp->flushq);
 915
 916                 lockinit(&hmp->vollk, "h2vol", 0, 0);
 917                 lockinit(&hmp->bulklk, "h2bulk", 0, 0);
 918
 919                 /*
 920                  * vchain setup. vchain.data is embedded.
 921                  * vchain.refs is initialized and will never drop to 0.
 922                  *
 923                  * NOTE! voldata is not yet loaded.
 924                  */
 925                 hmp->vchain.hmp = hmp;
 926                 hmp->vchain.refs = 1;
 927                 hmp->vchain.data = (void *)&hmp->voldata;
 928                 hmp->vchain.bref.type = HAMMER2_BREF_TYPE_VOLUME;
 929                 hmp->vchain.bref.data_off = 0 | HAMMER2_PBUFRADIX;
 930                 hmp->vchain.bref.mirror_tid = hmp->voldata.mirror_tid;
 931
 932                 hammer2_chain_core_init(&hmp->vchain);
 933                 /* hmp->vchain.u.xxx is left NULL */
 934
 935                 /*
 936                  * fchain setup.  fchain.data is embedded.
 937                  * fchain.refs is initialized and will never drop to 0.
 938                  *
 939                  * The data is not used but needs to be initialized to
 940                  * pass assertion muster.  We use this chain primarily
 941                  * as a placeholder for the freemap's top-level RBTREE
 942                  * so it does not interfere with the volume's topology
 943                  * RBTREE.
 944                  */
 945                 hmp->fchain.hmp = hmp;
 946                 hmp->fchain.refs = 1;
 947                 hmp->fchain.data = (void *)&hmp->voldata.freemap_blockset;
 948                 hmp->fchain.bref.type = HAMMER2_BREF_TYPE_FREEMAP;
 949                 hmp->fchain.bref.data_off = 0 | HAMMER2_PBUFRADIX;
 950                 hmp->fchain.bref.mirror_tid = hmp->voldata.freemap_tid;
 951                 hmp->fchain.bref.methods =
 952                         HAMMER2_ENC_CHECK(HAMMER2_CHECK_FREEMAP) |
 953                         HAMMER2_ENC_COMP(HAMMER2_COMP_NONE);
 954
 955                 hammer2_chain_core_init(&hmp->fchain);
 956                 /* hmp->fchain.u.xxx is left NULL */
 957
 958                 /*
 959                  * Install the volume header and initialize fields from
 960                  * voldata.
 961                  */
 962                 error = hammer2_install_volume_header(hmp);
 963                 if (error) {
 964                         hammer2_unmount_helper(mp, NULL, hmp);
 965                         lockmgr(&hammer2_mntlk, LK_RELEASE);
 966                         hammer2_vfs_unmount(mp, MNT_FORCE);
 967                         return error;
 968                 }
 969
 970                 /*
 971                  * Really important to get these right or flush will get
 972                  * confused.
 973                  */
 974                 hmp->spmp = hammer2_pfsalloc(NULL, NULL, 0);
 975                 kprintf("alloc spmp %p tid %016jx\n",
 976                         hmp->spmp, hmp->voldata.mirror_tid);
 977                 spmp = hmp->spmp;
 978
 979                 /*
 980                  * Dummy-up vchain and fchain's modify_tid.  mirror_tid
 981                  * is inherited from the volume header.
 982                  */
 983                 xid = 0;
 984                 hmp->vchain.bref.mirror_tid = hmp->voldata.mirror_tid;
 985                 hmp->vchain.bref.modify_tid = hmp->vchain.bref.mirror_tid;
 986                 hmp->vchain.pmp = spmp;
 987                 hmp->fchain.bref.mirror_tid = hmp->voldata.freemap_tid;
 988                 hmp->fchain.bref.modify_tid = hmp->fchain.bref.mirror_tid;
 989                 hmp->fchain.pmp = spmp;
 990
 991                 /*
 992                  * First locate the super-root inode, which is key 0
 993                  * relative to the volume header's blockset.
 994                  *
 995                  * Then locate the root inode by scanning the directory keyspace
 996                  * represented by the label.
 997                  */
 998                 parent = hammer2_chain_lookup_init(&hmp->vchain, 0);
 999                 schain = hammer2_chain_lookup(&parent, &key_dummy,
1000                                       HAMMER2_SROOT_KEY, HAMMER2_SROOT_KEY,
1001                                       &cache_index, 0);
1002                 hammer2_chain_lookup_done(parent);
1003                 if (schain == NULL) {
1004                         kprintf("hammer2_mount: invalid super-root\n");
1005                         hammer2_unmount_helper(mp, NULL, hmp);
1006                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1007                         hammer2_vfs_unmount(mp, MNT_FORCE);
1008                         return EINVAL;
1009                 }
1010                 if (schain->error) {
1011                         kprintf("hammer2_mount: error %s reading super-root\n",
1012                                 hammer2_error_str(schain->error));
1013                         hammer2_chain_unlock(schain);
1014                         hammer2_chain_drop(schain);
1015                         schain = NULL;
1016                         hammer2_unmount_helper(mp, NULL, hmp);
1017                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1018                         hammer2_vfs_unmount(mp, MNT_FORCE);
1019                         return EINVAL;
1020                 }
1021
1022                 /*
1023                  * The super-root always uses an inode_tid of 1 when
1024                  * creating PFSs.
1025                  */
1026                 spmp->inode_tid = 1;
1027                 spmp->modify_tid = schain->bref.modify_tid + 1;
1028
1029                 /*
1030                  * Sanity-check schain's pmp and finish initialization.
1031                  * Any chain belonging to the super-root topology should
1032                  * have a NULL pmp (not even set to spmp).
1033                  */
1034                 ripdata = &hammer2_chain_rdata(schain)->ipdata;
1035                 KKASSERT(schain->pmp == NULL);
1036                 spmp->pfs_clid = ripdata->meta.pfs_clid;
1037
1038                 /*
1039                  * Replace the dummy spmp->iroot with a real one.  It's
1040                  * easier to just do a wholesale replacement than to try
1041                  * to update the chain and fixup the iroot fields.
1042                  *
1043                  * The returned inode is locked with the supplied cluster.
1044                  */
1045                 cluster = hammer2_cluster_from_chain(schain);
1046                 hammer2_inode_drop(spmp->iroot);
1047                 spmp->iroot = NULL;
1048                 spmp->iroot = hammer2_inode_get(spmp, NULL, cluster, -1);
1049                 spmp->spmp_hmp = hmp;
1050                 spmp->pfs_types[0] = ripdata->meta.pfs_type;
1051                 spmp->pfs_hmps[0] = hmp;
1052                 hammer2_inode_ref(spmp->iroot);
1053                 hammer2_inode_unlock(spmp->iroot);
1054                 hammer2_cluster_unlock(cluster);
1055                 hammer2_cluster_drop(cluster);
1056                 schain = NULL;
1057                 /* leave spmp->iroot with one ref */
1058
1059                 if ((mp->mnt_flag & MNT_RDONLY) == 0) {
1060                         error = hammer2_recovery(hmp);
1061                         /* XXX do something with error */
1062                 }
1063                 hammer2_update_pmps(hmp);
1064                 hammer2_iocom_init(hmp);
1065
1066                 /*
1067                  * Ref the cluster management messaging descriptor.  The mount
1068                  * program deals with the other end of the communications pipe.
1069                  *
1070                  * Root mounts typically do not supply one.
1071                  */
1072                 if (info.cluster_fd >= 0) {
1073                         fp = holdfp(curproc->p_fd, info.cluster_fd, -1);
1074                         if (fp) {
1075                                 hammer2_cluster_reconnect(hmp, fp);
1076                         } else {
1077                                 kprintf("hammer2_mount: bad cluster_fd!\n");
1078                         }
1079                 }
1080         } else {
1081                 spmp = hmp->spmp;
1082         }
1083
1084         /*
1085          * Lookup the mount point under the media-localized super-root.
1086          * Scanning hammer2_pfslist doesn't help us because it represents
1087          * PFS cluster ids which can aggregate several named PFSs together.
1088          *
1089          * cluster->pmp will incorrectly point to spmp and must be fixed
1090          * up later on.
1091          */
1092         hammer2_inode_lock(spmp->iroot, 0);
1093         parent = hammer2_inode_chain(spmp->iroot, 0, HAMMER2_RESOLVE_ALWAYS);
1094         lhc = hammer2_dirhash(label, strlen(label));
1095         chain = hammer2_chain_lookup(&parent, &key_next,
1096                                      lhc, lhc + HAMMER2_DIRHASH_LOMASK,
1097                                      &cache_index, 0);
1098         while (chain) {
1099                 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
1100                     strcmp(label, chain->data->ipdata.filename) == 0) {
1101                         break;
1102                 }
1103                 chain = hammer2_chain_next(&parent, chain, &key_next,
1104                                             key_next,
1105                                             lhc + HAMMER2_DIRHASH_LOMASK,
1106                                             &cache_index, 0);
1107         }
1108         if (parent) {
1109                 hammer2_chain_unlock(parent);
1110                 hammer2_chain_drop(parent);
1111         }
1112         hammer2_inode_unlock(spmp->iroot);
1113
1114         /*
1115          * PFS could not be found?
1116          */
1117         if (chain == NULL) {
1118                 kprintf("hammer2_mount: PFS label not found\n");
1119                 hammer2_unmount_helper(mp, NULL, hmp);
1120                 lockmgr(&hammer2_mntlk, LK_RELEASE);
1121                 hammer2_vfs_unmount(mp, MNT_FORCE);
1122
1123                 return EINVAL;
1124         }
1125
1126         /*
1127          * Acquire the pmp structure (it should have already been allocated
1128          * via hammer2_update_pmps() so do not pass cluster in to add to
1129          * available chains).
1130          *
1131          * Check if the cluster has already been mounted.  A cluster can
1132          * only be mounted once, use null mounts to mount additional copies.
1133          */
1134         ripdata = &chain->data->ipdata;
1135         bref = chain->bref;
1136         pmp = hammer2_pfsalloc(NULL, ripdata, bref.modify_tid);
1137         hammer2_chain_unlock(chain);
1138         hammer2_chain_drop(chain);
1139
1140         if (pmp->mp) {
1141                 kprintf("hammer2_mount: PFS already mounted!\n");
1142                 hammer2_unmount_helper(mp, NULL, hmp);
1143                 lockmgr(&hammer2_mntlk, LK_RELEASE);
1144                 hammer2_vfs_unmount(mp, MNT_FORCE);
1145
1146                 return EBUSY;
1147         }
1148
1149         /*
1150          * Finish the mount
1151          */
1152         kprintf("hammer2_mount hmp=%p pmp=%p\n", hmp, pmp);
1153
1154         mp->mnt_flag = MNT_LOCAL;
1155         mp->mnt_kern_flag |= MNTK_ALL_MPSAFE;   /* all entry pts are SMP */
1156         mp->mnt_kern_flag |= MNTK_THR_SYNC;     /* new vsyncscan semantics */
1157
1158         /*
1159          * required mount structure initializations
1160          */
1161         mp->mnt_stat.f_iosize = HAMMER2_PBUFSIZE;
1162         mp->mnt_stat.f_bsize = HAMMER2_PBUFSIZE;
1163
1164         mp->mnt_vstat.f_frsize = HAMMER2_PBUFSIZE;
1165         mp->mnt_vstat.f_bsize = HAMMER2_PBUFSIZE;
1166
1167         /*
1168          * Optional fields
1169          */
1170         mp->mnt_iosize_max = MAXPHYS;
1171
1172         /*
1173          * Connect up mount pointers.
1174          */
1175         hammer2_mount_helper(mp, pmp);
1176
1177         lockmgr(&hammer2_mntlk, LK_RELEASE);
1178
1179         /*
1180          * Finish setup
1181          */
1182         vfs_getnewfsid(mp);
1183         vfs_add_vnodeops(mp, &hammer2_vnode_vops, &mp->mnt_vn_norm_ops);
1184         vfs_add_vnodeops(mp, &hammer2_spec_vops, &mp->mnt_vn_spec_ops);
1185         vfs_add_vnodeops(mp, &hammer2_fifo_vops, &mp->mnt_vn_fifo_ops);
1186
1187         if (path) {
1188                 copyinstr(info.volume, mp->mnt_stat.f_mntfromname,
1189                           MNAMELEN - 1, &size);
1190                 bzero(mp->mnt_stat.f_mntfromname + size, MNAMELEN - size);
1191         } /* else root mount, already in there */
1192
1193         bzero(mp->mnt_stat.f_mntonname, sizeof(mp->mnt_stat.f_mntonname));
1194         if (path) {
1195                 copyinstr(path, mp->mnt_stat.f_mntonname,
1196                           sizeof(mp->mnt_stat.f_mntonname) - 1,
1197                           &size);
1198         } else {
1199                 /* root mount */
1200                 mp->mnt_stat.f_mntonname[0] = '/';
1201         }
1202
1203         /*
1204          * Initial statfs to prime mnt_stat.
1205          */
1206         hammer2_vfs_statfs(mp, &mp->mnt_stat, cred);
1207
1208         return 0;
1209 }
1210
1211 /*
1212  * Scan PFSs under the super-root and create hammer2_pfs structures.
1213  */
1214 static
1215 void
1216 hammer2_update_pmps(hammer2_dev_t *hmp)
1217 {
1218         const hammer2_inode_data_t *ripdata;
1219         hammer2_chain_t *parent;
1220         hammer2_chain_t *chain;
1221         hammer2_blockref_t bref;
1222         hammer2_pfs_t *spmp;
1223         hammer2_pfs_t *pmp;
1224         hammer2_key_t key_next;
1225         int cache_index = -1;
1226
1227         /*
1228          * Lookup mount point under the media-localized super-root.
1229          *
1230          * cluster->pmp will incorrectly point to spmp and must be fixed
1231          * up later on.
1232          */
1233         spmp = hmp->spmp;
1234         hammer2_inode_lock(spmp->iroot, 0);
1235         parent = hammer2_inode_chain(spmp->iroot, 0, HAMMER2_RESOLVE_ALWAYS);
1236         chain = hammer2_chain_lookup(&parent, &key_next,
1237                                          HAMMER2_KEY_MIN, HAMMER2_KEY_MAX,
1238                                          &cache_index, 0);
1239         while (chain) {
1240                 if (chain->bref.type != HAMMER2_BREF_TYPE_INODE)
1241                         continue;
1242                 ripdata = &chain->data->ipdata;
1243                 bref = chain->bref;
1244                 kprintf("ADD LOCAL PFS: %s\n", ripdata->filename);
1245
1246                 pmp = hammer2_pfsalloc(chain, ripdata, bref.modify_tid);
1247                 chain = hammer2_chain_next(&parent, chain, &key_next,
1248                                            key_next, HAMMER2_KEY_MAX,
1249                                            &cache_index, 0);
1250         }
1251         if (parent) {
1252                 hammer2_chain_unlock(parent);
1253                 hammer2_chain_drop(parent);
1254         }
1255         hammer2_inode_unlock(spmp->iroot);
1256 }
1257
1258 static
1259 int
1260 hammer2_remount(hammer2_dev_t *hmp, struct mount *mp, char *path __unused,
1261                 struct vnode *devvp, struct ucred *cred)
1262 {
1263         int error;
1264
1265         if (hmp->ronly && (mp->mnt_kern_flag & MNTK_WANTRDWR)) {
1266                 error = hammer2_recovery(hmp);
1267         } else {
1268                 error = 0;
1269         }
1270         return error;
1271 }
1272
1273 static
1274 int
1275 hammer2_vfs_unmount(struct mount *mp, int mntflags)
1276 {
1277         hammer2_pfs_t *pmp;
1278         int flags;
1279         int error = 0;
1280
1281         pmp = MPTOPMP(mp);
1282
1283         if (pmp == NULL)
1284                 return(0);
1285
1286         lockmgr(&hammer2_mntlk, LK_EXCLUSIVE);
1287
1288         /*
1289          * If mount initialization proceeded far enough we must flush
1290          * its vnodes and sync the underlying mount points.  Three syncs
1291          * are required to fully flush the filesystem (freemap updates lag
1292          * by one flush, and one extra for safety).
1293          */
1294         if (mntflags & MNT_FORCE)
1295                 flags = FORCECLOSE;
1296         else
1297                 flags = 0;
1298         if (pmp->iroot) {
1299                 error = vflush(mp, 0, flags);
1300                 if (error)
1301                         goto failed;
1302                 hammer2_vfs_sync(mp, MNT_WAIT);
1303                 hammer2_vfs_sync(mp, MNT_WAIT);
1304                 hammer2_vfs_sync(mp, MNT_WAIT);
1305         }
1306
1307         /*
1308          * Cleanup the frontend support XOPS threads
1309          */
1310         hammer2_xop_helper_cleanup(pmp);
1311
1312         /*
1313          * Cleanup our reference on ihidden.
1314          */
1315         if (pmp->ihidden) {
1316                 hammer2_inode_drop(pmp->ihidden);
1317                 pmp->ihidden = NULL;
1318         }
1319         if (pmp->mp)
1320                 hammer2_unmount_helper(mp, pmp, NULL);
1321
1322         error = 0;
1323 failed:
1324         lockmgr(&hammer2_mntlk, LK_RELEASE);
1325
1326         return (error);
1327 }
1328
1329 /*
1330  * Mount helper, hook the system mount into our PFS.
1331  * The mount lock is held.
1332  *
1333  * We must bump the mount_count on related devices for any
1334  * mounted PFSs.
1335  */
1336 static
1337 void
1338 hammer2_mount_helper(struct mount *mp, hammer2_pfs_t *pmp)
1339 {
1340         hammer2_cluster_t *cluster;
1341         hammer2_chain_t *rchain;
1342         int i;
1343
1344         mp->mnt_data = (qaddr_t)pmp;
1345         pmp->mp = mp;
1346
1347         /*
1348          * After pmp->mp is set we have to adjust hmp->mount_count.
1349          */
1350         cluster = &pmp->iroot->cluster;
1351         for (i = 0; i < cluster->nchains; ++i) {
1352                 rchain = cluster->array[i].chain;
1353                 if (rchain == NULL)
1354                         continue;
1355                 ++rchain->hmp->mount_count;
1356                 kprintf("hammer2_mount hmp=%p ++mount_count=%d\n",
1357                         rchain->hmp, rchain->hmp->mount_count);
1358         }
1359
1360         /*
1361          * Create missing Xop threads
1362          */
1363         hammer2_xop_helper_create(pmp);
1364 }
1365
1366 /*
1367  * Mount helper, unhook the system mount from our PFS.
1368  * The mount lock is held.
1369  *
1370  * If hmp is supplied a mount responsible for being the first to open
1371  * the block device failed and the block device and all PFSs using the
1372  * block device must be cleaned up.
1373  *
1374  * If pmp is supplied multiple devices might be backing the PFS and each
1375  * must be disconnect.  This might not be the last PFS using some of the
1376  * underlying devices.  Also, we have to adjust our hmp->mount_count
1377  * accounting for the devices backing the pmp which is now undergoing an
1378  * unmount.
1379  */
1380 static
1381 void
1382 hammer2_unmount_helper(struct mount *mp, hammer2_pfs_t *pmp, hammer2_dev_t *hmp)
1383 {
1384         hammer2_cluster_t *cluster;
1385         hammer2_chain_t *rchain;
1386         struct vnode *devvp;
1387         int dumpcnt;
1388         int ronly = 0;
1389         int i;
1390
1391         /*
1392          * If no device supplied this is a high-level unmount and we have to
1393          * to disconnect the mount, adjust mount_count, and locate devices
1394          * that might now have no mounts.
1395          */
1396         if (pmp) {
1397                 KKASSERT(hmp == NULL);
1398                 KKASSERT((void *)(intptr_t)mp->mnt_data == pmp);
1399                 pmp->mp = NULL;
1400                 mp->mnt_data = NULL;
1401
1402                 /*
1403                  * After pmp->mp is cleared we have to account for
1404                  * mount_count.
1405                  */
1406                 cluster = &pmp->iroot->cluster;
1407                 for (i = 0; i < cluster->nchains; ++i) {
1408                         rchain = cluster->array[i].chain;
1409                         if (rchain == NULL)
1410                                 continue;
1411                         --rchain->hmp->mount_count;
1412                         kprintf("hammer2_unmount hmp=%p --mount_count=%d\n",
1413                                 rchain->hmp, rchain->hmp->mount_count);
1414                         /* scrapping hmp now may invalidate the pmp */
1415                 }
1416 again:
1417                 TAILQ_FOREACH(hmp, &hammer2_mntlist, mntentry) {
1418                         if (hmp->mount_count == 0) {
1419                                 hammer2_unmount_helper(NULL, NULL, hmp);
1420                                 goto again;
1421                         }
1422                 }
1423                 return;
1424         }
1425
1426         /*
1427          * Try to terminate the block device.  We can't terminate it if
1428          * there are still PFSs referencing it.
1429          */
1430         kprintf("hammer2_unmount hmp=%p mount_count=%d\n",
1431                 hmp, hmp->mount_count);
1432         if (hmp->mount_count)
1433                 return;
1434
1435         hammer2_pfsfree_scan(hmp);
1436         hammer2_dev_exlock(hmp);        /* XXX order */
1437
1438         /*
1439          * Cycle the volume data lock as a safety (probably not needed any
1440          * more).  To ensure everything is out we need to flush at least
1441          * three times.  (1) The running of the sideq can dirty the
1442          * filesystem, (2) A normal flush can dirty the freemap, and
1443          * (3) ensure that the freemap is fully synchronized.
1444          *
1445          * The next mount's recovery scan can clean everything up but we want
1446          * to leave the filesystem in a 100% clean state on a normal unmount.
1447          */
1448 #if 0
1449         hammer2_voldata_lock(hmp);
1450         hammer2_voldata_unlock(hmp);
1451 #endif
1452         hammer2_iocom_uninit(hmp);
1453
1454         if ((hmp->vchain.flags | hmp->fchain.flags) &
1455             HAMMER2_CHAIN_FLUSH_MASK) {
1456                 kprintf("hammer2_unmount: chains left over "
1457                         "after final sync\n");
1458                 kprintf("    vchain %08x\n", hmp->vchain.flags);
1459                 kprintf("    fchain %08x\n", hmp->fchain.flags);
1460
1461                 if (hammer2_debug & 0x0010)
1462                         Debugger("entered debugger");
1463         }
1464
1465         KKASSERT(hmp->spmp == NULL);
1466
1467         /*
1468          * Finish up with the device vnode
1469          */
1470         if ((devvp = hmp->devvp) != NULL) {
1471                 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1472                 vinvalbuf(devvp, (ronly ? 0 : V_SAVE), 0, 0);
1473                 hmp->devvp = NULL;
1474                 VOP_CLOSE(devvp, (ronly ? FREAD : FREAD|FWRITE), NULL);
1475                 vn_unlock(devvp);
1476                 vrele(devvp);
1477                 devvp = NULL;
1478         }
1479
1480         /*
1481          * Clear vchain/fchain flags that might prevent final cleanup
1482          * of these chains.
1483          */
1484         if (hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED) {
1485                 atomic_add_long(&hammer2_count_modified_chains, -1);
1486                 atomic_clear_int(&hmp->vchain.flags, HAMMER2_CHAIN_MODIFIED);
1487                 hammer2_pfs_memory_wakeup(hmp->vchain.pmp);
1488         }
1489         if (hmp->vchain.flags & HAMMER2_CHAIN_UPDATE) {
1490                 atomic_clear_int(&hmp->vchain.flags, HAMMER2_CHAIN_UPDATE);
1491         }
1492
1493         if (hmp->fchain.flags & HAMMER2_CHAIN_MODIFIED) {
1494                 atomic_add_long(&hammer2_count_modified_chains, -1);
1495                 atomic_clear_int(&hmp->fchain.flags, HAMMER2_CHAIN_MODIFIED);
1496                 hammer2_pfs_memory_wakeup(hmp->fchain.pmp);
1497         }
1498         if (hmp->fchain.flags & HAMMER2_CHAIN_UPDATE) {
1499                 atomic_clear_int(&hmp->fchain.flags, HAMMER2_CHAIN_UPDATE);
1500         }
1501
1502         /*
1503          * Final drop of embedded freemap root chain to
1504          * clean up fchain.core (fchain structure is not
1505          * flagged ALLOCATED so it is cleaned out and then
1506          * left to rot).
1507          */
1508         hammer2_chain_drop(&hmp->fchain);
1509
1510         /*
1511          * Final drop of embedded volume root chain to clean
1512          * up vchain.core (vchain structure is not flagged
1513          * ALLOCATED so it is cleaned out and then left to
1514          * rot).
1515          */
1516         dumpcnt = 50;
1517         hammer2_dump_chain(&hmp->vchain, 0, &dumpcnt, 'v');
1518         dumpcnt = 50;
1519         hammer2_dump_chain(&hmp->fchain, 0, &dumpcnt, 'f');
1520         hammer2_dev_unlock(hmp);
1521         hammer2_chain_drop(&hmp->vchain);
1522
1523         hammer2_io_cleanup(hmp, &hmp->iotree);
1524         if (hmp->iofree_count) {
1525                 kprintf("io_cleanup: %d I/O's left hanging\n",
1526                         hmp->iofree_count);
1527         }
1528
1529         TAILQ_REMOVE(&hammer2_mntlist, hmp, mntentry);
1530         kmalloc_destroy(&hmp->mchain);
1531         kfree(hmp, M_HAMMER2);
1532 }
1533
1534 static
1535 int
1536 hammer2_vfs_vget(struct mount *mp, struct vnode *dvp,
1537              ino_t ino, struct vnode **vpp)
1538 {
1539         kprintf("hammer2_vget\n");
1540         return (EOPNOTSUPP);
1541 }
1542
1543 static
1544 int
1545 hammer2_vfs_root(struct mount *mp, struct vnode **vpp)
1546 {
1547         hammer2_pfs_t *pmp;
1548         int error;
1549         struct vnode *vp;
1550
1551         pmp = MPTOPMP(mp);
1552         if (pmp->iroot == NULL) {
1553                 *vpp = NULL;
1554                 return EINVAL;
1555         }
1556
1557         error = 0;
1558         hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED);
1559
1560         while (pmp->inode_tid == 0) {
1561                 hammer2_xop_ipcluster_t *xop;
1562                 hammer2_inode_meta_t *meta;
1563
1564                 xop = hammer2_xop_alloc(pmp->iroot, HAMMER2_XOP_MODIFYING);
1565                 hammer2_xop_start(&xop->head, hammer2_xop_ipcluster);
1566                 error = hammer2_xop_collect(&xop->head, 0);
1567
1568                 if (error == 0) {
1569                         meta = &xop->head.cluster.focus->data->ipdata.meta;
1570                         pmp->iroot->meta = *meta;
1571                         pmp->inode_tid = meta->pfs_inum + 1;
1572                         if (pmp->inode_tid < HAMMER2_INODE_START)
1573                                 pmp->inode_tid = HAMMER2_INODE_START;
1574                         pmp->modify_tid =
1575                                 xop->head.cluster.focus->bref.modify_tid + 1;
1576                         kprintf("PFS: Starting inode %jd\n",
1577                                 (intmax_t)pmp->inode_tid);
1578                         kprintf("PMP focus good set nextino=%ld mod=%016jx\n",
1579                                 pmp->inode_tid, pmp->modify_tid);
1580                         wakeup(&pmp->iroot);
1581
1582                         hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
1583
1584                         /*
1585                          * Prime the mount info.
1586                          */
1587                         hammer2_vfs_statfs(mp, &mp->mnt_stat, NULL);
1588
1589                         /*
1590                          * With the cluster operational, check for and
1591                          * install ihidden if needed.  The install_hidden
1592                          * code needs to get a transaction so we must unlock
1593                          * iroot around it.
1594                          *
1595                          * This is only applicable PFS mounts, there is no
1596                          * hidden directory in the spmp.
1597                          */
1598                         hammer2_inode_unlock(pmp->iroot);
1599                         hammer2_inode_install_hidden(pmp);
1600                         hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED);
1601
1602                         break;
1603                 }
1604
1605                 /*
1606                  * Loop, try again
1607                  */
1608                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
1609                 hammer2_inode_unlock(pmp->iroot);
1610                 error = tsleep(&pmp->iroot, PCATCH, "h2root", hz);
1611                 hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED);
1612                 if (error == EINTR)
1613                         break;
1614         }
1615
1616         if (error) {
1617                 hammer2_inode_unlock(pmp->iroot);
1618                 *vpp = NULL;
1619         } else {
1620                 vp = hammer2_igetv(pmp->iroot, &error);
1621                 hammer2_inode_unlock(pmp->iroot);
1622                 *vpp = vp;
1623         }
1624
1625         return (error);
1626 }
1627
1628 /*
1629  * Filesystem status
1630  *
1631  * XXX incorporate ipdata->meta.inode_quota and data_quota
1632  */
1633 static
1634 int
1635 hammer2_vfs_statfs(struct mount *mp, struct statfs *sbp, struct ucred *cred)
1636 {
1637         hammer2_pfs_t *pmp;
1638         hammer2_dev_t *hmp;
1639         hammer2_blockref_t bref;
1640         int i;
1641
1642         /*
1643          * NOTE: iroot might not have validated the cluster yet.
1644          */
1645         pmp = MPTOPMP(mp);
1646
1647         mp->mnt_stat.f_files = 0;
1648         mp->mnt_stat.f_ffree = 0;
1649         mp->mnt_stat.f_blocks = 0;
1650         mp->mnt_stat.f_bfree = 0;
1651         mp->mnt_stat.f_bavail = 0;
1652
1653         for (i = 0; i < pmp->iroot->cluster.nchains; ++i) {
1654                 hmp = pmp->pfs_hmps[i];
1655                 if (hmp == NULL)
1656                         continue;
1657                 if (pmp->iroot->cluster.array[i].chain)
1658                         bref = pmp->iroot->cluster.array[i].chain->bref;
1659                 else
1660                         bzero(&bref, sizeof(bref));
1661
1662                 mp->mnt_stat.f_files = bref.inode_count;
1663                 mp->mnt_stat.f_ffree = 0;
1664                 mp->mnt_stat.f_blocks = (bref.data_count +
1665                                          hmp->voldata.allocator_free) /
1666                                         mp->mnt_vstat.f_bsize;
1667                 mp->mnt_stat.f_bfree =  hmp->voldata.allocator_free /
1668                                         mp->mnt_vstat.f_bsize;
1669                 mp->mnt_stat.f_bavail = mp->mnt_stat.f_bfree;
1670
1671                 *sbp = mp->mnt_stat;
1672         }
1673         return (0);
1674 }
1675
1676 static
1677 int
1678 hammer2_vfs_statvfs(struct mount *mp, struct statvfs *sbp, struct ucred *cred)
1679 {
1680         hammer2_pfs_t *pmp;
1681         hammer2_dev_t *hmp;
1682         hammer2_blockref_t bref;
1683         int i;
1684
1685         /*
1686          * NOTE: iroot might not have validated the cluster yet.
1687          */
1688         pmp = MPTOPMP(mp);
1689
1690         mp->mnt_vstat.f_bsize = 0;
1691         mp->mnt_vstat.f_files = 0;
1692         mp->mnt_vstat.f_ffree = 0;
1693         mp->mnt_vstat.f_blocks = 0;
1694         mp->mnt_vstat.f_bfree = 0;
1695         mp->mnt_vstat.f_bavail = 0;
1696
1697         for (i = 0; i < pmp->iroot->cluster.nchains; ++i) {
1698                 hmp = pmp->pfs_hmps[i];
1699                 if (hmp == NULL)
1700                         continue;
1701                 if (pmp->iroot->cluster.array[i].chain)
1702                         bref = pmp->iroot->cluster.array[i].chain->bref;
1703                 else
1704                         bzero(&bref, sizeof(bref));
1705
1706                 mp->mnt_vstat.f_bsize = HAMMER2_PBUFSIZE;
1707                 mp->mnt_vstat.f_files = bref.inode_count;
1708                 mp->mnt_vstat.f_ffree = 0;
1709                 mp->mnt_vstat.f_blocks = (bref.data_count +
1710                                          hmp->voldata.allocator_free) /
1711                                         mp->mnt_vstat.f_bsize;
1712                 mp->mnt_vstat.f_bfree = hmp->voldata.allocator_free /
1713                                         mp->mnt_vstat.f_bsize;
1714                 mp->mnt_vstat.f_bavail = mp->mnt_vstat.f_bfree;
1715
1716                 *sbp = mp->mnt_vstat;
1717         }
1718         return (0);
1719 }
1720
1721 /*
1722  * Mount-time recovery (RW mounts)
1723  *
1724  * Updates to the free block table are allowed to lag flushes by one
1725  * transaction.  In case of a crash, then on a fresh mount we must do an
1726  * incremental scan of the last committed transaction id and make sure that
1727  * all related blocks have been marked allocated.
1728  *
1729  * The super-root topology and each PFS has its own transaction id domain,
1730  * so we must track PFS boundary transitions.
1731  */
1732 struct hammer2_recovery_elm {
1733         TAILQ_ENTRY(hammer2_recovery_elm) entry;
1734         hammer2_chain_t *chain;
1735         hammer2_tid_t sync_tid;
1736 };
1737
1738 TAILQ_HEAD(hammer2_recovery_list, hammer2_recovery_elm);
1739
1740 struct hammer2_recovery_info {
1741         struct hammer2_recovery_list list;
1742         hammer2_tid_t   mtid;
1743         int     depth;
1744 };
1745
1746 static int hammer2_recovery_scan(hammer2_dev_t *hmp,
1747                         hammer2_chain_t *parent,
1748                         struct hammer2_recovery_info *info,
1749                         hammer2_tid_t sync_tid);
1750
1751 #define HAMMER2_RECOVERY_MAXDEPTH       10
1752
1753 static
1754 int
1755 hammer2_recovery(hammer2_dev_t *hmp)
1756 {
1757         struct hammer2_recovery_info info;
1758         struct hammer2_recovery_elm *elm;
1759         hammer2_chain_t *parent;
1760         hammer2_tid_t sync_tid;
1761         hammer2_tid_t mirror_tid;
1762         int error;
1763         int cumulative_error = 0;
1764
1765         hammer2_trans_init(hmp->spmp, 0);
1766
1767         sync_tid = hmp->voldata.freemap_tid;
1768         mirror_tid = hmp->voldata.mirror_tid;
1769
1770         kprintf("hammer2 mount \"%s\": ", hmp->devrepname);
1771         if (sync_tid >= mirror_tid) {
1772                 kprintf(" no recovery needed\n");
1773         } else {
1774                 kprintf(" freemap recovery %016jx-%016jx\n",
1775                         sync_tid + 1, mirror_tid);
1776         }
1777
1778         TAILQ_INIT(&info.list);
1779         info.depth = 0;
1780         parent = hammer2_chain_lookup_init(&hmp->vchain, 0);
1781         cumulative_error = hammer2_recovery_scan(hmp, parent, &info, sync_tid);
1782         hammer2_chain_lookup_done(parent);
1783
1784         while ((elm = TAILQ_FIRST(&info.list)) != NULL) {
1785                 TAILQ_REMOVE(&info.list, elm, entry);
1786                 parent = elm->chain;
1787                 sync_tid = elm->sync_tid;
1788                 kfree(elm, M_HAMMER2);
1789
1790                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
1791                 error = hammer2_recovery_scan(hmp, parent, &info,
1792                                               hmp->voldata.freemap_tid);
1793                 hammer2_chain_unlock(parent);
1794                 hammer2_chain_drop(parent);     /* drop elm->chain ref */
1795                 if (error)
1796                         cumulative_error = error;
1797         }
1798         hammer2_trans_done(hmp->spmp);
1799
1800         return cumulative_error;
1801 }
1802
1803 static
1804 int
1805 hammer2_recovery_scan(hammer2_dev_t *hmp, hammer2_chain_t *parent,
1806                       struct hammer2_recovery_info *info,
1807                       hammer2_tid_t sync_tid)
1808 {
1809         const hammer2_inode_data_t *ripdata;
1810         hammer2_chain_t *chain;
1811         hammer2_blockref_t bref;
1812         int cache_index;
1813         int cumulative_error = 0;
1814         int error;
1815         int first;
1816
1817         /*
1818          * Adjust freemap to ensure that the block(s) are marked allocated.
1819          */
1820         if (parent->bref.type != HAMMER2_BREF_TYPE_VOLUME) {
1821                 hammer2_freemap_adjust(hmp, &parent->bref,
1822                                        HAMMER2_FREEMAP_DORECOVER);
1823         }
1824
1825         /*
1826          * Check type for recursive scan
1827          */
1828         switch(parent->bref.type) {
1829         case HAMMER2_BREF_TYPE_VOLUME:
1830                 /* data already instantiated */
1831                 break;
1832         case HAMMER2_BREF_TYPE_INODE:
1833                 /*
1834                  * Must instantiate data for DIRECTDATA test and also
1835                  * for recursion.
1836                  */
1837                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
1838                 ripdata = &hammer2_chain_rdata(parent)->ipdata;
1839                 if (ripdata->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA) {
1840                         /* not applicable to recovery scan */
1841                         hammer2_chain_unlock(parent);
1842                         return 0;
1843                 }
1844                 hammer2_chain_unlock(parent);
1845                 break;
1846         case HAMMER2_BREF_TYPE_INDIRECT:
1847                 /*
1848                  * Must instantiate data for recursion
1849                  */
1850                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
1851                 hammer2_chain_unlock(parent);
1852                 break;
1853         case HAMMER2_BREF_TYPE_DATA:
1854         case HAMMER2_BREF_TYPE_FREEMAP:
1855         case HAMMER2_BREF_TYPE_FREEMAP_NODE:
1856         case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
1857                 /* not applicable to recovery scan */
1858                 return 0;
1859                 break;
1860         default:
1861                 return EDOM;
1862         }
1863
1864         /*
1865          * Defer operation if depth limit reached or if we are crossing a
1866          * PFS boundary.
1867          */
1868         if (info->depth >= HAMMER2_RECOVERY_MAXDEPTH) {
1869                 struct hammer2_recovery_elm *elm;
1870
1871                 elm = kmalloc(sizeof(*elm), M_HAMMER2, M_ZERO | M_WAITOK);
1872                 elm->chain = parent;
1873                 elm->sync_tid = sync_tid;
1874                 hammer2_chain_ref(parent);
1875                 TAILQ_INSERT_TAIL(&info->list, elm, entry);
1876                 /* unlocked by caller */
1877
1878                 return(0);
1879         }
1880
1881
1882         /*
1883          * Recursive scan of the last flushed transaction only.  We are
1884          * doing this without pmp assignments so don't leave the chains
1885          * hanging around after we are done with them.
1886          */
1887         cache_index = 0;
1888         chain = NULL;
1889         first = 1;
1890
1891         while (hammer2_chain_scan(parent, &chain, &bref,
1892                                   &first, &cache_index,
1893                                   HAMMER2_LOOKUP_NODATA) != NULL) {
1894                 /*
1895                  * If this is a leaf
1896                  */
1897                 if (chain == NULL) {
1898                         if (bref.mirror_tid > sync_tid) {
1899                                 hammer2_freemap_adjust(hmp, &bref,
1900                                                      HAMMER2_FREEMAP_DORECOVER);
1901                         }
1902                         continue;
1903                 }
1904
1905                 /*
1906                  * This may or may not be a recursive node.
1907                  */
1908                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
1909                 if (bref.mirror_tid > sync_tid) {
1910                         ++info->depth;
1911                         error = hammer2_recovery_scan(hmp, chain,
1912                                                       info, sync_tid);
1913                         --info->depth;
1914                         if (error)
1915                                 cumulative_error = error;
1916                 }
1917
1918                 /*
1919                  * Flush the recovery at the PFS boundary to stage it for
1920                  * the final flush of the super-root topology.
1921                  */
1922                 if ((bref.flags & HAMMER2_BREF_FLAG_PFSROOT) &&
1923                     (chain->flags & HAMMER2_CHAIN_ONFLUSH)) {
1924                         hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1925                 }
1926         }
1927
1928         return cumulative_error;
1929 }
1930
1931 /*
1932  * Sync a mount point; this is called on a per-mount basis from the
1933  * filesystem syncer process periodically and whenever a user issues
1934  * a sync.
1935  */
1936 int
1937 hammer2_vfs_sync(struct mount *mp, int waitfor)
1938 {
1939         hammer2_xop_flush_t *xop;
1940         struct hammer2_sync_info info;
1941         hammer2_inode_t *iroot;
1942         hammer2_pfs_t *pmp;
1943         int flags;
1944         int error;
1945
1946         pmp = MPTOPMP(mp);
1947         iroot = pmp->iroot;
1948         KKASSERT(iroot);
1949         KKASSERT(iroot->pmp == pmp);
1950
1951         /*
1952          * We can't acquire locks on existing vnodes while in a transaction
1953          * without risking a deadlock.  This assumes that vfsync() can be
1954          * called without the vnode locked (which it can in DragonFly).
1955          * Otherwise we'd have to implement a multi-pass or flag the lock
1956          * failures and retry.
1957          *
1958          * The reclamation code interlocks with the sync list's token
1959          * (by removing the vnode from the scan list) before unlocking
1960          * the inode, giving us time to ref the inode.
1961          */
1962         /*flags = VMSC_GETVP;*/
1963         flags = 0;
1964         if (waitfor & MNT_LAZY)
1965                 flags |= VMSC_ONEPASS;
1966
1967 #if 0
1968         /*
1969          * Preflush the vnodes using a normal transaction before interlocking
1970          * with a flush transaction.
1971          */
1972         hammer2_trans_init(pmp, 0);
1973         info.error = 0;
1974         info.waitfor = MNT_NOWAIT;
1975         vsyncscan(mp, flags | VMSC_NOWAIT, hammer2_sync_scan2, &info);
1976         hammer2_trans_done(pmp);
1977 #endif
1978
1979         /*
1980          * Start our flush transaction.  This does not return until all
1981          * concurrent transactions have completed and will prevent any
1982          * new transactions from running concurrently, except for the
1983          * buffer cache transactions.
1984          *
1985          * For efficiency do an async pass before making sure with a
1986          * synchronous pass on all related buffer cache buffers.  It
1987          * should theoretically not be possible for any new file buffers
1988          * to be instantiated during this sequence.
1989          */
1990         hammer2_trans_init(pmp, HAMMER2_TRANS_ISFLUSH |
1991                                 HAMMER2_TRANS_PREFLUSH);
1992         hammer2_inode_run_sideq(pmp);
1993
1994         info.error = 0;
1995         info.waitfor = MNT_NOWAIT;
1996         vsyncscan(mp, flags | VMSC_NOWAIT, hammer2_sync_scan2, &info);
1997         info.waitfor = MNT_WAIT;
1998         vsyncscan(mp, flags, hammer2_sync_scan2, &info);
1999
2000         /*
2001          * Clear PREFLUSH.  This prevents (or asserts on) any new logical
2002          * buffer cache flushes which occur during the flush.  Device buffers
2003          * are not affected.
2004          */
2005         hammer2_bioq_sync(pmp);
2006         hammer2_trans_clear_preflush(pmp);
2007
2008         /*
2009          * Use the XOP interface to concurrently flush all nodes to
2010          * synchronize the PFSROOT subtopology to the media.  A standard
2011          * end-of-scan ENOENT error indicates cluster sufficiency.
2012          *
2013          * Note that this flush will not be visible on crash recovery until
2014          * we flush the super-root topology in the next loop.
2015          *
2016          * XXX For now wait for all flushes to complete.
2017          */
2018         if (iroot) {
2019                 xop = hammer2_xop_alloc(iroot, HAMMER2_XOP_MODIFYING);
2020                 hammer2_xop_start(&xop->head, hammer2_inode_xop_flush);
2021                 error = hammer2_xop_collect(&xop->head,
2022                                             HAMMER2_XOP_COLLECT_WAITALL);
2023                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
2024                 if (error == ENOENT)
2025                         error = 0;
2026         } else {
2027                 error = 0;
2028         }
2029         hammer2_trans_done(pmp);
2030
2031         return (error);
2032 }
2033
2034 /*
2035  * Sync passes.
2036  *
2037  * Note that we ignore the tranasction mtid we got above.  Instead,
2038  * each vfsync below will ultimately get its own via TRANS_BUFCACHE
2039  * transactions.
2040  */
2041 static int
2042 hammer2_sync_scan2(struct mount *mp, struct vnode *vp, void *data)
2043 {
2044         struct hammer2_sync_info *info = data;
2045         hammer2_inode_t *ip;
2046         int error;
2047
2048         /*
2049          * Degenerate cases.  Note that ip == NULL typically means the
2050          * syncer vnode itself and we don't want to vclrisdirty() in that
2051          * situation.
2052          */
2053         ip = VTOI(vp);
2054         if (ip == NULL) {
2055                 return(0);
2056         }
2057         if (vp->v_type == VNON || vp->v_type == VBAD) {
2058                 vclrisdirty(vp);
2059                 return(0);
2060         }
2061
2062         /*
2063          * VOP_FSYNC will start a new transaction so replicate some code
2064          * here to do it inline (see hammer2_vop_fsync()).
2065          *
2066          * WARNING: The vfsync interacts with the buffer cache and might
2067          *          block, we can't hold the inode lock at that time.
2068          *          However, we MUST ref ip before blocking to ensure that
2069          *          it isn't ripped out from under us (since we do not
2070          *          hold a lock on the vnode).
2071          */
2072         hammer2_inode_ref(ip);
2073         if ((ip->flags & HAMMER2_INODE_MODIFIED) ||
2074             !RB_EMPTY(&vp->v_rbdirty_tree)) {
2075                 vfsync(vp, info->waitfor, 1, NULL, NULL);
2076                 if (ip->flags & (HAMMER2_INODE_RESIZED |
2077                                  HAMMER2_INODE_MODIFIED)) {
2078                         hammer2_inode_lock(ip, 0);
2079                         hammer2_inode_chain_sync(ip);
2080                         hammer2_inode_unlock(ip);
2081                 }
2082         }
2083         if ((ip->flags & HAMMER2_INODE_MODIFIED) == 0 &&
2084             RB_EMPTY(&vp->v_rbdirty_tree)) {
2085                 vclrisdirty(vp);
2086         }
2087
2088         hammer2_inode_drop(ip);
2089 #if 1
2090         error = 0;
2091         if (error)
2092                 info->error = error;
2093 #endif
2094         return(0);
2095 }
2096
2097 static
2098 int
2099 hammer2_vfs_vptofh(struct vnode *vp, struct fid *fhp)
2100 {
2101         return (0);
2102 }
2103
2104 static
2105 int
2106 hammer2_vfs_fhtovp(struct mount *mp, struct vnode *rootvp,
2107                struct fid *fhp, struct vnode **vpp)
2108 {
2109         return (0);
2110 }
2111
2112 static
2113 int
2114 hammer2_vfs_checkexp(struct mount *mp, struct sockaddr *nam,
2115                  int *exflagsp, struct ucred **credanonp)
2116 {
2117         return (0);
2118 }
2119
2120 /*
2121  * Support code for hammer2_vfs_mount().  Read, verify, and install the volume
2122  * header into the HMP
2123  *
2124  * XXX read four volhdrs and use the one with the highest TID whos CRC
2125  *     matches.
2126  *
2127  * XXX check iCRCs.
2128  *
2129  * XXX For filesystems w/ less than 4 volhdrs, make sure to not write to
2130  *     nonexistant locations.
2131  *
2132  * XXX Record selected volhdr and ring updates to each of 4 volhdrs
2133  */
2134 static
2135 int
2136 hammer2_install_volume_header(hammer2_dev_t *hmp)
2137 {
2138         hammer2_volume_data_t *vd;
2139         struct buf *bp;
2140         hammer2_crc32_t crc0, crc, bcrc0, bcrc;
2141         int error_reported;
2142         int error;
2143         int valid;
2144         int i;
2145
2146         error_reported = 0;
2147         error = 0;
2148         valid = 0;
2149         bp = NULL;
2150
2151         /*
2152          * There are up to 4 copies of the volume header (syncs iterate
2153          * between them so there is no single master).  We don't trust the
2154          * volu_size field so we don't know precisely how large the filesystem
2155          * is, so depend on the OS to return an error if we go beyond the
2156          * block device's EOF.
2157          */
2158         for (i = 0; i < HAMMER2_NUM_VOLHDRS; i++) {
2159                 error = bread(hmp->devvp, i * HAMMER2_ZONE_BYTES64,
2160                               HAMMER2_VOLUME_BYTES, &bp);
2161                 if (error) {
2162                         brelse(bp);
2163                         bp = NULL;
2164                         continue;
2165                 }
2166
2167                 vd = (struct hammer2_volume_data *) bp->b_data;
2168                 if ((vd->magic != HAMMER2_VOLUME_ID_HBO) &&
2169                     (vd->magic != HAMMER2_VOLUME_ID_ABO)) {
2170                         brelse(bp);
2171                         bp = NULL;
2172                         continue;
2173                 }
2174
2175                 if (vd->magic == HAMMER2_VOLUME_ID_ABO) {
2176                         /* XXX: Reversed-endianness filesystem */
2177                         kprintf("hammer2: reverse-endian filesystem detected");
2178                         brelse(bp);
2179                         bp = NULL;
2180                         continue;
2181                 }
2182
2183                 crc = vd->icrc_sects[HAMMER2_VOL_ICRC_SECT0];
2184                 crc0 = hammer2_icrc32(bp->b_data + HAMMER2_VOLUME_ICRC0_OFF,
2185                                       HAMMER2_VOLUME_ICRC0_SIZE);
2186                 bcrc = vd->icrc_sects[HAMMER2_VOL_ICRC_SECT1];
2187                 bcrc0 = hammer2_icrc32(bp->b_data + HAMMER2_VOLUME_ICRC1_OFF,
2188                                        HAMMER2_VOLUME_ICRC1_SIZE);
2189                 if ((crc0 != crc) || (bcrc0 != bcrc)) {
2190                         kprintf("hammer2 volume header crc "
2191                                 "mismatch copy #%d %08x/%08x\n",
2192                                 i, crc0, crc);
2193                         error_reported = 1;
2194                         brelse(bp);
2195                         bp = NULL;
2196                         continue;
2197                 }
2198                 if (valid == 0 || hmp->voldata.mirror_tid < vd->mirror_tid) {
2199                         valid = 1;
2200                         hmp->voldata = *vd;
2201                         hmp->volhdrno = i;
2202                 }
2203                 brelse(bp);
2204                 bp = NULL;
2205         }
2206         if (valid) {
2207                 hmp->volsync = hmp->voldata;
2208                 error = 0;
2209                 if (error_reported || bootverbose || 1) { /* 1/DEBUG */
2210                         kprintf("hammer2: using volume header #%d\n",
2211                                 hmp->volhdrno);
2212                 }
2213         } else {
2214                 error = EINVAL;
2215                 kprintf("hammer2: no valid volume headers found!\n");
2216         }
2217         return (error);
2218 }
2219
2220 /*
2221  * This handles hysteresis on regular file flushes.  Because the BIOs are
2222  * routed to a thread it is possible for an excessive number to build up
2223  * and cause long front-end stalls long before the runningbuffspace limit
2224  * is hit, so we implement hammer2_flush_pipe to control the
2225  * hysteresis.
2226  *
2227  * This is a particular problem when compression is used.
2228  */
2229 void
2230 hammer2_lwinprog_ref(hammer2_pfs_t *pmp)
2231 {
2232         atomic_add_int(&pmp->count_lwinprog, 1);
2233 }
2234
2235 void
2236 hammer2_lwinprog_drop(hammer2_pfs_t *pmp)
2237 {
2238         int lwinprog;
2239
2240         lwinprog = atomic_fetchadd_int(&pmp->count_lwinprog, -1);
2241         if ((lwinprog & HAMMER2_LWINPROG_WAITING) &&
2242             (lwinprog & HAMMER2_LWINPROG_MASK) <= hammer2_flush_pipe * 2 / 3) {
2243                 atomic_clear_int(&pmp->count_lwinprog,
2244                                  HAMMER2_LWINPROG_WAITING);
2245                 wakeup(&pmp->count_lwinprog);
2246         }
2247         if ((lwinprog & HAMMER2_LWINPROG_WAITING0) &&
2248             (lwinprog & HAMMER2_LWINPROG_MASK) <= 0) {
2249                 atomic_clear_int(&pmp->count_lwinprog,
2250                                  HAMMER2_LWINPROG_WAITING0);
2251                 wakeup(&pmp->count_lwinprog);
2252         }
2253 }
2254
2255 void
2256 hammer2_lwinprog_wait(hammer2_pfs_t *pmp, int flush_pipe)
2257 {
2258         int lwinprog;
2259         int lwflag = (flush_pipe) ? HAMMER2_LWINPROG_WAITING :
2260                                     HAMMER2_LWINPROG_WAITING0;
2261
2262         for (;;) {
2263                 lwinprog = pmp->count_lwinprog;
2264                 cpu_ccfence();
2265                 if ((lwinprog & HAMMER2_LWINPROG_MASK) <= flush_pipe)
2266                         break;
2267                 tsleep_interlock(&pmp->count_lwinprog, 0);
2268                 atomic_set_int(&pmp->count_lwinprog, lwflag);
2269                 lwinprog = pmp->count_lwinprog;
2270                 if ((lwinprog & HAMMER2_LWINPROG_MASK) <= flush_pipe)
2271                         break;
2272                 tsleep(&pmp->count_lwinprog, PINTERLOCKED, "h2wpipe", hz);
2273         }
2274 }
2275
2276 /*
2277  * Manage excessive memory resource use for chain and related
2278  * structures.
2279  */
2280 void
2281 hammer2_pfs_memory_wait(hammer2_pfs_t *pmp)
2282 {
2283         uint32_t waiting;
2284         uint32_t count;
2285         uint32_t limit;
2286 #if 0
2287         static int zzticks;
2288 #endif
2289
2290         /*
2291          * Atomic check condition and wait.  Also do an early speedup of
2292          * the syncer to try to avoid hitting the wait.
2293          */
2294         for (;;) {
2295                 waiting = pmp->inmem_dirty_chains;
2296                 cpu_ccfence();
2297                 count = waiting & HAMMER2_DIRTYCHAIN_MASK;
2298
2299                 limit = pmp->mp->mnt_nvnodelistsize / 10;
2300                 if (limit < hammer2_limit_dirty_chains)
2301                         limit = hammer2_limit_dirty_chains;
2302                 if (limit < 1000)
2303                         limit = 1000;
2304
2305 #if 0
2306                 if ((int)(ticks - zzticks) > hz) {
2307                         zzticks = ticks;
2308                         kprintf("count %ld %ld\n", count, limit);
2309                 }
2310 #endif
2311
2312                 /*
2313                  * Block if there are too many dirty chains present, wait
2314                  * for the flush to clean some out.
2315                  */
2316                 if (count > limit) {
2317                         tsleep_interlock(&pmp->inmem_dirty_chains, 0);
2318                         if (atomic_cmpset_int(&pmp->inmem_dirty_chains,
2319                                                waiting,
2320                                        waiting | HAMMER2_DIRTYCHAIN_WAITING)) {
2321                                 speedup_syncer(pmp->mp);
2322                                 tsleep(&pmp->inmem_dirty_chains, PINTERLOCKED,
2323                                        "chnmem", hz);
2324                         }
2325                         continue;       /* loop on success or fail */
2326                 }
2327
2328                 /*
2329                  * Try to start an early flush before we are forced to block.
2330                  */
2331                 if (count > limit * 7 / 10)
2332                         speedup_syncer(pmp->mp);
2333                 break;
2334         }
2335 }
2336
2337 void
2338 hammer2_pfs_memory_inc(hammer2_pfs_t *pmp)
2339 {
2340         if (pmp) {
2341                 atomic_add_int(&pmp->inmem_dirty_chains, 1);
2342         }
2343 }
2344
2345 void
2346 hammer2_pfs_memory_wakeup(hammer2_pfs_t *pmp)
2347 {
2348         uint32_t waiting;
2349
2350         if (pmp == NULL)
2351                 return;
2352
2353         for (;;) {
2354                 waiting = pmp->inmem_dirty_chains;
2355                 cpu_ccfence();
2356                 if (atomic_cmpset_int(&pmp->inmem_dirty_chains,
2357                                        waiting,
2358                                        (waiting - 1) &
2359                                         ~HAMMER2_DIRTYCHAIN_WAITING)) {
2360                         break;
2361                 }
2362         }
2363
2364         if (waiting & HAMMER2_DIRTYCHAIN_WAITING)
2365                 wakeup(&pmp->inmem_dirty_chains);
2366 }
2367
2368 /*
2369  * Debugging
2370  */
2371 void
2372 hammer2_dump_chain(hammer2_chain_t *chain, int tab, int *countp, char pfx)
2373 {
2374         hammer2_chain_t *scan;
2375         hammer2_chain_t *parent;
2376
2377         --*countp;
2378         if (*countp == 0) {
2379                 kprintf("%*.*s...\n", tab, tab, "");
2380                 return;
2381         }
2382         if (*countp < 0)
2383                 return;
2384         kprintf("%*.*s%c-chain %p.%d %016jx/%d mir=%016jx\n",
2385                 tab, tab, "", pfx,
2386                 chain, chain->bref.type,
2387                 chain->bref.key, chain->bref.keybits,
2388                 chain->bref.mirror_tid);
2389
2390         kprintf("%*.*s      [%08x] (%s) refs=%d",
2391                 tab, tab, "",
2392                 chain->flags,
2393                 ((chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
2394                 chain->data) ?  (char *)chain->data->ipdata.filename : "?"),
2395                 chain->refs);
2396
2397         parent = chain->parent;
2398         if (parent)
2399                 kprintf("\n%*.*s      p=%p [pflags %08x prefs %d",
2400                         tab, tab, "",
2401                         parent, parent->flags, parent->refs);
2402         if (RB_EMPTY(&chain->core.rbtree)) {
2403                 kprintf("\n");
2404         } else {
2405                 kprintf(" {\n");
2406                 RB_FOREACH(scan, hammer2_chain_tree, &chain->core.rbtree)
2407                         hammer2_dump_chain(scan, tab + 4, countp, 'a');
2408                 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE && chain->data)
2409                         kprintf("%*.*s}(%s)\n", tab, tab, "",
2410                                 chain->data->ipdata.filename);
2411                 else
2412                         kprintf("%*.*s}\n", tab, tab, "");
2413         }
2414 }