sys/vfs/hammer2/hammer2_vfsops.c

   1 /*
   2  * Copyright (c) 2011-2015 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  * by Daniel Flores (GSOC 2013 - mentored by Matthew Dillon, compression)
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in
  16  *    the documentation and/or other materials provided with the
  17  *    distribution.
  18  * 3. Neither the name of The DragonFly Project nor the names of its
  19  *    contributors may be used to endorse or promote products derived
  20  *    from this software without specific, prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  26  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  27  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  28  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  29  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  30  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  31  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  32  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  33  * SUCH DAMAGE.
  34  */
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/kernel.h>
  38 #include <sys/nlookup.h>
  39 #include <sys/vnode.h>
  40 #include <sys/mount.h>
  41 #include <sys/fcntl.h>
  42 #include <sys/buf.h>
  43 #include <sys/uuid.h>
  44 #include <sys/vfsops.h>
  45 #include <sys/sysctl.h>
  46 #include <sys/socket.h>
  47 #include <sys/objcache.h>
  48
  49 #include <sys/proc.h>
  50 #include <sys/namei.h>
  51 #include <sys/mountctl.h>
  52 #include <sys/dirent.h>
  53 #include <sys/uio.h>
  54
  55 #include <sys/mutex.h>
  56 #include <sys/mutex2.h>
  57
  58 #include "hammer2.h"
  59 #include "hammer2_disk.h"
  60 #include "hammer2_mount.h"
  61 #include "hammer2_lz4.h"
  62
  63 #include "zlib/hammer2_zlib.h"
  64
  65 #define REPORT_REFS_ERRORS 1    /* XXX remove me */
  66
  67 MALLOC_DEFINE(M_OBJCACHE, "objcache", "Object Cache");
  68
  69 struct hammer2_sync_info {
  70         int error;
  71         int waitfor;
  72 };
  73
  74 TAILQ_HEAD(hammer2_mntlist, hammer2_dev);
  75 static struct hammer2_mntlist hammer2_mntlist;
  76
  77 struct hammer2_pfslist hammer2_pfslist;
  78 struct lock hammer2_mntlk;
  79
  80 int hammer2_debug;
  81 int hammer2_cluster_meta_read = 1;      /* physical read-ahead */
  82 int hammer2_cluster_data_read = 4;      /* physical read-ahead */
  83 int hammer2_cluster_write = 0;          /* bdwrite() so later inval works */
  84 int hammer2_dedup_enable = 1;
  85 int hammer2_always_compress = 0;        /* always try to compress */
  86 int hammer2_inval_enable = 0;
  87 int hammer2_flush_pipe = 100;
  88 int hammer2_synchronous_flush = 1;
  89 int hammer2_dio_count;
  90 int hammer2_limit_dio = 256;
  91 int hammer2_bulkfree_tps = 5000;
  92 long hammer2_chain_allocs;
  93 long hammer2_chain_frees;
  94 long hammer2_limit_dirty_chains;
  95 long hammer2_count_modified_chains;
  96 long hammer2_iod_invals;
  97 long hammer2_iod_file_read;
  98 long hammer2_iod_meta_read;
  99 long hammer2_iod_indr_read;
 100 long hammer2_iod_fmap_read;
 101 long hammer2_iod_volu_read;
 102 long hammer2_iod_file_write;
 103 long hammer2_iod_file_wembed;
 104 long hammer2_iod_file_wzero;
 105 long hammer2_iod_file_wdedup;
 106 long hammer2_iod_meta_write;
 107 long hammer2_iod_indr_write;
 108 long hammer2_iod_fmap_write;
 109 long hammer2_iod_volu_write;
 110
 111 MALLOC_DECLARE(M_HAMMER2_CBUFFER);
 112 MALLOC_DEFINE(M_HAMMER2_CBUFFER, "HAMMER2-compbuffer",
 113                 "Buffer used for compression.");
 114
 115 MALLOC_DECLARE(M_HAMMER2_DEBUFFER);
 116 MALLOC_DEFINE(M_HAMMER2_DEBUFFER, "HAMMER2-decompbuffer",
 117                 "Buffer used for decompression.");
 118
 119 SYSCTL_NODE(_vfs, OID_AUTO, hammer2, CTLFLAG_RW, 0, "HAMMER2 filesystem");
 120
 121 SYSCTL_INT(_vfs_hammer2, OID_AUTO, debug, CTLFLAG_RW,
 122            &hammer2_debug, 0, "");
 123 SYSCTL_INT(_vfs_hammer2, OID_AUTO, cluster_meta_read, CTLFLAG_RW,
 124            &hammer2_cluster_meta_read, 0, "");
 125 SYSCTL_INT(_vfs_hammer2, OID_AUTO, cluster_data_read, CTLFLAG_RW,
 126            &hammer2_cluster_data_read, 0, "");
 127 SYSCTL_INT(_vfs_hammer2, OID_AUTO, cluster_write, CTLFLAG_RW,
 128            &hammer2_cluster_write, 0, "");
 129 SYSCTL_INT(_vfs_hammer2, OID_AUTO, dedup_enable, CTLFLAG_RW,
 130            &hammer2_dedup_enable, 0, "");
 131 SYSCTL_INT(_vfs_hammer2, OID_AUTO, always_compress, CTLFLAG_RW,
 132            &hammer2_always_compress, 0, "");
 133 SYSCTL_INT(_vfs_hammer2, OID_AUTO, inval_enable, CTLFLAG_RW,
 134            &hammer2_inval_enable, 0, "");
 135 SYSCTL_INT(_vfs_hammer2, OID_AUTO, flush_pipe, CTLFLAG_RW,
 136            &hammer2_flush_pipe, 0, "");
 137 SYSCTL_INT(_vfs_hammer2, OID_AUTO, synchronous_flush, CTLFLAG_RW,
 138            &hammer2_synchronous_flush, 0, "");
 139 SYSCTL_INT(_vfs_hammer2, OID_AUTO, bulkfree_tps, CTLFLAG_RW,
 140            &hammer2_bulkfree_tps, 0, "");
 141 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, chain_allocs, CTLFLAG_RW,
 142            &hammer2_chain_allocs, 0, "");
 143 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, chain_frees, CTLFLAG_RW,
 144            &hammer2_chain_frees, 0, "");
 145 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, limit_dirty_chains, CTLFLAG_RW,
 146            &hammer2_limit_dirty_chains, 0, "");
 147 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, count_modified_chains, CTLFLAG_RW,
 148            &hammer2_count_modified_chains, 0, "");
 149 SYSCTL_INT(_vfs_hammer2, OID_AUTO, dio_count, CTLFLAG_RD,
 150            &hammer2_dio_count, 0, "");
 151 SYSCTL_INT(_vfs_hammer2, OID_AUTO, limit_dio, CTLFLAG_RW,
 152            &hammer2_limit_dio, 0, "");
 153
 154 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_invals, CTLFLAG_RW,
 155            &hammer2_iod_invals, 0, "");
 156 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_read, CTLFLAG_RW,
 157            &hammer2_iod_file_read, 0, "");
 158 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_meta_read, CTLFLAG_RW,
 159            &hammer2_iod_meta_read, 0, "");
 160 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_indr_read, CTLFLAG_RW,
 161            &hammer2_iod_indr_read, 0, "");
 162 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_fmap_read, CTLFLAG_RW,
 163            &hammer2_iod_fmap_read, 0, "");
 164 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_volu_read, CTLFLAG_RW,
 165            &hammer2_iod_volu_read, 0, "");
 166
 167 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_write, CTLFLAG_RW,
 168            &hammer2_iod_file_write, 0, "");
 169 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_wembed, CTLFLAG_RW,
 170            &hammer2_iod_file_wembed, 0, "");
 171 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_wzero, CTLFLAG_RW,
 172            &hammer2_iod_file_wzero, 0, "");
 173 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_wdedup, CTLFLAG_RW,
 174            &hammer2_iod_file_wdedup, 0, "");
 175 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_meta_write, CTLFLAG_RW,
 176            &hammer2_iod_meta_write, 0, "");
 177 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_indr_write, CTLFLAG_RW,
 178            &hammer2_iod_indr_write, 0, "");
 179 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_fmap_write, CTLFLAG_RW,
 180            &hammer2_iod_fmap_write, 0, "");
 181 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_volu_write, CTLFLAG_RW,
 182            &hammer2_iod_volu_write, 0, "");
 183
 184 long hammer2_check_icrc32;
 185 long hammer2_check_xxhash64;
 186 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, check_icrc32, CTLFLAG_RW,
 187            &hammer2_check_icrc32, 0, "");
 188 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, check_xxhash64, CTLFLAG_RW,
 189            &hammer2_check_xxhash64, 0, "");
 190
 191 static int hammer2_vfs_init(struct vfsconf *conf);
 192 static int hammer2_vfs_uninit(struct vfsconf *vfsp);
 193 static int hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 194                                 struct ucred *cred);
 195 static int hammer2_remount(hammer2_dev_t *, struct mount *, char *,
 196                                 struct vnode *, struct ucred *);
 197 static int hammer2_recovery(hammer2_dev_t *hmp);
 198 static int hammer2_vfs_unmount(struct mount *mp, int mntflags);
 199 static int hammer2_vfs_root(struct mount *mp, struct vnode **vpp);
 200 static int hammer2_vfs_statfs(struct mount *mp, struct statfs *sbp,
 201                                 struct ucred *cred);
 202 static int hammer2_vfs_statvfs(struct mount *mp, struct statvfs *sbp,
 203                                 struct ucred *cred);
 204 static int hammer2_vfs_fhtovp(struct mount *mp, struct vnode *rootvp,
 205                                 struct fid *fhp, struct vnode **vpp);
 206 static int hammer2_vfs_vptofh(struct vnode *vp, struct fid *fhp);
 207 static int hammer2_vfs_checkexp(struct mount *mp, struct sockaddr *nam,
 208                                 int *exflagsp, struct ucred **credanonp);
 209
 210 static int hammer2_install_volume_header(hammer2_dev_t *hmp);
 211 static int hammer2_sync_scan2(struct mount *mp, struct vnode *vp, void *data);
 212
 213 static void hammer2_update_pmps(hammer2_dev_t *hmp);
 214
 215 static void hammer2_mount_helper(struct mount *mp, hammer2_pfs_t *pmp);
 216 static void hammer2_unmount_helper(struct mount *mp, hammer2_pfs_t *pmp,
 217                                 hammer2_dev_t *hmp);
 218
 219 /*
 220  * HAMMER2 vfs operations.
 221  */
 222 static struct vfsops hammer2_vfsops = {
 223         .vfs_init       = hammer2_vfs_init,
 224         .vfs_uninit     = hammer2_vfs_uninit,
 225         .vfs_sync       = hammer2_vfs_sync,
 226         .vfs_mount      = hammer2_vfs_mount,
 227         .vfs_unmount    = hammer2_vfs_unmount,
 228         .vfs_root       = hammer2_vfs_root,
 229         .vfs_statfs     = hammer2_vfs_statfs,
 230         .vfs_statvfs    = hammer2_vfs_statvfs,
 231         .vfs_vget       = hammer2_vfs_vget,
 232         .vfs_vptofh     = hammer2_vfs_vptofh,
 233         .vfs_fhtovp     = hammer2_vfs_fhtovp,
 234         .vfs_checkexp   = hammer2_vfs_checkexp
 235 };
 236
 237 MALLOC_DEFINE(M_HAMMER2, "HAMMER2-mount", "");
 238
 239 VFS_SET(hammer2_vfsops, hammer2, VFCF_MPSAFE);
 240 MODULE_VERSION(hammer2, 1);
 241
 242 static
 243 int
 244 hammer2_vfs_init(struct vfsconf *conf)
 245 {
 246         static struct objcache_malloc_args margs_read;
 247         static struct objcache_malloc_args margs_write;
 248         static struct objcache_malloc_args margs_vop;
 249
 250         int error;
 251
 252         error = 0;
 253
 254         /*
 255          * A large DIO cache is needed to retain dedup enablement masks.
 256          * The bulkfree code clears related masks as part of the disk block
 257          * recycling algorithm, preventing it from being used for a later
 258          * dedup.
 259          *
 260          * NOTE: A large buffer cache can actually interfere with dedup
 261          *       operation because we dedup based on media physical buffers
 262          *       and not logical buffers.  Try to make the DIO chace large
 263          *       enough to avoid this problem, but also cap it.
 264          */
 265         hammer2_limit_dio = nbuf * 2;
 266         if (hammer2_limit_dio > 100000)
 267                 hammer2_limit_dio = 100000;
 268
 269         if (HAMMER2_BLOCKREF_BYTES != sizeof(struct hammer2_blockref))
 270                 error = EINVAL;
 271         if (HAMMER2_INODE_BYTES != sizeof(struct hammer2_inode_data))
 272                 error = EINVAL;
 273         if (HAMMER2_VOLUME_BYTES != sizeof(struct hammer2_volume_data))
 274                 error = EINVAL;
 275
 276         if (error)
 277                 kprintf("HAMMER2 structure size mismatch; cannot continue.\n");
 278
 279         margs_read.objsize = 65536;
 280         margs_read.mtype = M_HAMMER2_DEBUFFER;
 281
 282         margs_write.objsize = 32768;
 283         margs_write.mtype = M_HAMMER2_CBUFFER;
 284
 285         margs_vop.objsize = sizeof(hammer2_xop_t);
 286         margs_vop.mtype = M_HAMMER2;
 287
 288         /*
 289          * Note thaht for the XOPS cache we want backing store allocations
 290          * to use M_ZERO.  This is not allowed in objcache_get() (to avoid
 291          * confusion), so use the backing store function that does it.  This
 292          * means that initial XOPS objects are zerod but REUSED objects are
 293          * not.  So we are responsible for cleaning the object up sufficiently
 294          * for our needs before objcache_put()ing it back (typically just the
 295          * FIFO indices).
 296          */
 297         cache_buffer_read = objcache_create(margs_read.mtype->ks_shortdesc,
 298                                 0, 1, NULL, NULL, NULL,
 299                                 objcache_malloc_alloc,
 300                                 objcache_malloc_free,
 301                                 &margs_read);
 302         cache_buffer_write = objcache_create(margs_write.mtype->ks_shortdesc,
 303                                 0, 1, NULL, NULL, NULL,
 304                                 objcache_malloc_alloc,
 305                                 objcache_malloc_free,
 306                                 &margs_write);
 307         cache_xops = objcache_create(margs_vop.mtype->ks_shortdesc,
 308                                 0, 1, NULL, NULL, NULL,
 309                                 objcache_malloc_alloc_zero,
 310                                 objcache_malloc_free,
 311                                 &margs_vop);
 312
 313
 314         lockinit(&hammer2_mntlk, "mntlk", 0, 0);
 315         TAILQ_INIT(&hammer2_mntlist);
 316         TAILQ_INIT(&hammer2_pfslist);
 317
 318         hammer2_limit_dirty_chains = maxvnodes / 10;
 319         if (hammer2_limit_dirty_chains > HAMMER2_LIMIT_DIRTY_CHAINS)
 320                 hammer2_limit_dirty_chains = HAMMER2_LIMIT_DIRTY_CHAINS;
 321
 322         return (error);
 323 }
 324
 325 static
 326 int
 327 hammer2_vfs_uninit(struct vfsconf *vfsp __unused)
 328 {
 329         objcache_destroy(cache_buffer_read);
 330         objcache_destroy(cache_buffer_write);
 331         objcache_destroy(cache_xops);
 332         return 0;
 333 }
 334
 335 /*
 336  * Core PFS allocator.  Used to allocate or reference the pmp structure
 337  * for PFS cluster mounts and the spmp structure for media (hmp) structures.
 338  * The pmp can be passed in or loaded by this function using the chain and
 339  * inode data.
 340  *
 341  * pmp->modify_tid tracks new modify_tid transaction ids for front-end
 342  * transactions.  Note that synchronization does not use this field.
 343  * (typically frontend operations and synchronization cannot run on the
 344  * same PFS node at the same time).
 345  *
 346  * XXX check locking
 347  */
 348 hammer2_pfs_t *
 349 hammer2_pfsalloc(hammer2_chain_t *chain,
 350                  const hammer2_inode_data_t *ripdata,
 351                  hammer2_tid_t modify_tid, hammer2_dev_t *force_local)
 352 {
 353         hammer2_pfs_t *pmp;
 354         hammer2_inode_t *iroot;
 355         int count;
 356         int i;
 357         int j;
 358
 359         pmp = NULL;
 360
 361         /*
 362          * Locate or create the PFS based on the cluster id.  If ripdata
 363          * is NULL this is a spmp which is unique and is always allocated.
 364          *
 365          * If the device is mounted in local mode all PFSs are considered
 366          * independent and not part of any cluster (for debugging only).
 367          */
 368         if (ripdata) {
 369                 TAILQ_FOREACH(pmp, &hammer2_pfslist, mntentry) {
 370                         if (force_local != pmp->force_local)
 371                                 continue;
 372                         if (force_local == NULL &&
 373                             bcmp(&pmp->pfs_clid, &ripdata->meta.pfs_clid,
 374                                  sizeof(pmp->pfs_clid)) == 0) {
 375                                         break;
 376                         } else if (force_local && pmp->pfs_names[0] &&
 377                             strcmp(pmp->pfs_names[0], ripdata->filename) == 0) {
 378                                         break;
 379                         }
 380                 }
 381         }
 382
 383         if (pmp == NULL) {
 384                 pmp = kmalloc(sizeof(*pmp), M_HAMMER2, M_WAITOK | M_ZERO);
 385                 pmp->force_local = force_local;
 386                 hammer2_trans_manage_init(pmp);
 387                 kmalloc_create(&pmp->minode, "HAMMER2-inodes");
 388                 kmalloc_create(&pmp->mmsg, "HAMMER2-pfsmsg");
 389                 lockinit(&pmp->lock, "pfslk", 0, 0);
 390                 lockinit(&pmp->lock_nlink, "h2nlink", 0, 0);
 391                 spin_init(&pmp->inum_spin, "hm2pfsalloc_inum");
 392                 spin_init(&pmp->xop_spin, "h2xop");
 393                 spin_init(&pmp->lru_spin, "h2lru");
 394                 RB_INIT(&pmp->inum_tree);
 395                 TAILQ_INIT(&pmp->sideq);
 396                 TAILQ_INIT(&pmp->lru_list);
 397                 spin_init(&pmp->list_spin, "hm2pfsalloc_list");
 398
 399                 /*
 400                  * Distribute backend operations to threads
 401                  */
 402                 for (i = 0; i < HAMMER2_XOPGROUPS; ++i)
 403                         hammer2_xop_group_init(pmp, &pmp->xop_groups[i]);
 404
 405                 /*
 406                  * Save the last media transaction id for the flusher.  Set
 407                  * initial
 408                  */
 409                 if (ripdata)
 410                         pmp->pfs_clid = ripdata->meta.pfs_clid;
 411                 TAILQ_INSERT_TAIL(&hammer2_pfslist, pmp, mntentry);
 412
 413                 /*
 414                  * The synchronization thread may start too early, make
 415                  * sure it stays frozen until we are ready to let it go.
 416                  * XXX
 417                  */
 418                 /*
 419                 pmp->primary_thr.flags = HAMMER2_THREAD_FROZEN |
 420                                          HAMMER2_THREAD_REMASTER;
 421                 */
 422         }
 423
 424         /*
 425          * Create the PFS's root inode and any missing XOP helper threads.
 426          */
 427         if ((iroot = pmp->iroot) == NULL) {
 428                 iroot = hammer2_inode_get(pmp, NULL, NULL, -1);
 429                 if (ripdata)
 430                         iroot->meta = ripdata->meta;
 431                 pmp->iroot = iroot;
 432                 hammer2_inode_ref(iroot);
 433                 hammer2_inode_unlock(iroot);
 434         }
 435
 436         /*
 437          * Stop here if no chain is passed in.
 438          */
 439         if (chain == NULL)
 440                 goto done;
 441
 442         /*
 443          * When a chain is passed in we must add it to the PFS's root
 444          * inode, update pmp->pfs_types[], and update the syncronization
 445          * threads.
 446          *
 447          * When forcing local mode, mark the PFS as a MASTER regardless.
 448          *
 449          * At the moment empty spots can develop due to removals or failures.
 450          * Ultimately we want to re-fill these spots but doing so might
 451          * confused running code. XXX
 452          */
 453         hammer2_inode_ref(iroot);
 454         hammer2_mtx_ex(&iroot->lock);
 455         j = iroot->cluster.nchains;
 456
 457         kprintf("add PFS to pmp %p[%d]\n", pmp, j);
 458
 459         if (j == HAMMER2_MAXCLUSTER) {
 460                 kprintf("hammer2_mount: cluster full!\n");
 461                 /* XXX fatal error? */
 462         } else {
 463                 KKASSERT(chain->pmp == NULL);
 464                 chain->pmp = pmp;
 465                 hammer2_chain_ref(chain);
 466                 iroot->cluster.array[j].chain = chain;
 467                 if (force_local)
 468                         pmp->pfs_types[j] = HAMMER2_PFSTYPE_MASTER;
 469                 else
 470                         pmp->pfs_types[j] = ripdata->meta.pfs_type;
 471                 pmp->pfs_names[j] = kstrdup(ripdata->filename, M_HAMMER2);
 472                 pmp->pfs_hmps[j] = chain->hmp;
 473
 474                 /*
 475                  * If the PFS is already mounted we must account
 476                  * for the mount_count here.
 477                  */
 478                 if (pmp->mp)
 479                         ++chain->hmp->mount_count;
 480
 481                 /*
 482                  * May have to fixup dirty chain tracking.  Previous
 483                  * pmp was NULL so nothing to undo.
 484                  */
 485                 if (chain->flags & HAMMER2_CHAIN_MODIFIED)
 486                         hammer2_pfs_memory_inc(pmp);
 487                 ++j;
 488         }
 489         iroot->cluster.nchains = j;
 490
 491         /*
 492          * Update nmasters from any PFS inode which is part of the cluster.
 493          * It is possible that this will result in a value which is too
 494          * high.  MASTER PFSs are authoritative for pfs_nmasters and will
 495          * override this value later on.
 496          *
 497          * (This informs us of masters that might not currently be
 498          *  discoverable by this mount).
 499          */
 500         if (ripdata && pmp->pfs_nmasters < ripdata->meta.pfs_nmasters) {
 501                 pmp->pfs_nmasters = ripdata->meta.pfs_nmasters;
 502         }
 503
 504         /*
 505          * Count visible masters.  Masters are usually added with
 506          * ripdata->meta.pfs_nmasters set to 1.  This detects when there
 507          * are more (XXX and must update the master inodes).
 508          */
 509         count = 0;
 510         for (i = 0; i < iroot->cluster.nchains; ++i) {
 511                 if (pmp->pfs_types[i] == HAMMER2_PFSTYPE_MASTER)
 512                         ++count;
 513         }
 514         if (pmp->pfs_nmasters < count)
 515                 pmp->pfs_nmasters = count;
 516
 517         /*
 518          * Create missing synchronization and support threads.
 519          *
 520          * Single-node masters (including snapshots) have nothing to
 521          * synchronize and do not require this thread.
 522          *
 523          * Multi-node masters or any number of soft masters, slaves, copy,
 524          * or other PFS types need the thread.
 525          *
 526          * Each thread is responsible for its particular cluster index.
 527          * We use independent threads so stalls or mismatches related to
 528          * any given target do not affect other targets.
 529          */
 530         for (i = 0; i < iroot->cluster.nchains; ++i) {
 531                 /*
 532                  * Single-node masters (including snapshots) have nothing
 533                  * to synchronize and will make direct xops support calls,
 534                  * thus they do not require this thread.
 535                  *
 536                  * Note that there can be thousands of snapshots.  We do not
 537                  * want to create thousands of threads.
 538                  */
 539                 if (pmp->pfs_nmasters <= 1 &&
 540                     pmp->pfs_types[i] == HAMMER2_PFSTYPE_MASTER) {
 541                         continue;
 542                 }
 543
 544                 /*
 545                  * Sync support thread
 546                  */
 547                 if (pmp->sync_thrs[i].td == NULL) {
 548                         hammer2_thr_create(&pmp->sync_thrs[i], pmp, NULL,
 549                                            "h2nod", i, -1,
 550                                            hammer2_primary_sync_thread);
 551                 }
 552         }
 553
 554         /*
 555          * Create missing Xop threads
 556          *
 557          * NOTE: We create helper threads for all mounted PFSs or any
 558          *       PFSs with 2+ nodes (so the sync thread can update them,
 559          *       even if not mounted).
 560          */
 561         if (pmp->mp || iroot->cluster.nchains >= 2)
 562                 hammer2_xop_helper_create(pmp);
 563
 564         hammer2_mtx_unlock(&iroot->lock);
 565         hammer2_inode_drop(iroot);
 566 done:
 567         return pmp;
 568 }
 569
 570 /*
 571  * Deallocate an element of a probed PFS.  If destroying and this is a
 572  * MASTER, adjust nmasters.
 573  *
 574  * This function does not physically destroy the PFS element in its device
 575  * under the super-root  (see hammer2_ioctl_pfs_delete()).
 576  */
 577 void
 578 hammer2_pfsdealloc(hammer2_pfs_t *pmp, int clindex, int destroying)
 579 {
 580         hammer2_inode_t *iroot;
 581         hammer2_chain_t *chain;
 582         int j;
 583
 584         /*
 585          * Cleanup our reference on iroot.  iroot is (should) not be needed
 586          * by the flush code.
 587          */
 588         iroot = pmp->iroot;
 589         if (iroot) {
 590                 /*
 591                  * Stop synchronizing
 592                  *
 593                  * XXX flush after acquiring the iroot lock.
 594                  * XXX clean out the cluster index from all inode structures.
 595                  */
 596                 hammer2_thr_delete(&pmp->sync_thrs[clindex]);
 597
 598                 /*
 599                  * Remove the cluster index from the group.  If destroying
 600                  * the PFS and this is a master, adjust pfs_nmasters.
 601                  */
 602                 hammer2_mtx_ex(&iroot->lock);
 603                 chain = iroot->cluster.array[clindex].chain;
 604                 iroot->cluster.array[clindex].chain = NULL;
 605
 606                 switch(pmp->pfs_types[clindex]) {
 607                 case HAMMER2_PFSTYPE_MASTER:
 608                         if (destroying && pmp->pfs_nmasters > 0)
 609                                 --pmp->pfs_nmasters;
 610                         /* XXX adjust ripdata->meta.pfs_nmasters */
 611                         break;
 612                 default:
 613                         break;
 614                 }
 615                 pmp->pfs_types[clindex] = HAMMER2_PFSTYPE_NONE;
 616
 617                 hammer2_mtx_unlock(&iroot->lock);
 618
 619                 /*
 620                  * Release the chain.
 621                  */
 622                 if (chain) {
 623                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
 624                         hammer2_chain_drop(chain);
 625                 }
 626
 627                 /*
 628                  * Terminate all XOP threads for the cluster index.
 629                  */
 630                 for (j = 0; j < HAMMER2_XOPGROUPS; ++j)
 631                         hammer2_thr_delete(&pmp->xop_groups[j].thrs[clindex]);
 632         }
 633 }
 634
 635 /*
 636  * Destroy a PFS, typically only occurs after the last mount on a device
 637  * has gone away.
 638  */
 639 static void
 640 hammer2_pfsfree(hammer2_pfs_t *pmp)
 641 {
 642         hammer2_inode_t *iroot;
 643         hammer2_chain_t *chain;
 644         int i;
 645         int j;
 646
 647         /*
 648          * Cleanup our reference on iroot.  iroot is (should) not be needed
 649          * by the flush code.
 650          */
 651         TAILQ_REMOVE(&hammer2_pfslist, pmp, mntentry);
 652
 653         iroot = pmp->iroot;
 654         if (iroot) {
 655                 for (i = 0; i < iroot->cluster.nchains; ++i) {
 656                         hammer2_thr_delete(&pmp->sync_thrs[i]);
 657                         for (j = 0; j < HAMMER2_XOPGROUPS; ++j)
 658                                 hammer2_thr_delete(&pmp->xop_groups[j].thrs[i]);
 659                 }
 660 #if REPORT_REFS_ERRORS
 661                 if (pmp->iroot->refs != 1)
 662                         kprintf("PMP->IROOT %p REFS WRONG %d\n",
 663                                 pmp->iroot, pmp->iroot->refs);
 664 #else
 665                 KKASSERT(pmp->iroot->refs == 1);
 666 #endif
 667                 /* ref for pmp->iroot */
 668                 hammer2_inode_drop(pmp->iroot);
 669                 pmp->iroot = NULL;
 670         }
 671
 672         /*
 673          * Cleanup chains remaining on LRU list.
 674          */
 675         kprintf("pfsfree: %p lrucount=%d\n", pmp, pmp->lru_count);
 676         while ((chain = TAILQ_FIRST(&pmp->lru_list)) != NULL) {
 677                 hammer2_chain_ref(chain);
 678                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
 679                 hammer2_chain_drop(chain);
 680         }
 681
 682         /*
 683          * Free remaining pmp resources
 684          */
 685         kmalloc_destroy(&pmp->mmsg);
 686         kmalloc_destroy(&pmp->minode);
 687
 688         kfree(pmp, M_HAMMER2);
 689 }
 690
 691 /*
 692  * Remove all references to hmp from the pfs list.  Any PFS which becomes
 693  * empty is terminated and freed.
 694  *
 695  * XXX inefficient.
 696  */
 697 static void
 698 hammer2_pfsfree_scan(hammer2_dev_t *hmp)
 699 {
 700         hammer2_pfs_t *pmp;
 701         hammer2_inode_t *iroot;
 702         hammer2_chain_t *rchain;
 703         int didfreeze;
 704         int i;
 705         int j;
 706
 707 again:
 708         TAILQ_FOREACH(pmp, &hammer2_pfslist, mntentry) {
 709                 if ((iroot = pmp->iroot) == NULL)
 710                         continue;
 711                 if (hmp->spmp == pmp) {
 712                         kprintf("unmount hmp %p remove spmp %p\n",
 713                                 hmp, pmp);
 714                         hmp->spmp = NULL;
 715                 }
 716
 717                 /*
 718                  * Determine if this PFS is affected.  If it is we must
 719                  * freeze all management threads and lock its iroot.
 720                  *
 721                  * Freezing a management thread forces it idle, operations
 722                  * in-progress will be aborted and it will have to start
 723                  * over again when unfrozen, or exit if told to exit.
 724                  */
 725                 for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 726                         if (pmp->pfs_hmps[i] == hmp)
 727                                 break;
 728                 }
 729                 if (i != HAMMER2_MAXCLUSTER) {
 730                         /*
 731                          * Make sure all synchronization threads are locked
 732                          * down.
 733                          */
 734                         for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 735                                 if (pmp->pfs_hmps[i] == NULL)
 736                                         continue;
 737                                 hammer2_thr_freeze_async(&pmp->sync_thrs[i]);
 738                                 for (j = 0; j < HAMMER2_XOPGROUPS; ++j) {
 739                                         hammer2_thr_freeze_async(
 740                                                 &pmp->xop_groups[j].thrs[i]);
 741                                 }
 742                         }
 743                         for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 744                                 if (pmp->pfs_hmps[i] == NULL)
 745                                         continue;
 746                                 hammer2_thr_freeze(&pmp->sync_thrs[i]);
 747                                 for (j = 0; j < HAMMER2_XOPGROUPS; ++j) {
 748                                         hammer2_thr_freeze(
 749                                                 &pmp->xop_groups[j].thrs[i]);
 750                                 }
 751                         }
 752
 753                         /*
 754                          * Lock the inode and clean out matching chains.
 755                          * Note that we cannot use hammer2_inode_lock_*()
 756                          * here because that would attempt to validate the
 757                          * cluster that we are in the middle of ripping
 758                          * apart.
 759                          *
 760                          * WARNING! We are working directly on the inodes
 761                          *          embedded cluster.
 762                          */
 763                         hammer2_mtx_ex(&iroot->lock);
 764
 765                         /*
 766                          * Remove the chain from matching elements of the PFS.
 767                          */
 768                         for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 769                                 if (pmp->pfs_hmps[i] != hmp)
 770                                         continue;
 771                                 hammer2_thr_delete(&pmp->sync_thrs[i]);
 772                                 for (j = 0; j < HAMMER2_XOPGROUPS; ++j) {
 773                                         hammer2_thr_delete(
 774                                                 &pmp->xop_groups[j].thrs[i]);
 775                                 }
 776                                 rchain = iroot->cluster.array[i].chain;
 777                                 iroot->cluster.array[i].chain = NULL;
 778                                 pmp->pfs_types[i] = 0;
 779                                 if (pmp->pfs_names[i]) {
 780                                         kfree(pmp->pfs_names[i], M_HAMMER2);
 781                                         pmp->pfs_names[i] = NULL;
 782                                 }
 783                                 if (rchain) {
 784                                         hammer2_chain_drop(rchain);
 785                                         /* focus hint */
 786                                         if (iroot->cluster.focus == rchain)
 787                                                 iroot->cluster.focus = NULL;
 788                                 }
 789                                 pmp->pfs_hmps[i] = NULL;
 790                         }
 791                         hammer2_mtx_unlock(&iroot->lock);
 792                         didfreeze = 1;  /* remaster, unfreeze down below */
 793                 } else {
 794                         didfreeze = 0;
 795                 }
 796
 797                 /*
 798                  * Cleanup trailing chains.  Gaps may remain.
 799                  */
 800                 for (i = HAMMER2_MAXCLUSTER - 1; i >= 0; --i) {
 801                         if (pmp->pfs_hmps[i])
 802                                 break;
 803                 }
 804                 iroot->cluster.nchains = i + 1;
 805
 806                 /*
 807                  * If the PMP has no elements remaining we can destroy it.
 808                  * (this will transition management threads from frozen->exit).
 809                  */
 810                 if (iroot->cluster.nchains == 0) {
 811                         kprintf("unmount hmp %p last ref to PMP=%p\n",
 812                                 hmp, pmp);
 813                         hammer2_pfsfree(pmp);
 814                         goto again;
 815                 }
 816
 817                 /*
 818                  * If elements still remain we need to set the REMASTER
 819                  * flag and unfreeze it.
 820                  */
 821                 if (didfreeze) {
 822                         for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 823                                 if (pmp->pfs_hmps[i] == NULL)
 824                                         continue;
 825                                 hammer2_thr_remaster(&pmp->sync_thrs[i]);
 826                                 hammer2_thr_unfreeze(&pmp->sync_thrs[i]);
 827                                 for (j = 0; j < HAMMER2_XOPGROUPS; ++j) {
 828                                         hammer2_thr_remaster(
 829                                                 &pmp->xop_groups[j].thrs[i]);
 830                                         hammer2_thr_unfreeze(
 831                                                 &pmp->xop_groups[j].thrs[i]);
 832                                 }
 833                         }
 834                 }
 835         }
 836 }
 837
 838 /*
 839  * Mount or remount HAMMER2 fileystem from physical media
 840  *
 841  *      mountroot
 842  *              mp              mount point structure
 843  *              path            NULL
 844  *              data            <unused>
 845  *              cred            <unused>
 846  *
 847  *      mount
 848  *              mp              mount point structure
 849  *              path            path to mount point
 850  *              data            pointer to argument structure in user space
 851  *                      volume  volume path (device@LABEL form)
 852  *                      hflags  user mount flags
 853  *              cred            user credentials
 854  *
 855  * RETURNS:     0       Success
 856  *              !0      error number
 857  */
 858 static
 859 int
 860 hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 861                   struct ucred *cred)
 862 {
 863         struct hammer2_mount_info info;
 864         hammer2_pfs_t *pmp;
 865         hammer2_pfs_t *spmp;
 866         hammer2_dev_t *hmp;
 867         hammer2_dev_t *force_local;
 868         hammer2_key_t key_next;
 869         hammer2_key_t key_dummy;
 870         hammer2_key_t lhc;
 871         struct vnode *devvp;
 872         struct nlookupdata nd;
 873         hammer2_chain_t *parent;
 874         hammer2_chain_t *chain;
 875         hammer2_cluster_t *cluster;
 876         const hammer2_inode_data_t *ripdata;
 877         hammer2_blockref_t bref;
 878         struct file *fp;
 879         char devstr[MNAMELEN];
 880         size_t size;
 881         size_t done;
 882         char *dev;
 883         char *label;
 884         int ronly = 1;
 885         int error;
 886         int i;
 887
 888         hmp = NULL;
 889         pmp = NULL;
 890         dev = NULL;
 891         label = NULL;
 892         devvp = NULL;
 893
 894         kprintf("hammer2_mount\n");
 895
 896         if (path == NULL) {
 897                 /*
 898                  * Root mount
 899                  */
 900                 bzero(&info, sizeof(info));
 901                 info.cluster_fd = -1;
 902                 ksnprintf(devstr, sizeof(devstr), "%s",
 903                           mp->mnt_stat.f_mntfromname);
 904                 kprintf("hammer2_mount: root '%s'\n", devstr);
 905         } else {
 906                 /*
 907                  * Non-root mount or updating a mount
 908                  */
 909                 error = copyin(data, &info, sizeof(info));
 910                 if (error)
 911                         return (error);
 912
 913                 error = copyinstr(info.volume, devstr, MNAMELEN - 1, &done);
 914                 if (error)
 915                         return (error);
 916         }
 917
 918         /*
 919          * Extract device and label, automatically mount @BOOT, @ROOT, or @DATA
 920          * if no label specified, based on the partition id.  Error out if no
 921          * label or device (with partition id) is specified.  This is strictly
 922          * a convenience to match the default label created by newfs_hammer2,
 923          * our preference is that a label always be specified.
 924          *
 925          * NOTE: We allow 'mount @LABEL <blah>'... that is, a mount command
 926          *       that does not specify a device, as long as some H2 label
 927          *       has already been mounted from that device.  This makes
 928          *       mounting snapshots a lot easier.
 929          */
 930         dev = devstr;
 931         label = strchr(devstr, '@');
 932         if (label && ((label + 1) - dev) > done)
 933                 return (EINVAL);
 934         if (label == NULL || label[1] == 0) {
 935                 char slice;
 936
 937                 if (label == NULL)
 938                         label = devstr + strlen(devstr);
 939                 slice = label[-1];
 940                 switch(slice) {
 941                 case 'a':
 942                         label = "BOOT";
 943                         break;
 944                 case 'd':
 945                         label = "ROOT";
 946                         break;
 947                 default:
 948                         label = "DATA";
 949                         break;
 950                 }
 951         } else {
 952                 *label = '\0';
 953                 label++;
 954         }
 955
 956         kprintf("hammer2_mount: dev=\"%s\" label=\"%s\" rdonly=%d\n",
 957                 dev, label, (mp->mnt_flag & MNT_RDONLY));
 958
 959         if (mp->mnt_flag & MNT_UPDATE) {
 960                 /*
 961                  * Update mount.  Note that pmp->iroot->cluster is
 962                  * an inode-embedded cluster and thus cannot be
 963                  * directly locked.
 964                  *
 965                  * XXX HAMMER2 needs to implement NFS export via
 966                  *     mountctl.
 967                  */
 968                 pmp = MPTOPMP(mp);
 969                 pmp->hflags = info.hflags;
 970                 cluster = &pmp->iroot->cluster;
 971                 for (i = 0; i < cluster->nchains; ++i) {
 972                         if (cluster->array[i].chain == NULL)
 973                                 continue;
 974                         hmp = cluster->array[i].chain->hmp;
 975                         devvp = hmp->devvp;
 976                         error = hammer2_remount(hmp, mp, path,
 977                                                 devvp, cred);
 978                         if (error)
 979                                 break;
 980                 }
 981
 982                 return error;
 983         }
 984
 985         /*
 986          * HMP device mount
 987          *
 988          * If a path is specified and dev is not an empty string, lookup the
 989          * name and verify that it referes to a block device.
 990          *
 991          * If a path is specified and dev is an empty string we fall through
 992          * and locate the label in the hmp search.
 993          */
 994         if (path && *dev != 0) {
 995                 error = nlookup_init(&nd, dev, UIO_SYSSPACE, NLC_FOLLOW);
 996                 if (error == 0)
 997                         error = nlookup(&nd);
 998                 if (error == 0)
 999                         error = cache_vref(&nd.nl_nch, nd.nl_cred, &devvp);
1000                 nlookup_done(&nd);
1001         } else if (path == NULL) {
1002                 /* root mount */
1003                 cdev_t cdev = kgetdiskbyname(dev);
1004                 error = bdevvp(cdev, &devvp);
1005                 if (error)
1006                         kprintf("hammer2: cannot find '%s'\n", dev);
1007         } else {
1008                 /*
1009                  * We will locate the hmp using the label in the hmp loop.
1010                  */
1011                 error = 0;
1012         }
1013
1014         /*
1015          * Make sure its a block device.  Do not check to see if it is
1016          * already mounted until we determine that its a fresh H2 device.
1017          */
1018         if (error == 0 && devvp) {
1019                 vn_isdisk(devvp, &error);
1020         }
1021
1022         /*
1023          * Determine if the device has already been mounted.  After this
1024          * check hmp will be non-NULL if we are doing the second or more
1025          * hammer2 mounts from the same device.
1026          */
1027         lockmgr(&hammer2_mntlk, LK_EXCLUSIVE);
1028         if (devvp) {
1029                 /*
1030                  * Match the device.  Due to the way devfs works,
1031                  * we may not be able to directly match the vnode pointer,
1032                  * so also check to see if the underlying device matches.
1033                  */
1034                 TAILQ_FOREACH(hmp, &hammer2_mntlist, mntentry) {
1035                         if (hmp->devvp == devvp)
1036                                 break;
1037                         if (devvp->v_rdev &&
1038                             hmp->devvp->v_rdev == devvp->v_rdev) {
1039                                 break;
1040                         }
1041                 }
1042
1043                 /*
1044                  * If no match this may be a fresh H2 mount, make sure
1045                  * the device is not mounted on anything else.
1046                  */
1047                 if (hmp == NULL)
1048                         error = vfs_mountedon(devvp);
1049         } else if (error == 0) {
1050                 /*
1051                  * Match the label to a pmp already probed.
1052                  */
1053                 TAILQ_FOREACH(pmp, &hammer2_pfslist, mntentry) {
1054                         for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
1055                                 if (pmp->pfs_names[i] &&
1056                                     strcmp(pmp->pfs_names[i], label) == 0) {
1057                                         hmp = pmp->pfs_hmps[i];
1058                                         break;
1059                                 }
1060                         }
1061                         if (hmp)
1062                                 break;
1063                 }
1064                 if (hmp == NULL)
1065                         error = ENOENT;
1066         }
1067
1068         /*
1069          * Open the device if this isn't a secondary mount and construct
1070          * the H2 device mount (hmp).
1071          */
1072         if (hmp == NULL) {
1073                 hammer2_chain_t *schain;
1074                 hammer2_xid_t xid;
1075
1076                 if (error == 0 && vcount(devvp) > 0) {
1077                         kprintf("Primary device already has references\n");
1078                         error = EBUSY;
1079                 }
1080
1081                 /*
1082                  * Now open the device
1083                  */
1084                 if (error == 0) {
1085                         ronly = ((mp->mnt_flag & MNT_RDONLY) != 0);
1086                         vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1087                         error = vinvalbuf(devvp, V_SAVE, 0, 0);
1088                         if (error == 0) {
1089                                 error = VOP_OPEN(devvp,
1090                                              (ronly ? FREAD : FREAD | FWRITE),
1091                                              FSCRED, NULL);
1092                         }
1093                         vn_unlock(devvp);
1094                 }
1095                 if (error && devvp) {
1096                         vrele(devvp);
1097                         devvp = NULL;
1098                 }
1099                 if (error) {
1100                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1101                         return error;
1102                 }
1103                 hmp = kmalloc(sizeof(*hmp), M_HAMMER2, M_WAITOK | M_ZERO);
1104                 ksnprintf(hmp->devrepname, sizeof(hmp->devrepname), "%s", dev);
1105                 hmp->ronly = ronly;
1106                 hmp->devvp = devvp;
1107                 hmp->hflags = info.hflags & HMNT2_DEVFLAGS;
1108                 kmalloc_create(&hmp->mchain, "HAMMER2-chains");
1109                 TAILQ_INSERT_TAIL(&hammer2_mntlist, hmp, mntentry);
1110                 RB_INIT(&hmp->iotree);
1111                 spin_init(&hmp->io_spin, "hm2mount_io");
1112                 spin_init(&hmp->list_spin, "hm2mount_list");
1113                 TAILQ_INIT(&hmp->flushq);
1114
1115                 lockinit(&hmp->vollk, "h2vol", 0, 0);
1116                 lockinit(&hmp->bulklk, "h2bulk", 0, 0);
1117                 lockinit(&hmp->bflock, "h2bflk", 0, 0);
1118
1119                 /*
1120                  * vchain setup. vchain.data is embedded.
1121                  * vchain.refs is initialized and will never drop to 0.
1122                  *
1123                  * NOTE! voldata is not yet loaded.
1124                  */
1125                 hmp->vchain.hmp = hmp;
1126                 hmp->vchain.refs = 1;
1127                 hmp->vchain.data = (void *)&hmp->voldata;
1128                 hmp->vchain.bref.type = HAMMER2_BREF_TYPE_VOLUME;
1129                 hmp->vchain.bref.data_off = 0 | HAMMER2_PBUFRADIX;
1130                 hmp->vchain.bref.mirror_tid = hmp->voldata.mirror_tid;
1131
1132                 hammer2_chain_core_init(&hmp->vchain);
1133                 /* hmp->vchain.u.xxx is left NULL */
1134
1135                 /*
1136                  * fchain setup.  fchain.data is embedded.
1137                  * fchain.refs is initialized and will never drop to 0.
1138                  *
1139                  * The data is not used but needs to be initialized to
1140                  * pass assertion muster.  We use this chain primarily
1141                  * as a placeholder for the freemap's top-level RBTREE
1142                  * so it does not interfere with the volume's topology
1143                  * RBTREE.
1144                  */
1145                 hmp->fchain.hmp = hmp;
1146                 hmp->fchain.refs = 1;
1147                 hmp->fchain.data = (void *)&hmp->voldata.freemap_blockset;
1148                 hmp->fchain.bref.type = HAMMER2_BREF_TYPE_FREEMAP;
1149                 hmp->fchain.bref.data_off = 0 | HAMMER2_PBUFRADIX;
1150                 hmp->fchain.bref.mirror_tid = hmp->voldata.freemap_tid;
1151                 hmp->fchain.bref.methods =
1152                         HAMMER2_ENC_CHECK(HAMMER2_CHECK_FREEMAP) |
1153                         HAMMER2_ENC_COMP(HAMMER2_COMP_NONE);
1154
1155                 hammer2_chain_core_init(&hmp->fchain);
1156                 /* hmp->fchain.u.xxx is left NULL */
1157
1158                 /*
1159                  * Install the volume header and initialize fields from
1160                  * voldata.
1161                  */
1162                 error = hammer2_install_volume_header(hmp);
1163                 if (error) {
1164                         hammer2_unmount_helper(mp, NULL, hmp);
1165                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1166                         hammer2_vfs_unmount(mp, MNT_FORCE);
1167                         return error;
1168                 }
1169
1170                 /*
1171                  * Really important to get these right or flush will get
1172                  * confused.
1173                  */
1174                 hmp->spmp = hammer2_pfsalloc(NULL, NULL, 0, NULL);
1175                 kprintf("alloc spmp %p tid %016jx\n",
1176                         hmp->spmp, hmp->voldata.mirror_tid);
1177                 spmp = hmp->spmp;
1178
1179                 /*
1180                  * Dummy-up vchain and fchain's modify_tid.  mirror_tid
1181                  * is inherited from the volume header.
1182                  */
1183                 xid = 0;
1184                 hmp->vchain.bref.mirror_tid = hmp->voldata.mirror_tid;
1185                 hmp->vchain.bref.modify_tid = hmp->vchain.bref.mirror_tid;
1186                 hmp->vchain.pmp = spmp;
1187                 hmp->fchain.bref.mirror_tid = hmp->voldata.freemap_tid;
1188                 hmp->fchain.bref.modify_tid = hmp->fchain.bref.mirror_tid;
1189                 hmp->fchain.pmp = spmp;
1190
1191                 /*
1192                  * First locate the super-root inode, which is key 0
1193                  * relative to the volume header's blockset.
1194                  *
1195                  * Then locate the root inode by scanning the directory keyspace
1196                  * represented by the label.
1197                  */
1198                 parent = hammer2_chain_lookup_init(&hmp->vchain, 0);
1199                 schain = hammer2_chain_lookup(&parent, &key_dummy,
1200                                       HAMMER2_SROOT_KEY, HAMMER2_SROOT_KEY,
1201                                       &error, 0);
1202                 hammer2_chain_lookup_done(parent);
1203                 if (schain == NULL) {
1204                         kprintf("hammer2_mount: invalid super-root\n");
1205                         hammer2_unmount_helper(mp, NULL, hmp);
1206                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1207                         hammer2_vfs_unmount(mp, MNT_FORCE);
1208                         return EINVAL;
1209                 }
1210                 if (schain->error) {
1211                         kprintf("hammer2_mount: error %s reading super-root\n",
1212                                 hammer2_error_str(schain->error));
1213                         hammer2_chain_unlock(schain);
1214                         hammer2_chain_drop(schain);
1215                         schain = NULL;
1216                         hammer2_unmount_helper(mp, NULL, hmp);
1217                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1218                         hammer2_vfs_unmount(mp, MNT_FORCE);
1219                         return EINVAL;
1220                 }
1221
1222                 /*
1223                  * The super-root always uses an inode_tid of 1 when
1224                  * creating PFSs.
1225                  */
1226                 spmp->inode_tid = 1;
1227                 spmp->modify_tid = schain->bref.modify_tid + 1;
1228
1229                 /*
1230                  * Sanity-check schain's pmp and finish initialization.
1231                  * Any chain belonging to the super-root topology should
1232                  * have a NULL pmp (not even set to spmp).
1233                  */
1234                 ripdata = &hammer2_chain_rdata(schain)->ipdata;
1235                 KKASSERT(schain->pmp == NULL);
1236                 spmp->pfs_clid = ripdata->meta.pfs_clid;
1237
1238                 /*
1239                  * Replace the dummy spmp->iroot with a real one.  It's
1240                  * easier to just do a wholesale replacement than to try
1241                  * to update the chain and fixup the iroot fields.
1242                  *
1243                  * The returned inode is locked with the supplied cluster.
1244                  */
1245                 cluster = hammer2_cluster_from_chain(schain);
1246                 hammer2_inode_drop(spmp->iroot);
1247                 spmp->iroot = NULL;
1248                 spmp->iroot = hammer2_inode_get(spmp, NULL, cluster, -1);
1249                 spmp->spmp_hmp = hmp;
1250                 spmp->pfs_types[0] = ripdata->meta.pfs_type;
1251                 spmp->pfs_hmps[0] = hmp;
1252                 hammer2_inode_ref(spmp->iroot);
1253                 hammer2_inode_unlock(spmp->iroot);
1254                 hammer2_cluster_unlock(cluster);
1255                 hammer2_cluster_drop(cluster);
1256                 schain = NULL;
1257                 /* leave spmp->iroot with one ref */
1258
1259                 if ((mp->mnt_flag & MNT_RDONLY) == 0) {
1260                         error = hammer2_recovery(hmp);
1261                         /* XXX do something with error */
1262                 }
1263                 hammer2_update_pmps(hmp);
1264                 hammer2_iocom_init(hmp);
1265                 hammer2_bulkfree_init(hmp);
1266
1267                 /*
1268                  * Ref the cluster management messaging descriptor.  The mount
1269                  * program deals with the other end of the communications pipe.
1270                  *
1271                  * Root mounts typically do not supply one.
1272                  */
1273                 if (info.cluster_fd >= 0) {
1274                         fp = holdfp(curproc->p_fd, info.cluster_fd, -1);
1275                         if (fp) {
1276                                 hammer2_cluster_reconnect(hmp, fp);
1277                         } else {
1278                                 kprintf("hammer2_mount: bad cluster_fd!\n");
1279                         }
1280                 }
1281         } else {
1282                 spmp = hmp->spmp;
1283                 if (info.hflags & HMNT2_DEVFLAGS) {
1284                         kprintf("hammer2: Warning: mount flags pertaining "
1285                                 "to the whole device may only be specified "
1286                                 "on the first mount of the device: %08x\n",
1287                                 info.hflags & HMNT2_DEVFLAGS);
1288                 }
1289         }
1290
1291         /*
1292          * Force local mount (disassociate all PFSs from their clusters).
1293          * Used primarily for debugging.
1294          */
1295         force_local = (hmp->hflags & HMNT2_LOCAL) ? hmp : NULL;
1296
1297         /*
1298          * Lookup the mount point under the media-localized super-root.
1299          * Scanning hammer2_pfslist doesn't help us because it represents
1300          * PFS cluster ids which can aggregate several named PFSs together.
1301          *
1302          * cluster->pmp will incorrectly point to spmp and must be fixed
1303          * up later on.
1304          */
1305         hammer2_inode_lock(spmp->iroot, 0);
1306         parent = hammer2_inode_chain(spmp->iroot, 0, HAMMER2_RESOLVE_ALWAYS);
1307         lhc = hammer2_dirhash(label, strlen(label));
1308         chain = hammer2_chain_lookup(&parent, &key_next,
1309                                      lhc, lhc + HAMMER2_DIRHASH_LOMASK,
1310                                      &error, 0);
1311         while (chain) {
1312                 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
1313                     strcmp(label, chain->data->ipdata.filename) == 0) {
1314                         break;
1315                 }
1316                 chain = hammer2_chain_next(&parent, chain, &key_next,
1317                                             key_next,
1318                                             lhc + HAMMER2_DIRHASH_LOMASK,
1319                                             &error, 0);
1320         }
1321         if (parent) {
1322                 hammer2_chain_unlock(parent);
1323                 hammer2_chain_drop(parent);
1324         }
1325         hammer2_inode_unlock(spmp->iroot);
1326
1327         /*
1328          * PFS could not be found?
1329          */
1330         if (chain == NULL) {
1331                 if (error)
1332                         kprintf("hammer2_mount: PFS label I/O error\n");
1333                 else
1334                         kprintf("hammer2_mount: PFS label not found\n");
1335                 hammer2_unmount_helper(mp, NULL, hmp);
1336                 lockmgr(&hammer2_mntlk, LK_RELEASE);
1337                 hammer2_vfs_unmount(mp, MNT_FORCE);
1338
1339                 return EINVAL;
1340         }
1341
1342         /*
1343          * Acquire the pmp structure (it should have already been allocated
1344          * via hammer2_update_pmps() so do not pass cluster in to add to
1345          * available chains).
1346          *
1347          * Check if the cluster has already been mounted.  A cluster can
1348          * only be mounted once, use null mounts to mount additional copies.
1349          */
1350         if (chain->error) {
1351                 kprintf("hammer2_mount: PFS label I/O error\n");
1352         } else {
1353                 ripdata = &chain->data->ipdata;
1354                 bref = chain->bref;
1355                 pmp = hammer2_pfsalloc(NULL, ripdata,
1356                                        bref.modify_tid, force_local);
1357         }
1358         hammer2_chain_unlock(chain);
1359         hammer2_chain_drop(chain);
1360
1361         /*
1362          * Finish the mount
1363          */
1364         kprintf("hammer2_mount hmp=%p pmp=%p\n", hmp, pmp);
1365
1366         if (pmp->mp) {
1367                 kprintf("hammer2_mount: PFS already mounted!\n");
1368                 hammer2_unmount_helper(mp, NULL, hmp);
1369                 lockmgr(&hammer2_mntlk, LK_RELEASE);
1370                 hammer2_vfs_unmount(mp, MNT_FORCE);
1371
1372                 return EBUSY;
1373         }
1374
1375         pmp->hflags = info.hflags;
1376         mp->mnt_flag |= MNT_LOCAL;
1377         mp->mnt_kern_flag |= MNTK_ALL_MPSAFE;   /* all entry pts are SMP */
1378         mp->mnt_kern_flag |= MNTK_THR_SYNC;     /* new vsyncscan semantics */
1379
1380         /*
1381          * required mount structure initializations
1382          */
1383         mp->mnt_stat.f_iosize = HAMMER2_PBUFSIZE;
1384         mp->mnt_stat.f_bsize = HAMMER2_PBUFSIZE;
1385
1386         mp->mnt_vstat.f_frsize = HAMMER2_PBUFSIZE;
1387         mp->mnt_vstat.f_bsize = HAMMER2_PBUFSIZE;
1388
1389         /*
1390          * Optional fields
1391          */
1392         mp->mnt_iosize_max = MAXPHYS;
1393
1394         /*
1395          * Connect up mount pointers.
1396          */
1397         hammer2_mount_helper(mp, pmp);
1398
1399         lockmgr(&hammer2_mntlk, LK_RELEASE);
1400
1401         /*
1402          * Finish setup
1403          */
1404         vfs_getnewfsid(mp);
1405         vfs_add_vnodeops(mp, &hammer2_vnode_vops, &mp->mnt_vn_norm_ops);
1406         vfs_add_vnodeops(mp, &hammer2_spec_vops, &mp->mnt_vn_spec_ops);
1407         vfs_add_vnodeops(mp, &hammer2_fifo_vops, &mp->mnt_vn_fifo_ops);
1408
1409         if (path) {
1410                 copyinstr(info.volume, mp->mnt_stat.f_mntfromname,
1411                           MNAMELEN - 1, &size);
1412                 bzero(mp->mnt_stat.f_mntfromname + size, MNAMELEN - size);
1413         } /* else root mount, already in there */
1414
1415         bzero(mp->mnt_stat.f_mntonname, sizeof(mp->mnt_stat.f_mntonname));
1416         if (path) {
1417                 copyinstr(path, mp->mnt_stat.f_mntonname,
1418                           sizeof(mp->mnt_stat.f_mntonname) - 1,
1419                           &size);
1420         } else {
1421                 /* root mount */
1422                 mp->mnt_stat.f_mntonname[0] = '/';
1423         }
1424
1425         /*
1426          * Initial statfs to prime mnt_stat.
1427          */
1428         hammer2_vfs_statfs(mp, &mp->mnt_stat, cred);
1429
1430         return 0;
1431 }
1432
1433 /*
1434  * Scan PFSs under the super-root and create hammer2_pfs structures.
1435  */
1436 static
1437 void
1438 hammer2_update_pmps(hammer2_dev_t *hmp)
1439 {
1440         const hammer2_inode_data_t *ripdata;
1441         hammer2_chain_t *parent;
1442         hammer2_chain_t *chain;
1443         hammer2_blockref_t bref;
1444         hammer2_dev_t *force_local;
1445         hammer2_pfs_t *spmp;
1446         hammer2_pfs_t *pmp;
1447         hammer2_key_t key_next;
1448         int error;
1449
1450         /*
1451          * Force local mount (disassociate all PFSs from their clusters).
1452          * Used primarily for debugging.
1453          */
1454         force_local = (hmp->hflags & HMNT2_LOCAL) ? hmp : NULL;
1455
1456         /*
1457          * Lookup mount point under the media-localized super-root.
1458          *
1459          * cluster->pmp will incorrectly point to spmp and must be fixed
1460          * up later on.
1461          */
1462         spmp = hmp->spmp;
1463         hammer2_inode_lock(spmp->iroot, 0);
1464         parent = hammer2_inode_chain(spmp->iroot, 0, HAMMER2_RESOLVE_ALWAYS);
1465         chain = hammer2_chain_lookup(&parent, &key_next,
1466                                          HAMMER2_KEY_MIN, HAMMER2_KEY_MAX,
1467                                          &error, 0);
1468         while (chain) {
1469                 if (chain->bref.type != HAMMER2_BREF_TYPE_INODE)
1470                         continue;
1471                 if (chain->error) {
1472                         kprintf("I/O error scanning PFS labels\n");
1473                 } else {
1474                         ripdata = &chain->data->ipdata;
1475                         bref = chain->bref;
1476                         kprintf("ADD LOCAL PFS: %s\n", ripdata->filename);
1477
1478                         pmp = hammer2_pfsalloc(chain, ripdata,
1479                                                bref.modify_tid, force_local);
1480                 }
1481                 chain = hammer2_chain_next(&parent, chain, &key_next,
1482                                            key_next, HAMMER2_KEY_MAX,
1483                                            &error, 0);
1484         }
1485         if (parent) {
1486                 hammer2_chain_unlock(parent);
1487                 hammer2_chain_drop(parent);
1488         }
1489         hammer2_inode_unlock(spmp->iroot);
1490 }
1491
1492 static
1493 int
1494 hammer2_remount(hammer2_dev_t *hmp, struct mount *mp, char *path __unused,
1495                 struct vnode *devvp, struct ucred *cred)
1496 {
1497         int error;
1498
1499         if (hmp->ronly && (mp->mnt_kern_flag & MNTK_WANTRDWR)) {
1500                 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1501                 VOP_OPEN(devvp, FREAD | FWRITE, FSCRED, NULL);
1502                 vn_unlock(devvp);
1503                 error = hammer2_recovery(hmp);
1504                 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1505                 if (error == 0) {
1506                         VOP_CLOSE(devvp, FREAD, NULL);
1507                         hmp->ronly = 0;
1508                 } else {
1509                         VOP_CLOSE(devvp, FREAD | FWRITE, NULL);
1510                 }
1511                 vn_unlock(devvp);
1512         } else {
1513                 error = 0;
1514         }
1515         return error;
1516 }
1517
1518 static
1519 int
1520 hammer2_vfs_unmount(struct mount *mp, int mntflags)
1521 {
1522         hammer2_pfs_t *pmp;
1523         int flags;
1524         int error = 0;
1525
1526         pmp = MPTOPMP(mp);
1527
1528         if (pmp == NULL)
1529                 return(0);
1530
1531         lockmgr(&hammer2_mntlk, LK_EXCLUSIVE);
1532
1533         /*
1534          * If mount initialization proceeded far enough we must flush
1535          * its vnodes and sync the underlying mount points.  Three syncs
1536          * are required to fully flush the filesystem (freemap updates lag
1537          * by one flush, and one extra for safety).
1538          */
1539         if (mntflags & MNT_FORCE)
1540                 flags = FORCECLOSE;
1541         else
1542                 flags = 0;
1543         if (pmp->iroot) {
1544                 error = vflush(mp, 0, flags);
1545                 if (error)
1546                         goto failed;
1547                 hammer2_vfs_sync(mp, MNT_WAIT);
1548                 hammer2_vfs_sync(mp, MNT_WAIT);
1549                 hammer2_vfs_sync(mp, MNT_WAIT);
1550         }
1551
1552         /*
1553          * Cleanup the frontend support XOPS threads
1554          */
1555         hammer2_xop_helper_cleanup(pmp);
1556
1557         if (pmp->mp)
1558                 hammer2_unmount_helper(mp, pmp, NULL);
1559
1560         error = 0;
1561 failed:
1562         lockmgr(&hammer2_mntlk, LK_RELEASE);
1563
1564         return (error);
1565 }
1566
1567 /*
1568  * Mount helper, hook the system mount into our PFS.
1569  * The mount lock is held.
1570  *
1571  * We must bump the mount_count on related devices for any
1572  * mounted PFSs.
1573  */
1574 static
1575 void
1576 hammer2_mount_helper(struct mount *mp, hammer2_pfs_t *pmp)
1577 {
1578         hammer2_cluster_t *cluster;
1579         hammer2_chain_t *rchain;
1580         int i;
1581
1582         mp->mnt_data = (qaddr_t)pmp;
1583         pmp->mp = mp;
1584
1585         /*
1586          * After pmp->mp is set we have to adjust hmp->mount_count.
1587          */
1588         cluster = &pmp->iroot->cluster;
1589         for (i = 0; i < cluster->nchains; ++i) {
1590                 rchain = cluster->array[i].chain;
1591                 if (rchain == NULL)
1592                         continue;
1593                 ++rchain->hmp->mount_count;
1594                 kprintf("hammer2_mount hmp=%p ++mount_count=%d\n",
1595                         rchain->hmp, rchain->hmp->mount_count);
1596         }
1597
1598         /*
1599          * Create missing Xop threads
1600          */
1601         hammer2_xop_helper_create(pmp);
1602 }
1603
1604 /*
1605  * Mount helper, unhook the system mount from our PFS.
1606  * The mount lock is held.
1607  *
1608  * If hmp is supplied a mount responsible for being the first to open
1609  * the block device failed and the block device and all PFSs using the
1610  * block device must be cleaned up.
1611  *
1612  * If pmp is supplied multiple devices might be backing the PFS and each
1613  * must be disconnected.  This might not be the last PFS using some of the
1614  * underlying devices.  Also, we have to adjust our hmp->mount_count
1615  * accounting for the devices backing the pmp which is now undergoing an
1616  * unmount.
1617  */
1618 static
1619 void
1620 hammer2_unmount_helper(struct mount *mp, hammer2_pfs_t *pmp, hammer2_dev_t *hmp)
1621 {
1622         hammer2_cluster_t *cluster;
1623         hammer2_chain_t *rchain;
1624         struct vnode *devvp;
1625         int dumpcnt;
1626         int ronly;
1627         int i;
1628
1629         /*
1630          * If no device supplied this is a high-level unmount and we have to
1631          * to disconnect the mount, adjust mount_count, and locate devices
1632          * that might now have no mounts.
1633          */
1634         if (pmp) {
1635                 KKASSERT(hmp == NULL);
1636                 KKASSERT((void *)(intptr_t)mp->mnt_data == pmp);
1637                 pmp->mp = NULL;
1638                 mp->mnt_data = NULL;
1639
1640                 /*
1641                  * After pmp->mp is cleared we have to account for
1642                  * mount_count.
1643                  */
1644                 cluster = &pmp->iroot->cluster;
1645                 for (i = 0; i < cluster->nchains; ++i) {
1646                         rchain = cluster->array[i].chain;
1647                         if (rchain == NULL)
1648                                 continue;
1649                         --rchain->hmp->mount_count;
1650                         kprintf("hammer2_unmount hmp=%p --mount_count=%d\n",
1651                                 rchain->hmp, rchain->hmp->mount_count);
1652                         /* scrapping hmp now may invalidate the pmp */
1653                 }
1654 again:
1655                 TAILQ_FOREACH(hmp, &hammer2_mntlist, mntentry) {
1656                         if (hmp->mount_count == 0) {
1657                                 hammer2_unmount_helper(NULL, NULL, hmp);
1658                                 goto again;
1659                         }
1660                 }
1661                 return;
1662         }
1663
1664         /*
1665          * Try to terminate the block device.  We can't terminate it if
1666          * there are still PFSs referencing it.
1667          */
1668         kprintf("hammer2_unmount hmp=%p mount_count=%d\n",
1669                 hmp, hmp->mount_count);
1670         if (hmp->mount_count)
1671                 return;
1672
1673         hammer2_bulkfree_uninit(hmp);
1674         hammer2_pfsfree_scan(hmp);
1675         hammer2_dev_exlock(hmp);        /* XXX order */
1676
1677         /*
1678          * Cycle the volume data lock as a safety (probably not needed any
1679          * more).  To ensure everything is out we need to flush at least
1680          * three times.  (1) The running of the sideq can dirty the
1681          * filesystem, (2) A normal flush can dirty the freemap, and
1682          * (3) ensure that the freemap is fully synchronized.
1683          *
1684          * The next mount's recovery scan can clean everything up but we want
1685          * to leave the filesystem in a 100% clean state on a normal unmount.
1686          */
1687 #if 0
1688         hammer2_voldata_lock(hmp);
1689         hammer2_voldata_unlock(hmp);
1690 #endif
1691         hammer2_iocom_uninit(hmp);
1692
1693         if ((hmp->vchain.flags | hmp->fchain.flags) &
1694             HAMMER2_CHAIN_FLUSH_MASK) {
1695                 kprintf("hammer2_unmount: chains left over "
1696                         "after final sync\n");
1697                 kprintf("    vchain %08x\n", hmp->vchain.flags);
1698                 kprintf("    fchain %08x\n", hmp->fchain.flags);
1699
1700                 if (hammer2_debug & 0x0010)
1701                         Debugger("entered debugger");
1702         }
1703
1704         KKASSERT(hmp->spmp == NULL);
1705
1706         /*
1707          * Finish up with the device vnode
1708          */
1709         if ((devvp = hmp->devvp) != NULL) {
1710                 ronly = hmp->ronly;
1711                 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1712                 kprintf("hammer2_unmount(A): devvp %s rbdirty %p ronly=%d\n",
1713                         hmp->devrepname, RB_ROOT(&devvp->v_rbdirty_tree),
1714                         ronly);
1715                 vinvalbuf(devvp, (ronly ? 0 : V_SAVE), 0, 0);
1716                 kprintf("hammer2_unmount(B): devvp %s rbdirty %p\n",
1717                         hmp->devrepname, RB_ROOT(&devvp->v_rbdirty_tree));
1718                 hmp->devvp = NULL;
1719                 VOP_CLOSE(devvp, (ronly ? FREAD : FREAD|FWRITE), NULL);
1720                 vn_unlock(devvp);
1721                 vrele(devvp);
1722                 devvp = NULL;
1723         }
1724
1725         /*
1726          * Clear vchain/fchain flags that might prevent final cleanup
1727          * of these chains.
1728          */
1729         if (hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED) {
1730                 atomic_add_long(&hammer2_count_modified_chains, -1);
1731                 atomic_clear_int(&hmp->vchain.flags, HAMMER2_CHAIN_MODIFIED);
1732                 hammer2_pfs_memory_wakeup(hmp->vchain.pmp);
1733         }
1734         if (hmp->vchain.flags & HAMMER2_CHAIN_UPDATE) {
1735                 atomic_clear_int(&hmp->vchain.flags, HAMMER2_CHAIN_UPDATE);
1736         }
1737
1738         if (hmp->fchain.flags & HAMMER2_CHAIN_MODIFIED) {
1739                 atomic_add_long(&hammer2_count_modified_chains, -1);
1740                 atomic_clear_int(&hmp->fchain.flags, HAMMER2_CHAIN_MODIFIED);
1741                 hammer2_pfs_memory_wakeup(hmp->fchain.pmp);
1742         }
1743         if (hmp->fchain.flags & HAMMER2_CHAIN_UPDATE) {
1744                 atomic_clear_int(&hmp->fchain.flags, HAMMER2_CHAIN_UPDATE);
1745         }
1746
1747         /*
1748          * Final drop of embedded freemap root chain to
1749          * clean up fchain.core (fchain structure is not
1750          * flagged ALLOCATED so it is cleaned out and then
1751          * left to rot).
1752          */
1753         hammer2_chain_drop(&hmp->fchain);
1754
1755         /*
1756          * Final drop of embedded volume root chain to clean
1757          * up vchain.core (vchain structure is not flagged
1758          * ALLOCATED so it is cleaned out and then left to
1759          * rot).
1760          */
1761         dumpcnt = 50;
1762         hammer2_dump_chain(&hmp->vchain, 0, &dumpcnt, 'v');
1763         dumpcnt = 50;
1764         hammer2_dump_chain(&hmp->fchain, 0, &dumpcnt, 'f');
1765         hammer2_dev_unlock(hmp);
1766         hammer2_chain_drop(&hmp->vchain);
1767
1768         hammer2_io_cleanup(hmp, &hmp->iotree);
1769         if (hmp->iofree_count) {
1770                 kprintf("io_cleanup: %d I/O's left hanging\n",
1771                         hmp->iofree_count);
1772         }
1773
1774         TAILQ_REMOVE(&hammer2_mntlist, hmp, mntentry);
1775         kmalloc_destroy(&hmp->mchain);
1776         kfree(hmp, M_HAMMER2);
1777 }
1778
1779 int
1780 hammer2_vfs_vget(struct mount *mp, struct vnode *dvp,
1781                  ino_t ino, struct vnode **vpp)
1782 {
1783         hammer2_xop_lookup_t *xop;
1784         hammer2_pfs_t *pmp;
1785         hammer2_inode_t *ip;
1786         hammer2_tid_t inum;
1787         int error;
1788
1789         inum = (hammer2_tid_t)ino & HAMMER2_DIRHASH_USERMSK;
1790
1791         error = 0;
1792         pmp = MPTOPMP(mp);
1793
1794         /*
1795          * Easy if we already have it cached
1796          */
1797         ip = hammer2_inode_lookup(pmp, inum);
1798         if (ip) {
1799                 hammer2_inode_lock(ip, HAMMER2_RESOLVE_SHARED);
1800                 *vpp = hammer2_igetv(ip, &error);
1801                 hammer2_inode_unlock(ip);
1802                 hammer2_inode_drop(ip);         /* from lookup */
1803
1804                 return error;
1805         }
1806
1807         /*
1808          * Otherwise we have to find the inode
1809          */
1810         xop = hammer2_xop_alloc(pmp->iroot, 0);
1811         xop->lhc = inum;
1812         hammer2_xop_start(&xop->head, hammer2_xop_lookup);
1813         error = hammer2_xop_collect(&xop->head, 0);
1814
1815         if (error == 0) {
1816                 if (hammer2_cluster_rdata(&xop->head.cluster) == NULL) {
1817                         kprintf("vget: no collect error but also no rdata\n");
1818                         kprintf("xop %p\n", xop);
1819                         while ((hammer2_debug & 0x80000) == 0) {
1820                                 tsleep(xop, PCATCH, "wait", hz * 10);
1821                         }
1822                         ip = NULL;
1823                 } else {
1824                         ip = hammer2_inode_get(pmp, NULL, &xop->head.cluster, -1);
1825                 }
1826         }
1827         hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
1828
1829         if (ip) {
1830                 *vpp = hammer2_igetv(ip, &error);
1831                 hammer2_inode_unlock(ip);
1832         } else {
1833                 *vpp = NULL;
1834                 error = ENOENT;
1835         }
1836         return (error);
1837 }
1838
1839 static
1840 int
1841 hammer2_vfs_root(struct mount *mp, struct vnode **vpp)
1842 {
1843         hammer2_pfs_t *pmp;
1844         struct vnode *vp;
1845         int error;
1846
1847         pmp = MPTOPMP(mp);
1848         if (pmp->iroot == NULL) {
1849                 *vpp = NULL;
1850                 return EINVAL;
1851         }
1852
1853         error = 0;
1854         hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED);
1855
1856         while (pmp->inode_tid == 0) {
1857                 hammer2_xop_ipcluster_t *xop;
1858                 hammer2_inode_meta_t *meta;
1859
1860                 xop = hammer2_xop_alloc(pmp->iroot, HAMMER2_XOP_MODIFYING);
1861                 hammer2_xop_start(&xop->head, hammer2_xop_ipcluster);
1862                 error = hammer2_xop_collect(&xop->head, 0);
1863
1864                 if (error == 0) {
1865                         meta = &xop->head.cluster.focus->data->ipdata.meta;
1866                         pmp->iroot->meta = *meta;
1867                         pmp->inode_tid = meta->pfs_inum + 1;
1868                         if (pmp->inode_tid < HAMMER2_INODE_START)
1869                                 pmp->inode_tid = HAMMER2_INODE_START;
1870                         pmp->modify_tid =
1871                                 xop->head.cluster.focus->bref.modify_tid + 1;
1872                         kprintf("PFS: Starting inode %jd\n",
1873                                 (intmax_t)pmp->inode_tid);
1874                         kprintf("PMP focus good set nextino=%ld mod=%016jx\n",
1875                                 pmp->inode_tid, pmp->modify_tid);
1876                         wakeup(&pmp->iroot);
1877
1878                         hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
1879
1880                         /*
1881                          * Prime the mount info.
1882                          */
1883                         hammer2_vfs_statfs(mp, &mp->mnt_stat, NULL);
1884                         break;
1885                 }
1886
1887                 /*
1888                  * Loop, try again
1889                  */
1890                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
1891                 hammer2_inode_unlock(pmp->iroot);
1892                 error = tsleep(&pmp->iroot, PCATCH, "h2root", hz);
1893                 hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED);
1894                 if (error == EINTR)
1895                         break;
1896         }
1897
1898         if (error) {
1899                 hammer2_inode_unlock(pmp->iroot);
1900                 *vpp = NULL;
1901         } else {
1902                 vp = hammer2_igetv(pmp->iroot, &error);
1903                 hammer2_inode_unlock(pmp->iroot);
1904                 *vpp = vp;
1905         }
1906
1907         return (error);
1908 }
1909
1910 /*
1911  * Filesystem status
1912  *
1913  * XXX incorporate ipdata->meta.inode_quota and data_quota
1914  */
1915 static
1916 int
1917 hammer2_vfs_statfs(struct mount *mp, struct statfs *sbp, struct ucred *cred)
1918 {
1919         hammer2_pfs_t *pmp;
1920         hammer2_dev_t *hmp;
1921         hammer2_blockref_t bref;
1922         int i;
1923
1924         /*
1925          * NOTE: iroot might not have validated the cluster yet.
1926          */
1927         pmp = MPTOPMP(mp);
1928
1929         mp->mnt_stat.f_files = 0;
1930         mp->mnt_stat.f_ffree = 0;
1931         mp->mnt_stat.f_blocks = 0;
1932         mp->mnt_stat.f_bfree = 0;
1933         mp->mnt_stat.f_bavail = 0;
1934
1935         for (i = 0; i < pmp->iroot->cluster.nchains; ++i) {
1936                 hmp = pmp->pfs_hmps[i];
1937                 if (hmp == NULL)
1938                         continue;
1939                 if (pmp->iroot->cluster.array[i].chain)
1940                         bref = pmp->iroot->cluster.array[i].chain->bref;
1941                 else
1942                         bzero(&bref, sizeof(bref));
1943
1944                 mp->mnt_stat.f_files = bref.embed.stats.inode_count;
1945                 mp->mnt_stat.f_ffree = 0;
1946                 mp->mnt_stat.f_blocks = hmp->voldata.allocator_size /
1947                                         mp->mnt_vstat.f_bsize;
1948                 mp->mnt_stat.f_bfree = hmp->voldata.allocator_free /
1949                                         mp->mnt_vstat.f_bsize;
1950                 mp->mnt_stat.f_bavail = mp->mnt_stat.f_bfree;
1951
1952                 if (cred && cred->cr_uid != 0) {
1953                         uint64_t adj;
1954
1955                         /* 5% */
1956                         adj = hmp->free_reserved / mp->mnt_vstat.f_bsize;
1957                         mp->mnt_stat.f_blocks -= adj;
1958                         mp->mnt_stat.f_bfree -= adj;
1959                         mp->mnt_stat.f_bavail -= adj;
1960                 }
1961
1962                 *sbp = mp->mnt_stat;
1963         }
1964         return (0);
1965 }
1966
1967 static
1968 int
1969 hammer2_vfs_statvfs(struct mount *mp, struct statvfs *sbp, struct ucred *cred)
1970 {
1971         hammer2_pfs_t *pmp;
1972         hammer2_dev_t *hmp;
1973         hammer2_blockref_t bref;
1974         int i;
1975
1976         /*
1977          * NOTE: iroot might not have validated the cluster yet.
1978          */
1979         pmp = MPTOPMP(mp);
1980
1981         mp->mnt_vstat.f_bsize = 0;
1982         mp->mnt_vstat.f_files = 0;
1983         mp->mnt_vstat.f_ffree = 0;
1984         mp->mnt_vstat.f_blocks = 0;
1985         mp->mnt_vstat.f_bfree = 0;
1986         mp->mnt_vstat.f_bavail = 0;
1987
1988         for (i = 0; i < pmp->iroot->cluster.nchains; ++i) {
1989                 hmp = pmp->pfs_hmps[i];
1990                 if (hmp == NULL)
1991                         continue;
1992                 if (pmp->iroot->cluster.array[i].chain)
1993                         bref = pmp->iroot->cluster.array[i].chain->bref;
1994                 else
1995                         bzero(&bref, sizeof(bref));
1996
1997                 mp->mnt_vstat.f_bsize = HAMMER2_PBUFSIZE;
1998                 mp->mnt_vstat.f_files = bref.embed.stats.inode_count;
1999                 mp->mnt_vstat.f_ffree = 0;
2000                 mp->mnt_vstat.f_blocks = hmp->voldata.allocator_size /
2001                                         mp->mnt_vstat.f_bsize;
2002                 mp->mnt_vstat.f_bfree = hmp->voldata.allocator_free /
2003                                         mp->mnt_vstat.f_bsize;
2004                 mp->mnt_vstat.f_bavail = mp->mnt_vstat.f_bfree;
2005
2006                 if (cred && cred->cr_uid != 0) {
2007                         uint64_t adj;
2008
2009                         /* 5% */
2010                         adj = hmp->free_reserved / mp->mnt_vstat.f_bsize;
2011                         mp->mnt_vstat.f_blocks -= adj;
2012                         mp->mnt_vstat.f_bfree -= adj;
2013                         mp->mnt_vstat.f_bavail -= adj;
2014                 }
2015
2016                 *sbp = mp->mnt_vstat;
2017         }
2018         return (0);
2019 }
2020
2021 /*
2022  * Mount-time recovery (RW mounts)
2023  *
2024  * Updates to the free block table are allowed to lag flushes by one
2025  * transaction.  In case of a crash, then on a fresh mount we must do an
2026  * incremental scan of the last committed transaction id and make sure that
2027  * all related blocks have been marked allocated.
2028  *
2029  * The super-root topology and each PFS has its own transaction id domain,
2030  * so we must track PFS boundary transitions.
2031  */
2032 struct hammer2_recovery_elm {
2033         TAILQ_ENTRY(hammer2_recovery_elm) entry;
2034         hammer2_chain_t *chain;
2035         hammer2_tid_t sync_tid;
2036 };
2037
2038 TAILQ_HEAD(hammer2_recovery_list, hammer2_recovery_elm);
2039
2040 struct hammer2_recovery_info {
2041         struct hammer2_recovery_list list;
2042         hammer2_tid_t   mtid;
2043         int     depth;
2044 };
2045
2046 static int hammer2_recovery_scan(hammer2_dev_t *hmp,
2047                         hammer2_chain_t *parent,
2048                         struct hammer2_recovery_info *info,
2049                         hammer2_tid_t sync_tid);
2050
2051 #define HAMMER2_RECOVERY_MAXDEPTH       10
2052
2053 static
2054 int
2055 hammer2_recovery(hammer2_dev_t *hmp)
2056 {
2057         struct hammer2_recovery_info info;
2058         struct hammer2_recovery_elm *elm;
2059         hammer2_chain_t *parent;
2060         hammer2_tid_t sync_tid;
2061         hammer2_tid_t mirror_tid;
2062         int error;
2063
2064         hammer2_trans_init(hmp->spmp, 0);
2065
2066         sync_tid = hmp->voldata.freemap_tid;
2067         mirror_tid = hmp->voldata.mirror_tid;
2068
2069         kprintf("hammer2 mount \"%s\": ", hmp->devrepname);
2070         if (sync_tid >= mirror_tid) {
2071                 kprintf(" no recovery needed\n");
2072         } else {
2073                 kprintf(" freemap recovery %016jx-%016jx\n",
2074                         sync_tid + 1, mirror_tid);
2075         }
2076
2077         TAILQ_INIT(&info.list);
2078         info.depth = 0;
2079         parent = hammer2_chain_lookup_init(&hmp->vchain, 0);
2080         error = hammer2_recovery_scan(hmp, parent, &info, sync_tid);
2081         hammer2_chain_lookup_done(parent);
2082
2083         while ((elm = TAILQ_FIRST(&info.list)) != NULL) {
2084                 TAILQ_REMOVE(&info.list, elm, entry);
2085                 parent = elm->chain;
2086                 sync_tid = elm->sync_tid;
2087                 kfree(elm, M_HAMMER2);
2088
2089                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
2090                 error |= hammer2_recovery_scan(hmp, parent, &info,
2091                                               hmp->voldata.freemap_tid);
2092                 hammer2_chain_unlock(parent);
2093                 hammer2_chain_drop(parent);     /* drop elm->chain ref */
2094         }
2095         hammer2_trans_done(hmp->spmp);
2096
2097         return error;
2098 }
2099
2100 static
2101 int
2102 hammer2_recovery_scan(hammer2_dev_t *hmp, hammer2_chain_t *parent,
2103                       struct hammer2_recovery_info *info,
2104                       hammer2_tid_t sync_tid)
2105 {
2106         const hammer2_inode_data_t *ripdata;
2107         hammer2_chain_t *chain;
2108         hammer2_blockref_t bref;
2109         int tmp_error;
2110         int rup_error;
2111         int error;
2112         int first;
2113
2114         /*
2115          * Adjust freemap to ensure that the block(s) are marked allocated.
2116          */
2117         if (parent->bref.type != HAMMER2_BREF_TYPE_VOLUME) {
2118                 hammer2_freemap_adjust(hmp, &parent->bref,
2119                                        HAMMER2_FREEMAP_DORECOVER);
2120         }
2121
2122         /*
2123          * Check type for recursive scan
2124          */
2125         switch(parent->bref.type) {
2126         case HAMMER2_BREF_TYPE_VOLUME:
2127                 /* data already instantiated */
2128                 break;
2129         case HAMMER2_BREF_TYPE_INODE:
2130                 /*
2131                  * Must instantiate data for DIRECTDATA test and also
2132                  * for recursion.
2133                  */
2134                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
2135                 ripdata = &hammer2_chain_rdata(parent)->ipdata;
2136                 if (ripdata->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA) {
2137                         /* not applicable to recovery scan */
2138                         hammer2_chain_unlock(parent);
2139                         return 0;
2140                 }
2141                 hammer2_chain_unlock(parent);
2142                 break;
2143         case HAMMER2_BREF_TYPE_INDIRECT:
2144                 /*
2145                  * Must instantiate data for recursion
2146                  */
2147                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
2148                 hammer2_chain_unlock(parent);
2149                 break;
2150         case HAMMER2_BREF_TYPE_DIRENT:
2151         case HAMMER2_BREF_TYPE_DATA:
2152         case HAMMER2_BREF_TYPE_FREEMAP:
2153         case HAMMER2_BREF_TYPE_FREEMAP_NODE:
2154         case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
2155                 /* not applicable to recovery scan */
2156                 return 0;
2157                 break;
2158         default:
2159                 return HAMMER2_ERROR_BADBREF;
2160         }
2161
2162         /*
2163          * Defer operation if depth limit reached or if we are crossing a
2164          * PFS boundary.
2165          */
2166         if (info->depth >= HAMMER2_RECOVERY_MAXDEPTH) {
2167                 struct hammer2_recovery_elm *elm;
2168
2169                 elm = kmalloc(sizeof(*elm), M_HAMMER2, M_ZERO | M_WAITOK);
2170                 elm->chain = parent;
2171                 elm->sync_tid = sync_tid;
2172                 hammer2_chain_ref(parent);
2173                 TAILQ_INSERT_TAIL(&info->list, elm, entry);
2174                 /* unlocked by caller */
2175
2176                 return(0);
2177         }
2178
2179
2180         /*
2181          * Recursive scan of the last flushed transaction only.  We are
2182          * doing this without pmp assignments so don't leave the chains
2183          * hanging around after we are done with them.
2184          *
2185          * error        Cumulative error this level only
2186          * rup_error    Cumulative error for recursion
2187          * tmp_error    Specific non-cumulative recursion error
2188          */
2189         chain = NULL;
2190         first = 1;
2191         rup_error = 0;
2192         error = 0;
2193
2194         for (;;) {
2195                 error |= hammer2_chain_scan(parent, &chain, &bref,
2196                                             &first,
2197                                             HAMMER2_LOOKUP_NODATA);
2198
2199                 /*
2200                  * Problem during scan or EOF
2201                  */
2202                 if (error)
2203                         break;
2204
2205                 /*
2206                  * If this is a leaf
2207                  */
2208                 if (chain == NULL) {
2209                         if (bref.mirror_tid > sync_tid) {
2210                                 hammer2_freemap_adjust(hmp, &bref,
2211                                                      HAMMER2_FREEMAP_DORECOVER);
2212                         }
2213                         continue;
2214                 }
2215
2216                 /*
2217                  * This may or may not be a recursive node.
2218                  */
2219                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
2220                 if (bref.mirror_tid > sync_tid) {
2221                         ++info->depth;
2222                         tmp_error = hammer2_recovery_scan(hmp, chain,
2223                                                            info, sync_tid);
2224                         --info->depth;
2225                 } else {
2226                         tmp_error = 0;
2227                 }
2228
2229                 /*
2230                  * Flush the recovery at the PFS boundary to stage it for
2231                  * the final flush of the super-root topology.
2232                  */
2233                 if (tmp_error == 0 &&
2234                     (bref.flags & HAMMER2_BREF_FLAG_PFSROOT) &&
2235                     (chain->flags & HAMMER2_CHAIN_ONFLUSH)) {
2236                         hammer2_flush(chain, HAMMER2_FLUSH_TOP);
2237                 }
2238                 rup_error |= tmp_error;
2239         }
2240         return ((error | rup_error) & ~HAMMER2_ERROR_EOF);
2241 }
2242
2243 /*
2244  * Sync a mount point; this is called on a per-mount basis from the
2245  * filesystem syncer process periodically and whenever a user issues
2246  * a sync.
2247  */
2248 int
2249 hammer2_vfs_sync(struct mount *mp, int waitfor)
2250 {
2251         hammer2_xop_flush_t *xop;
2252         struct hammer2_sync_info info;
2253         hammer2_inode_t *iroot;
2254         hammer2_pfs_t *pmp;
2255         int flags;
2256         int error;
2257
2258         pmp = MPTOPMP(mp);
2259         iroot = pmp->iroot;
2260         KKASSERT(iroot);
2261         KKASSERT(iroot->pmp == pmp);
2262
2263         /*
2264          * We can't acquire locks on existing vnodes while in a transaction
2265          * without risking a deadlock.  This assumes that vfsync() can be
2266          * called without the vnode locked (which it can in DragonFly).
2267          * Otherwise we'd have to implement a multi-pass or flag the lock
2268          * failures and retry.
2269          *
2270          * The reclamation code interlocks with the sync list's token
2271          * (by removing the vnode from the scan list) before unlocking
2272          * the inode, giving us time to ref the inode.
2273          */
2274         /*flags = VMSC_GETVP;*/
2275         flags = 0;
2276         if (waitfor & MNT_LAZY)
2277                 flags |= VMSC_ONEPASS;
2278
2279         /*
2280          * Preflush the vnodes using a normal transaction before interlocking
2281          * with a flush transaction.  We do this to try to run as much of
2282          * the compression as possible outside the flush transaction.
2283          *
2284          * For efficiency do an async pass before making sure with a
2285          * synchronous pass on all related buffer cache buffers.
2286          */
2287         hammer2_trans_init(pmp, 0);
2288         info.error = 0;
2289         info.waitfor = MNT_NOWAIT;
2290         vsyncscan(mp, flags | VMSC_NOWAIT, hammer2_sync_scan2, &info);
2291         info.waitfor = MNT_WAIT;
2292         vsyncscan(mp, flags, hammer2_sync_scan2, &info);
2293         hammer2_trans_done(pmp);
2294
2295         /*
2296          * Start our flush transaction.  This does not return until all
2297          * concurrent transactions have completed and will prevent any
2298          * new transactions from running concurrently, except for the
2299          * buffer cache transactions.
2300          *
2301          * (1) vfsync() all dirty vnodes via vfsyncscan().
2302          *
2303          * (2) Flush any remaining dirty inodes (the sideq), including any
2304          *     which may have been created during or raced against the
2305          *     vfsync().  To catch all cases this must be done after the
2306          *     vfsync().
2307          *
2308          * (3) Wait for any pending BIO I/O to complete (hammer2_bioq_sync()).
2309          *
2310          * NOTE! It is still possible for the paging code to push pages
2311          *       out via a UIO_NOCOPY hammer2_vop_write() during the main
2312          *       flush.
2313          */
2314         hammer2_trans_init(pmp, HAMMER2_TRANS_ISFLUSH);
2315
2316         info.error = 0;
2317         info.waitfor = MNT_NOWAIT;
2318         vsyncscan(mp, flags | VMSC_NOWAIT, hammer2_sync_scan2, &info);
2319         info.waitfor = MNT_WAIT;
2320         vsyncscan(mp, flags, hammer2_sync_scan2, &info);
2321         hammer2_inode_run_sideq(pmp);
2322         hammer2_bioq_sync(pmp);
2323
2324         /*
2325          * Use the XOP interface to concurrently flush all nodes to
2326          * synchronize the PFSROOT subtopology to the media.  A standard
2327          * end-of-scan ENOENT error indicates cluster sufficiency.
2328          *
2329          * Note that this flush will not be visible on crash recovery until
2330          * we flush the super-root topology in the next loop.
2331          *
2332          * XXX For now wait for all flushes to complete.
2333          */
2334         if (iroot) {
2335                 xop = hammer2_xop_alloc(iroot, HAMMER2_XOP_MODIFYING);
2336                 hammer2_xop_start(&xop->head, hammer2_inode_xop_flush);
2337                 error = hammer2_xop_collect(&xop->head,
2338                                             HAMMER2_XOP_COLLECT_WAITALL);
2339                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
2340                 if (error == HAMMER2_ERROR_ENOENT)
2341                         error = 0;
2342                 else
2343                         error = hammer2_error_to_errno(error);
2344         } else {
2345                 error = 0;
2346         }
2347         hammer2_trans_done(pmp);
2348
2349         return (error);
2350 }
2351
2352 /*
2353  * Sync passes.
2354  *
2355  * Note that we ignore the tranasction mtid we got above.  Instead,
2356  * each vfsync below will ultimately get its own via TRANS_BUFCACHE
2357  * transactions.
2358  */
2359 static int
2360 hammer2_sync_scan2(struct mount *mp, struct vnode *vp, void *data)
2361 {
2362         struct hammer2_sync_info *info = data;
2363         hammer2_inode_t *ip;
2364         int error;
2365
2366         /*
2367          * Degenerate cases.  Note that ip == NULL typically means the
2368          * syncer vnode itself and we don't want to vclrisdirty() in that
2369          * situation.
2370          */
2371         ip = VTOI(vp);
2372         if (ip == NULL) {
2373                 return(0);
2374         }
2375         if (vp->v_type == VNON || vp->v_type == VBAD) {
2376                 vclrisdirty(vp);
2377                 return(0);
2378         }
2379
2380         /*
2381          * VOP_FSYNC will start a new transaction so replicate some code
2382          * here to do it inline (see hammer2_vop_fsync()).
2383          *
2384          * WARNING: The vfsync interacts with the buffer cache and might
2385          *          block, we can't hold the inode lock at that time.
2386          *          However, we MUST ref ip before blocking to ensure that
2387          *          it isn't ripped out from under us (since we do not
2388          *          hold a lock on the vnode).
2389          */
2390         hammer2_inode_ref(ip);
2391         if ((ip->flags & HAMMER2_INODE_MODIFIED) ||
2392             !RB_EMPTY(&vp->v_rbdirty_tree)) {
2393                 vfsync(vp, info->waitfor, 1, NULL, NULL);
2394                 if (ip->flags & (HAMMER2_INODE_RESIZED |
2395                                  HAMMER2_INODE_MODIFIED)) {
2396                         hammer2_inode_lock(ip, 0);
2397                         if (ip->flags & (HAMMER2_INODE_RESIZED |
2398                                          HAMMER2_INODE_MODIFIED)) {
2399                                 hammer2_inode_chain_sync(ip);
2400                         }
2401                         hammer2_inode_unlock(ip);
2402                 }
2403         }
2404         if ((ip->flags & HAMMER2_INODE_MODIFIED) == 0 &&
2405             RB_EMPTY(&vp->v_rbdirty_tree)) {
2406                 vclrisdirty(vp);
2407         }
2408
2409         hammer2_inode_drop(ip);
2410 #if 1
2411         error = 0;
2412         if (error)
2413                 info->error = error;
2414 #endif
2415         return(0);
2416 }
2417
2418 static
2419 int
2420 hammer2_vfs_vptofh(struct vnode *vp, struct fid *fhp)
2421 {
2422         hammer2_inode_t *ip;
2423
2424         KKASSERT(MAXFIDSZ >= 16);
2425         ip = VTOI(vp);
2426         fhp->fid_len = offsetof(struct fid, fid_data[16]);
2427         fhp->fid_ext = 0;
2428         ((hammer2_tid_t *)fhp->fid_data)[0] = ip->meta.inum;
2429         ((hammer2_tid_t *)fhp->fid_data)[1] = 0;
2430
2431         return 0;
2432 }
2433
2434 static
2435 int
2436 hammer2_vfs_fhtovp(struct mount *mp, struct vnode *rootvp,
2437                struct fid *fhp, struct vnode **vpp)
2438 {
2439         hammer2_pfs_t *pmp;
2440         hammer2_tid_t inum;
2441         int error;
2442
2443         pmp = MPTOPMP(mp);
2444         inum = ((hammer2_tid_t *)fhp->fid_data)[0] & HAMMER2_DIRHASH_USERMSK;
2445         if (vpp) {
2446                 if (inum == 1)
2447                         error = hammer2_vfs_root(mp, vpp);
2448                 else
2449                         error = hammer2_vfs_vget(mp, NULL, inum, vpp);
2450         } else {
2451                 error = 0;
2452         }
2453         if (error)
2454                 kprintf("fhtovp: %016jx -> %p, %d\n", inum, *vpp, error);
2455         return error;
2456 }
2457
2458 static
2459 int
2460 hammer2_vfs_checkexp(struct mount *mp, struct sockaddr *nam,
2461                  int *exflagsp, struct ucred **credanonp)
2462 {
2463         hammer2_pfs_t *pmp;
2464         struct netcred *np;
2465         int error;
2466
2467         pmp = MPTOPMP(mp);
2468         np = vfs_export_lookup(mp, &pmp->export, nam);
2469         if (np) {
2470                 *exflagsp = np->netc_exflags;
2471                 *credanonp = &np->netc_anon;
2472                 error = 0;
2473         } else {
2474                 error = EACCES;
2475         }
2476         return error;
2477 }
2478
2479 /*
2480  * Support code for hammer2_vfs_mount().  Read, verify, and install the volume
2481  * header into the HMP
2482  *
2483  * XXX read four volhdrs and use the one with the highest TID whos CRC
2484  *     matches.
2485  *
2486  * XXX check iCRCs.
2487  *
2488  * XXX For filesystems w/ less than 4 volhdrs, make sure to not write to
2489  *     nonexistant locations.
2490  *
2491  * XXX Record selected volhdr and ring updates to each of 4 volhdrs
2492  */
2493 static
2494 int
2495 hammer2_install_volume_header(hammer2_dev_t *hmp)
2496 {
2497         hammer2_volume_data_t *vd;
2498         struct buf *bp;
2499         hammer2_crc32_t crc0, crc, bcrc0, bcrc;
2500         int error_reported;
2501         int error;
2502         int valid;
2503         int i;
2504
2505         error_reported = 0;
2506         error = 0;
2507         valid = 0;
2508         bp = NULL;
2509
2510         /*
2511          * There are up to 4 copies of the volume header (syncs iterate
2512          * between them so there is no single master).  We don't trust the
2513          * volu_size field so we don't know precisely how large the filesystem
2514          * is, so depend on the OS to return an error if we go beyond the
2515          * block device's EOF.
2516          */
2517         for (i = 0; i < HAMMER2_NUM_VOLHDRS; i++) {
2518                 error = bread(hmp->devvp, i * HAMMER2_ZONE_BYTES64,
2519                               HAMMER2_VOLUME_BYTES, &bp);
2520                 if (error) {
2521                         brelse(bp);
2522                         bp = NULL;
2523                         continue;
2524                 }
2525
2526                 vd = (struct hammer2_volume_data *) bp->b_data;
2527                 if ((vd->magic != HAMMER2_VOLUME_ID_HBO) &&
2528                     (vd->magic != HAMMER2_VOLUME_ID_ABO)) {
2529                         brelse(bp);
2530                         bp = NULL;
2531                         continue;
2532                 }
2533
2534                 if (vd->magic == HAMMER2_VOLUME_ID_ABO) {
2535                         /* XXX: Reversed-endianness filesystem */
2536                         kprintf("hammer2: reverse-endian filesystem detected");
2537                         brelse(bp);
2538                         bp = NULL;
2539                         continue;
2540                 }
2541
2542                 crc = vd->icrc_sects[HAMMER2_VOL_ICRC_SECT0];
2543                 crc0 = hammer2_icrc32(bp->b_data + HAMMER2_VOLUME_ICRC0_OFF,
2544                                       HAMMER2_VOLUME_ICRC0_SIZE);
2545                 bcrc = vd->icrc_sects[HAMMER2_VOL_ICRC_SECT1];
2546                 bcrc0 = hammer2_icrc32(bp->b_data + HAMMER2_VOLUME_ICRC1_OFF,
2547                                        HAMMER2_VOLUME_ICRC1_SIZE);
2548                 if ((crc0 != crc) || (bcrc0 != bcrc)) {
2549                         kprintf("hammer2 volume header crc "
2550                                 "mismatch copy #%d %08x/%08x\n",
2551                                 i, crc0, crc);
2552                         error_reported = 1;
2553                         brelse(bp);
2554                         bp = NULL;
2555                         continue;
2556                 }
2557                 if (valid == 0 || hmp->voldata.mirror_tid < vd->mirror_tid) {
2558                         valid = 1;
2559                         hmp->voldata = *vd;
2560                         hmp->volhdrno = i;
2561                 }
2562                 brelse(bp);
2563                 bp = NULL;
2564         }
2565         if (valid) {
2566                 hmp->volsync = hmp->voldata;
2567                 hmp->free_reserved = hmp->voldata.allocator_size / 20;
2568                 error = 0;
2569                 if (error_reported || bootverbose || 1) { /* 1/DEBUG */
2570                         kprintf("hammer2: using volume header #%d\n",
2571                                 hmp->volhdrno);
2572                 }
2573         } else {
2574                 error = EINVAL;
2575                 kprintf("hammer2: no valid volume headers found!\n");
2576         }
2577         return (error);
2578 }
2579
2580 /*
2581  * This handles hysteresis on regular file flushes.  Because the BIOs are
2582  * routed to a thread it is possible for an excessive number to build up
2583  * and cause long front-end stalls long before the runningbuffspace limit
2584  * is hit, so we implement hammer2_flush_pipe to control the
2585  * hysteresis.
2586  *
2587  * This is a particular problem when compression is used.
2588  */
2589 void
2590 hammer2_lwinprog_ref(hammer2_pfs_t *pmp)
2591 {
2592         atomic_add_int(&pmp->count_lwinprog, 1);
2593 }
2594
2595 void
2596 hammer2_lwinprog_drop(hammer2_pfs_t *pmp)
2597 {
2598         int lwinprog;
2599
2600         lwinprog = atomic_fetchadd_int(&pmp->count_lwinprog, -1);
2601         if ((lwinprog & HAMMER2_LWINPROG_WAITING) &&
2602             (lwinprog & HAMMER2_LWINPROG_MASK) <= hammer2_flush_pipe * 2 / 3) {
2603                 atomic_clear_int(&pmp->count_lwinprog,
2604                                  HAMMER2_LWINPROG_WAITING);
2605                 wakeup(&pmp->count_lwinprog);
2606         }
2607         if ((lwinprog & HAMMER2_LWINPROG_WAITING0) &&
2608             (lwinprog & HAMMER2_LWINPROG_MASK) <= 0) {
2609                 atomic_clear_int(&pmp->count_lwinprog,
2610                                  HAMMER2_LWINPROG_WAITING0);
2611                 wakeup(&pmp->count_lwinprog);
2612         }
2613 }
2614
2615 void
2616 hammer2_lwinprog_wait(hammer2_pfs_t *pmp, int flush_pipe)
2617 {
2618         int lwinprog;
2619         int lwflag = (flush_pipe) ? HAMMER2_LWINPROG_WAITING :
2620                                     HAMMER2_LWINPROG_WAITING0;
2621
2622         for (;;) {
2623                 lwinprog = pmp->count_lwinprog;
2624                 cpu_ccfence();
2625                 if ((lwinprog & HAMMER2_LWINPROG_MASK) <= flush_pipe)
2626                         break;
2627                 tsleep_interlock(&pmp->count_lwinprog, 0);
2628                 atomic_set_int(&pmp->count_lwinprog, lwflag);
2629                 lwinprog = pmp->count_lwinprog;
2630                 if ((lwinprog & HAMMER2_LWINPROG_MASK) <= flush_pipe)
2631                         break;
2632                 tsleep(&pmp->count_lwinprog, PINTERLOCKED, "h2wpipe", hz);
2633         }
2634 }
2635
2636 /*
2637  * Manage excessive memory resource use for chain and related
2638  * structures.
2639  */
2640 void
2641 hammer2_pfs_memory_wait(hammer2_pfs_t *pmp)
2642 {
2643         uint32_t waiting;
2644         uint32_t count;
2645         uint32_t limit;
2646 #if 0
2647         static int zzticks;
2648 #endif
2649
2650         /*
2651          * Atomic check condition and wait.  Also do an early speedup of
2652          * the syncer to try to avoid hitting the wait.
2653          */
2654         for (;;) {
2655                 waiting = pmp->inmem_dirty_chains;
2656                 cpu_ccfence();
2657                 count = waiting & HAMMER2_DIRTYCHAIN_MASK;
2658
2659                 limit = pmp->mp->mnt_nvnodelistsize / 10;
2660                 if (limit < hammer2_limit_dirty_chains)
2661                         limit = hammer2_limit_dirty_chains;
2662                 if (limit < 1000)
2663                         limit = 1000;
2664
2665 #if 0
2666                 if ((int)(ticks - zzticks) > hz) {
2667                         zzticks = ticks;
2668                         kprintf("count %ld %ld\n", count, limit);
2669                 }
2670 #endif
2671
2672                 /*
2673                  * Block if there are too many dirty chains present, wait
2674                  * for the flush to clean some out.
2675                  */
2676                 if (count > limit) {
2677                         tsleep_interlock(&pmp->inmem_dirty_chains, 0);
2678                         if (atomic_cmpset_int(&pmp->inmem_dirty_chains,
2679                                                waiting,
2680                                        waiting | HAMMER2_DIRTYCHAIN_WAITING)) {
2681                                 speedup_syncer(pmp->mp);
2682                                 tsleep(&pmp->inmem_dirty_chains, PINTERLOCKED,
2683                                        "chnmem", hz);
2684                         }
2685                         continue;       /* loop on success or fail */
2686                 }
2687
2688                 /*
2689                  * Try to start an early flush before we are forced to block.
2690                  */
2691                 if (count > limit * 7 / 10)
2692                         speedup_syncer(pmp->mp);
2693                 break;
2694         }
2695 }
2696
2697 void
2698 hammer2_pfs_memory_inc(hammer2_pfs_t *pmp)
2699 {
2700         if (pmp) {
2701                 atomic_add_int(&pmp->inmem_dirty_chains, 1);
2702         }
2703 }
2704
2705 void
2706 hammer2_pfs_memory_wakeup(hammer2_pfs_t *pmp)
2707 {
2708         uint32_t waiting;
2709
2710         if (pmp == NULL)
2711                 return;
2712
2713         for (;;) {
2714                 waiting = pmp->inmem_dirty_chains;
2715                 cpu_ccfence();
2716                 if (atomic_cmpset_int(&pmp->inmem_dirty_chains,
2717                                        waiting,
2718                                        (waiting - 1) &
2719                                         ~HAMMER2_DIRTYCHAIN_WAITING)) {
2720                         break;
2721                 }
2722         }
2723
2724         if (waiting & HAMMER2_DIRTYCHAIN_WAITING)
2725                 wakeup(&pmp->inmem_dirty_chains);
2726 }
2727
2728 /*
2729  * Returns 0 if the filesystem has tons of free space
2730  * Returns 1 if the filesystem has less than 10% remaining
2731  * Returns 2 if the filesystem has less than 2%/5% (user/root) remaining.
2732  */
2733 int
2734 hammer2_vfs_enospace(hammer2_inode_t *ip, off_t bytes, struct ucred *cred)
2735 {
2736         hammer2_pfs_t *pmp;
2737         hammer2_dev_t *hmp;
2738         hammer2_off_t free_reserved;
2739         hammer2_off_t free_nominal;
2740         int i;
2741
2742         pmp = ip->pmp;
2743
2744         if (pmp->free_ticks == 0 || pmp->free_ticks != ticks) {
2745                 free_reserved = HAMMER2_SEGSIZE;
2746                 free_nominal = 0x7FFFFFFFFFFFFFFFLLU;
2747                 for (i = 0; i < pmp->iroot->cluster.nchains; ++i) {
2748                         hmp = pmp->pfs_hmps[i];
2749                         if (hmp == NULL)
2750                                 continue;
2751                         if (pmp->pfs_types[i] != HAMMER2_PFSTYPE_MASTER &&
2752                             pmp->pfs_types[i] != HAMMER2_PFSTYPE_SOFT_MASTER)
2753                                 continue;
2754
2755                         if (free_nominal > hmp->voldata.allocator_free)
2756                                 free_nominal = hmp->voldata.allocator_free;
2757                         if (free_reserved < hmp->free_reserved)
2758                                 free_reserved = hmp->free_reserved;
2759                 }
2760
2761                 /*
2762                  * SMP races ok
2763                  */
2764                 pmp->free_reserved = free_reserved;
2765                 pmp->free_nominal = free_nominal;
2766                 pmp->free_ticks = ticks;
2767         } else {
2768                 free_reserved = pmp->free_reserved;
2769                 free_nominal = pmp->free_nominal;
2770         }
2771         if (cred && cred->cr_uid != 0) {
2772                 if ((int64_t)(free_nominal - bytes) <
2773                     (int64_t)free_reserved) {
2774                         return 2;
2775                 }
2776         } else {
2777                 if ((int64_t)(free_nominal - bytes) <
2778                     (int64_t)free_reserved / 2) {
2779                         return 2;
2780                 }
2781         }
2782         if ((int64_t)(free_nominal - bytes) < (int64_t)free_reserved * 2)
2783                 return 1;
2784         return 0;
2785 }
2786
2787 /*
2788  * Debugging
2789  */
2790 void
2791 hammer2_dump_chain(hammer2_chain_t *chain, int tab, int *countp, char pfx)
2792 {
2793         hammer2_chain_t *scan;
2794         hammer2_chain_t *parent;
2795
2796         --*countp;
2797         if (*countp == 0) {
2798                 kprintf("%*.*s...\n", tab, tab, "");
2799                 return;
2800         }
2801         if (*countp < 0)
2802                 return;
2803         kprintf("%*.*s%c-chain %p.%d %016jx/%d mir=%016jx\n",
2804                 tab, tab, "", pfx,
2805                 chain, chain->bref.type,
2806                 chain->bref.key, chain->bref.keybits,
2807                 chain->bref.mirror_tid);
2808
2809         kprintf("%*.*s      [%08x] (%s) refs=%d",
2810                 tab, tab, "",
2811                 chain->flags,
2812                 ((chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
2813                 chain->data) ?  (char *)chain->data->ipdata.filename : "?"),
2814                 chain->refs);
2815
2816         parent = chain->parent;
2817         if (parent)
2818                 kprintf("\n%*.*s      p=%p [pflags %08x prefs %d",
2819                         tab, tab, "",
2820                         parent, parent->flags, parent->refs);
2821         if (RB_EMPTY(&chain->core.rbtree)) {
2822                 kprintf("\n");
2823         } else {
2824                 kprintf(" {\n");
2825                 RB_FOREACH(scan, hammer2_chain_tree, &chain->core.rbtree)
2826                         hammer2_dump_chain(scan, tab + 4, countp, 'a');
2827                 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE && chain->data)
2828                         kprintf("%*.*s}(%s)\n", tab, tab, "",
2829                                 chain->data->ipdata.filename);
2830                 else
2831                         kprintf("%*.*s}\n", tab, tab, "");
2832         }
2833 }