sys/vfs/hammer2/hammer2_vfsops.c

   1 /*
   2  * Copyright (c) 2011-2018 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  * by Daniel Flores (GSOC 2013 - mentored by Matthew Dillon, compression)
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in
  16  *    the documentation and/or other materials provided with the
  17  *    distribution.
  18  * 3. Neither the name of The DragonFly Project nor the names of its
  19  *    contributors may be used to endorse or promote products derived
  20  *    from this software without specific, prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  26  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  27  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  28  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  29  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  30  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  31  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  32  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  33  * SUCH DAMAGE.
  34  */
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/kernel.h>
  38 #include <sys/nlookup.h>
  39 #include <sys/vnode.h>
  40 #include <sys/mount.h>
  41 #include <sys/fcntl.h>
  42 #include <sys/buf.h>
  43 #include <sys/uuid.h>
  44 #include <sys/vfsops.h>
  45 #include <sys/sysctl.h>
  46 #include <sys/socket.h>
  47 #include <sys/objcache.h>
  48
  49 #include <sys/proc.h>
  50 #include <sys/mountctl.h>
  51 #include <sys/dirent.h>
  52 #include <sys/uio.h>
  53
  54 #include "hammer2.h"
  55 #include "hammer2_disk.h"
  56 #include "hammer2_mount.h"
  57 #include "hammer2_lz4.h"
  58
  59 #include "zlib/hammer2_zlib.h"
  60
  61 #define REPORT_REFS_ERRORS 1    /* XXX remove me */
  62
  63 MALLOC_DEFINE(M_OBJCACHE, "objcache", "Object Cache");
  64
  65 struct hammer2_sync_info {
  66         int error;
  67         int waitfor;
  68         int pass;
  69 };
  70
  71 TAILQ_HEAD(hammer2_mntlist, hammer2_dev);
  72 static struct hammer2_mntlist hammer2_mntlist;
  73
  74 struct hammer2_pfslist hammer2_pfslist;
  75 struct hammer2_pfslist hammer2_spmplist;
  76 struct lock hammer2_mntlk;
  77
  78 int hammer2_supported_version = HAMMER2_VOL_VERSION_DEFAULT;
  79 int hammer2_debug;
  80 int hammer2_xopgroups;
  81 long hammer2_debug_inode;
  82 int hammer2_cluster_meta_read = 1;      /* physical read-ahead */
  83 int hammer2_cluster_data_read = 4;      /* physical read-ahead */
  84 int hammer2_cluster_write = 0;          /* physical write clustering */
  85 int hammer2_dedup_enable = 1;
  86 int hammer2_always_compress = 0;        /* always try to compress */
  87 int hammer2_flush_pipe = 100;
  88 int hammer2_dio_count;
  89 int hammer2_dio_limit = 256;
  90 int hammer2_bulkfree_tps = 5000;
  91 int hammer2_worker_rmask = 3;
  92 long hammer2_chain_allocs;
  93 long hammer2_limit_dirty_chains;
  94 long hammer2_limit_dirty_inodes;
  95 long hammer2_count_modified_chains;
  96 long hammer2_iod_file_read;
  97 long hammer2_iod_meta_read;
  98 long hammer2_iod_indr_read;
  99 long hammer2_iod_fmap_read;
 100 long hammer2_iod_volu_read;
 101 long hammer2_iod_file_write;
 102 long hammer2_iod_file_wembed;
 103 long hammer2_iod_file_wzero;
 104 long hammer2_iod_file_wdedup;
 105 long hammer2_iod_meta_write;
 106 long hammer2_iod_indr_write;
 107 long hammer2_iod_fmap_write;
 108 long hammer2_iod_volu_write;
 109 static long hammer2_iod_inode_creates;
 110 static long hammer2_iod_inode_deletes;
 111
 112 long hammer2_process_icrc32;
 113 long hammer2_process_xxhash64;
 114
 115 MALLOC_DECLARE(M_HAMMER2_CBUFFER);
 116 MALLOC_DEFINE(M_HAMMER2_CBUFFER, "HAMMER2-compbuffer",
 117                 "Buffer used for compression.");
 118
 119 MALLOC_DECLARE(M_HAMMER2_DEBUFFER);
 120 MALLOC_DEFINE(M_HAMMER2_DEBUFFER, "HAMMER2-decompbuffer",
 121                 "Buffer used for decompression.");
 122
 123 SYSCTL_NODE(_vfs, OID_AUTO, hammer2, CTLFLAG_RW, 0, "HAMMER2 filesystem");
 124
 125 SYSCTL_INT(_vfs_hammer2, OID_AUTO, supported_version, CTLFLAG_RD,
 126            &hammer2_supported_version, 0, "");
 127 SYSCTL_INT(_vfs_hammer2, OID_AUTO, debug, CTLFLAG_RW,
 128            &hammer2_debug, 0, "");
 129 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, debug_inode, CTLFLAG_RW,
 130            &hammer2_debug_inode, 0, "");
 131 SYSCTL_INT(_vfs_hammer2, OID_AUTO, cluster_meta_read, CTLFLAG_RW,
 132            &hammer2_cluster_meta_read, 0, "");
 133 SYSCTL_INT(_vfs_hammer2, OID_AUTO, cluster_data_read, CTLFLAG_RW,
 134            &hammer2_cluster_data_read, 0, "");
 135 SYSCTL_INT(_vfs_hammer2, OID_AUTO, cluster_write, CTLFLAG_RW,
 136            &hammer2_cluster_write, 0, "");
 137 SYSCTL_INT(_vfs_hammer2, OID_AUTO, dedup_enable, CTLFLAG_RW,
 138            &hammer2_dedup_enable, 0, "");
 139 SYSCTL_INT(_vfs_hammer2, OID_AUTO, always_compress, CTLFLAG_RW,
 140            &hammer2_always_compress, 0, "");
 141 SYSCTL_INT(_vfs_hammer2, OID_AUTO, flush_pipe, CTLFLAG_RW,
 142            &hammer2_flush_pipe, 0, "");
 143 SYSCTL_INT(_vfs_hammer2, OID_AUTO, worker_rmask, CTLFLAG_RW,
 144            &hammer2_worker_rmask, 0, "");
 145 SYSCTL_INT(_vfs_hammer2, OID_AUTO, bulkfree_tps, CTLFLAG_RW,
 146            &hammer2_bulkfree_tps, 0, "");
 147 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, chain_allocs, CTLFLAG_RW,
 148            &hammer2_chain_allocs, 0, "");
 149 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, limit_dirty_chains, CTLFLAG_RW,
 150            &hammer2_limit_dirty_chains, 0, "");
 151 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, limit_dirty_inodes, CTLFLAG_RW,
 152            &hammer2_limit_dirty_inodes, 0, "");
 153 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, count_modified_chains, CTLFLAG_RW,
 154            &hammer2_count_modified_chains, 0, "");
 155 SYSCTL_INT(_vfs_hammer2, OID_AUTO, dio_count, CTLFLAG_RD,
 156            &hammer2_dio_count, 0, "");
 157 SYSCTL_INT(_vfs_hammer2, OID_AUTO, dio_limit, CTLFLAG_RW,
 158            &hammer2_dio_limit, 0, "");
 159
 160 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_read, CTLFLAG_RW,
 161            &hammer2_iod_file_read, 0, "");
 162 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_meta_read, CTLFLAG_RW,
 163            &hammer2_iod_meta_read, 0, "");
 164 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_indr_read, CTLFLAG_RW,
 165            &hammer2_iod_indr_read, 0, "");
 166 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_fmap_read, CTLFLAG_RW,
 167            &hammer2_iod_fmap_read, 0, "");
 168 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_volu_read, CTLFLAG_RW,
 169            &hammer2_iod_volu_read, 0, "");
 170
 171 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_write, CTLFLAG_RW,
 172            &hammer2_iod_file_write, 0, "");
 173 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_wembed, CTLFLAG_RW,
 174            &hammer2_iod_file_wembed, 0, "");
 175 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_wzero, CTLFLAG_RW,
 176            &hammer2_iod_file_wzero, 0, "");
 177 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_wdedup, CTLFLAG_RW,
 178            &hammer2_iod_file_wdedup, 0, "");
 179 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_meta_write, CTLFLAG_RW,
 180            &hammer2_iod_meta_write, 0, "");
 181 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_indr_write, CTLFLAG_RW,
 182            &hammer2_iod_indr_write, 0, "");
 183 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_fmap_write, CTLFLAG_RW,
 184            &hammer2_iod_fmap_write, 0, "");
 185 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_volu_write, CTLFLAG_RW,
 186            &hammer2_iod_volu_write, 0, "");
 187 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_inode_creates, CTLFLAG_RW,
 188            &hammer2_iod_inode_creates, 0, "");
 189 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_inode_deletes, CTLFLAG_RW,
 190            &hammer2_iod_inode_deletes, 0, "");
 191
 192 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, process_icrc32, CTLFLAG_RW,
 193            &hammer2_process_icrc32, 0, "");
 194 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, process_xxhash64, CTLFLAG_RW,
 195            &hammer2_process_xxhash64, 0, "");
 196
 197 static int hammer2_vfs_init(struct vfsconf *conf);
 198 static int hammer2_vfs_uninit(struct vfsconf *vfsp);
 199 static int hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 200                                 struct ucred *cred);
 201 static int hammer2_remount(hammer2_dev_t *, struct mount *, char *,
 202                                 struct vnode *, struct ucred *);
 203 static int hammer2_recovery(hammer2_dev_t *hmp);
 204 static int hammer2_vfs_unmount(struct mount *mp, int mntflags);
 205 static int hammer2_vfs_root(struct mount *mp, struct vnode **vpp);
 206 static int hammer2_vfs_statfs(struct mount *mp, struct statfs *sbp,
 207                                 struct ucred *cred);
 208 static int hammer2_vfs_statvfs(struct mount *mp, struct statvfs *sbp,
 209                                 struct ucred *cred);
 210 static int hammer2_vfs_fhtovp(struct mount *mp, struct vnode *rootvp,
 211                                 struct fid *fhp, struct vnode **vpp);
 212 static int hammer2_vfs_vptofh(struct vnode *vp, struct fid *fhp);
 213 static int hammer2_vfs_checkexp(struct mount *mp, struct sockaddr *nam,
 214                                 int *exflagsp, struct ucred **credanonp);
 215 static int hammer2_vfs_modifying(struct mount *mp);
 216
 217 static int hammer2_install_volume_header(hammer2_dev_t *hmp);
 218 #if 0
 219 static int hammer2_sync_scan2(struct mount *mp, struct vnode *vp, void *data);
 220 #endif
 221
 222 static void hammer2_update_pmps(hammer2_dev_t *hmp);
 223
 224 static void hammer2_mount_helper(struct mount *mp, hammer2_pfs_t *pmp);
 225 static void hammer2_unmount_helper(struct mount *mp, hammer2_pfs_t *pmp,
 226                                 hammer2_dev_t *hmp);
 227 static int hammer2_fixup_pfses(hammer2_dev_t *hmp);
 228
 229 /*
 230  * HAMMER2 vfs operations.
 231  */
 232 static struct vfsops hammer2_vfsops = {
 233         .vfs_flags      = 0,
 234         .vfs_init       = hammer2_vfs_init,
 235         .vfs_uninit     = hammer2_vfs_uninit,
 236         .vfs_sync       = hammer2_vfs_sync,
 237         .vfs_mount      = hammer2_vfs_mount,
 238         .vfs_unmount    = hammer2_vfs_unmount,
 239         .vfs_root       = hammer2_vfs_root,
 240         .vfs_statfs     = hammer2_vfs_statfs,
 241         .vfs_statvfs    = hammer2_vfs_statvfs,
 242         .vfs_vget       = hammer2_vfs_vget,
 243         .vfs_vptofh     = hammer2_vfs_vptofh,
 244         .vfs_fhtovp     = hammer2_vfs_fhtovp,
 245         .vfs_checkexp   = hammer2_vfs_checkexp,
 246         .vfs_modifying  = hammer2_vfs_modifying
 247 };
 248
 249 MALLOC_DEFINE(M_HAMMER2, "HAMMER2-mount", "");
 250
 251 VFS_SET(hammer2_vfsops, hammer2, VFCF_MPSAFE);
 252 MODULE_VERSION(hammer2, 1);
 253
 254 static
 255 int
 256 hammer2_vfs_init(struct vfsconf *conf)
 257 {
 258         static struct objcache_malloc_args margs_read;
 259         static struct objcache_malloc_args margs_write;
 260         static struct objcache_malloc_args margs_vop;
 261
 262         int error;
 263
 264         error = 0;
 265         kmalloc_raise_limit(M_HAMMER2, 0);      /* unlimited */
 266
 267         /*
 268          * hammer2_xopgroups must be even and is most optimal if
 269          * 2 x ncpus so strategy functions can be queued to the same
 270          * cpu.
 271          */
 272         hammer2_xopgroups = HAMMER2_XOPGROUPS_MIN;
 273         if (hammer2_xopgroups < ncpus * 2)
 274                 hammer2_xopgroups = ncpus * 2;
 275
 276         /*
 277          * A large DIO cache is needed to retain dedup enablement masks.
 278          * The bulkfree code clears related masks as part of the disk block
 279          * recycling algorithm, preventing it from being used for a later
 280          * dedup.
 281          *
 282          * NOTE: A large buffer cache can actually interfere with dedup
 283          *       operation because we dedup based on media physical buffers
 284          *       and not logical buffers.  Try to make the DIO case large
 285          *       enough to avoid this problem, but also cap it.
 286          */
 287         hammer2_dio_limit = nbuf * 2;
 288         if (hammer2_dio_limit > 100000)
 289                 hammer2_dio_limit = 100000;
 290
 291         if (HAMMER2_BLOCKREF_BYTES != sizeof(struct hammer2_blockref))
 292                 error = EINVAL;
 293         if (HAMMER2_INODE_BYTES != sizeof(struct hammer2_inode_data))
 294                 error = EINVAL;
 295         if (HAMMER2_VOLUME_BYTES != sizeof(struct hammer2_volume_data))
 296                 error = EINVAL;
 297
 298         if (error)
 299                 kprintf("HAMMER2 structure size mismatch; cannot continue.\n");
 300
 301         margs_read.objsize = 65536;
 302         margs_read.mtype = M_HAMMER2_DEBUFFER;
 303
 304         margs_write.objsize = 32768;
 305         margs_write.mtype = M_HAMMER2_CBUFFER;
 306
 307         margs_vop.objsize = sizeof(hammer2_xop_t);
 308         margs_vop.mtype = M_HAMMER2;
 309
 310         /*
 311          * Note thaht for the XOPS cache we want backing store allocations
 312          * to use M_ZERO.  This is not allowed in objcache_get() (to avoid
 313          * confusion), so use the backing store function that does it.  This
 314          * means that initial XOPS objects are zerod but REUSED objects are
 315          * not.  So we are responsible for cleaning the object up sufficiently
 316          * for our needs before objcache_put()ing it back (typically just the
 317          * FIFO indices).
 318          */
 319         cache_buffer_read = objcache_create(margs_read.mtype->ks_shortdesc,
 320                                 0, 1, NULL, NULL, NULL,
 321                                 objcache_malloc_alloc,
 322                                 objcache_malloc_free,
 323                                 &margs_read);
 324         cache_buffer_write = objcache_create(margs_write.mtype->ks_shortdesc,
 325                                 0, 1, NULL, NULL, NULL,
 326                                 objcache_malloc_alloc,
 327                                 objcache_malloc_free,
 328                                 &margs_write);
 329         cache_xops = objcache_create(margs_vop.mtype->ks_shortdesc,
 330                                 0, 1, NULL, NULL, NULL,
 331                                 objcache_malloc_alloc_zero,
 332                                 objcache_malloc_free,
 333                                 &margs_vop);
 334
 335
 336         lockinit(&hammer2_mntlk, "mntlk", 0, 0);
 337         TAILQ_INIT(&hammer2_mntlist);
 338         TAILQ_INIT(&hammer2_pfslist);
 339         TAILQ_INIT(&hammer2_spmplist);
 340
 341         hammer2_limit_dirty_chains = maxvnodes / 10;
 342         if (hammer2_limit_dirty_chains > HAMMER2_LIMIT_DIRTY_CHAINS)
 343                 hammer2_limit_dirty_chains = HAMMER2_LIMIT_DIRTY_CHAINS;
 344         if (hammer2_limit_dirty_chains < 1000)
 345                 hammer2_limit_dirty_chains = 1000;
 346
 347         hammer2_limit_dirty_inodes = maxvnodes / 25;
 348         if (hammer2_limit_dirty_inodes < 100)
 349                 hammer2_limit_dirty_inodes = 100;
 350         if (hammer2_limit_dirty_inodes > HAMMER2_LIMIT_DIRTY_INODES)
 351                 hammer2_limit_dirty_inodes = HAMMER2_LIMIT_DIRTY_INODES;
 352
 353         return (error);
 354 }
 355
 356 static
 357 int
 358 hammer2_vfs_uninit(struct vfsconf *vfsp __unused)
 359 {
 360         objcache_destroy(cache_buffer_read);
 361         objcache_destroy(cache_buffer_write);
 362         objcache_destroy(cache_xops);
 363         return 0;
 364 }
 365
 366 /*
 367  * Core PFS allocator.  Used to allocate or reference the pmp structure
 368  * for PFS cluster mounts and the spmp structure for media (hmp) structures.
 369  * The pmp can be passed in or loaded by this function using the chain and
 370  * inode data.
 371  *
 372  * pmp->modify_tid tracks new modify_tid transaction ids for front-end
 373  * transactions.  Note that synchronization does not use this field.
 374  * (typically frontend operations and synchronization cannot run on the
 375  * same PFS node at the same time).
 376  *
 377  * XXX check locking
 378  */
 379 hammer2_pfs_t *
 380 hammer2_pfsalloc(hammer2_chain_t *chain,
 381                  const hammer2_inode_data_t *ripdata,
 382                  hammer2_tid_t modify_tid, hammer2_dev_t *force_local)
 383 {
 384         hammer2_pfs_t *pmp;
 385         hammer2_inode_t *iroot;
 386         int count;
 387         int i;
 388         int j;
 389
 390         pmp = NULL;
 391
 392         /*
 393          * Locate or create the PFS based on the cluster id.  If ripdata
 394          * is NULL this is a spmp which is unique and is always allocated.
 395          *
 396          * If the device is mounted in local mode all PFSs are considered
 397          * independent and not part of any cluster (for debugging only).
 398          */
 399         if (ripdata) {
 400                 TAILQ_FOREACH(pmp, &hammer2_pfslist, mntentry) {
 401                         if (force_local != pmp->force_local)
 402                                 continue;
 403                         if (force_local == NULL &&
 404                             bcmp(&pmp->pfs_clid, &ripdata->meta.pfs_clid,
 405                                  sizeof(pmp->pfs_clid)) == 0) {
 406                                         break;
 407                         } else if (force_local && pmp->pfs_names[0] &&
 408                             strcmp(pmp->pfs_names[0], ripdata->filename) == 0) {
 409                                         break;
 410                         }
 411                 }
 412         }
 413
 414         if (pmp == NULL) {
 415                 pmp = kmalloc(sizeof(*pmp), M_HAMMER2, M_WAITOK | M_ZERO);
 416                 pmp->force_local = force_local;
 417                 hammer2_trans_manage_init(pmp);
 418                 kmalloc_create(&pmp->minode, "HAMMER2-inodes");
 419                 kmalloc_create(&pmp->mmsg, "HAMMER2-pfsmsg");
 420                 lockinit(&pmp->lock, "pfslk", 0, 0);
 421                 lockinit(&pmp->lock_nlink, "h2nlink", 0, 0);
 422                 spin_init(&pmp->inum_spin, "hm2pfsalloc_inum");
 423                 spin_init(&pmp->xop_spin, "h2xop");
 424                 spin_init(&pmp->lru_spin, "h2lru");
 425                 RB_INIT(&pmp->inum_tree);
 426                 TAILQ_INIT(&pmp->syncq);
 427                 TAILQ_INIT(&pmp->depq);
 428                 TAILQ_INIT(&pmp->lru_list);
 429                 spin_init(&pmp->list_spin, "h2pfsalloc_list");
 430
 431                 /*
 432                  * Save the last media transaction id for the flusher.  Set
 433                  * initial
 434                  */
 435                 if (ripdata) {
 436                         pmp->pfs_clid = ripdata->meta.pfs_clid;
 437                         TAILQ_INSERT_TAIL(&hammer2_pfslist, pmp, mntentry);
 438                 } else {
 439                         pmp->flags |= HAMMER2_PMPF_SPMP;
 440                         TAILQ_INSERT_TAIL(&hammer2_spmplist, pmp, mntentry);
 441                 }
 442
 443                 /*
 444                  * The synchronization thread may start too early, make
 445                  * sure it stays frozen until we are ready to let it go.
 446                  * XXX
 447                  */
 448                 /*
 449                 pmp->primary_thr.flags = HAMMER2_THREAD_FROZEN |
 450                                          HAMMER2_THREAD_REMASTER;
 451                 */
 452         }
 453
 454         /*
 455          * Create the PFS's root inode and any missing XOP helper threads.
 456          */
 457         if ((iroot = pmp->iroot) == NULL) {
 458                 iroot = hammer2_inode_get(pmp, NULL, 1, -1);
 459                 if (ripdata)
 460                         iroot->meta = ripdata->meta;
 461                 pmp->iroot = iroot;
 462                 hammer2_inode_ref(iroot);
 463                 hammer2_inode_unlock(iroot);
 464         }
 465
 466         /*
 467          * Stop here if no chain is passed in.
 468          */
 469         if (chain == NULL)
 470                 goto done;
 471
 472         /*
 473          * When a chain is passed in we must add it to the PFS's root
 474          * inode, update pmp->pfs_types[], and update the syncronization
 475          * threads.
 476          *
 477          * When forcing local mode, mark the PFS as a MASTER regardless.
 478          *
 479          * At the moment empty spots can develop due to removals or failures.
 480          * Ultimately we want to re-fill these spots but doing so might
 481          * confused running code. XXX
 482          */
 483         hammer2_inode_ref(iroot);
 484         hammer2_mtx_ex(&iroot->lock);
 485         j = iroot->cluster.nchains;
 486
 487         if (j == HAMMER2_MAXCLUSTER) {
 488                 kprintf("hammer2_pfsalloc: cluster full!\n");
 489                 /* XXX fatal error? */
 490         } else {
 491                 KKASSERT(chain->pmp == NULL);
 492                 chain->pmp = pmp;
 493                 hammer2_chain_ref(chain);
 494                 iroot->cluster.array[j].chain = chain;
 495                 if (force_local)
 496                         pmp->pfs_types[j] = HAMMER2_PFSTYPE_MASTER;
 497                 else
 498                         pmp->pfs_types[j] = ripdata->meta.pfs_type;
 499                 pmp->pfs_names[j] = kstrdup(ripdata->filename, M_HAMMER2);
 500                 pmp->pfs_hmps[j] = chain->hmp;
 501                 hammer2_spin_ex(&pmp->inum_spin);
 502                 pmp->pfs_iroot_blocksets[j] = chain->data->ipdata.u.blockset;
 503                 hammer2_spin_unex(&pmp->inum_spin);
 504
 505                 /*
 506                  * If the PFS is already mounted we must account
 507                  * for the mount_count here.
 508                  */
 509                 if (pmp->mp)
 510                         ++chain->hmp->mount_count;
 511
 512                 /*
 513                  * May have to fixup dirty chain tracking.  Previous
 514                  * pmp was NULL so nothing to undo.
 515                  */
 516                 if (chain->flags & HAMMER2_CHAIN_MODIFIED)
 517                         hammer2_pfs_memory_inc(pmp);
 518                 ++j;
 519         }
 520         iroot->cluster.nchains = j;
 521
 522         /*
 523          * Update nmasters from any PFS inode which is part of the cluster.
 524          * It is possible that this will result in a value which is too
 525          * high.  MASTER PFSs are authoritative for pfs_nmasters and will
 526          * override this value later on.
 527          *
 528          * (This informs us of masters that might not currently be
 529          *  discoverable by this mount).
 530          */
 531         if (ripdata && pmp->pfs_nmasters < ripdata->meta.pfs_nmasters) {
 532                 pmp->pfs_nmasters = ripdata->meta.pfs_nmasters;
 533         }
 534
 535         /*
 536          * Count visible masters.  Masters are usually added with
 537          * ripdata->meta.pfs_nmasters set to 1.  This detects when there
 538          * are more (XXX and must update the master inodes).
 539          */
 540         count = 0;
 541         for (i = 0; i < iroot->cluster.nchains; ++i) {
 542                 if (pmp->pfs_types[i] == HAMMER2_PFSTYPE_MASTER)
 543                         ++count;
 544         }
 545         if (pmp->pfs_nmasters < count)
 546                 pmp->pfs_nmasters = count;
 547
 548         /*
 549          * Create missing synchronization and support threads.
 550          *
 551          * Single-node masters (including snapshots) have nothing to
 552          * synchronize and do not require this thread.
 553          *
 554          * Multi-node masters or any number of soft masters, slaves, copy,
 555          * or other PFS types need the thread.
 556          *
 557          * Each thread is responsible for its particular cluster index.
 558          * We use independent threads so stalls or mismatches related to
 559          * any given target do not affect other targets.
 560          */
 561         for (i = 0; i < iroot->cluster.nchains; ++i) {
 562                 /*
 563                  * Single-node masters (including snapshots) have nothing
 564                  * to synchronize and will make direct xops support calls,
 565                  * thus they do not require this thread.
 566                  *
 567                  * Note that there can be thousands of snapshots.  We do not
 568                  * want to create thousands of threads.
 569                  */
 570                 if (pmp->pfs_nmasters <= 1 &&
 571                     pmp->pfs_types[i] == HAMMER2_PFSTYPE_MASTER) {
 572                         continue;
 573                 }
 574
 575                 /*
 576                  * Sync support thread
 577                  */
 578                 if (pmp->sync_thrs[i].td == NULL) {
 579                         hammer2_thr_create(&pmp->sync_thrs[i], pmp, NULL,
 580                                            "h2nod", i, -1,
 581                                            hammer2_primary_sync_thread);
 582                 }
 583         }
 584
 585         /*
 586          * Create missing Xop threads
 587          *
 588          * NOTE: We create helper threads for all mounted PFSs or any
 589          *       PFSs with 2+ nodes (so the sync thread can update them,
 590          *       even if not mounted).
 591          */
 592         if (pmp->mp || iroot->cluster.nchains >= 2)
 593                 hammer2_xop_helper_create(pmp);
 594
 595         hammer2_mtx_unlock(&iroot->lock);
 596         hammer2_inode_drop(iroot);
 597 done:
 598         return pmp;
 599 }
 600
 601 /*
 602  * Deallocate an element of a probed PFS.  If destroying and this is a
 603  * MASTER, adjust nmasters.
 604  *
 605  * This function does not physically destroy the PFS element in its device
 606  * under the super-root  (see hammer2_ioctl_pfs_delete()).
 607  */
 608 void
 609 hammer2_pfsdealloc(hammer2_pfs_t *pmp, int clindex, int destroying)
 610 {
 611         hammer2_inode_t *iroot;
 612         hammer2_chain_t *chain;
 613         int j;
 614
 615         /*
 616          * Cleanup our reference on iroot.  iroot is (should) not be needed
 617          * by the flush code.
 618          */
 619         iroot = pmp->iroot;
 620         if (iroot) {
 621                 /*
 622                  * Stop synchronizing
 623                  *
 624                  * XXX flush after acquiring the iroot lock.
 625                  * XXX clean out the cluster index from all inode structures.
 626                  */
 627                 hammer2_thr_delete(&pmp->sync_thrs[clindex]);
 628
 629                 /*
 630                  * Remove the cluster index from the group.  If destroying
 631                  * the PFS and this is a master, adjust pfs_nmasters.
 632                  */
 633                 hammer2_mtx_ex(&iroot->lock);
 634                 chain = iroot->cluster.array[clindex].chain;
 635                 iroot->cluster.array[clindex].chain = NULL;
 636
 637                 switch(pmp->pfs_types[clindex]) {
 638                 case HAMMER2_PFSTYPE_MASTER:
 639                         if (destroying && pmp->pfs_nmasters > 0)
 640                                 --pmp->pfs_nmasters;
 641                         /* XXX adjust ripdata->meta.pfs_nmasters */
 642                         break;
 643                 default:
 644                         break;
 645                 }
 646                 pmp->pfs_types[clindex] = HAMMER2_PFSTYPE_NONE;
 647
 648                 hammer2_mtx_unlock(&iroot->lock);
 649
 650                 /*
 651                  * Release the chain.
 652                  */
 653                 if (chain) {
 654                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
 655                         hammer2_chain_drop(chain);
 656                 }
 657
 658                 /*
 659                  * Terminate all XOP threads for the cluster index.
 660                  */
 661                 if (pmp->xop_groups) {
 662                         for (j = 0; j < hammer2_xopgroups; ++j) {
 663                                 hammer2_thr_delete(
 664                                         &pmp->xop_groups[j].thrs[clindex]);
 665                         }
 666                 }
 667         }
 668 }
 669
 670 /*
 671  * Destroy a PFS, typically only occurs after the last mount on a device
 672  * has gone away.
 673  */
 674 static void
 675 hammer2_pfsfree(hammer2_pfs_t *pmp)
 676 {
 677         hammer2_inode_t *iroot;
 678         hammer2_chain_t *chain;
 679         int chains_still_present = 0;
 680         int i;
 681         int j;
 682
 683         /*
 684          * Cleanup our reference on iroot.  iroot is (should) not be needed
 685          * by the flush code.
 686          */
 687         if (pmp->flags & HAMMER2_PMPF_SPMP)
 688                 TAILQ_REMOVE(&hammer2_spmplist, pmp, mntentry);
 689         else
 690                 TAILQ_REMOVE(&hammer2_pfslist, pmp, mntentry);
 691
 692         /*
 693          * Cleanup chains remaining on LRU list.
 694          */
 695         hammer2_spin_ex(&pmp->lru_spin);
 696         while ((chain = TAILQ_FIRST(&pmp->lru_list)) != NULL) {
 697                 KKASSERT(chain->flags & HAMMER2_CHAIN_ONLRU);
 698                 atomic_add_int(&pmp->lru_count, -1);
 699                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONLRU);
 700                 TAILQ_REMOVE(&pmp->lru_list, chain, lru_node);
 701                 hammer2_chain_ref(chain);
 702                 hammer2_spin_unex(&pmp->lru_spin);
 703                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
 704                 hammer2_chain_drop(chain);
 705                 hammer2_spin_ex(&pmp->lru_spin);
 706         }
 707         hammer2_spin_unex(&pmp->lru_spin);
 708
 709         /*
 710          * Clean up iroot
 711          */
 712         iroot = pmp->iroot;
 713         if (iroot) {
 714                 for (i = 0; i < iroot->cluster.nchains; ++i) {
 715                         hammer2_thr_delete(&pmp->sync_thrs[i]);
 716                         if (pmp->xop_groups) {
 717                                 for (j = 0; j < hammer2_xopgroups; ++j)
 718                                         hammer2_thr_delete(
 719                                                 &pmp->xop_groups[j].thrs[i]);
 720                         }
 721                         chain = iroot->cluster.array[i].chain;
 722                         if (chain && !RB_EMPTY(&chain->core.rbtree)) {
 723                                 kprintf("hammer2: Warning pmp %p still "
 724                                         "has active chains\n", pmp);
 725                                 chains_still_present = 1;
 726                         }
 727                 }
 728 #if REPORT_REFS_ERRORS
 729                 if (iroot->refs != 1)
 730                         kprintf("PMP->IROOT %p REFS WRONG %d\n",
 731                                 iroot, iroot->refs);
 732 #else
 733                 KKASSERT(iroot->refs == 1);
 734 #endif
 735                 /* ref for iroot */
 736                 hammer2_inode_drop(iroot);
 737                 pmp->iroot = NULL;
 738         }
 739
 740         /*
 741          * Free remaining pmp resources
 742          */
 743         if (chains_still_present) {
 744                 kprintf("hammer2: cannot free pmp %p, still in use\n", pmp);
 745         } else {
 746                 kmalloc_destroy(&pmp->mmsg);
 747                 kmalloc_destroy(&pmp->minode);
 748                 kfree(pmp, M_HAMMER2);
 749         }
 750 }
 751
 752 /*
 753  * Remove all references to hmp from the pfs list.  Any PFS which becomes
 754  * empty is terminated and freed.
 755  *
 756  * XXX inefficient.
 757  */
 758 static void
 759 hammer2_pfsfree_scan(hammer2_dev_t *hmp, int which)
 760 {
 761         hammer2_pfs_t *pmp;
 762         hammer2_inode_t *iroot;
 763         hammer2_chain_t *rchain;
 764         int i;
 765         int j;
 766         struct hammer2_pfslist *wlist;
 767
 768         if (which == 0)
 769                 wlist = &hammer2_pfslist;
 770         else
 771                 wlist = &hammer2_spmplist;
 772 again:
 773         TAILQ_FOREACH(pmp, wlist, mntentry) {
 774                 if ((iroot = pmp->iroot) == NULL)
 775                         continue;
 776
 777                 /*
 778                  * Determine if this PFS is affected.  If it is we must
 779                  * freeze all management threads and lock its iroot.
 780                  *
 781                  * Freezing a management thread forces it idle, operations
 782                  * in-progress will be aborted and it will have to start
 783                  * over again when unfrozen, or exit if told to exit.
 784                  */
 785                 for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 786                         if (pmp->pfs_hmps[i] == hmp)
 787                                 break;
 788                 }
 789                 if (i == HAMMER2_MAXCLUSTER)
 790                         continue;
 791
 792                 hammer2_vfs_sync_pmp(pmp, MNT_WAIT);
 793
 794                 /*
 795                  * Make sure all synchronization threads are locked
 796                  * down.
 797                  */
 798                 for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 799                         if (pmp->pfs_hmps[i] == NULL)
 800                                 continue;
 801                         hammer2_thr_freeze_async(&pmp->sync_thrs[i]);
 802                         if (pmp->xop_groups) {
 803                                 for (j = 0; j < hammer2_xopgroups; ++j) {
 804                                         hammer2_thr_freeze_async(
 805                                                 &pmp->xop_groups[j].thrs[i]);
 806                                 }
 807                         }
 808                 }
 809                 for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 810                         if (pmp->pfs_hmps[i] == NULL)
 811                                 continue;
 812                         hammer2_thr_freeze(&pmp->sync_thrs[i]);
 813                         if (pmp->xop_groups) {
 814                                 for (j = 0; j < hammer2_xopgroups; ++j) {
 815                                         hammer2_thr_freeze(
 816                                                 &pmp->xop_groups[j].thrs[i]);
 817                                 }
 818                         }
 819                 }
 820
 821                 /*
 822                  * Lock the inode and clean out matching chains.
 823                  * Note that we cannot use hammer2_inode_lock_*()
 824                  * here because that would attempt to validate the
 825                  * cluster that we are in the middle of ripping
 826                  * apart.
 827                  *
 828                  * WARNING! We are working directly on the inodes
 829                  *          embedded cluster.
 830                  */
 831                 hammer2_mtx_ex(&iroot->lock);
 832
 833                 /*
 834                  * Remove the chain from matching elements of the PFS.
 835                  */
 836                 for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 837                         if (pmp->pfs_hmps[i] != hmp)
 838                                 continue;
 839                         hammer2_thr_delete(&pmp->sync_thrs[i]);
 840                         if (pmp->xop_groups) {
 841                                 for (j = 0; j < hammer2_xopgroups; ++j) {
 842                                         hammer2_thr_delete(
 843                                                 &pmp->xop_groups[j].thrs[i]);
 844                                 }
 845                         }
 846                         rchain = iroot->cluster.array[i].chain;
 847                         iroot->cluster.array[i].chain = NULL;
 848                         pmp->pfs_types[i] = 0;
 849                         if (pmp->pfs_names[i]) {
 850                                 kfree(pmp->pfs_names[i], M_HAMMER2);
 851                                 pmp->pfs_names[i] = NULL;
 852                         }
 853                         if (rchain) {
 854                                 hammer2_chain_drop(rchain);
 855                                 /* focus hint */
 856                                 if (iroot->cluster.focus == rchain)
 857                                         iroot->cluster.focus = NULL;
 858                         }
 859                         pmp->pfs_hmps[i] = NULL;
 860                 }
 861                 hammer2_mtx_unlock(&iroot->lock);
 862
 863                 /*
 864                  * Cleanup trailing chains.  Gaps may remain.
 865                  */
 866                 for (i = HAMMER2_MAXCLUSTER - 1; i >= 0; --i) {
 867                         if (pmp->pfs_hmps[i])
 868                                 break;
 869                 }
 870                 iroot->cluster.nchains = i + 1;
 871
 872                 /*
 873                  * If the PMP has no elements remaining we can destroy it.
 874                  * (this will transition management threads from frozen->exit).
 875                  */
 876                 if (iroot->cluster.nchains == 0) {
 877                         /*
 878                          * If this was the hmp's spmp, we need to clean
 879                          * a little more stuff out.
 880                          */
 881                         if (hmp->spmp == pmp) {
 882                                 hmp->spmp = NULL;
 883                                 hmp->vchain.pmp = NULL;
 884                                 hmp->fchain.pmp = NULL;
 885                         }
 886
 887                         /*
 888                          * Free the pmp and restart the loop
 889                          */
 890                         KKASSERT(TAILQ_EMPTY(&pmp->syncq));
 891                         KKASSERT(TAILQ_EMPTY(&pmp->depq));
 892                         hammer2_pfsfree(pmp);
 893                         goto again;
 894                 }
 895
 896                 /*
 897                  * If elements still remain we need to set the REMASTER
 898                  * flag and unfreeze it.
 899                  */
 900                 for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 901                         if (pmp->pfs_hmps[i] == NULL)
 902                                 continue;
 903                         hammer2_thr_remaster(&pmp->sync_thrs[i]);
 904                         hammer2_thr_unfreeze(&pmp->sync_thrs[i]);
 905                         if (pmp->xop_groups) {
 906                                 for (j = 0; j < hammer2_xopgroups; ++j) {
 907                                         hammer2_thr_remaster(
 908                                                 &pmp->xop_groups[j].thrs[i]);
 909                                         hammer2_thr_unfreeze(
 910                                                 &pmp->xop_groups[j].thrs[i]);
 911                                 }
 912                         }
 913                 }
 914         }
 915 }
 916
 917 /*
 918  * Mount or remount HAMMER2 fileystem from physical media
 919  *
 920  *      mountroot
 921  *              mp              mount point structure
 922  *              path            NULL
 923  *              data            <unused>
 924  *              cred            <unused>
 925  *
 926  *      mount
 927  *              mp              mount point structure
 928  *              path            path to mount point
 929  *              data            pointer to argument structure in user space
 930  *                      volume  volume path (device@LABEL form)
 931  *                      hflags  user mount flags
 932  *              cred            user credentials
 933  *
 934  * RETURNS:     0       Success
 935  *              !0      error number
 936  */
 937 static
 938 int
 939 hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 940                   struct ucred *cred)
 941 {
 942         struct hammer2_mount_info info;
 943         hammer2_pfs_t *pmp;
 944         hammer2_pfs_t *spmp;
 945         hammer2_dev_t *hmp;
 946         hammer2_dev_t *force_local;
 947         hammer2_key_t key_next;
 948         hammer2_key_t key_dummy;
 949         hammer2_key_t lhc;
 950         struct vnode *devvp;
 951         struct nlookupdata nd;
 952         hammer2_chain_t *parent;
 953         hammer2_chain_t *chain;
 954         const hammer2_inode_data_t *ripdata;
 955         hammer2_blockref_t bref;
 956         struct file *fp;
 957         char devstr[MNAMELEN];
 958         size_t size;
 959         size_t done;
 960         char *dev;
 961         char *label;
 962         int ronly = 1;
 963         int error;
 964         int i;
 965
 966         hmp = NULL;
 967         pmp = NULL;
 968         dev = NULL;
 969         label = NULL;
 970         devvp = NULL;
 971         bzero(&info, sizeof(info));
 972
 973         if (path) {
 974                 /*
 975                  * Non-root mount or updating a mount
 976                  */
 977                 error = copyin(data, &info, sizeof(info));
 978                 if (error)
 979                         return (error);
 980         }
 981
 982         if (mp->mnt_flag & MNT_UPDATE) {
 983                 /*
 984                  * Update mount.  Note that pmp->iroot->cluster is
 985                  * an inode-embedded cluster and thus cannot be
 986                  * directly locked.
 987                  *
 988                  * XXX HAMMER2 needs to implement NFS export via
 989                  *     mountctl.
 990                  */
 991                 hammer2_cluster_t *cluster;
 992
 993                 pmp = MPTOPMP(mp);
 994                 pmp->hflags = info.hflags;
 995                 cluster = &pmp->iroot->cluster;
 996                 for (i = 0; i < cluster->nchains; ++i) {
 997                         if (cluster->array[i].chain == NULL)
 998                                 continue;
 999                         hmp = cluster->array[i].chain->hmp;
1000                         devvp = hmp->devvp;
1001                         error = hammer2_remount(hmp, mp, path,
1002                                                 devvp, cred);
1003                         if (error)
1004                                 break;
1005                 }
1006
1007                 return error;
1008         }
1009
1010         if (path == NULL) {
1011                 /*
1012                  * Root mount
1013                  */
1014                 info.cluster_fd = -1;
1015                 ksnprintf(devstr, sizeof(devstr), "%s",
1016                           mp->mnt_stat.f_mntfromname);
1017                 done = strlen(devstr) + 1;
1018                 kprintf("hammer2_mount: root devstr=\"%s\"\n", devstr);
1019         } else {
1020                 error = copyinstr(info.volume, devstr, MNAMELEN - 1, &done);
1021                 if (error)
1022                         return (error);
1023                 kprintf("hammer2_mount: devstr=\"%s\"\n", devstr);
1024         }
1025
1026         /*
1027          * Extract device and label, automatically mount @BOOT, @ROOT, or @DATA
1028          * if no label specified, based on the partition id.  Error out if no
1029          * label or device (with partition id) is specified.  This is strictly
1030          * a convenience to match the default label created by newfs_hammer2,
1031          * our preference is that a label always be specified.
1032          *
1033          * NOTE: We allow 'mount @LABEL <blah>'... that is, a mount command
1034          *       that does not specify a device, as long as some H2 label
1035          *       has already been mounted from that device.  This makes
1036          *       mounting snapshots a lot easier.
1037          */
1038         dev = devstr;
1039         label = strchr(devstr, '@');
1040         if (label && ((label + 1) - dev) > done) {
1041                 kprintf("hammer2: mount: bad label %s/%zd\n", devstr, done);
1042                 return (EINVAL);
1043         }
1044         if (label == NULL || label[1] == 0) {
1045                 char slice;
1046
1047                 if (label == NULL)
1048                         label = devstr + strlen(devstr);
1049                 else
1050                         *label = '\0';          /* clean up trailing @ */
1051
1052                 slice = label[-1];
1053                 switch(slice) {
1054                 case 'a':
1055                         label = "BOOT";
1056                         break;
1057                 case 'd':
1058                         label = "ROOT";
1059                         break;
1060                 default:
1061                         label = "DATA";
1062                         break;
1063                 }
1064         } else {
1065                 *label = '\0';
1066                 label++;
1067         }
1068
1069         kprintf("hammer2_mount: dev=\"%s\" label=\"%s\" rdonly=%d\n",
1070                 dev, label, (mp->mnt_flag & MNT_RDONLY));
1071
1072         /*
1073          * HMP device mount
1074          *
1075          * If a path is specified and dev is not an empty string, lookup the
1076          * name and verify that it referes to a block device.
1077          *
1078          * If a path is specified and dev is an empty string we fall through
1079          * and locate the label in the hmp search.
1080          */
1081         if (path && *dev != 0) {
1082                 error = nlookup_init(&nd, dev, UIO_SYSSPACE, NLC_FOLLOW);
1083                 if (error == 0)
1084                         error = nlookup(&nd);
1085                 if (error == 0)
1086                         error = cache_vref(&nd.nl_nch, nd.nl_cred, &devvp);
1087                 nlookup_done(&nd);
1088         } else if (path == NULL) {
1089                 /* root mount */
1090                 cdev_t cdev = kgetdiskbyname(dev);
1091                 error = bdevvp(cdev, &devvp);
1092                 if (error)
1093                         kprintf("hammer2: cannot find '%s'\n", dev);
1094         } else {
1095                 /*
1096                  * We will locate the hmp using the label in the hmp loop.
1097                  */
1098                 error = 0;
1099         }
1100
1101         /*
1102          * Make sure its a block device.  Do not check to see if it is
1103          * already mounted until we determine that its a fresh H2 device.
1104          */
1105         if (error == 0 && devvp) {
1106                 vn_isdisk(devvp, &error);
1107         }
1108
1109         /*
1110          * Determine if the device has already been mounted.  After this
1111          * check hmp will be non-NULL if we are doing the second or more
1112          * hammer2 mounts from the same device.
1113          */
1114         lockmgr(&hammer2_mntlk, LK_EXCLUSIVE);
1115         if (devvp) {
1116                 /*
1117                  * Match the device.  Due to the way devfs works,
1118                  * we may not be able to directly match the vnode pointer,
1119                  * so also check to see if the underlying device matches.
1120                  */
1121                 TAILQ_FOREACH(hmp, &hammer2_mntlist, mntentry) {
1122                         if (hmp->devvp == devvp)
1123                                 break;
1124                         if (devvp->v_rdev &&
1125                             hmp->devvp->v_rdev == devvp->v_rdev) {
1126                                 break;
1127                         }
1128                 }
1129
1130                 /*
1131                  * If no match this may be a fresh H2 mount, make sure
1132                  * the device is not mounted on anything else.
1133                  */
1134                 if (hmp == NULL)
1135                         error = vfs_mountedon(devvp);
1136         } else if (error == 0) {
1137                 /*
1138                  * Match the label to a pmp already probed.
1139                  */
1140                 TAILQ_FOREACH(pmp, &hammer2_pfslist, mntentry) {
1141                         for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
1142                                 if (pmp->pfs_names[i] &&
1143                                     strcmp(pmp->pfs_names[i], label) == 0) {
1144                                         hmp = pmp->pfs_hmps[i];
1145                                         break;
1146                                 }
1147                         }
1148                         if (hmp)
1149                                 break;
1150                 }
1151                 if (hmp == NULL) {
1152                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1153                         kprintf("hammer2_mount: PFS label \"%s\" not found\n",
1154                                 label);
1155                         return ENOENT;
1156                 }
1157         }
1158
1159         /*
1160          * Open the device if this isn't a secondary mount and construct
1161          * the H2 device mount (hmp).
1162          */
1163         if (hmp == NULL) {
1164                 hammer2_chain_t *schain;
1165                 hammer2_xid_t xid;
1166                 hammer2_xop_head_t xop;
1167
1168                 if (error == 0 && vcount(devvp) > 0) {
1169                         kprintf("Primary device already has references\n");
1170                         error = EBUSY;
1171                 }
1172
1173                 /*
1174                  * Now open the device
1175                  */
1176                 if (error == 0) {
1177                         ronly = ((mp->mnt_flag & MNT_RDONLY) != 0);
1178                         vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1179                         error = vinvalbuf(devvp, V_SAVE, 0, 0);
1180                         if (error == 0) {
1181                                 error = VOP_OPEN(devvp,
1182                                              (ronly ? FREAD : FREAD | FWRITE),
1183                                              FSCRED, NULL);
1184                         }
1185                         vn_unlock(devvp);
1186                 }
1187                 if (error && devvp) {
1188                         vrele(devvp);
1189                         devvp = NULL;
1190                 }
1191                 if (error) {
1192                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1193                         return error;
1194                 }
1195                 hmp = kmalloc(sizeof(*hmp), M_HAMMER2, M_WAITOK | M_ZERO);
1196                 ksnprintf(hmp->devrepname, sizeof(hmp->devrepname), "%s", dev);
1197                 hmp->ronly = ronly;
1198                 hmp->devvp = devvp;
1199                 hmp->hflags = info.hflags & HMNT2_DEVFLAGS;
1200                 kmalloc_create(&hmp->mchain, "HAMMER2-chains");
1201                 TAILQ_INSERT_TAIL(&hammer2_mntlist, hmp, mntentry);
1202                 RB_INIT(&hmp->iotree);
1203                 spin_init(&hmp->io_spin, "h2mount_io");
1204                 spin_init(&hmp->list_spin, "h2mount_list");
1205
1206                 lockinit(&hmp->vollk, "h2vol", 0, 0);
1207                 lockinit(&hmp->bulklk, "h2bulk", 0, 0);
1208                 lockinit(&hmp->bflock, "h2bflk", 0, 0);
1209
1210                 /*
1211                  * vchain setup. vchain.data is embedded.
1212                  * vchain.refs is initialized and will never drop to 0.
1213                  *
1214                  * NOTE! voldata is not yet loaded.
1215                  */
1216                 hmp->vchain.hmp = hmp;
1217                 hmp->vchain.refs = 1;
1218                 hmp->vchain.data = (void *)&hmp->voldata;
1219                 hmp->vchain.bref.type = HAMMER2_BREF_TYPE_VOLUME;
1220                 hmp->vchain.bref.data_off = 0 | HAMMER2_PBUFRADIX;
1221                 hmp->vchain.bref.mirror_tid = hmp->voldata.mirror_tid;
1222                 hammer2_chain_core_init(&hmp->vchain);
1223
1224                 /*
1225                  * fchain setup.  fchain.data is embedded.
1226                  * fchain.refs is initialized and will never drop to 0.
1227                  *
1228                  * The data is not used but needs to be initialized to
1229                  * pass assertion muster.  We use this chain primarily
1230                  * as a placeholder for the freemap's top-level RBTREE
1231                  * so it does not interfere with the volume's topology
1232                  * RBTREE.
1233                  */
1234                 hmp->fchain.hmp = hmp;
1235                 hmp->fchain.refs = 1;
1236                 hmp->fchain.data = (void *)&hmp->voldata.freemap_blockset;
1237                 hmp->fchain.bref.type = HAMMER2_BREF_TYPE_FREEMAP;
1238                 hmp->fchain.bref.data_off = 0 | HAMMER2_PBUFRADIX;
1239                 hmp->fchain.bref.mirror_tid = hmp->voldata.freemap_tid;
1240                 hmp->fchain.bref.methods =
1241                         HAMMER2_ENC_CHECK(HAMMER2_CHECK_FREEMAP) |
1242                         HAMMER2_ENC_COMP(HAMMER2_COMP_NONE);
1243                 hammer2_chain_core_init(&hmp->fchain);
1244
1245                 /*
1246                  * Install the volume header and initialize fields from
1247                  * voldata.
1248                  */
1249                 error = hammer2_install_volume_header(hmp);
1250                 if (error) {
1251                         hammer2_unmount_helper(mp, NULL, hmp);
1252                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1253                         hammer2_vfs_unmount(mp, MNT_FORCE);
1254                         return error;
1255                 }
1256
1257                 /*
1258                  * Really important to get these right or the flush and
1259                  * teardown code will get confused.
1260                  */
1261                 hmp->spmp = hammer2_pfsalloc(NULL, NULL, 0, NULL);
1262                 spmp = hmp->spmp;
1263                 spmp->pfs_hmps[0] = hmp;
1264
1265                 /*
1266                  * Dummy-up vchain and fchain's modify_tid.  mirror_tid
1267                  * is inherited from the volume header.
1268                  */
1269                 xid = 0;
1270                 hmp->vchain.bref.mirror_tid = hmp->voldata.mirror_tid;
1271                 hmp->vchain.bref.modify_tid = hmp->vchain.bref.mirror_tid;
1272                 hmp->vchain.pmp = spmp;
1273                 hmp->fchain.bref.mirror_tid = hmp->voldata.freemap_tid;
1274                 hmp->fchain.bref.modify_tid = hmp->fchain.bref.mirror_tid;
1275                 hmp->fchain.pmp = spmp;
1276
1277                 /*
1278                  * First locate the super-root inode, which is key 0
1279                  * relative to the volume header's blockset.
1280                  *
1281                  * Then locate the root inode by scanning the directory keyspace
1282                  * represented by the label.
1283                  */
1284                 parent = hammer2_chain_lookup_init(&hmp->vchain, 0);
1285                 schain = hammer2_chain_lookup(&parent, &key_dummy,
1286                                       HAMMER2_SROOT_KEY, HAMMER2_SROOT_KEY,
1287                                       &error, 0);
1288                 hammer2_chain_lookup_done(parent);
1289                 if (schain == NULL) {
1290                         kprintf("hammer2_mount: invalid super-root\n");
1291                         hammer2_unmount_helper(mp, NULL, hmp);
1292                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1293                         hammer2_vfs_unmount(mp, MNT_FORCE);
1294                         return EINVAL;
1295                 }
1296                 if (schain->error) {
1297                         kprintf("hammer2_mount: error %s reading super-root\n",
1298                                 hammer2_error_str(schain->error));
1299                         hammer2_chain_unlock(schain);
1300                         hammer2_chain_drop(schain);
1301                         schain = NULL;
1302                         hammer2_unmount_helper(mp, NULL, hmp);
1303                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1304                         hammer2_vfs_unmount(mp, MNT_FORCE);
1305                         return EINVAL;
1306                 }
1307
1308                 /*
1309                  * The super-root always uses an inode_tid of 1 when
1310                  * creating PFSs.
1311                  */
1312                 spmp->inode_tid = 1;
1313                 spmp->modify_tid = schain->bref.modify_tid + 1;
1314
1315                 /*
1316                  * Sanity-check schain's pmp and finish initialization.
1317                  * Any chain belonging to the super-root topology should
1318                  * have a NULL pmp (not even set to spmp).
1319                  */
1320                 ripdata = &hammer2_chain_rdata(schain)->ipdata;
1321                 KKASSERT(schain->pmp == NULL);
1322                 spmp->pfs_clid = ripdata->meta.pfs_clid;
1323
1324                 /*
1325                  * Replace the dummy spmp->iroot with a real one.  It's
1326                  * easier to just do a wholesale replacement than to try
1327                  * to update the chain and fixup the iroot fields.
1328                  *
1329                  * The returned inode is locked with the supplied cluster.
1330                  */
1331                 hammer2_dummy_xop_from_chain(&xop, schain);
1332                 hammer2_inode_drop(spmp->iroot);
1333                 spmp->iroot = NULL;
1334                 spmp->iroot = hammer2_inode_get(spmp, &xop, -1, -1);
1335                 spmp->spmp_hmp = hmp;
1336                 spmp->pfs_types[0] = ripdata->meta.pfs_type;
1337                 spmp->pfs_hmps[0] = hmp;
1338                 hammer2_inode_ref(spmp->iroot);
1339                 hammer2_inode_unlock(spmp->iroot);
1340                 hammer2_cluster_unlock(&xop.cluster);
1341                 hammer2_chain_drop(schain);
1342                 /* do not call hammer2_cluster_drop() on an embedded cluster */
1343                 schain = NULL;  /* now invalid */
1344                 /* leave spmp->iroot with one ref */
1345
1346                 if ((mp->mnt_flag & MNT_RDONLY) == 0) {
1347                         error = hammer2_recovery(hmp);
1348                         if (error == 0)
1349                                 error |= hammer2_fixup_pfses(hmp);
1350                         /* XXX do something with error */
1351                 }
1352                 hammer2_update_pmps(hmp);
1353                 hammer2_iocom_init(hmp);
1354                 hammer2_bulkfree_init(hmp);
1355
1356                 /*
1357                  * Ref the cluster management messaging descriptor.  The mount
1358                  * program deals with the other end of the communications pipe.
1359                  *
1360                  * Root mounts typically do not supply one.
1361                  */
1362                 if (info.cluster_fd >= 0) {
1363                         fp = holdfp(curthread, info.cluster_fd, -1);
1364                         if (fp) {
1365                                 hammer2_cluster_reconnect(hmp, fp);
1366                         } else {
1367                                 kprintf("hammer2_mount: bad cluster_fd!\n");
1368                         }
1369                 }
1370         } else {
1371                 spmp = hmp->spmp;
1372                 if (info.hflags & HMNT2_DEVFLAGS) {
1373                         kprintf("hammer2: Warning: mount flags pertaining "
1374                                 "to the whole device may only be specified "
1375                                 "on the first mount of the device: %08x\n",
1376                                 info.hflags & HMNT2_DEVFLAGS);
1377                 }
1378         }
1379
1380         /*
1381          * Force local mount (disassociate all PFSs from their clusters).
1382          * Used primarily for debugging.
1383          */
1384         force_local = (hmp->hflags & HMNT2_LOCAL) ? hmp : NULL;
1385
1386         /*
1387          * Lookup the mount point under the media-localized super-root.
1388          * Scanning hammer2_pfslist doesn't help us because it represents
1389          * PFS cluster ids which can aggregate several named PFSs together.
1390          *
1391          * cluster->pmp will incorrectly point to spmp and must be fixed
1392          * up later on.
1393          */
1394         hammer2_inode_lock(spmp->iroot, 0);
1395         parent = hammer2_inode_chain(spmp->iroot, 0, HAMMER2_RESOLVE_ALWAYS);
1396         lhc = hammer2_dirhash(label, strlen(label));
1397         chain = hammer2_chain_lookup(&parent, &key_next,
1398                                      lhc, lhc + HAMMER2_DIRHASH_LOMASK,
1399                                      &error, 0);
1400         while (chain) {
1401                 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
1402                     strcmp(label, chain->data->ipdata.filename) == 0) {
1403                         break;
1404                 }
1405                 chain = hammer2_chain_next(&parent, chain, &key_next,
1406                                             key_next,
1407                                             lhc + HAMMER2_DIRHASH_LOMASK,
1408                                             &error, 0);
1409         }
1410         if (parent) {
1411                 hammer2_chain_unlock(parent);
1412                 hammer2_chain_drop(parent);
1413         }
1414         hammer2_inode_unlock(spmp->iroot);
1415
1416         /*
1417          * PFS could not be found?
1418          */
1419         if (chain == NULL) {
1420                 hammer2_unmount_helper(mp, NULL, hmp);
1421                 lockmgr(&hammer2_mntlk, LK_RELEASE);
1422                 hammer2_vfs_unmount(mp, MNT_FORCE);
1423
1424                 if (error) {
1425                         kprintf("hammer2_mount: PFS label I/O error\n");
1426                         return EINVAL;
1427                 } else {
1428                         kprintf("hammer2_mount: PFS label \"%s\" not found\n",
1429                                 label);
1430                         return ENOENT;
1431                 }
1432         }
1433
1434         /*
1435          * Acquire the pmp structure (it should have already been allocated
1436          * via hammer2_update_pmps() so do not pass cluster in to add to
1437          * available chains).
1438          *
1439          * Check if the cluster has already been mounted.  A cluster can
1440          * only be mounted once, use null mounts to mount additional copies.
1441          */
1442         if (chain->error) {
1443                 kprintf("hammer2_mount: PFS label I/O error\n");
1444         } else {
1445                 ripdata = &chain->data->ipdata;
1446                 bref = chain->bref;
1447                 pmp = hammer2_pfsalloc(NULL, ripdata,
1448                                        bref.modify_tid, force_local);
1449         }
1450         hammer2_chain_unlock(chain);
1451         hammer2_chain_drop(chain);
1452
1453         /*
1454          * Finish the mount
1455          */
1456         kprintf("hammer2_mount hmp=%p pmp=%p\n", hmp, pmp);
1457
1458         if (pmp->mp) {
1459                 kprintf("hammer2_mount: PFS already mounted!\n");
1460                 hammer2_unmount_helper(mp, NULL, hmp);
1461                 lockmgr(&hammer2_mntlk, LK_RELEASE);
1462                 hammer2_vfs_unmount(mp, MNT_FORCE);
1463
1464                 return EBUSY;
1465         }
1466
1467         pmp->hflags = info.hflags;
1468         mp->mnt_flag |= MNT_LOCAL;
1469         mp->mnt_kern_flag |= MNTK_ALL_MPSAFE;   /* all entry pts are SMP */
1470         mp->mnt_kern_flag |= MNTK_THR_SYNC;     /* new vsyncscan semantics */
1471
1472         /*
1473          * required mount structure initializations
1474          */
1475         mp->mnt_stat.f_iosize = HAMMER2_PBUFSIZE;
1476         mp->mnt_stat.f_bsize = HAMMER2_PBUFSIZE;
1477
1478         mp->mnt_vstat.f_frsize = HAMMER2_PBUFSIZE;
1479         mp->mnt_vstat.f_bsize = HAMMER2_PBUFSIZE;
1480
1481         /*
1482          * Optional fields
1483          */
1484         mp->mnt_iosize_max = MAXPHYS;
1485
1486         /*
1487          * Connect up mount pointers.
1488          */
1489         hammer2_mount_helper(mp, pmp);
1490         hmp->devvp->v_rdev->si_mountpoint = mp;
1491
1492         lockmgr(&hammer2_mntlk, LK_RELEASE);
1493
1494         /*
1495          * Finish setup
1496          */
1497         vfs_getnewfsid(mp);
1498         vfs_add_vnodeops(mp, &hammer2_vnode_vops, &mp->mnt_vn_norm_ops);
1499         vfs_add_vnodeops(mp, &hammer2_spec_vops, &mp->mnt_vn_spec_ops);
1500         vfs_add_vnodeops(mp, &hammer2_fifo_vops, &mp->mnt_vn_fifo_ops);
1501
1502         if (path) {
1503                 copyinstr(info.volume, mp->mnt_stat.f_mntfromname,
1504                           MNAMELEN - 1, &size);
1505                 bzero(mp->mnt_stat.f_mntfromname + size, MNAMELEN - size);
1506         } /* else root mount, already in there */
1507
1508         bzero(mp->mnt_stat.f_mntonname, sizeof(mp->mnt_stat.f_mntonname));
1509         if (path) {
1510                 copyinstr(path, mp->mnt_stat.f_mntonname,
1511                           sizeof(mp->mnt_stat.f_mntonname) - 1,
1512                           &size);
1513         } else {
1514                 /* root mount */
1515                 mp->mnt_stat.f_mntonname[0] = '/';
1516         }
1517
1518         /*
1519          * Initial statfs to prime mnt_stat.
1520          */
1521         hammer2_vfs_statfs(mp, &mp->mnt_stat, cred);
1522
1523         return 0;
1524 }
1525
1526 /*
1527  * Scan PFSs under the super-root and create hammer2_pfs structures.
1528  */
1529 static
1530 void
1531 hammer2_update_pmps(hammer2_dev_t *hmp)
1532 {
1533         const hammer2_inode_data_t *ripdata;
1534         hammer2_chain_t *parent;
1535         hammer2_chain_t *chain;
1536         hammer2_blockref_t bref;
1537         hammer2_dev_t *force_local;
1538         hammer2_pfs_t *spmp;
1539         hammer2_pfs_t *pmp;
1540         hammer2_key_t key_next;
1541         int error;
1542
1543         /*
1544          * Force local mount (disassociate all PFSs from their clusters).
1545          * Used primarily for debugging.
1546          */
1547         force_local = (hmp->hflags & HMNT2_LOCAL) ? hmp : NULL;
1548
1549         /*
1550          * Lookup mount point under the media-localized super-root.
1551          *
1552          * cluster->pmp will incorrectly point to spmp and must be fixed
1553          * up later on.
1554          */
1555         spmp = hmp->spmp;
1556         hammer2_inode_lock(spmp->iroot, 0);
1557         parent = hammer2_inode_chain(spmp->iroot, 0, HAMMER2_RESOLVE_ALWAYS);
1558         chain = hammer2_chain_lookup(&parent, &key_next,
1559                                          HAMMER2_KEY_MIN, HAMMER2_KEY_MAX,
1560                                          &error, 0);
1561         while (chain) {
1562                 if (chain->error) {
1563                         kprintf("I/O error scanning PFS labels\n");
1564                 } else if (chain->bref.type != HAMMER2_BREF_TYPE_INODE) {
1565                         kprintf("Non inode chain type %d under super-root\n",
1566                                 chain->bref.type);
1567                 } else {
1568                         ripdata = &chain->data->ipdata;
1569                         bref = chain->bref;
1570                         pmp = hammer2_pfsalloc(chain, ripdata,
1571                                                bref.modify_tid, force_local);
1572                 }
1573                 chain = hammer2_chain_next(&parent, chain, &key_next,
1574                                            key_next, HAMMER2_KEY_MAX,
1575                                            &error, 0);
1576         }
1577         if (parent) {
1578                 hammer2_chain_unlock(parent);
1579                 hammer2_chain_drop(parent);
1580         }
1581         hammer2_inode_unlock(spmp->iroot);
1582 }
1583
1584 static
1585 int
1586 hammer2_remount(hammer2_dev_t *hmp, struct mount *mp, char *path __unused,
1587                 struct vnode *devvp, struct ucred *cred)
1588 {
1589         int error;
1590
1591         if (hmp->ronly && (mp->mnt_kern_flag & MNTK_WANTRDWR)) {
1592                 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1593                 VOP_OPEN(devvp, FREAD | FWRITE, FSCRED, NULL);
1594                 vn_unlock(devvp);
1595                 error = hammer2_recovery(hmp);
1596                 if (error == 0)
1597                         error |= hammer2_fixup_pfses(hmp);
1598                 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1599                 if (error == 0) {
1600                         VOP_CLOSE(devvp, FREAD, NULL);
1601                         hmp->ronly = 0;
1602                 } else {
1603                         VOP_CLOSE(devvp, FREAD | FWRITE, NULL);
1604                 }
1605                 vn_unlock(devvp);
1606         } else {
1607                 error = 0;
1608         }
1609         return error;
1610 }
1611
1612 static
1613 int
1614 hammer2_vfs_unmount(struct mount *mp, int mntflags)
1615 {
1616         hammer2_pfs_t *pmp;
1617         int flags;
1618         int error = 0;
1619
1620         pmp = MPTOPMP(mp);
1621
1622         if (pmp == NULL)
1623                 return(0);
1624
1625         lockmgr(&hammer2_mntlk, LK_EXCLUSIVE);
1626
1627         /*
1628          * If mount initialization proceeded far enough we must flush
1629          * its vnodes and sync the underlying mount points.  Three syncs
1630          * are required to fully flush the filesystem (freemap updates lag
1631          * by one flush, and one extra for safety).
1632          */
1633         if (mntflags & MNT_FORCE)
1634                 flags = FORCECLOSE;
1635         else
1636                 flags = 0;
1637         if (pmp->iroot) {
1638                 error = vflush(mp, 0, flags);
1639                 if (error)
1640                         goto failed;
1641                 hammer2_vfs_sync(mp, MNT_WAIT);
1642                 hammer2_vfs_sync(mp, MNT_WAIT);
1643                 hammer2_vfs_sync(mp, MNT_WAIT);
1644         }
1645
1646         /*
1647          * Cleanup the frontend support XOPS threads
1648          */
1649         hammer2_xop_helper_cleanup(pmp);
1650
1651         if (pmp->mp)
1652                 hammer2_unmount_helper(mp, pmp, NULL);
1653
1654         error = 0;
1655 failed:
1656         lockmgr(&hammer2_mntlk, LK_RELEASE);
1657
1658         return (error);
1659 }
1660
1661 /*
1662  * Mount helper, hook the system mount into our PFS.
1663  * The mount lock is held.
1664  *
1665  * We must bump the mount_count on related devices for any
1666  * mounted PFSs.
1667  */
1668 static
1669 void
1670 hammer2_mount_helper(struct mount *mp, hammer2_pfs_t *pmp)
1671 {
1672         hammer2_cluster_t *cluster;
1673         hammer2_chain_t *rchain;
1674         int i;
1675
1676         mp->mnt_data = (qaddr_t)pmp;
1677         pmp->mp = mp;
1678
1679         /*
1680          * After pmp->mp is set we have to adjust hmp->mount_count.
1681          */
1682         cluster = &pmp->iroot->cluster;
1683         for (i = 0; i < cluster->nchains; ++i) {
1684                 rchain = cluster->array[i].chain;
1685                 if (rchain == NULL)
1686                         continue;
1687                 ++rchain->hmp->mount_count;
1688         }
1689
1690         /*
1691          * Create missing Xop threads
1692          */
1693         hammer2_xop_helper_create(pmp);
1694 }
1695
1696 /*
1697  * Mount helper, unhook the system mount from our PFS.
1698  * The mount lock is held.
1699  *
1700  * If hmp is supplied a mount responsible for being the first to open
1701  * the block device failed and the block device and all PFSs using the
1702  * block device must be cleaned up.
1703  *
1704  * If pmp is supplied multiple devices might be backing the PFS and each
1705  * must be disconnected.  This might not be the last PFS using some of the
1706  * underlying devices.  Also, we have to adjust our hmp->mount_count
1707  * accounting for the devices backing the pmp which is now undergoing an
1708  * unmount.
1709  */
1710 static
1711 void
1712 hammer2_unmount_helper(struct mount *mp, hammer2_pfs_t *pmp, hammer2_dev_t *hmp)
1713 {
1714         hammer2_cluster_t *cluster;
1715         hammer2_chain_t *rchain;
1716         struct vnode *devvp;
1717         int dumpcnt;
1718         int ronly;
1719         int i;
1720
1721         /*
1722          * If no device supplied this is a high-level unmount and we have to
1723          * to disconnect the mount, adjust mount_count, and locate devices
1724          * that might now have no mounts.
1725          */
1726         if (pmp) {
1727                 KKASSERT(hmp == NULL);
1728                 KKASSERT((void *)(intptr_t)mp->mnt_data == pmp);
1729                 pmp->mp = NULL;
1730                 mp->mnt_data = NULL;
1731
1732                 /*
1733                  * After pmp->mp is cleared we have to account for
1734                  * mount_count.
1735                  */
1736                 cluster = &pmp->iroot->cluster;
1737                 for (i = 0; i < cluster->nchains; ++i) {
1738                         rchain = cluster->array[i].chain;
1739                         if (rchain == NULL)
1740                                 continue;
1741                         --rchain->hmp->mount_count;
1742                         /* scrapping hmp now may invalidate the pmp */
1743                 }
1744 again:
1745                 TAILQ_FOREACH(hmp, &hammer2_mntlist, mntentry) {
1746                         if (hmp->mount_count == 0) {
1747                                 hammer2_unmount_helper(NULL, NULL, hmp);
1748                                 goto again;
1749                         }
1750                 }
1751                 return;
1752         }
1753
1754         /*
1755          * Try to terminate the block device.  We can't terminate it if
1756          * there are still PFSs referencing it.
1757          */
1758         if (hmp->mount_count)
1759                 return;
1760
1761         /*
1762          * Decomission the network before we start messing with the
1763          * device and PFS.
1764          */
1765         hammer2_iocom_uninit(hmp);
1766
1767         hammer2_bulkfree_uninit(hmp);
1768         hammer2_pfsfree_scan(hmp, 0);
1769 #if 0
1770         hammer2_dev_exlock(hmp);        /* XXX order */
1771 #endif
1772
1773         /*
1774          * Cycle the volume data lock as a safety (probably not needed any
1775          * more).  To ensure everything is out we need to flush at least
1776          * three times.  (1) The running of the sideq can dirty the
1777          * filesystem, (2) A normal flush can dirty the freemap, and
1778          * (3) ensure that the freemap is fully synchronized.
1779          *
1780          * The next mount's recovery scan can clean everything up but we want
1781          * to leave the filesystem in a 100% clean state on a normal unmount.
1782          */
1783 #if 0
1784         hammer2_voldata_lock(hmp);
1785         hammer2_voldata_unlock(hmp);
1786 #endif
1787
1788         /*
1789          * Flush whatever is left.  Unmounted but modified PFS's might still
1790          * have some dirty chains on them.
1791          */
1792         hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1793         hammer2_chain_lock(&hmp->fchain, HAMMER2_RESOLVE_ALWAYS);
1794
1795         if (hmp->fchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1796                 hammer2_voldata_modify(hmp);
1797                 hammer2_flush(&hmp->fchain, HAMMER2_FLUSH_TOP |
1798                                             HAMMER2_FLUSH_ALL);
1799         }
1800         hammer2_chain_unlock(&hmp->fchain);
1801
1802         if (hmp->vchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1803                 hammer2_flush(&hmp->vchain, HAMMER2_FLUSH_TOP |
1804                                             HAMMER2_FLUSH_ALL);
1805         }
1806         hammer2_chain_unlock(&hmp->vchain);
1807
1808         if ((hmp->vchain.flags | hmp->fchain.flags) &
1809             HAMMER2_CHAIN_FLUSH_MASK) {
1810                 kprintf("hammer2_unmount: chains left over after final sync\n");
1811                 kprintf("    vchain %08x\n", hmp->vchain.flags);
1812                 kprintf("    fchain %08x\n", hmp->fchain.flags);
1813
1814                 if (hammer2_debug & 0x0010)
1815                         Debugger("entered debugger");
1816         }
1817
1818         hammer2_pfsfree_scan(hmp, 1);
1819
1820         KKASSERT(hmp->spmp == NULL);
1821
1822         /*
1823          * Finish up with the device vnode
1824          */
1825         if ((devvp = hmp->devvp) != NULL) {
1826                 ronly = hmp->ronly;
1827                 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1828                 kprintf("hammer2_unmount(A): devvp %s rbdirty %p ronly=%d\n",
1829                         hmp->devrepname, RB_ROOT(&devvp->v_rbdirty_tree),
1830                         ronly);
1831                 vinvalbuf(devvp, (ronly ? 0 : V_SAVE), 0, 0);
1832                 kprintf("hammer2_unmount(B): devvp %s rbdirty %p\n",
1833                         hmp->devrepname, RB_ROOT(&devvp->v_rbdirty_tree));
1834                 devvp->v_rdev->si_mountpoint = NULL;
1835                 hmp->devvp = NULL;
1836                 VOP_CLOSE(devvp, (ronly ? FREAD : FREAD|FWRITE), NULL);
1837                 vn_unlock(devvp);
1838                 vrele(devvp);
1839                 devvp = NULL;
1840         }
1841
1842         /*
1843          * Clear vchain/fchain flags that might prevent final cleanup
1844          * of these chains.
1845          */
1846         if (hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED) {
1847                 atomic_add_long(&hammer2_count_modified_chains, -1);
1848                 atomic_clear_int(&hmp->vchain.flags, HAMMER2_CHAIN_MODIFIED);
1849                 hammer2_pfs_memory_wakeup(hmp->vchain.pmp, -1);
1850         }
1851         if (hmp->vchain.flags & HAMMER2_CHAIN_UPDATE) {
1852                 atomic_clear_int(&hmp->vchain.flags, HAMMER2_CHAIN_UPDATE);
1853         }
1854
1855         if (hmp->fchain.flags & HAMMER2_CHAIN_MODIFIED) {
1856                 atomic_add_long(&hammer2_count_modified_chains, -1);
1857                 atomic_clear_int(&hmp->fchain.flags, HAMMER2_CHAIN_MODIFIED);
1858                 hammer2_pfs_memory_wakeup(hmp->fchain.pmp, -1);
1859         }
1860         if (hmp->fchain.flags & HAMMER2_CHAIN_UPDATE) {
1861                 atomic_clear_int(&hmp->fchain.flags, HAMMER2_CHAIN_UPDATE);
1862         }
1863
1864         /*
1865          * Final drop of embedded freemap root chain to
1866          * clean up fchain.core (fchain structure is not
1867          * flagged ALLOCATED so it is cleaned out and then
1868          * left to rot).
1869          */
1870         hammer2_chain_drop(&hmp->fchain);
1871
1872         /*
1873          * Final drop of embedded volume root chain to clean
1874          * up vchain.core (vchain structure is not flagged
1875          * ALLOCATED so it is cleaned out and then left to
1876          * rot).
1877          */
1878         dumpcnt = 50;
1879         hammer2_dump_chain(&hmp->vchain, 0, 0, &dumpcnt, 'v', (u_int)-1);
1880         dumpcnt = 50;
1881         hammer2_dump_chain(&hmp->fchain, 0, 0, &dumpcnt, 'f', (u_int)-1);
1882 #if 0
1883         hammer2_dev_unlock(hmp);
1884 #endif
1885         hammer2_chain_drop(&hmp->vchain);
1886
1887         hammer2_io_cleanup(hmp, &hmp->iotree);
1888         if (hmp->iofree_count) {
1889                 kprintf("io_cleanup: %d I/O's left hanging\n",
1890                         hmp->iofree_count);
1891         }
1892
1893         TAILQ_REMOVE(&hammer2_mntlist, hmp, mntentry);
1894         kmalloc_destroy(&hmp->mchain);
1895         kfree(hmp, M_HAMMER2);
1896 }
1897
1898 int
1899 hammer2_vfs_vget(struct mount *mp, struct vnode *dvp,
1900                  ino_t ino, struct vnode **vpp)
1901 {
1902         hammer2_xop_lookup_t *xop;
1903         hammer2_pfs_t *pmp;
1904         hammer2_inode_t *ip;
1905         hammer2_tid_t inum;
1906         int error;
1907
1908         inum = (hammer2_tid_t)ino & HAMMER2_DIRHASH_USERMSK;
1909
1910         error = 0;
1911         pmp = MPTOPMP(mp);
1912
1913         /*
1914          * Easy if we already have it cached
1915          */
1916         ip = hammer2_inode_lookup(pmp, inum);
1917         if (ip) {
1918                 hammer2_inode_lock(ip, HAMMER2_RESOLVE_SHARED);
1919                 *vpp = hammer2_igetv(ip, &error);
1920                 hammer2_inode_unlock(ip);
1921                 hammer2_inode_drop(ip);         /* from lookup */
1922
1923                 return error;
1924         }
1925
1926         /*
1927          * Otherwise we have to find the inode
1928          */
1929         xop = hammer2_xop_alloc(pmp->iroot, 0);
1930         xop->lhc = inum;
1931         hammer2_xop_start(&xop->head, &hammer2_lookup_desc);
1932         error = hammer2_xop_collect(&xop->head, 0);
1933
1934         if (error == 0)
1935                 ip = hammer2_inode_get(pmp, &xop->head, -1, -1);
1936         hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
1937
1938         if (ip) {
1939                 *vpp = hammer2_igetv(ip, &error);
1940                 hammer2_inode_unlock(ip);
1941         } else {
1942                 *vpp = NULL;
1943                 error = ENOENT;
1944         }
1945         return (error);
1946 }
1947
1948 static
1949 int
1950 hammer2_vfs_root(struct mount *mp, struct vnode **vpp)
1951 {
1952         hammer2_pfs_t *pmp;
1953         struct vnode *vp;
1954         int error;
1955
1956         pmp = MPTOPMP(mp);
1957         if (pmp->iroot == NULL) {
1958                 kprintf("hammer2 (%s): no root inode\n",
1959                         mp->mnt_stat.f_mntfromname);
1960                 *vpp = NULL;
1961                 return EINVAL;
1962         }
1963
1964         error = 0;
1965         hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED);
1966
1967         while (pmp->inode_tid == 0) {
1968                 hammer2_xop_ipcluster_t *xop;
1969                 const hammer2_inode_meta_t *meta;
1970
1971                 xop = hammer2_xop_alloc(pmp->iroot, HAMMER2_XOP_MODIFYING);
1972                 hammer2_xop_start(&xop->head, &hammer2_ipcluster_desc);
1973                 error = hammer2_xop_collect(&xop->head, 0);
1974
1975                 if (error == 0) {
1976                         meta = &hammer2_xop_gdata(&xop->head)->ipdata.meta;
1977                         pmp->iroot->meta = *meta;
1978                         pmp->inode_tid = meta->pfs_inum + 1;
1979                         hammer2_xop_pdata(&xop->head);
1980                         /* meta invalid */
1981
1982                         if (pmp->inode_tid < HAMMER2_INODE_START)
1983                                 pmp->inode_tid = HAMMER2_INODE_START;
1984                         pmp->modify_tid =
1985                                 xop->head.cluster.focus->bref.modify_tid + 1;
1986 #if 0
1987                         kprintf("PFS: Starting inode %jd\n",
1988                                 (intmax_t)pmp->inode_tid);
1989                         kprintf("PMP focus good set nextino=%ld mod=%016jx\n",
1990                                 pmp->inode_tid, pmp->modify_tid);
1991 #endif
1992                         wakeup(&pmp->iroot);
1993
1994                         hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
1995
1996                         /*
1997                          * Prime the mount info.
1998                          */
1999                         hammer2_vfs_statfs(mp, &mp->mnt_stat, NULL);
2000                         break;
2001                 }
2002
2003                 /*
2004                  * Loop, try again
2005                  */
2006                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
2007                 hammer2_inode_unlock(pmp->iroot);
2008                 error = tsleep(&pmp->iroot, PCATCH, "h2root", hz);
2009                 hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED);
2010                 if (error == EINTR)
2011                         break;
2012         }
2013
2014         if (error) {
2015                 hammer2_inode_unlock(pmp->iroot);
2016                 *vpp = NULL;
2017         } else {
2018                 vp = hammer2_igetv(pmp->iroot, &error);
2019                 hammer2_inode_unlock(pmp->iroot);
2020                 *vpp = vp;
2021         }
2022
2023         return (error);
2024 }
2025
2026 /*
2027  * Filesystem status
2028  *
2029  * XXX incorporate ipdata->meta.inode_quota and data_quota
2030  */
2031 static
2032 int
2033 hammer2_vfs_statfs(struct mount *mp, struct statfs *sbp, struct ucred *cred)
2034 {
2035         hammer2_pfs_t *pmp;
2036         hammer2_dev_t *hmp;
2037         hammer2_blockref_t bref;
2038         struct statfs tmp;
2039         int i;
2040
2041         /*
2042          * NOTE: iroot might not have validated the cluster yet.
2043          */
2044         pmp = MPTOPMP(mp);
2045
2046         bzero(&tmp, sizeof(tmp));
2047
2048         for (i = 0; i < pmp->iroot->cluster.nchains; ++i) {
2049                 hmp = pmp->pfs_hmps[i];
2050                 if (hmp == NULL)
2051                         continue;
2052                 if (pmp->iroot->cluster.array[i].chain)
2053                         bref = pmp->iroot->cluster.array[i].chain->bref;
2054                 else
2055                         bzero(&bref, sizeof(bref));
2056
2057                 tmp.f_files = bref.embed.stats.inode_count;
2058                 tmp.f_ffree = 0;
2059                 tmp.f_blocks = hmp->voldata.allocator_size /
2060                                mp->mnt_vstat.f_bsize;
2061                 tmp.f_bfree = hmp->voldata.allocator_free /
2062                               mp->mnt_vstat.f_bsize;
2063                 tmp.f_bavail = tmp.f_bfree;
2064
2065                 if (cred && cred->cr_uid != 0) {
2066                         uint64_t adj;
2067
2068                         /* 5% */
2069                         adj = hmp->free_reserved / mp->mnt_vstat.f_bsize;
2070                         tmp.f_blocks -= adj;
2071                         tmp.f_bfree -= adj;
2072                         tmp.f_bavail -= adj;
2073                 }
2074
2075                 mp->mnt_stat.f_blocks = tmp.f_blocks;
2076                 mp->mnt_stat.f_bfree = tmp.f_bfree;
2077                 mp->mnt_stat.f_bavail = tmp.f_bavail;
2078                 mp->mnt_stat.f_files = tmp.f_files;
2079                 mp->mnt_stat.f_ffree = tmp.f_ffree;
2080
2081                 *sbp = mp->mnt_stat;
2082         }
2083         return (0);
2084 }
2085
2086 static
2087 int
2088 hammer2_vfs_statvfs(struct mount *mp, struct statvfs *sbp, struct ucred *cred)
2089 {
2090         hammer2_pfs_t *pmp;
2091         hammer2_dev_t *hmp;
2092         hammer2_blockref_t bref;
2093         struct statvfs tmp;
2094         int i;
2095
2096         /*
2097          * NOTE: iroot might not have validated the cluster yet.
2098          */
2099         pmp = MPTOPMP(mp);
2100         bzero(&tmp, sizeof(tmp));
2101
2102         for (i = 0; i < pmp->iroot->cluster.nchains; ++i) {
2103                 hmp = pmp->pfs_hmps[i];
2104                 if (hmp == NULL)
2105                         continue;
2106                 if (pmp->iroot->cluster.array[i].chain)
2107                         bref = pmp->iroot->cluster.array[i].chain->bref;
2108                 else
2109                         bzero(&bref, sizeof(bref));
2110
2111                 tmp.f_files = bref.embed.stats.inode_count;
2112                 tmp.f_ffree = 0;
2113                 tmp.f_blocks = hmp->voldata.allocator_size /
2114                                mp->mnt_vstat.f_bsize;
2115                 tmp.f_bfree = hmp->voldata.allocator_free /
2116                               mp->mnt_vstat.f_bsize;
2117                 tmp.f_bavail = tmp.f_bfree;
2118
2119                 if (cred && cred->cr_uid != 0) {
2120                         uint64_t adj;
2121
2122                         /* 5% */
2123                         adj = hmp->free_reserved / mp->mnt_vstat.f_bsize;
2124                         tmp.f_blocks -= adj;
2125                         tmp.f_bfree -= adj;
2126                         tmp.f_bavail -= adj;
2127                 }
2128
2129                 mp->mnt_vstat.f_blocks = tmp.f_blocks;
2130                 mp->mnt_vstat.f_bfree = tmp.f_bfree;
2131                 mp->mnt_vstat.f_bavail = tmp.f_bavail;
2132                 mp->mnt_vstat.f_files = tmp.f_files;
2133                 mp->mnt_vstat.f_ffree = tmp.f_ffree;
2134
2135                 *sbp = mp->mnt_vstat;
2136         }
2137         return (0);
2138 }
2139
2140 /*
2141  * Mount-time recovery (RW mounts)
2142  *
2143  * Updates to the free block table are allowed to lag flushes by one
2144  * transaction.  In case of a crash, then on a fresh mount we must do an
2145  * incremental scan of the last committed transaction id and make sure that
2146  * all related blocks have been marked allocated.
2147  */
2148 struct hammer2_recovery_elm {
2149         TAILQ_ENTRY(hammer2_recovery_elm) entry;
2150         hammer2_chain_t *chain;
2151         hammer2_tid_t sync_tid;
2152 };
2153
2154 TAILQ_HEAD(hammer2_recovery_list, hammer2_recovery_elm);
2155
2156 struct hammer2_recovery_info {
2157         struct hammer2_recovery_list list;
2158         hammer2_tid_t   mtid;
2159         int     depth;
2160 };
2161
2162 static int hammer2_recovery_scan(hammer2_dev_t *hmp,
2163                         hammer2_chain_t *parent,
2164                         struct hammer2_recovery_info *info,
2165                         hammer2_tid_t sync_tid);
2166
2167 #define HAMMER2_RECOVERY_MAXDEPTH       10
2168
2169 static
2170 int
2171 hammer2_recovery(hammer2_dev_t *hmp)
2172 {
2173         struct hammer2_recovery_info info;
2174         struct hammer2_recovery_elm *elm;
2175         hammer2_chain_t *parent;
2176         hammer2_tid_t sync_tid;
2177         hammer2_tid_t mirror_tid;
2178         int error;
2179
2180         hammer2_trans_init(hmp->spmp, 0);
2181
2182         sync_tid = hmp->voldata.freemap_tid;
2183         mirror_tid = hmp->voldata.mirror_tid;
2184
2185         kprintf("hammer2_mount \"%s\": ", hmp->devrepname);
2186         if (sync_tid >= mirror_tid) {
2187                 kprintf("no recovery needed\n");
2188         } else {
2189                 kprintf("freemap recovery %016jx-%016jx\n",
2190                         sync_tid + 1, mirror_tid);
2191         }
2192
2193         TAILQ_INIT(&info.list);
2194         info.depth = 0;
2195         parent = hammer2_chain_lookup_init(&hmp->vchain, 0);
2196         error = hammer2_recovery_scan(hmp, parent, &info, sync_tid);
2197         hammer2_chain_lookup_done(parent);
2198
2199         while ((elm = TAILQ_FIRST(&info.list)) != NULL) {
2200                 TAILQ_REMOVE(&info.list, elm, entry);
2201                 parent = elm->chain;
2202                 sync_tid = elm->sync_tid;
2203                 kfree(elm, M_HAMMER2);
2204
2205                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
2206                 error |= hammer2_recovery_scan(hmp, parent, &info,
2207                                               hmp->voldata.freemap_tid);
2208                 hammer2_chain_unlock(parent);
2209                 hammer2_chain_drop(parent);     /* drop elm->chain ref */
2210         }
2211
2212         hammer2_trans_done(hmp->spmp, 0);
2213
2214         return error;
2215 }
2216
2217 static
2218 int
2219 hammer2_recovery_scan(hammer2_dev_t *hmp, hammer2_chain_t *parent,
2220                       struct hammer2_recovery_info *info,
2221                       hammer2_tid_t sync_tid)
2222 {
2223         const hammer2_inode_data_t *ripdata;
2224         hammer2_chain_t *chain;
2225         hammer2_blockref_t bref;
2226         int tmp_error;
2227         int rup_error;
2228         int error;
2229         int first;
2230
2231         /*
2232          * Adjust freemap to ensure that the block(s) are marked allocated.
2233          */
2234         if (parent->bref.type != HAMMER2_BREF_TYPE_VOLUME) {
2235                 hammer2_freemap_adjust(hmp, &parent->bref,
2236                                        HAMMER2_FREEMAP_DORECOVER);
2237         }
2238
2239         /*
2240          * Check type for recursive scan
2241          */
2242         switch(parent->bref.type) {
2243         case HAMMER2_BREF_TYPE_VOLUME:
2244                 /* data already instantiated */
2245                 break;
2246         case HAMMER2_BREF_TYPE_INODE:
2247                 /*
2248                  * Must instantiate data for DIRECTDATA test and also
2249                  * for recursion.
2250                  */
2251                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
2252                 ripdata = &hammer2_chain_rdata(parent)->ipdata;
2253                 if (ripdata->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA) {
2254                         /* not applicable to recovery scan */
2255                         hammer2_chain_unlock(parent);
2256                         return 0;
2257                 }
2258                 hammer2_chain_unlock(parent);
2259                 break;
2260         case HAMMER2_BREF_TYPE_INDIRECT:
2261                 /*
2262                  * Must instantiate data for recursion
2263                  */
2264                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
2265                 hammer2_chain_unlock(parent);
2266                 break;
2267         case HAMMER2_BREF_TYPE_DIRENT:
2268         case HAMMER2_BREF_TYPE_DATA:
2269         case HAMMER2_BREF_TYPE_FREEMAP:
2270         case HAMMER2_BREF_TYPE_FREEMAP_NODE:
2271         case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
2272                 /* not applicable to recovery scan */
2273                 return 0;
2274                 break;
2275         default:
2276                 return HAMMER2_ERROR_BADBREF;
2277         }
2278
2279         /*
2280          * Defer operation if depth limit reached.
2281          */
2282         if (info->depth >= HAMMER2_RECOVERY_MAXDEPTH) {
2283                 struct hammer2_recovery_elm *elm;
2284
2285                 elm = kmalloc(sizeof(*elm), M_HAMMER2, M_ZERO | M_WAITOK);
2286                 elm->chain = parent;
2287                 elm->sync_tid = sync_tid;
2288                 hammer2_chain_ref(parent);
2289                 TAILQ_INSERT_TAIL(&info->list, elm, entry);
2290                 /* unlocked by caller */
2291
2292                 return(0);
2293         }
2294
2295
2296         /*
2297          * Recursive scan of the last flushed transaction only.  We are
2298          * doing this without pmp assignments so don't leave the chains
2299          * hanging around after we are done with them.
2300          *
2301          * error        Cumulative error this level only
2302          * rup_error    Cumulative error for recursion
2303          * tmp_error    Specific non-cumulative recursion error
2304          */
2305         chain = NULL;
2306         first = 1;
2307         rup_error = 0;
2308         error = 0;
2309
2310         for (;;) {
2311                 error |= hammer2_chain_scan(parent, &chain, &bref,
2312                                             &first,
2313                                             HAMMER2_LOOKUP_NODATA);
2314
2315                 /*
2316                  * Problem during scan or EOF
2317                  */
2318                 if (error)
2319                         break;
2320
2321                 /*
2322                  * If this is a leaf
2323                  */
2324                 if (chain == NULL) {
2325                         if (bref.mirror_tid > sync_tid) {
2326                                 hammer2_freemap_adjust(hmp, &bref,
2327                                                      HAMMER2_FREEMAP_DORECOVER);
2328                         }
2329                         continue;
2330                 }
2331
2332                 /*
2333                  * This may or may not be a recursive node.
2334                  */
2335                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
2336                 if (bref.mirror_tid > sync_tid) {
2337                         ++info->depth;
2338                         tmp_error = hammer2_recovery_scan(hmp, chain,
2339                                                            info, sync_tid);
2340                         --info->depth;
2341                 } else {
2342                         tmp_error = 0;
2343                 }
2344
2345                 /*
2346                  * Flush the recovery at the PFS boundary to stage it for
2347                  * the final flush of the super-root topology.
2348                  */
2349                 if (tmp_error == 0 &&
2350                     (bref.flags & HAMMER2_BREF_FLAG_PFSROOT) &&
2351                     (chain->flags & HAMMER2_CHAIN_ONFLUSH)) {
2352                         hammer2_flush(chain, HAMMER2_FLUSH_TOP |
2353                                              HAMMER2_FLUSH_ALL);
2354                 }
2355                 rup_error |= tmp_error;
2356         }
2357         return ((error | rup_error) & ~HAMMER2_ERROR_EOF);
2358 }
2359
2360 /*
2361  * This fixes up an error introduced in earlier H2 implementations where
2362  * moving a PFS inode into an indirect block wound up causing the
2363  * HAMMER2_BREF_FLAG_PFSROOT flag in the bref to get cleared.
2364  */
2365 static
2366 int
2367 hammer2_fixup_pfses(hammer2_dev_t *hmp)
2368 {
2369         const hammer2_inode_data_t *ripdata;
2370         hammer2_chain_t *parent;
2371         hammer2_chain_t *chain;
2372         hammer2_key_t key_next;
2373         hammer2_pfs_t *spmp;
2374         int error;
2375
2376         error = 0;
2377
2378         /*
2379          * Lookup mount point under the media-localized super-root.
2380          *
2381          * cluster->pmp will incorrectly point to spmp and must be fixed
2382          * up later on.
2383          */
2384         spmp = hmp->spmp;
2385         hammer2_inode_lock(spmp->iroot, 0);
2386         parent = hammer2_inode_chain(spmp->iroot, 0, HAMMER2_RESOLVE_ALWAYS);
2387         chain = hammer2_chain_lookup(&parent, &key_next,
2388                                          HAMMER2_KEY_MIN, HAMMER2_KEY_MAX,
2389                                          &error, 0);
2390         while (chain) {
2391                 if (chain->bref.type != HAMMER2_BREF_TYPE_INODE)
2392                         continue;
2393                 if (chain->error) {
2394                         kprintf("I/O error scanning PFS labels\n");
2395                         error |= chain->error;
2396                 } else if ((chain->bref.flags &
2397                             HAMMER2_BREF_FLAG_PFSROOT) == 0) {
2398                         int error2;
2399
2400                         ripdata = &chain->data->ipdata;
2401                         hammer2_trans_init(hmp->spmp, 0);
2402                         error2 = hammer2_chain_modify(chain,
2403                                                       chain->bref.modify_tid,
2404                                                       0, 0);
2405                         if (error2 == 0) {
2406                                 kprintf("hammer2: Correct mis-flagged PFS %s\n",
2407                                         ripdata->filename);
2408                                 chain->bref.flags |= HAMMER2_BREF_FLAG_PFSROOT;
2409                         } else {
2410                                 error |= error2;
2411                         }
2412                         hammer2_flush(chain, HAMMER2_FLUSH_TOP |
2413                                              HAMMER2_FLUSH_ALL);
2414                         hammer2_trans_done(hmp->spmp, 0);
2415                 }
2416                 chain = hammer2_chain_next(&parent, chain, &key_next,
2417                                            key_next, HAMMER2_KEY_MAX,
2418                                            &error, 0);
2419         }
2420         if (parent) {
2421                 hammer2_chain_unlock(parent);
2422                 hammer2_chain_drop(parent);
2423         }
2424         hammer2_inode_unlock(spmp->iroot);
2425
2426         return error;
2427 }
2428
2429 /*
2430  * Sync a mount point; this is called periodically on a per-mount basis from
2431  * the filesystem syncer, and whenever a user issues a sync.
2432  */
2433 int
2434 hammer2_vfs_sync(struct mount *mp, int waitfor)
2435 {
2436         int error;
2437
2438         error = hammer2_vfs_sync_pmp(MPTOPMP(mp), waitfor);
2439
2440         return error;
2441 }
2442
2443 /*
2444  * Because frontend operations lock vnodes before we get a chance to
2445  * lock the related inode, we can't just acquire a vnode lock without
2446  * risking a deadlock.  The frontend may be holding a vnode lock while
2447  * also blocked on our SYNCQ flag while trying to get the inode lock.
2448  *
2449  * To deal with this situation we can check the vnode lock situation
2450  * after locking the inode and perform a work-around.
2451  */
2452 int
2453 hammer2_vfs_sync_pmp(hammer2_pfs_t *pmp, int waitfor)
2454 {
2455         struct mount *mp;
2456         /*hammer2_xop_flush_t *xop;*/
2457         /*struct hammer2_sync_info info;*/
2458         hammer2_inode_t *ip;
2459         hammer2_depend_t *depend;
2460         hammer2_depend_t *depend_next;
2461         struct vnode *vp;
2462         uint32_t pass2;
2463         int error;
2464         int wakecount;
2465         int dorestart;
2466
2467         mp = pmp->mp;
2468
2469         /*
2470          * Move all inodes on sideq to syncq.  This will clear sideq.
2471          * This should represent all flushable inodes.  These inodes
2472          * will already have refs due to being on syncq or sideq.  We
2473          * must do this all at once with the spinlock held to ensure that
2474          * all inode dependencies are part of the same flush.
2475          *
2476          * We should be able to do this asynchronously from frontend
2477          * operations because we will be locking the inodes later on
2478          * to actually flush them, and that will partition any frontend
2479          * op using the same inode.  Either it has already locked the
2480          * inode and we will block, or it has not yet locked the inode
2481          * and it will block until we are finished flushing that inode.
2482          *
2483          * When restarting, only move the inodes flagged as PASS2 from
2484          * SIDEQ to SYNCQ.  PASS2 propagation by inode_lock4() and
2485          * inode_depend() are atomic with the spin-lock.
2486          */
2487         hammer2_trans_init(pmp, HAMMER2_TRANS_ISFLUSH);
2488 #ifdef HAMMER2_DEBUG_SYNC
2489         kprintf("FILESYSTEM SYNC BOUNDARY\n");
2490 #endif
2491         dorestart = 0;
2492
2493         /*
2494          * Move inodes from depq to syncq, releasing the related
2495          * depend structures.
2496          */
2497 restart:
2498 #ifdef HAMMER2_DEBUG_SYNC
2499         kprintf("FILESYSTEM SYNC RESTART (%d)\n", dorestart);
2500 #endif
2501         hammer2_trans_setflags(pmp, 0/*HAMMER2_TRANS_COPYQ*/);
2502         hammer2_trans_clearflags(pmp, HAMMER2_TRANS_RESCAN);
2503
2504         /*
2505          * Move inodes from depq to syncq.  When restarting, only depq's
2506          * marked pass2 are moved.
2507          */
2508         hammer2_spin_ex(&pmp->list_spin);
2509         depend_next = TAILQ_FIRST(&pmp->depq);
2510         wakecount = 0;
2511
2512         while ((depend = depend_next) != NULL) {
2513                 depend_next = TAILQ_NEXT(depend, entry);
2514                 if (dorestart && depend->pass2 == 0)
2515                         continue;
2516                 TAILQ_FOREACH(ip, &depend->sideq, entry) {
2517                         KKASSERT(ip->flags & HAMMER2_INODE_SIDEQ);
2518                         atomic_set_int(&ip->flags, HAMMER2_INODE_SYNCQ);
2519                         atomic_clear_int(&ip->flags, HAMMER2_INODE_SIDEQ);
2520                         ip->depend = NULL;
2521                 }
2522
2523                 /*
2524                  * NOTE: pmp->sideq_count includes both sideq and syncq
2525                  */
2526                 TAILQ_CONCAT(&pmp->syncq, &depend->sideq, entry);
2527
2528                 depend->count = 0;
2529                 depend->pass2 = 0;
2530                 TAILQ_REMOVE(&pmp->depq, depend, entry);
2531         }
2532
2533         hammer2_spin_unex(&pmp->list_spin);
2534         hammer2_trans_clearflags(pmp, /*HAMMER2_TRANS_COPYQ |*/
2535                                       HAMMER2_TRANS_WAITING);
2536         dorestart = 0;
2537
2538         /*
2539          * sideq_count may have dropped enough to allow us to unstall
2540          * the frontend.
2541          */
2542         hammer2_pfs_memory_wakeup(pmp, 0);
2543
2544         /*
2545          * Now run through all inodes on syncq.
2546          *
2547          * Flush transactions only interlock with other flush transactions.
2548          * Any conflicting frontend operations will block on the inode, but
2549          * may hold a vnode lock while doing so.
2550          */
2551         hammer2_spin_ex(&pmp->list_spin);
2552         while ((ip = TAILQ_FIRST(&pmp->syncq)) != NULL) {
2553                 /*
2554                  * Remove the inode from the SYNCQ, transfer the syncq ref
2555                  * to us.  We must clear SYNCQ to allow any potential
2556                  * front-end deadlock to proceed.  We must set PASS2 so
2557                  * the dependency code knows what to do.
2558                  */
2559                 pass2 = ip->flags;
2560                 cpu_ccfence();
2561                 if (atomic_cmpset_int(&ip->flags,
2562                               pass2,
2563                               (pass2 & ~(HAMMER2_INODE_SYNCQ |
2564                                          HAMMER2_INODE_SYNCQ_WAKEUP)) |
2565                               HAMMER2_INODE_SYNCQ_PASS2) == 0) {
2566                         continue;
2567                 }
2568                 TAILQ_REMOVE(&pmp->syncq, ip, entry);
2569                 --pmp->sideq_count;
2570                 hammer2_spin_unex(&pmp->list_spin);
2571
2572                 /*
2573                  * Tickle anyone waiting on ip->flags or the hysteresis
2574                  * on the dirty inode count.
2575                  */
2576                 if (pass2 & HAMMER2_INODE_SYNCQ_WAKEUP)
2577                         wakeup(&ip->flags);
2578                 if (++wakecount >= hammer2_limit_dirty_inodes / 20 + 1) {
2579                         wakecount = 0;
2580                         hammer2_pfs_memory_wakeup(pmp, 0);
2581                 }
2582
2583                 /*
2584                  * Relock the inode, and we inherit a ref from the above.
2585                  * We will check for a race after we acquire the vnode.
2586                  */
2587                 hammer2_mtx_ex(&ip->lock);
2588
2589                 /*
2590                  * We need the vp in order to vfsync() dirty buffers, so if
2591                  * one isn't attached we can skip it.
2592                  *
2593                  * Ordering the inode lock and then the vnode lock has the
2594                  * potential to deadlock.  If we had left SYNCQ set that could
2595                  * also deadlock us against the frontend even if we don't hold
2596                  * any locks, but the latter is not a problem now since we
2597                  * cleared it.  igetv will temporarily release the inode lock
2598                  * in a safe manner to work-around the deadlock.
2599                  *
2600                  * Unfortunately it is still possible to deadlock when the
2601                  * frontend obtains multiple inode locks, because all the
2602                  * related vnodes are already locked (nor can the vnode locks
2603                  * be released and reacquired without messing up RECLAIM and
2604                  * INACTIVE sequencing).
2605                  *
2606                  * The solution for now is to move the vp back onto SIDEQ
2607                  * and set dorestart, which will restart the flush after we
2608                  * exhaust the current SYNCQ.  Note that additional
2609                  * dependencies may build up, so we definitely need to move
2610                  * the whole SIDEQ back to SYNCQ when we restart.
2611                  */
2612                 vp = ip->vp;
2613                 if (vp) {
2614                         if (vget(vp, LK_EXCLUSIVE|LK_NOWAIT)) {
2615                                 /*
2616                                  * Failed to get the vnode, requeue the inode
2617                                  * (PASS2 is already set so it will be found
2618                                  * again on the restart).
2619                                  *
2620                                  * Then unlock, possibly sleep, and retry
2621                                  * later.  We sleep if PASS2 was *previously*
2622                                  * set, before we set it again above.
2623                                  */
2624                                 vp = NULL;
2625                                 dorestart = 1;
2626 #ifdef HAMMER2_DEBUG_SYNC
2627                                 kprintf("inum %ld (sync delayed by vnode)\n",
2628                                         (long)ip->meta.inum);
2629 #endif
2630                                 hammer2_inode_delayed_sideq(ip);
2631
2632                                 hammer2_mtx_unlock(&ip->lock);
2633                                 hammer2_inode_drop(ip);
2634
2635                                 if (pass2 & HAMMER2_INODE_SYNCQ_PASS2) {
2636                                         tsleep(&dorestart, 0, "h2syndel", 2);
2637                                 }
2638                                 hammer2_spin_ex(&pmp->list_spin);
2639                                 continue;
2640                         }
2641                 } else {
2642                         vp = NULL;
2643                 }
2644
2645                 /*
2646                  * If the inode wound up on a SIDEQ again it will already be
2647                  * prepped for another PASS2.  In this situation if we flush
2648                  * it now we will just wind up flushing it again in the same
2649                  * syncer run, so we might as well not flush it now.
2650                  */
2651                 if (ip->flags & HAMMER2_INODE_SIDEQ) {
2652                         hammer2_mtx_unlock(&ip->lock);
2653                         hammer2_inode_drop(ip);
2654                         if (vp)
2655                                 vput(vp);
2656                         dorestart = 1;
2657                         hammer2_spin_ex(&pmp->list_spin);
2658                         continue;
2659                 }
2660
2661                 /*
2662                  * Ok we have the inode exclusively locked and if vp is
2663                  * not NULL that will also be exclusively locked.  Do the
2664                  * meat of the flush.
2665                  *
2666                  * vp token needed for v_rbdirty_tree check / vclrisdirty
2667                  * sequencing.  Though we hold the vnode exclusively so
2668                  * we shouldn't need to hold the token also in this case.
2669                  */
2670                 if (vp) {
2671                         vfsync(vp, MNT_WAIT, 1, NULL, NULL);
2672                         bio_track_wait(&vp->v_track_write, 0, 0); /* XXX */
2673                 }
2674
2675                 /*
2676                  * If the inode has not yet been inserted into the tree
2677                  * we must do so.  Then sync and flush it.  The flush should
2678                  * update the parent.
2679                  */
2680                 if (ip->flags & HAMMER2_INODE_DELETING) {
2681 #ifdef HAMMER2_DEBUG_SYNC
2682                         kprintf("inum %ld destroy\n", (long)ip->meta.inum);
2683 #endif
2684                         hammer2_inode_chain_des(ip);
2685                         atomic_add_long(&hammer2_iod_inode_deletes, 1);
2686                 } else if (ip->flags & HAMMER2_INODE_CREATING) {
2687 #ifdef HAMMER2_DEBUG_SYNC
2688                         kprintf("inum %ld insert\n", (long)ip->meta.inum);
2689 #endif
2690                         hammer2_inode_chain_ins(ip);
2691                         atomic_add_long(&hammer2_iod_inode_creates, 1);
2692                 }
2693 #ifdef HAMMER2_DEBUG_SYNC
2694                 kprintf("inum %ld chain-sync\n", (long)ip->meta.inum);
2695 #endif
2696
2697                 /*
2698                  * Because I kinda messed up the design and index the inodes
2699                  * under the root inode, along side the directory entries,
2700                  * we can't flush the inode index under the iroot until the
2701                  * end.  If we do it now we might miss effects created by
2702                  * other inodes on the SYNCQ.
2703                  *
2704                  * Do a normal (non-FSSYNC) flush instead, which allows the
2705                  * vnode code to work the same.  We don't want to force iroot
2706                  * back onto the SIDEQ, and we also don't want the flush code
2707                  * to update pfs_iroot_blocksets until the final flush later.
2708                  *
2709                  * XXX at the moment this will likely result in a double-flush
2710                  * of the iroot chain.
2711                  */
2712                 hammer2_inode_chain_sync(ip);
2713                 if (ip == pmp->iroot) {
2714                         hammer2_inode_chain_flush(ip, HAMMER2_XOP_INODE_STOP);
2715                 } else {
2716                         hammer2_inode_chain_flush(ip, HAMMER2_XOP_INODE_STOP |
2717                                                       HAMMER2_XOP_FSSYNC);
2718                 }
2719                 if (vp) {
2720                         lwkt_gettoken(&vp->v_token);
2721                         if ((ip->flags & (HAMMER2_INODE_MODIFIED |
2722                                           HAMMER2_INODE_RESIZED |
2723                                           HAMMER2_INODE_DIRTYDATA)) == 0 &&
2724                             RB_EMPTY(&vp->v_rbdirty_tree) &&
2725                             !bio_track_active(&vp->v_track_write)) {
2726                                 vclrisdirty(vp);
2727                         } else {
2728                                 hammer2_inode_delayed_sideq(ip);
2729                         }
2730                         lwkt_reltoken(&vp->v_token);
2731                         vput(vp);
2732                         vp = NULL;      /* safety */
2733                 }
2734                 atomic_clear_int(&ip->flags, HAMMER2_INODE_SYNCQ_PASS2);
2735                 hammer2_inode_unlock(ip);       /* unlock+drop */
2736                 /* ip pointer invalid */
2737
2738                 /*
2739                  * If the inode got dirted after we dropped our locks,
2740                  * it will have already been moved back to the SIDEQ.
2741                  */
2742                 hammer2_spin_ex(&pmp->list_spin);
2743         }
2744         hammer2_spin_unex(&pmp->list_spin);
2745         hammer2_pfs_memory_wakeup(pmp, 0);
2746
2747         if (dorestart || (pmp->trans.flags & HAMMER2_TRANS_RESCAN)) {
2748 #ifdef HAMMER2_DEBUG_SYNC
2749                 kprintf("FILESYSTEM SYNC STAGE 1 RESTART\n");
2750                 /*tsleep(&dorestart, 0, "h2STG1-R", hz*20);*/
2751 #endif
2752                 dorestart = 1;
2753                 goto restart;
2754         }
2755 #ifdef HAMMER2_DEBUG_SYNC
2756         kprintf("FILESYSTEM SYNC STAGE 2 BEGIN\n");
2757         /*tsleep(&dorestart, 0, "h2STG2", hz*20);*/
2758 #endif
2759
2760         /*
2761          * We have to flush the PFS root last, even if it does not appear to
2762          * be dirty, because all the inodes in the PFS are indexed under it.
2763          * The normal flushing of iroot above would only occur if directory
2764          * entries under the root were changed.
2765          *
2766          * Specifying VOLHDR will cause an additionl flush of hmp->spmp
2767          * for the media making up the cluster.
2768          */
2769         if ((ip = pmp->iroot) != NULL) {
2770                 hammer2_inode_ref(ip);
2771                 hammer2_mtx_ex(&ip->lock);
2772                 hammer2_inode_chain_sync(ip);
2773                 hammer2_inode_chain_flush(ip, HAMMER2_XOP_INODE_STOP |
2774                                               HAMMER2_XOP_FSSYNC |
2775                                               HAMMER2_XOP_VOLHDR);
2776                 hammer2_inode_unlock(ip);       /* unlock+drop */
2777         }
2778 #ifdef HAMMER2_DEBUG_SYNC
2779         kprintf("FILESYSTEM SYNC STAGE 2 DONE\n");
2780 #endif
2781
2782         /*
2783          * device bioq sync
2784          */
2785         hammer2_bioq_sync(pmp);
2786
2787 #if 0
2788         info.pass = 1;
2789         info.waitfor = MNT_WAIT;
2790         vsyncscan(mp, flags, hammer2_sync_scan2, &info);
2791
2792         info.pass = 2;
2793         info.waitfor = MNT_WAIT;
2794         vsyncscan(mp, flags, hammer2_sync_scan2, &info);
2795 #endif
2796 #if 0
2797         /*
2798          * Generally speaking we now want to flush the media topology from
2799          * the iroot through to the inodes.  The flush stops at any inode
2800          * boundary, which allows the frontend to continue running concurrent
2801          * modifying operations on inodes (including kernel flushes of
2802          * buffers) without interfering with the main sync.
2803          *
2804          * Use the XOP interface to concurrently flush all nodes to
2805          * synchronize the PFSROOT subtopology to the media.  A standard
2806          * end-of-scan ENOENT error indicates cluster sufficiency.
2807          *
2808          * Note that this flush will not be visible on crash recovery until
2809          * we flush the super-root topology in the next loop.
2810          *
2811          * XXX For now wait for all flushes to complete.
2812          */
2813         if (mp && (ip = pmp->iroot) != NULL) {
2814                 /*
2815                  * If unmounting try to flush everything including any
2816                  * sub-trees under inodes, just in case there is dangling
2817                  * modified data, as a safety.  Otherwise just flush up to
2818                  * the inodes in this stage.
2819                  */
2820                 kprintf("MP & IROOT\n");
2821 #ifdef HAMMER2_DEBUG_SYNC
2822                 kprintf("FILESYSTEM SYNC STAGE 3 IROOT BEGIN\n");
2823 #endif
2824                 if (mp->mnt_kern_flag & MNTK_UNMOUNT) {
2825                         xop = hammer2_xop_alloc(ip, HAMMER2_XOP_MODIFYING |
2826                                                     HAMMER2_XOP_VOLHDR |
2827                                                     HAMMER2_XOP_FSSYNC |
2828                                                     HAMMER2_XOP_INODE_STOP);
2829                 } else {
2830                         xop = hammer2_xop_alloc(ip, HAMMER2_XOP_MODIFYING |
2831                                                     HAMMER2_XOP_INODE_STOP |
2832                                                     HAMMER2_XOP_VOLHDR |
2833                                                     HAMMER2_XOP_FSSYNC |
2834                                                     HAMMER2_XOP_INODE_STOP);
2835                 }
2836                 hammer2_xop_start(&xop->head, &hammer2_inode_flush_desc);
2837                 error = hammer2_xop_collect(&xop->head,
2838                                             HAMMER2_XOP_COLLECT_WAITALL);
2839                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
2840 #ifdef HAMMER2_DEBUG_SYNC
2841                 kprintf("FILESYSTEM SYNC STAGE 3 IROOT END\n");
2842 #endif
2843                 if (error == HAMMER2_ERROR_ENOENT)
2844                         error = 0;
2845                 else
2846                         error = hammer2_error_to_errno(error);
2847         } else {
2848                 error = 0;
2849         }
2850 #endif
2851         error = 0;      /* XXX */
2852         hammer2_trans_done(pmp, HAMMER2_TRANS_ISFLUSH);
2853
2854         return (error);
2855 }
2856
2857 static
2858 int
2859 hammer2_vfs_vptofh(struct vnode *vp, struct fid *fhp)
2860 {
2861         hammer2_inode_t *ip;
2862
2863         KKASSERT(MAXFIDSZ >= 16);
2864         ip = VTOI(vp);
2865         fhp->fid_len = offsetof(struct fid, fid_data[16]);
2866         fhp->fid_ext = 0;
2867         ((hammer2_tid_t *)fhp->fid_data)[0] = ip->meta.inum;
2868         ((hammer2_tid_t *)fhp->fid_data)[1] = 0;
2869
2870         return 0;
2871 }
2872
2873 static
2874 int
2875 hammer2_vfs_fhtovp(struct mount *mp, struct vnode *rootvp,
2876                struct fid *fhp, struct vnode **vpp)
2877 {
2878         hammer2_pfs_t *pmp;
2879         hammer2_tid_t inum;
2880         int error;
2881
2882         pmp = MPTOPMP(mp);
2883         inum = ((hammer2_tid_t *)fhp->fid_data)[0] & HAMMER2_DIRHASH_USERMSK;
2884         if (vpp) {
2885                 if (inum == 1)
2886                         error = hammer2_vfs_root(mp, vpp);
2887                 else
2888                         error = hammer2_vfs_vget(mp, NULL, inum, vpp);
2889         } else {
2890                 error = 0;
2891         }
2892         if (error)
2893                 kprintf("fhtovp: %016jx -> %p, %d\n", inum, *vpp, error);
2894         return error;
2895 }
2896
2897 static
2898 int
2899 hammer2_vfs_checkexp(struct mount *mp, struct sockaddr *nam,
2900                  int *exflagsp, struct ucred **credanonp)
2901 {
2902         hammer2_pfs_t *pmp;
2903         struct netcred *np;
2904         int error;
2905
2906         pmp = MPTOPMP(mp);
2907         np = vfs_export_lookup(mp, &pmp->export, nam);
2908         if (np) {
2909                 *exflagsp = np->netc_exflags;
2910                 *credanonp = &np->netc_anon;
2911                 error = 0;
2912         } else {
2913                 error = EACCES;
2914         }
2915         return error;
2916 }
2917
2918 /*
2919  * Support code for hammer2_vfs_mount().  Read, verify, and install the volume
2920  * header into the HMP
2921  */
2922 static
2923 int
2924 hammer2_install_volume_header(hammer2_dev_t *hmp)
2925 {
2926         hammer2_volume_data_t *vd;
2927         struct buf *bp;
2928         hammer2_crc32_t crc0, crc, bcrc0, bcrc;
2929         int error_reported;
2930         int error;
2931         int valid;
2932         int i;
2933
2934         error_reported = 0;
2935         error = 0;
2936         valid = 0;
2937         bp = NULL;
2938
2939         /*
2940          * There are up to 4 copies of the volume header (syncs iterate
2941          * between them so there is no single master).  We don't trust the
2942          * volu_size field so we don't know precisely how large the filesystem
2943          * is, so depend on the OS to return an error if we go beyond the
2944          * block device's EOF.
2945          */
2946         for (i = 0; i < HAMMER2_NUM_VOLHDRS; i++) {
2947                 error = bread(hmp->devvp, i * HAMMER2_ZONE_BYTES64,
2948                               HAMMER2_VOLUME_BYTES, &bp);
2949                 if (error) {
2950                         brelse(bp);
2951                         bp = NULL;
2952                         continue;
2953                 }
2954
2955                 vd = (struct hammer2_volume_data *) bp->b_data;
2956                 if ((vd->magic != HAMMER2_VOLUME_ID_HBO) &&
2957                     (vd->magic != HAMMER2_VOLUME_ID_ABO)) {
2958                         kprintf("hammer2: volume header #%d: bad magic\n", i);
2959                         brelse(bp);
2960                         bp = NULL;
2961                         continue;
2962                 }
2963
2964                 if (vd->magic == HAMMER2_VOLUME_ID_ABO) {
2965                         /* XXX: Reversed-endianness filesystem */
2966                         kprintf("hammer2: volume header #%d: reverse-endian "
2967                                 "filesystem detected\n", i);
2968                         brelse(bp);
2969                         bp = NULL;
2970                         continue;
2971                 }
2972
2973                 crc = vd->icrc_sects[HAMMER2_VOL_ICRC_SECT0];
2974                 crc0 = hammer2_icrc32(bp->b_data + HAMMER2_VOLUME_ICRC0_OFF,
2975                                       HAMMER2_VOLUME_ICRC0_SIZE);
2976                 bcrc = vd->icrc_sects[HAMMER2_VOL_ICRC_SECT1];
2977                 bcrc0 = hammer2_icrc32(bp->b_data + HAMMER2_VOLUME_ICRC1_OFF,
2978                                        HAMMER2_VOLUME_ICRC1_SIZE);
2979                 if ((crc0 != crc) || (bcrc0 != bcrc)) {
2980                         kprintf("hammer2: volume header #%d: volume header crc "
2981                                 "mismatch %08x/%08x\n",
2982                                 i, crc0, crc);
2983                         error_reported = 1;
2984                         brelse(bp);
2985                         bp = NULL;
2986                         continue;
2987                 }
2988                 if (valid == 0 || hmp->voldata.mirror_tid < vd->mirror_tid) {
2989                         valid = 1;
2990                         hmp->voldata = *vd;
2991                         hmp->volhdrno = i;
2992                 }
2993                 brelse(bp);
2994                 bp = NULL;
2995         }
2996         if (valid) {
2997                 hmp->volsync = hmp->voldata;
2998                 hmp->free_reserved = hmp->voldata.allocator_size / 20;
2999                 error = 0;
3000                 if (error_reported || bootverbose || 1) { /* 1/DEBUG */
3001                         kprintf("hammer2: using volume header #%d\n",
3002                                 hmp->volhdrno);
3003                 }
3004         } else {
3005                 error = EINVAL;
3006                 kprintf("hammer2: no valid volume headers found!\n");
3007         }
3008         return (error);
3009 }
3010
3011 /*
3012  * This handles hysteresis on regular file flushes.  Because the BIOs are
3013  * routed to a thread it is possible for an excessive number to build up
3014  * and cause long front-end stalls long before the runningbuffspace limit
3015  * is hit, so we implement hammer2_flush_pipe to control the
3016  * hysteresis.
3017  *
3018  * This is a particular problem when compression is used.
3019  */
3020 void
3021 hammer2_lwinprog_ref(hammer2_pfs_t *pmp)
3022 {
3023         atomic_add_int(&pmp->count_lwinprog, 1);
3024 }
3025
3026 void
3027 hammer2_lwinprog_drop(hammer2_pfs_t *pmp)
3028 {
3029         int lwinprog;
3030
3031         lwinprog = atomic_fetchadd_int(&pmp->count_lwinprog, -1);
3032         if ((lwinprog & HAMMER2_LWINPROG_WAITING) &&
3033             (lwinprog & HAMMER2_LWINPROG_MASK) <= hammer2_flush_pipe * 2 / 3) {
3034                 atomic_clear_int(&pmp->count_lwinprog,
3035                                  HAMMER2_LWINPROG_WAITING);
3036                 wakeup(&pmp->count_lwinprog);
3037         }
3038         if ((lwinprog & HAMMER2_LWINPROG_WAITING0) &&
3039             (lwinprog & HAMMER2_LWINPROG_MASK) <= 0) {
3040                 atomic_clear_int(&pmp->count_lwinprog,
3041                                  HAMMER2_LWINPROG_WAITING0);
3042                 wakeup(&pmp->count_lwinprog);
3043         }
3044 }
3045
3046 void
3047 hammer2_lwinprog_wait(hammer2_pfs_t *pmp, int flush_pipe)
3048 {
3049         int lwinprog;
3050         int lwflag = (flush_pipe) ? HAMMER2_LWINPROG_WAITING :
3051                                     HAMMER2_LWINPROG_WAITING0;
3052
3053         for (;;) {
3054                 lwinprog = pmp->count_lwinprog;
3055                 cpu_ccfence();
3056                 if ((lwinprog & HAMMER2_LWINPROG_MASK) <= flush_pipe)
3057                         break;
3058                 tsleep_interlock(&pmp->count_lwinprog, 0);
3059                 atomic_set_int(&pmp->count_lwinprog, lwflag);
3060                 lwinprog = pmp->count_lwinprog;
3061                 if ((lwinprog & HAMMER2_LWINPROG_MASK) <= flush_pipe)
3062                         break;
3063                 tsleep(&pmp->count_lwinprog, PINTERLOCKED, "h2wpipe", hz);
3064         }
3065 }
3066
3067 /*
3068  * It is possible for an excessive number of dirty chains or dirty inodes
3069  * to build up.  When this occurs we start an asynchronous filesystem sync.
3070  * If the level continues to build up, we stall, waiting for it to drop,
3071  * with some hysteresis.
3072  *
3073  * This relies on the kernel calling hammer2_vfs_modifying() prior to
3074  * obtaining any vnode locks before making a modifying VOP call.
3075  */
3076 static int
3077 hammer2_vfs_modifying(struct mount *mp)
3078 {
3079         if (mp->mnt_flag & MNT_RDONLY)
3080                 return EROFS;
3081         hammer2_pfs_memory_wait(MPTOPMP(mp));
3082
3083         return 0;
3084 }
3085
3086 /*
3087  * Initiate an asynchronous filesystem sync and, with hysteresis,
3088  * stall if the internal data structure count becomes too bloated.
3089  */
3090 void
3091 hammer2_pfs_memory_wait(hammer2_pfs_t *pmp)
3092 {
3093         uint32_t waiting;
3094         int pcatch;
3095         int error;
3096
3097         if (pmp == NULL || pmp->mp == NULL)
3098                 return;
3099
3100         for (;;) {
3101                 waiting = pmp->inmem_dirty_chains & HAMMER2_DIRTYCHAIN_MASK;
3102                 cpu_ccfence();
3103
3104                 /*
3105                  * Start the syncer running at 1/2 the limit
3106                  */
3107                 if (waiting > hammer2_limit_dirty_chains / 2 ||
3108                     pmp->sideq_count > hammer2_limit_dirty_inodes / 2) {
3109                         trigger_syncer(pmp->mp);
3110                 }
3111
3112                 /*
3113                  * Stall at the limit waiting for the counts to drop.
3114                  * This code will typically be woken up once the count
3115                  * drops below 3/4 the limit, or in one second.
3116                  */
3117                 if (waiting < hammer2_limit_dirty_chains &&
3118                     pmp->sideq_count < hammer2_limit_dirty_inodes) {
3119                         break;
3120                 }
3121
3122                 pcatch = curthread->td_proc ? PCATCH : 0;
3123
3124                 tsleep_interlock(&pmp->inmem_dirty_chains, pcatch);
3125                 atomic_set_int(&pmp->inmem_dirty_chains,
3126                                HAMMER2_DIRTYCHAIN_WAITING);
3127                 if (waiting < hammer2_limit_dirty_chains &&
3128                     pmp->sideq_count < hammer2_limit_dirty_inodes) {
3129                         break;
3130                 }
3131                 trigger_syncer(pmp->mp);
3132                 error = tsleep(&pmp->inmem_dirty_chains, PINTERLOCKED | pcatch,
3133                                "h2memw", hz);
3134                 if (error == ERESTART)
3135                         break;
3136         }
3137 }
3138
3139 /*
3140  * Wake up any stalled frontend ops waiting, with hysteresis, using
3141  * 2/3 of the limit.
3142  */
3143 void
3144 hammer2_pfs_memory_wakeup(hammer2_pfs_t *pmp, int count)
3145 {
3146         uint32_t waiting;
3147
3148         if (pmp) {
3149                 waiting = atomic_fetchadd_int(&pmp->inmem_dirty_chains, count);
3150                 /* don't need --waiting to test flag */
3151
3152                 if ((waiting & HAMMER2_DIRTYCHAIN_WAITING) &&
3153                     (pmp->inmem_dirty_chains & HAMMER2_DIRTYCHAIN_MASK) <=
3154                     hammer2_limit_dirty_chains * 2 / 3 &&
3155                     pmp->sideq_count <= hammer2_limit_dirty_inodes * 2 / 3) {
3156                         atomic_clear_int(&pmp->inmem_dirty_chains,
3157                                          HAMMER2_DIRTYCHAIN_WAITING);
3158                         wakeup(&pmp->inmem_dirty_chains);
3159                 }
3160         }
3161 }
3162
3163 void
3164 hammer2_pfs_memory_inc(hammer2_pfs_t *pmp)
3165 {
3166         if (pmp) {
3167                 atomic_add_int(&pmp->inmem_dirty_chains, 1);
3168         }
3169 }
3170
3171 /*
3172  * Returns 0 if the filesystem has tons of free space
3173  * Returns 1 if the filesystem has less than 10% remaining
3174  * Returns 2 if the filesystem has less than 2%/5% (user/root) remaining.
3175  */
3176 int
3177 hammer2_vfs_enospace(hammer2_inode_t *ip, off_t bytes, struct ucred *cred)
3178 {
3179         hammer2_pfs_t *pmp;
3180         hammer2_dev_t *hmp;
3181         hammer2_off_t free_reserved;
3182         hammer2_off_t free_nominal;
3183         int i;
3184
3185         pmp = ip->pmp;
3186
3187         if (pmp->free_ticks == 0 || pmp->free_ticks != ticks) {
3188                 free_reserved = HAMMER2_SEGSIZE;
3189                 free_nominal = 0x7FFFFFFFFFFFFFFFLLU;
3190                 for (i = 0; i < pmp->iroot->cluster.nchains; ++i) {
3191                         hmp = pmp->pfs_hmps[i];
3192                         if (hmp == NULL)
3193                                 continue;
3194                         if (pmp->pfs_types[i] != HAMMER2_PFSTYPE_MASTER &&
3195                             pmp->pfs_types[i] != HAMMER2_PFSTYPE_SOFT_MASTER)
3196                                 continue;
3197
3198                         if (free_nominal > hmp->voldata.allocator_free)
3199                                 free_nominal = hmp->voldata.allocator_free;
3200                         if (free_reserved < hmp->free_reserved)
3201                                 free_reserved = hmp->free_reserved;
3202                 }
3203
3204                 /*
3205                  * SMP races ok
3206                  */
3207                 pmp->free_reserved = free_reserved;
3208                 pmp->free_nominal = free_nominal;
3209                 pmp->free_ticks = ticks;
3210         } else {
3211                 free_reserved = pmp->free_reserved;
3212                 free_nominal = pmp->free_nominal;
3213         }
3214         if (cred && cred->cr_uid != 0) {
3215                 if ((int64_t)(free_nominal - bytes) <
3216                     (int64_t)free_reserved) {
3217                         return 2;
3218                 }
3219         } else {
3220                 if ((int64_t)(free_nominal - bytes) <
3221                     (int64_t)free_reserved / 2) {
3222                         return 2;
3223                 }
3224         }
3225         if ((int64_t)(free_nominal - bytes) < (int64_t)free_reserved * 2)
3226                 return 1;
3227         return 0;
3228 }
3229
3230 /*
3231  * Debugging
3232  */
3233 void
3234 hammer2_dump_chain(hammer2_chain_t *chain, int tab, int bi, int *countp,
3235                    char pfx, u_int flags)
3236 {
3237         hammer2_chain_t *scan;
3238         hammer2_chain_t *parent;
3239
3240         --*countp;
3241         if (*countp == 0) {
3242                 kprintf("%*.*s...\n", tab, tab, "");
3243                 return;
3244         }
3245         if (*countp < 0)
3246                 return;
3247         kprintf("%*.*s%c-chain %p %s.%-3d %016jx %016jx/%-2d mir=%016jx\n",
3248                 tab, tab, "", pfx, chain,
3249                 hammer2_bref_type_str(chain->bref.type), bi,
3250                 chain->bref.data_off, chain->bref.key, chain->bref.keybits,
3251                 chain->bref.mirror_tid);
3252
3253         kprintf("%*.*s      [%08x] (%s) refs=%d",
3254                 tab, tab, "",
3255                 chain->flags,
3256                 ((chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
3257                 chain->data) ?  (char *)chain->data->ipdata.filename : "?"),
3258                 chain->refs);
3259
3260         parent = chain->parent;
3261         if (parent)
3262                 kprintf("\n%*.*s      p=%p [pflags %08x prefs %d]",
3263                         tab, tab, "",
3264                         parent, parent->flags, parent->refs);
3265         if (RB_EMPTY(&chain->core.rbtree)) {
3266                 kprintf("\n");
3267         } else {
3268                 int bi = 0;
3269                 kprintf(" {\n");
3270                 RB_FOREACH(scan, hammer2_chain_tree, &chain->core.rbtree) {
3271                         if ((scan->flags & flags) || flags == (u_int)-1) {
3272                                 hammer2_dump_chain(scan, tab + 4, bi, countp,
3273                                                    'a', flags);
3274                         }
3275                         bi++;
3276                 }
3277                 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE && chain->data)
3278                         kprintf("%*.*s}(%s)\n", tab, tab, "",
3279                                 chain->data->ipdata.filename);
3280                 else
3281                         kprintf("%*.*s}\n", tab, tab, "");
3282         }
3283 }