sys/vfs/hammer2/hammer2_vfsops.c

   1 /*
   2  * Copyright (c) 2011-2018 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  * by Daniel Flores (GSOC 2013 - mentored by Matthew Dillon, compression)
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in
  16  *    the documentation and/or other materials provided with the
  17  *    distribution.
  18  * 3. Neither the name of The DragonFly Project nor the names of its
  19  *    contributors may be used to endorse or promote products derived
  20  *    from this software without specific, prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  26  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  27  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  28  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  29  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  30  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  31  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  32  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  33  * SUCH DAMAGE.
  34  */
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/kernel.h>
  38 #include <sys/nlookup.h>
  39 #include <sys/vnode.h>
  40 #include <sys/mount.h>
  41 #include <sys/fcntl.h>
  42 #include <sys/buf.h>
  43 #include <sys/uuid.h>
  44 #include <sys/vfsops.h>
  45 #include <sys/sysctl.h>
  46 #include <sys/socket.h>
  47 #include <sys/objcache.h>
  48
  49 #include <sys/proc.h>
  50 #include <sys/mountctl.h>
  51 #include <sys/dirent.h>
  52 #include <sys/uio.h>
  53
  54 #include "hammer2.h"
  55 #include "hammer2_disk.h"
  56 #include "hammer2_mount.h"
  57 #include "hammer2_lz4.h"
  58
  59 #include "zlib/hammer2_zlib.h"
  60
  61 #define REPORT_REFS_ERRORS 1    /* XXX remove me */
  62
  63 MALLOC_DEFINE(M_OBJCACHE, "objcache", "Object Cache");
  64
  65 struct hammer2_sync_info {
  66         int error;
  67         int waitfor;
  68         int pass;
  69 };
  70
  71 TAILQ_HEAD(hammer2_mntlist, hammer2_dev);
  72 static struct hammer2_mntlist hammer2_mntlist;
  73
  74 struct hammer2_pfslist hammer2_pfslist;
  75 struct hammer2_pfslist hammer2_spmplist;
  76 struct lock hammer2_mntlk;
  77
  78 int hammer2_supported_version = HAMMER2_VOL_VERSION_DEFAULT;
  79 int hammer2_debug;
  80 int hammer2_xopgroups;
  81 long hammer2_debug_inode;
  82 int hammer2_cluster_meta_read = 1;      /* physical read-ahead */
  83 int hammer2_cluster_data_read = 4;      /* physical read-ahead */
  84 int hammer2_cluster_write = 0;          /* physical write clustering */
  85 int hammer2_dedup_enable = 1;
  86 int hammer2_always_compress = 0;        /* always try to compress */
  87 int hammer2_flush_pipe = 100;
  88 int hammer2_dio_count;
  89 int hammer2_dio_limit = 256;
  90 int hammer2_bulkfree_tps = 5000;
  91 int hammer2_worker_rmask = 3;
  92 long hammer2_chain_allocs;
  93 long hammer2_limit_dirty_chains;
  94 long hammer2_limit_dirty_inodes;
  95 long hammer2_count_modified_chains;
  96 long hammer2_iod_file_read;
  97 long hammer2_iod_meta_read;
  98 long hammer2_iod_indr_read;
  99 long hammer2_iod_fmap_read;
 100 long hammer2_iod_volu_read;
 101 long hammer2_iod_file_write;
 102 long hammer2_iod_file_wembed;
 103 long hammer2_iod_file_wzero;
 104 long hammer2_iod_file_wdedup;
 105 long hammer2_iod_meta_write;
 106 long hammer2_iod_indr_write;
 107 long hammer2_iod_fmap_write;
 108 long hammer2_iod_volu_write;
 109 static long hammer2_iod_inode_creates;
 110 static long hammer2_iod_inode_deletes;
 111
 112 long hammer2_process_icrc32;
 113 long hammer2_process_xxhash64;
 114
 115 MALLOC_DECLARE(M_HAMMER2_CBUFFER);
 116 MALLOC_DEFINE(M_HAMMER2_CBUFFER, "HAMMER2-compbuffer",
 117                 "Buffer used for compression.");
 118
 119 MALLOC_DECLARE(M_HAMMER2_DEBUFFER);
 120 MALLOC_DEFINE(M_HAMMER2_DEBUFFER, "HAMMER2-decompbuffer",
 121                 "Buffer used for decompression.");
 122
 123 SYSCTL_NODE(_vfs, OID_AUTO, hammer2, CTLFLAG_RW, 0, "HAMMER2 filesystem");
 124
 125 SYSCTL_INT(_vfs_hammer2, OID_AUTO, supported_version, CTLFLAG_RD,
 126            &hammer2_supported_version, 0, "");
 127 SYSCTL_INT(_vfs_hammer2, OID_AUTO, debug, CTLFLAG_RW,
 128            &hammer2_debug, 0, "");
 129 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, debug_inode, CTLFLAG_RW,
 130            &hammer2_debug_inode, 0, "");
 131 SYSCTL_INT(_vfs_hammer2, OID_AUTO, cluster_meta_read, CTLFLAG_RW,
 132            &hammer2_cluster_meta_read, 0, "");
 133 SYSCTL_INT(_vfs_hammer2, OID_AUTO, cluster_data_read, CTLFLAG_RW,
 134            &hammer2_cluster_data_read, 0, "");
 135 SYSCTL_INT(_vfs_hammer2, OID_AUTO, cluster_write, CTLFLAG_RW,
 136            &hammer2_cluster_write, 0, "");
 137 SYSCTL_INT(_vfs_hammer2, OID_AUTO, dedup_enable, CTLFLAG_RW,
 138            &hammer2_dedup_enable, 0, "");
 139 SYSCTL_INT(_vfs_hammer2, OID_AUTO, always_compress, CTLFLAG_RW,
 140            &hammer2_always_compress, 0, "");
 141 SYSCTL_INT(_vfs_hammer2, OID_AUTO, flush_pipe, CTLFLAG_RW,
 142            &hammer2_flush_pipe, 0, "");
 143 SYSCTL_INT(_vfs_hammer2, OID_AUTO, worker_rmask, CTLFLAG_RW,
 144            &hammer2_worker_rmask, 0, "");
 145 SYSCTL_INT(_vfs_hammer2, OID_AUTO, bulkfree_tps, CTLFLAG_RW,
 146            &hammer2_bulkfree_tps, 0, "");
 147 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, chain_allocs, CTLFLAG_RW,
 148            &hammer2_chain_allocs, 0, "");
 149 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, limit_dirty_chains, CTLFLAG_RW,
 150            &hammer2_limit_dirty_chains, 0, "");
 151 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, limit_dirty_inodes, CTLFLAG_RW,
 152            &hammer2_limit_dirty_inodes, 0, "");
 153 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, count_modified_chains, CTLFLAG_RW,
 154            &hammer2_count_modified_chains, 0, "");
 155 SYSCTL_INT(_vfs_hammer2, OID_AUTO, dio_count, CTLFLAG_RD,
 156            &hammer2_dio_count, 0, "");
 157 SYSCTL_INT(_vfs_hammer2, OID_AUTO, dio_limit, CTLFLAG_RW,
 158            &hammer2_dio_limit, 0, "");
 159
 160 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_read, CTLFLAG_RW,
 161            &hammer2_iod_file_read, 0, "");
 162 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_meta_read, CTLFLAG_RW,
 163            &hammer2_iod_meta_read, 0, "");
 164 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_indr_read, CTLFLAG_RW,
 165            &hammer2_iod_indr_read, 0, "");
 166 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_fmap_read, CTLFLAG_RW,
 167            &hammer2_iod_fmap_read, 0, "");
 168 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_volu_read, CTLFLAG_RW,
 169            &hammer2_iod_volu_read, 0, "");
 170
 171 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_write, CTLFLAG_RW,
 172            &hammer2_iod_file_write, 0, "");
 173 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_wembed, CTLFLAG_RW,
 174            &hammer2_iod_file_wembed, 0, "");
 175 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_wzero, CTLFLAG_RW,
 176            &hammer2_iod_file_wzero, 0, "");
 177 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_file_wdedup, CTLFLAG_RW,
 178            &hammer2_iod_file_wdedup, 0, "");
 179 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_meta_write, CTLFLAG_RW,
 180            &hammer2_iod_meta_write, 0, "");
 181 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_indr_write, CTLFLAG_RW,
 182            &hammer2_iod_indr_write, 0, "");
 183 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_fmap_write, CTLFLAG_RW,
 184            &hammer2_iod_fmap_write, 0, "");
 185 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_volu_write, CTLFLAG_RW,
 186            &hammer2_iod_volu_write, 0, "");
 187 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_inode_creates, CTLFLAG_RW,
 188            &hammer2_iod_inode_creates, 0, "");
 189 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, iod_inode_deletes, CTLFLAG_RW,
 190            &hammer2_iod_inode_deletes, 0, "");
 191
 192 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, process_icrc32, CTLFLAG_RW,
 193            &hammer2_process_icrc32, 0, "");
 194 SYSCTL_LONG(_vfs_hammer2, OID_AUTO, process_xxhash64, CTLFLAG_RW,
 195            &hammer2_process_xxhash64, 0, "");
 196
 197 static int hammer2_vfs_init(struct vfsconf *conf);
 198 static int hammer2_vfs_uninit(struct vfsconf *vfsp);
 199 static int hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 200                                 struct ucred *cred);
 201 static int hammer2_remount(hammer2_dev_t *, struct mount *, char *,
 202                                 struct ucred *);
 203 static int hammer2_recovery(hammer2_dev_t *hmp);
 204 static int hammer2_vfs_unmount(struct mount *mp, int mntflags);
 205 static int hammer2_vfs_root(struct mount *mp, struct vnode **vpp);
 206 static int hammer2_vfs_statfs(struct mount *mp, struct statfs *sbp,
 207                                 struct ucred *cred);
 208 static int hammer2_vfs_statvfs(struct mount *mp, struct statvfs *sbp,
 209                                 struct ucred *cred);
 210 static int hammer2_vfs_fhtovp(struct mount *mp, struct vnode *rootvp,
 211                                 struct fid *fhp, struct vnode **vpp);
 212 static int hammer2_vfs_vptofh(struct vnode *vp, struct fid *fhp);
 213 static int hammer2_vfs_checkexp(struct mount *mp, struct sockaddr *nam,
 214                                 int *exflagsp, struct ucred **credanonp);
 215 static int hammer2_vfs_modifying(struct mount *mp);
 216
 217 static void hammer2_update_pmps(hammer2_dev_t *hmp);
 218
 219 static void hammer2_mount_helper(struct mount *mp, hammer2_pfs_t *pmp);
 220 static void hammer2_unmount_helper(struct mount *mp, hammer2_pfs_t *pmp,
 221                                 hammer2_dev_t *hmp);
 222 static int hammer2_fixup_pfses(hammer2_dev_t *hmp);
 223
 224 /*
 225  * HAMMER2 vfs operations.
 226  */
 227 static struct vfsops hammer2_vfsops = {
 228         .vfs_flags      = 0,
 229         .vfs_init       = hammer2_vfs_init,
 230         .vfs_uninit     = hammer2_vfs_uninit,
 231         .vfs_sync       = hammer2_vfs_sync,
 232         .vfs_mount      = hammer2_vfs_mount,
 233         .vfs_unmount    = hammer2_vfs_unmount,
 234         .vfs_root       = hammer2_vfs_root,
 235         .vfs_statfs     = hammer2_vfs_statfs,
 236         .vfs_statvfs    = hammer2_vfs_statvfs,
 237         .vfs_vget       = hammer2_vfs_vget,
 238         .vfs_vptofh     = hammer2_vfs_vptofh,
 239         .vfs_fhtovp     = hammer2_vfs_fhtovp,
 240         .vfs_checkexp   = hammer2_vfs_checkexp,
 241         .vfs_modifying  = hammer2_vfs_modifying
 242 };
 243
 244 MALLOC_DEFINE(M_HAMMER2, "HAMMER2-mount", "");
 245
 246 VFS_SET(hammer2_vfsops, hammer2, VFCF_MPSAFE);
 247 MODULE_VERSION(hammer2, 1);
 248
 249 static
 250 int
 251 hammer2_vfs_init(struct vfsconf *conf)
 252 {
 253         static struct objcache_malloc_args margs_read;
 254         static struct objcache_malloc_args margs_write;
 255         static struct objcache_malloc_args margs_vop;
 256
 257         int error;
 258
 259         error = 0;
 260         kmalloc_raise_limit(M_HAMMER2, 0);      /* unlimited */
 261
 262         /*
 263          * hammer2_xopgroups must be even and is most optimal if
 264          * 2 x ncpus so strategy functions can be queued to the same
 265          * cpu.
 266          */
 267         hammer2_xopgroups = HAMMER2_XOPGROUPS_MIN;
 268         if (hammer2_xopgroups < ncpus * 2)
 269                 hammer2_xopgroups = ncpus * 2;
 270
 271         /*
 272          * A large DIO cache is needed to retain dedup enablement masks.
 273          * The bulkfree code clears related masks as part of the disk block
 274          * recycling algorithm, preventing it from being used for a later
 275          * dedup.
 276          *
 277          * NOTE: A large buffer cache can actually interfere with dedup
 278          *       operation because we dedup based on media physical buffers
 279          *       and not logical buffers.  Try to make the DIO case large
 280          *       enough to avoid this problem, but also cap it.
 281          */
 282         hammer2_dio_limit = nbuf * 2;
 283         if (hammer2_dio_limit > 100000)
 284                 hammer2_dio_limit = 100000;
 285
 286         if (HAMMER2_BLOCKREF_BYTES != sizeof(struct hammer2_blockref))
 287                 error = EINVAL;
 288         if (HAMMER2_INODE_BYTES != sizeof(struct hammer2_inode_data))
 289                 error = EINVAL;
 290         if (HAMMER2_VOLUME_BYTES != sizeof(struct hammer2_volume_data))
 291                 error = EINVAL;
 292
 293         if (error)
 294                 kprintf("HAMMER2 structure size mismatch; cannot continue.\n");
 295
 296         margs_read.objsize = 65536;
 297         margs_read.mtype = M_HAMMER2_DEBUFFER;
 298
 299         margs_write.objsize = 32768;
 300         margs_write.mtype = M_HAMMER2_CBUFFER;
 301
 302         margs_vop.objsize = sizeof(hammer2_xop_t);
 303         margs_vop.mtype = M_HAMMER2;
 304
 305         /*
 306          * Note thaht for the XOPS cache we want backing store allocations
 307          * to use M_ZERO.  This is not allowed in objcache_get() (to avoid
 308          * confusion), so use the backing store function that does it.  This
 309          * means that initial XOPS objects are zerod but REUSED objects are
 310          * not.  So we are responsible for cleaning the object up sufficiently
 311          * for our needs before objcache_put()ing it back (typically just the
 312          * FIFO indices).
 313          */
 314         cache_buffer_read = objcache_create(margs_read.mtype->ks_shortdesc,
 315                                 0, 1, NULL, NULL, NULL,
 316                                 objcache_malloc_alloc,
 317                                 objcache_malloc_free,
 318                                 &margs_read);
 319         cache_buffer_write = objcache_create(margs_write.mtype->ks_shortdesc,
 320                                 0, 1, NULL, NULL, NULL,
 321                                 objcache_malloc_alloc,
 322                                 objcache_malloc_free,
 323                                 &margs_write);
 324         cache_xops = objcache_create(margs_vop.mtype->ks_shortdesc,
 325                                 0, 1, NULL, NULL, NULL,
 326                                 objcache_malloc_alloc_zero,
 327                                 objcache_malloc_free,
 328                                 &margs_vop);
 329
 330
 331         lockinit(&hammer2_mntlk, "mntlk", 0, 0);
 332         TAILQ_INIT(&hammer2_mntlist);
 333         TAILQ_INIT(&hammer2_pfslist);
 334         TAILQ_INIT(&hammer2_spmplist);
 335
 336         hammer2_limit_dirty_chains = maxvnodes / 10;
 337         if (hammer2_limit_dirty_chains > HAMMER2_LIMIT_DIRTY_CHAINS)
 338                 hammer2_limit_dirty_chains = HAMMER2_LIMIT_DIRTY_CHAINS;
 339         if (hammer2_limit_dirty_chains < 1000)
 340                 hammer2_limit_dirty_chains = 1000;
 341
 342         hammer2_limit_dirty_inodes = maxvnodes / 25;
 343         if (hammer2_limit_dirty_inodes < 100)
 344                 hammer2_limit_dirty_inodes = 100;
 345         if (hammer2_limit_dirty_inodes > HAMMER2_LIMIT_DIRTY_INODES)
 346                 hammer2_limit_dirty_inodes = HAMMER2_LIMIT_DIRTY_INODES;
 347
 348         return (error);
 349 }
 350
 351 static
 352 int
 353 hammer2_vfs_uninit(struct vfsconf *vfsp __unused)
 354 {
 355         objcache_destroy(cache_buffer_read);
 356         objcache_destroy(cache_buffer_write);
 357         objcache_destroy(cache_xops);
 358         return 0;
 359 }
 360
 361 /*
 362  * Core PFS allocator.  Used to allocate or reference the pmp structure
 363  * for PFS cluster mounts and the spmp structure for media (hmp) structures.
 364  * The pmp can be passed in or loaded by this function using the chain and
 365  * inode data.
 366  *
 367  * pmp->modify_tid tracks new modify_tid transaction ids for front-end
 368  * transactions.  Note that synchronization does not use this field.
 369  * (typically frontend operations and synchronization cannot run on the
 370  * same PFS node at the same time).
 371  *
 372  * XXX check locking
 373  */
 374 hammer2_pfs_t *
 375 hammer2_pfsalloc(hammer2_chain_t *chain,
 376                  const hammer2_inode_data_t *ripdata,
 377                  hammer2_tid_t modify_tid, hammer2_dev_t *force_local)
 378 {
 379         hammer2_pfs_t *pmp;
 380         hammer2_inode_t *iroot;
 381         int count;
 382         int i;
 383         int j;
 384
 385         pmp = NULL;
 386
 387         /*
 388          * Locate or create the PFS based on the cluster id.  If ripdata
 389          * is NULL this is a spmp which is unique and is always allocated.
 390          *
 391          * If the device is mounted in local mode all PFSs are considered
 392          * independent and not part of any cluster (for debugging only).
 393          */
 394         if (ripdata) {
 395                 TAILQ_FOREACH(pmp, &hammer2_pfslist, mntentry) {
 396                         if (force_local != pmp->force_local)
 397                                 continue;
 398                         if (force_local == NULL &&
 399                             bcmp(&pmp->pfs_clid, &ripdata->meta.pfs_clid,
 400                                  sizeof(pmp->pfs_clid)) == 0) {
 401                                         break;
 402                         } else if (force_local && pmp->pfs_names[0] &&
 403                             strcmp(pmp->pfs_names[0], ripdata->filename) == 0) {
 404                                         break;
 405                         }
 406                 }
 407         }
 408
 409         if (pmp == NULL) {
 410                 pmp = kmalloc(sizeof(*pmp), M_HAMMER2, M_WAITOK | M_ZERO);
 411                 pmp->force_local = force_local;
 412                 hammer2_trans_manage_init(pmp);
 413                 kmalloc_create_obj(&pmp->minode, "HAMMER2-inodes",
 414                                    sizeof(struct hammer2_inode));
 415                 lockinit(&pmp->lock, "pfslk", 0, 0);
 416                 lockinit(&pmp->lock_nlink, "h2nlink", 0, 0);
 417                 spin_init(&pmp->inum_spin, "hm2pfsalloc_inum");
 418                 spin_init(&pmp->xop_spin, "h2xop");
 419                 spin_init(&pmp->lru_spin, "h2lru");
 420                 RB_INIT(&pmp->inum_tree);
 421                 TAILQ_INIT(&pmp->syncq);
 422                 TAILQ_INIT(&pmp->depq);
 423                 TAILQ_INIT(&pmp->lru_list);
 424                 spin_init(&pmp->list_spin, "h2pfsalloc_list");
 425
 426                 /*
 427                  * Save the last media transaction id for the flusher.  Set
 428                  * initial
 429                  */
 430                 if (ripdata) {
 431                         pmp->pfs_clid = ripdata->meta.pfs_clid;
 432                         TAILQ_INSERT_TAIL(&hammer2_pfslist, pmp, mntentry);
 433                 } else {
 434                         pmp->flags |= HAMMER2_PMPF_SPMP;
 435                         TAILQ_INSERT_TAIL(&hammer2_spmplist, pmp, mntentry);
 436                 }
 437
 438                 /*
 439                  * The synchronization thread may start too early, make
 440                  * sure it stays frozen until we are ready to let it go.
 441                  * XXX
 442                  */
 443                 /*
 444                 pmp->primary_thr.flags = HAMMER2_THREAD_FROZEN |
 445                                          HAMMER2_THREAD_REMASTER;
 446                 */
 447         }
 448
 449         /*
 450          * Create the PFS's root inode and any missing XOP helper threads.
 451          */
 452         if ((iroot = pmp->iroot) == NULL) {
 453                 iroot = hammer2_inode_get(pmp, NULL, 1, -1);
 454                 if (ripdata)
 455                         iroot->meta = ripdata->meta;
 456                 pmp->iroot = iroot;
 457                 hammer2_inode_ref(iroot);
 458                 hammer2_inode_unlock(iroot);
 459         }
 460
 461         /*
 462          * Stop here if no chain is passed in.
 463          */
 464         if (chain == NULL)
 465                 goto done;
 466
 467         /*
 468          * When a chain is passed in we must add it to the PFS's root
 469          * inode, update pmp->pfs_types[], and update the syncronization
 470          * threads.
 471          *
 472          * When forcing local mode, mark the PFS as a MASTER regardless.
 473          *
 474          * At the moment empty spots can develop due to removals or failures.
 475          * Ultimately we want to re-fill these spots but doing so might
 476          * confused running code. XXX
 477          */
 478         hammer2_inode_ref(iroot);
 479         hammer2_mtx_ex(&iroot->lock);
 480         j = iroot->cluster.nchains;
 481
 482         if (j == HAMMER2_MAXCLUSTER) {
 483                 kprintf("hammer2_pfsalloc: cluster full!\n");
 484                 /* XXX fatal error? */
 485         } else {
 486                 KKASSERT(chain->pmp == NULL);
 487                 chain->pmp = pmp;
 488                 hammer2_chain_ref(chain);
 489                 iroot->cluster.array[j].chain = chain;
 490                 if (force_local)
 491                         pmp->pfs_types[j] = HAMMER2_PFSTYPE_MASTER;
 492                 else
 493                         pmp->pfs_types[j] = ripdata->meta.pfs_type;
 494                 pmp->pfs_names[j] = kstrdup(ripdata->filename, M_HAMMER2);
 495                 pmp->pfs_hmps[j] = chain->hmp;
 496                 hammer2_spin_ex(&pmp->inum_spin);
 497                 pmp->pfs_iroot_blocksets[j] = chain->data->ipdata.u.blockset;
 498                 hammer2_spin_unex(&pmp->inum_spin);
 499
 500                 /*
 501                  * If the PFS is already mounted we must account
 502                  * for the mount_count here.
 503                  */
 504                 if (pmp->mp)
 505                         ++chain->hmp->mount_count;
 506
 507                 /*
 508                  * May have to fixup dirty chain tracking.  Previous
 509                  * pmp was NULL so nothing to undo.
 510                  */
 511                 if (chain->flags & HAMMER2_CHAIN_MODIFIED)
 512                         hammer2_pfs_memory_inc(pmp);
 513                 ++j;
 514         }
 515         iroot->cluster.nchains = j;
 516
 517         /*
 518          * Update nmasters from any PFS inode which is part of the cluster.
 519          * It is possible that this will result in a value which is too
 520          * high.  MASTER PFSs are authoritative for pfs_nmasters and will
 521          * override this value later on.
 522          *
 523          * (This informs us of masters that might not currently be
 524          *  discoverable by this mount).
 525          */
 526         if (ripdata && pmp->pfs_nmasters < ripdata->meta.pfs_nmasters) {
 527                 pmp->pfs_nmasters = ripdata->meta.pfs_nmasters;
 528         }
 529
 530         /*
 531          * Count visible masters.  Masters are usually added with
 532          * ripdata->meta.pfs_nmasters set to 1.  This detects when there
 533          * are more (XXX and must update the master inodes).
 534          */
 535         count = 0;
 536         for (i = 0; i < iroot->cluster.nchains; ++i) {
 537                 if (pmp->pfs_types[i] == HAMMER2_PFSTYPE_MASTER)
 538                         ++count;
 539         }
 540         if (pmp->pfs_nmasters < count)
 541                 pmp->pfs_nmasters = count;
 542
 543         /*
 544          * Create missing synchronization and support threads.
 545          *
 546          * Single-node masters (including snapshots) have nothing to
 547          * synchronize and do not require this thread.
 548          *
 549          * Multi-node masters or any number of soft masters, slaves, copy,
 550          * or other PFS types need the thread.
 551          *
 552          * Each thread is responsible for its particular cluster index.
 553          * We use independent threads so stalls or mismatches related to
 554          * any given target do not affect other targets.
 555          */
 556         for (i = 0; i < iroot->cluster.nchains; ++i) {
 557                 /*
 558                  * Single-node masters (including snapshots) have nothing
 559                  * to synchronize and will make direct xops support calls,
 560                  * thus they do not require this thread.
 561                  *
 562                  * Note that there can be thousands of snapshots.  We do not
 563                  * want to create thousands of threads.
 564                  */
 565                 if (pmp->pfs_nmasters <= 1 &&
 566                     pmp->pfs_types[i] == HAMMER2_PFSTYPE_MASTER) {
 567                         continue;
 568                 }
 569
 570                 /*
 571                  * Sync support thread
 572                  */
 573                 if (pmp->sync_thrs[i].td == NULL) {
 574                         hammer2_thr_create(&pmp->sync_thrs[i], pmp, NULL,
 575                                            "h2nod", i, -1,
 576                                            hammer2_primary_sync_thread);
 577                 }
 578         }
 579
 580         /*
 581          * Create missing Xop threads
 582          *
 583          * NOTE: We create helper threads for all mounted PFSs or any
 584          *       PFSs with 2+ nodes (so the sync thread can update them,
 585          *       even if not mounted).
 586          */
 587         if (pmp->mp || iroot->cluster.nchains >= 2)
 588                 hammer2_xop_helper_create(pmp);
 589
 590         hammer2_mtx_unlock(&iroot->lock);
 591         hammer2_inode_drop(iroot);
 592 done:
 593         return pmp;
 594 }
 595
 596 /*
 597  * Deallocate an element of a probed PFS.  If destroying and this is a
 598  * MASTER, adjust nmasters.
 599  *
 600  * This function does not physically destroy the PFS element in its device
 601  * under the super-root  (see hammer2_ioctl_pfs_delete()).
 602  */
 603 void
 604 hammer2_pfsdealloc(hammer2_pfs_t *pmp, int clindex, int destroying)
 605 {
 606         hammer2_inode_t *iroot;
 607         hammer2_chain_t *chain;
 608         int j;
 609
 610         /*
 611          * Cleanup our reference on iroot.  iroot is (should) not be needed
 612          * by the flush code.
 613          */
 614         iroot = pmp->iroot;
 615         if (iroot) {
 616                 /*
 617                  * Stop synchronizing
 618                  *
 619                  * XXX flush after acquiring the iroot lock.
 620                  * XXX clean out the cluster index from all inode structures.
 621                  */
 622                 hammer2_thr_delete(&pmp->sync_thrs[clindex]);
 623
 624                 /*
 625                  * Remove the cluster index from the group.  If destroying
 626                  * the PFS and this is a master, adjust pfs_nmasters.
 627                  */
 628                 hammer2_mtx_ex(&iroot->lock);
 629                 chain = iroot->cluster.array[clindex].chain;
 630                 iroot->cluster.array[clindex].chain = NULL;
 631
 632                 switch(pmp->pfs_types[clindex]) {
 633                 case HAMMER2_PFSTYPE_MASTER:
 634                         if (destroying && pmp->pfs_nmasters > 0)
 635                                 --pmp->pfs_nmasters;
 636                         /* XXX adjust ripdata->meta.pfs_nmasters */
 637                         break;
 638                 default:
 639                         break;
 640                 }
 641                 pmp->pfs_types[clindex] = HAMMER2_PFSTYPE_NONE;
 642
 643                 hammer2_mtx_unlock(&iroot->lock);
 644
 645                 /*
 646                  * Release the chain.
 647                  */
 648                 if (chain) {
 649                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
 650                         hammer2_chain_drop(chain);
 651                 }
 652
 653                 /*
 654                  * Terminate all XOP threads for the cluster index.
 655                  */
 656                 if (pmp->xop_groups) {
 657                         for (j = 0; j < hammer2_xopgroups; ++j) {
 658                                 hammer2_thr_delete(
 659                                         &pmp->xop_groups[j].thrs[clindex]);
 660                         }
 661                 }
 662         }
 663 }
 664
 665 /*
 666  * Destroy a PFS, typically only occurs after the last mount on a device
 667  * has gone away.
 668  */
 669 static void
 670 hammer2_pfsfree(hammer2_pfs_t *pmp)
 671 {
 672         hammer2_inode_t *iroot;
 673         hammer2_chain_t *chain;
 674         int chains_still_present = 0;
 675         int i;
 676         int j;
 677
 678         /*
 679          * Cleanup our reference on iroot.  iroot is (should) not be needed
 680          * by the flush code.
 681          */
 682         if (pmp->flags & HAMMER2_PMPF_SPMP)
 683                 TAILQ_REMOVE(&hammer2_spmplist, pmp, mntentry);
 684         else
 685                 TAILQ_REMOVE(&hammer2_pfslist, pmp, mntentry);
 686
 687         /*
 688          * Cleanup chains remaining on LRU list.
 689          */
 690         hammer2_spin_ex(&pmp->lru_spin);
 691         while ((chain = TAILQ_FIRST(&pmp->lru_list)) != NULL) {
 692                 KKASSERT(chain->flags & HAMMER2_CHAIN_ONLRU);
 693                 atomic_add_int(&pmp->lru_count, -1);
 694                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONLRU);
 695                 TAILQ_REMOVE(&pmp->lru_list, chain, lru_node);
 696                 hammer2_chain_ref(chain);
 697                 hammer2_spin_unex(&pmp->lru_spin);
 698                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
 699                 hammer2_chain_drop(chain);
 700                 hammer2_spin_ex(&pmp->lru_spin);
 701         }
 702         hammer2_spin_unex(&pmp->lru_spin);
 703
 704         /*
 705          * Clean up iroot
 706          */
 707         iroot = pmp->iroot;
 708         if (iroot) {
 709                 for (i = 0; i < iroot->cluster.nchains; ++i) {
 710                         hammer2_thr_delete(&pmp->sync_thrs[i]);
 711                         if (pmp->xop_groups) {
 712                                 for (j = 0; j < hammer2_xopgroups; ++j)
 713                                         hammer2_thr_delete(
 714                                                 &pmp->xop_groups[j].thrs[i]);
 715                         }
 716                         chain = iroot->cluster.array[i].chain;
 717                         if (chain && !RB_EMPTY(&chain->core.rbtree)) {
 718                                 kprintf("hammer2: Warning pmp %p still "
 719                                         "has active chains\n", pmp);
 720                                 chains_still_present = 1;
 721                         }
 722                 }
 723 #if REPORT_REFS_ERRORS
 724                 if (iroot->refs != 1)
 725                         kprintf("PMP->IROOT %p REFS WRONG %d\n",
 726                                 iroot, iroot->refs);
 727 #else
 728                 KKASSERT(iroot->refs == 1);
 729 #endif
 730                 /* ref for iroot */
 731                 hammer2_inode_drop(iroot);
 732                 pmp->iroot = NULL;
 733         }
 734
 735         /*
 736          * Free remaining pmp resources
 737          */
 738         if (chains_still_present) {
 739                 kprintf("hammer2: cannot free pmp %p, still in use\n", pmp);
 740         } else {
 741                 kmalloc_destroy_obj(&pmp->minode);
 742                 kfree(pmp, M_HAMMER2);
 743         }
 744 }
 745
 746 /*
 747  * Remove all references to hmp from the pfs list.  Any PFS which becomes
 748  * empty is terminated and freed.
 749  *
 750  * XXX inefficient.
 751  */
 752 static void
 753 hammer2_pfsfree_scan(hammer2_dev_t *hmp, int which)
 754 {
 755         hammer2_pfs_t *pmp;
 756         hammer2_inode_t *iroot;
 757         hammer2_chain_t *rchain;
 758         int i;
 759         int j;
 760         struct hammer2_pfslist *wlist;
 761
 762         if (which == 0)
 763                 wlist = &hammer2_pfslist;
 764         else
 765                 wlist = &hammer2_spmplist;
 766 again:
 767         TAILQ_FOREACH(pmp, wlist, mntentry) {
 768                 if ((iroot = pmp->iroot) == NULL)
 769                         continue;
 770
 771                 /*
 772                  * Determine if this PFS is affected.  If it is we must
 773                  * freeze all management threads and lock its iroot.
 774                  *
 775                  * Freezing a management thread forces it idle, operations
 776                  * in-progress will be aborted and it will have to start
 777                  * over again when unfrozen, or exit if told to exit.
 778                  */
 779                 for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 780                         if (pmp->pfs_hmps[i] == hmp)
 781                                 break;
 782                 }
 783                 if (i == HAMMER2_MAXCLUSTER)
 784                         continue;
 785
 786                 hammer2_vfs_sync_pmp(pmp, MNT_WAIT);
 787
 788                 /*
 789                  * Make sure all synchronization threads are locked
 790                  * down.
 791                  */
 792                 for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 793                         if (pmp->pfs_hmps[i] == NULL)
 794                                 continue;
 795                         hammer2_thr_freeze_async(&pmp->sync_thrs[i]);
 796                         if (pmp->xop_groups) {
 797                                 for (j = 0; j < hammer2_xopgroups; ++j) {
 798                                         hammer2_thr_freeze_async(
 799                                                 &pmp->xop_groups[j].thrs[i]);
 800                                 }
 801                         }
 802                 }
 803                 for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 804                         if (pmp->pfs_hmps[i] == NULL)
 805                                 continue;
 806                         hammer2_thr_freeze(&pmp->sync_thrs[i]);
 807                         if (pmp->xop_groups) {
 808                                 for (j = 0; j < hammer2_xopgroups; ++j) {
 809                                         hammer2_thr_freeze(
 810                                                 &pmp->xop_groups[j].thrs[i]);
 811                                 }
 812                         }
 813                 }
 814
 815                 /*
 816                  * Lock the inode and clean out matching chains.
 817                  * Note that we cannot use hammer2_inode_lock_*()
 818                  * here because that would attempt to validate the
 819                  * cluster that we are in the middle of ripping
 820                  * apart.
 821                  *
 822                  * WARNING! We are working directly on the inodes
 823                  *          embedded cluster.
 824                  */
 825                 hammer2_mtx_ex(&iroot->lock);
 826
 827                 /*
 828                  * Remove the chain from matching elements of the PFS.
 829                  */
 830                 for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 831                         if (pmp->pfs_hmps[i] != hmp)
 832                                 continue;
 833                         hammer2_thr_delete(&pmp->sync_thrs[i]);
 834                         if (pmp->xop_groups) {
 835                                 for (j = 0; j < hammer2_xopgroups; ++j) {
 836                                         hammer2_thr_delete(
 837                                                 &pmp->xop_groups[j].thrs[i]);
 838                                 }
 839                         }
 840                         rchain = iroot->cluster.array[i].chain;
 841                         iroot->cluster.array[i].chain = NULL;
 842                         pmp->pfs_types[i] = 0;
 843                         if (pmp->pfs_names[i]) {
 844                                 kfree(pmp->pfs_names[i], M_HAMMER2);
 845                                 pmp->pfs_names[i] = NULL;
 846                         }
 847                         if (rchain) {
 848                                 hammer2_chain_drop(rchain);
 849                                 /* focus hint */
 850                                 if (iroot->cluster.focus == rchain)
 851                                         iroot->cluster.focus = NULL;
 852                         }
 853                         pmp->pfs_hmps[i] = NULL;
 854                 }
 855                 hammer2_mtx_unlock(&iroot->lock);
 856
 857                 /*
 858                  * Cleanup trailing chains.  Gaps may remain.
 859                  */
 860                 for (i = HAMMER2_MAXCLUSTER - 1; i >= 0; --i) {
 861                         if (pmp->pfs_hmps[i])
 862                                 break;
 863                 }
 864                 iroot->cluster.nchains = i + 1;
 865
 866                 /*
 867                  * If the PMP has no elements remaining we can destroy it.
 868                  * (this will transition management threads from frozen->exit).
 869                  */
 870                 if (iroot->cluster.nchains == 0) {
 871                         /*
 872                          * If this was the hmp's spmp, we need to clean
 873                          * a little more stuff out.
 874                          */
 875                         if (hmp->spmp == pmp) {
 876                                 hmp->spmp = NULL;
 877                                 hmp->vchain.pmp = NULL;
 878                                 hmp->fchain.pmp = NULL;
 879                         }
 880
 881                         /*
 882                          * Free the pmp and restart the loop
 883                          */
 884                         KKASSERT(TAILQ_EMPTY(&pmp->syncq));
 885                         KKASSERT(TAILQ_EMPTY(&pmp->depq));
 886                         hammer2_pfsfree(pmp);
 887                         goto again;
 888                 }
 889
 890                 /*
 891                  * If elements still remain we need to set the REMASTER
 892                  * flag and unfreeze it.
 893                  */
 894                 for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
 895                         if (pmp->pfs_hmps[i] == NULL)
 896                                 continue;
 897                         hammer2_thr_remaster(&pmp->sync_thrs[i]);
 898                         hammer2_thr_unfreeze(&pmp->sync_thrs[i]);
 899                         if (pmp->xop_groups) {
 900                                 for (j = 0; j < hammer2_xopgroups; ++j) {
 901                                         hammer2_thr_remaster(
 902                                                 &pmp->xop_groups[j].thrs[i]);
 903                                         hammer2_thr_unfreeze(
 904                                                 &pmp->xop_groups[j].thrs[i]);
 905                                 }
 906                         }
 907                 }
 908         }
 909 }
 910
 911 /*
 912  * Mount or remount HAMMER2 fileystem from physical media
 913  *
 914  *      mountroot
 915  *              mp              mount point structure
 916  *              path            NULL
 917  *              data            <unused>
 918  *              cred            <unused>
 919  *
 920  *      mount
 921  *              mp              mount point structure
 922  *              path            path to mount point
 923  *              data            pointer to argument structure in user space
 924  *                      volume  volume path (device@LABEL form)
 925  *                      hflags  user mount flags
 926  *              cred            user credentials
 927  *
 928  * RETURNS:     0       Success
 929  *              !0      error number
 930  */
 931 static
 932 int
 933 hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 934                   struct ucred *cred)
 935 {
 936         struct hammer2_mount_info info;
 937         hammer2_pfs_t *pmp;
 938         hammer2_pfs_t *spmp;
 939         hammer2_dev_t *hmp, *hmp_tmp;
 940         hammer2_dev_t *force_local;
 941         hammer2_key_t key_next;
 942         hammer2_key_t key_dummy;
 943         hammer2_key_t lhc;
 944         hammer2_chain_t *parent;
 945         hammer2_chain_t *chain;
 946         const hammer2_inode_data_t *ripdata;
 947         hammer2_blockref_t bref;
 948         hammer2_devvp_list_t devvpl;
 949         hammer2_devvp_t *e, *e_tmp;
 950         struct file *fp;
 951         char devstr[MNAMELEN];
 952         size_t size;
 953         size_t done;
 954         char *dev;
 955         char *label;
 956         int ronly = ((mp->mnt_flag & MNT_RDONLY) != 0);
 957         int error;
 958         int i;
 959
 960         hmp = NULL;
 961         pmp = NULL;
 962         dev = NULL;
 963         label = NULL;
 964         bzero(&info, sizeof(info));
 965
 966         if (path) {
 967                 /*
 968                  * Non-root mount or updating a mount
 969                  */
 970                 error = copyin(data, &info, sizeof(info));
 971                 if (error)
 972                         return (error);
 973         }
 974
 975         if (mp->mnt_flag & MNT_UPDATE) {
 976                 /*
 977                  * Update mount.  Note that pmp->iroot->cluster is
 978                  * an inode-embedded cluster and thus cannot be
 979                  * directly locked.
 980                  *
 981                  * XXX HAMMER2 needs to implement NFS export via
 982                  *     mountctl.
 983                  */
 984                 hammer2_cluster_t *cluster;
 985
 986                 pmp = MPTOPMP(mp);
 987                 pmp->hflags = info.hflags;
 988                 cluster = &pmp->iroot->cluster;
 989                 for (i = 0; i < cluster->nchains; ++i) {
 990                         if (cluster->array[i].chain == NULL)
 991                                 continue;
 992                         hmp = cluster->array[i].chain->hmp;
 993                         error = hammer2_remount(hmp, mp, path, cred);
 994                         if (error)
 995                                 break;
 996                 }
 997
 998                 return error;
 999         }
1000
1001         if (path == NULL) {
1002                 /*
1003                  * Root mount
1004                  */
1005                 info.cluster_fd = -1;
1006                 ksnprintf(devstr, sizeof(devstr), "%s",
1007                           mp->mnt_stat.f_mntfromname);
1008                 done = strlen(devstr) + 1;
1009                 kprintf("hammer2_mount: root devstr=\"%s\"\n", devstr);
1010         } else {
1011                 error = copyinstr(info.volume, devstr, MNAMELEN - 1, &done);
1012                 if (error)
1013                         return (error);
1014                 kprintf("hammer2_mount: devstr=\"%s\"\n", devstr);
1015         }
1016
1017         /*
1018          * Extract device and label, automatically mount @BOOT, @ROOT, or @DATA
1019          * if no label specified, based on the partition id.  Error out if no
1020          * label or device (with partition id) is specified.  This is strictly
1021          * a convenience to match the default label created by newfs_hammer2,
1022          * our preference is that a label always be specified.
1023          *
1024          * NOTE: We allow 'mount @LABEL <blah>'... that is, a mount command
1025          *       that does not specify a device, as long as some H2 label
1026          *       has already been mounted from that device.  This makes
1027          *       mounting snapshots a lot easier.
1028          */
1029         dev = devstr;
1030         label = strchr(devstr, '@');
1031         if (label && ((label + 1) - dev) > done) {
1032                 kprintf("hammer2_mount: bad label %s/%zd\n", devstr, done);
1033                 return (EINVAL);
1034         }
1035         if (label == NULL || label[1] == 0) {
1036                 char slice;
1037
1038                 if (label == NULL)
1039                         label = devstr + strlen(devstr);
1040                 else
1041                         *label = '\0';          /* clean up trailing @ */
1042
1043                 slice = label[-1];
1044                 switch(slice) {
1045                 case 'a':
1046                         label = "BOOT";
1047                         break;
1048                 case 'd':
1049                         label = "ROOT";
1050                         break;
1051                 default:
1052                         label = "DATA";
1053                         break;
1054                 }
1055         } else {
1056                 *label = '\0';
1057                 label++;
1058         }
1059
1060         kprintf("hammer2_mount: dev=\"%s\" label=\"%s\" rdonly=%d\n",
1061                 dev, label, ronly);
1062
1063         /*
1064          * Initialize all device vnodes.
1065          */
1066         TAILQ_INIT(&devvpl);
1067         error = hammer2_init_devvp(dev, path == NULL, &devvpl);
1068         if (error) {
1069                 kprintf("hammer2: failed to initialize devvp in %s\n", dev);
1070                 hammer2_cleanup_devvp(&devvpl);
1071                 return error;
1072         }
1073
1074         /*
1075          * Determine if the device has already been mounted.  After this
1076          * check hmp will be non-NULL if we are doing the second or more
1077          * hammer2 mounts from the same device.
1078          */
1079         lockmgr(&hammer2_mntlk, LK_EXCLUSIVE);
1080         if (!TAILQ_EMPTY(&devvpl)) {
1081                 /*
1082                  * Match the device.  Due to the way devfs works,
1083                  * we may not be able to directly match the vnode pointer,
1084                  * so also check to see if the underlying device matches.
1085                  */
1086                 TAILQ_FOREACH(hmp_tmp, &hammer2_mntlist, mntentry) {
1087                         TAILQ_FOREACH(e_tmp, &hmp_tmp->devvpl, entry) {
1088                                 int devvp_found = 0;
1089                                 TAILQ_FOREACH(e, &devvpl, entry) {
1090                                         KKASSERT(e->devvp);
1091                                         if (e_tmp->devvp == e->devvp)
1092                                                 devvp_found = 1;
1093                                         if (e_tmp->devvp->v_rdev &&
1094                                             e_tmp->devvp->v_rdev == e->devvp->v_rdev)
1095                                                 devvp_found = 1;
1096                                 }
1097                                 if (!devvp_found)
1098                                         goto next_hmp;
1099                         }
1100                         hmp = hmp_tmp;
1101                         kprintf("hammer2_mount: hmp=%p matched\n", hmp);
1102                         break;
1103 next_hmp:
1104                         continue;
1105                 }
1106
1107                 /*
1108                  * If no match this may be a fresh H2 mount, make sure
1109                  * the device is not mounted on anything else.
1110                  */
1111                 if (hmp == NULL) {
1112                         TAILQ_FOREACH(e, &devvpl, entry) {
1113                                 struct vnode *devvp = e->devvp;
1114                                 KKASSERT(devvp);
1115                                 error = vfs_mountedon(devvp);
1116                                 if (error) {
1117                                         kprintf("hammer2_mount: %s mounted %d\n",
1118                                                 e->path, error);
1119                                         hammer2_cleanup_devvp(&devvpl);
1120                                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1121                                         return error;
1122                                 }
1123                         }
1124                 }
1125         } else {
1126                 /*
1127                  * Match the label to a pmp already probed.
1128                  */
1129                 TAILQ_FOREACH(pmp, &hammer2_pfslist, mntentry) {
1130                         for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) {
1131                                 if (pmp->pfs_names[i] &&
1132                                     strcmp(pmp->pfs_names[i], label) == 0) {
1133                                         hmp = pmp->pfs_hmps[i];
1134                                         break;
1135                                 }
1136                         }
1137                         if (hmp)
1138                                 break;
1139                 }
1140                 if (hmp == NULL) {
1141                         kprintf("hammer2_mount: PFS label \"%s\" not found\n",
1142                                 label);
1143                         hammer2_cleanup_devvp(&devvpl);
1144                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1145                         return ENOENT;
1146                 }
1147         }
1148
1149         /*
1150          * Open the device if this isn't a secondary mount and construct
1151          * the H2 device mount (hmp).
1152          */
1153         if (hmp == NULL) {
1154                 hammer2_chain_t *schain;
1155                 hammer2_xid_t xid;
1156                 hammer2_xop_head_t xop;
1157
1158                 /*
1159                  * Now open the device
1160                  */
1161                 KKASSERT(!TAILQ_EMPTY(&devvpl));
1162                 if (error == 0) {
1163                         error = hammer2_open_devvp(&devvpl, ronly);
1164                         if (error) {
1165                                 hammer2_close_devvp(&devvpl, ronly);
1166                                 hammer2_cleanup_devvp(&devvpl);
1167                                 lockmgr(&hammer2_mntlk, LK_RELEASE);
1168                                 return error;
1169                         }
1170                 }
1171
1172                 /*
1173                  * Construct volumes and link with device vnodes.
1174                  */
1175                 hmp = kmalloc(sizeof(*hmp), M_HAMMER2, M_WAITOK | M_ZERO);
1176                 hmp->devvp = NULL;
1177                 error = hammer2_init_volumes(mp, &devvpl, hmp->volumes,
1178                                              &hmp->voldata, &hmp->devvp);
1179                 if (error) {
1180                         hammer2_close_devvp(&devvpl, ronly);
1181                         hammer2_cleanup_devvp(&devvpl);
1182                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1183                         kfree(hmp, M_HAMMER2);
1184                         return error;
1185                 }
1186                 if (!hmp->devvp) {
1187                         kprintf("hammer2: failed to initialize root volume\n");
1188                         hammer2_unmount_helper(mp, NULL, hmp);
1189                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1190                         hammer2_vfs_unmount(mp, MNT_FORCE);
1191                         return EINVAL;
1192                 }
1193
1194                 ksnprintf(hmp->devrepname, sizeof(hmp->devrepname), "%s", dev);
1195                 hmp->ronly = ronly;
1196                 hmp->hflags = info.hflags & HMNT2_DEVFLAGS;
1197                 kmalloc_create_obj(&hmp->mchain, "HAMMER2-chains",
1198                                    sizeof(struct hammer2_chain));
1199                 kmalloc_create_obj(&hmp->mio, "HAMMER2-dio",
1200                                    sizeof(struct hammer2_io));
1201                 kmalloc_create(&hmp->mmsg, "HAMMER2-msg");
1202                 TAILQ_INSERT_TAIL(&hammer2_mntlist, hmp, mntentry);
1203                 RB_INIT(&hmp->iotree);
1204                 spin_init(&hmp->io_spin, "h2mount_io");
1205                 spin_init(&hmp->list_spin, "h2mount_list");
1206
1207                 lockinit(&hmp->vollk, "h2vol", 0, 0);
1208                 lockinit(&hmp->bulklk, "h2bulk", 0, 0);
1209                 lockinit(&hmp->bflock, "h2bflk", 0, 0);
1210
1211                 /*
1212                  * vchain setup. vchain.data is embedded.
1213                  * vchain.refs is initialized and will never drop to 0.
1214                  *
1215                  * NOTE! voldata is not yet loaded.
1216                  */
1217                 hmp->vchain.hmp = hmp;
1218                 hmp->vchain.refs = 1;
1219                 hmp->vchain.data = (void *)&hmp->voldata;
1220                 hmp->vchain.bref.type = HAMMER2_BREF_TYPE_VOLUME;
1221                 hmp->vchain.bref.data_off = 0 | HAMMER2_PBUFRADIX;
1222                 hmp->vchain.bref.mirror_tid = hmp->voldata.mirror_tid;
1223                 hammer2_chain_core_init(&hmp->vchain);
1224
1225                 /*
1226                  * fchain setup.  fchain.data is embedded.
1227                  * fchain.refs is initialized and will never drop to 0.
1228                  *
1229                  * The data is not used but needs to be initialized to
1230                  * pass assertion muster.  We use this chain primarily
1231                  * as a placeholder for the freemap's top-level radix tree
1232                  * so it does not interfere with the volume's topology
1233                  * radix tree.
1234                  */
1235                 hmp->fchain.hmp = hmp;
1236                 hmp->fchain.refs = 1;
1237                 hmp->fchain.data = (void *)&hmp->voldata.freemap_blockset;
1238                 hmp->fchain.bref.type = HAMMER2_BREF_TYPE_FREEMAP;
1239                 hmp->fchain.bref.data_off = 0 | HAMMER2_PBUFRADIX;
1240                 hmp->fchain.bref.mirror_tid = hmp->voldata.freemap_tid;
1241                 hmp->fchain.bref.methods =
1242                         HAMMER2_ENC_CHECK(HAMMER2_CHECK_FREEMAP) |
1243                         HAMMER2_ENC_COMP(HAMMER2_COMP_NONE);
1244                 hammer2_chain_core_init(&hmp->fchain);
1245
1246                 /*
1247                  * Initialize volume header related fields.
1248                  */
1249                 KKASSERT(hmp->voldata.magic == HAMMER2_VOLUME_ID_HBO ||
1250                          hmp->voldata.magic == HAMMER2_VOLUME_ID_ABO);
1251                 hmp->volhdrno = error;
1252                 hmp->volsync = hmp->voldata;
1253                 hmp->free_reserved = hmp->voldata.allocator_size / 20;
1254                 /*
1255                  * Must use hmp instead of volume header for these two
1256                  * in order to handle volume versions transparently.
1257                  */
1258                 if (hmp->voldata.version >= HAMMER2_VOL_VERSION_MULTI_VOLUMES) {
1259                         hmp->nvolumes = hmp->voldata.nvolumes;
1260                         hmp->total_size = hmp->voldata.total_size;
1261                 } else {
1262                         hmp->nvolumes = 1;
1263                         hmp->total_size = hmp->voldata.volu_size;
1264                 }
1265                 KKASSERT(hmp->nvolumes > 0);
1266
1267                 /*
1268                  * Move devvpl entries to hmp.
1269                  */
1270                 TAILQ_INIT(&hmp->devvpl);
1271                 while ((e = TAILQ_FIRST(&devvpl)) != NULL) {
1272                         TAILQ_REMOVE(&devvpl, e, entry);
1273                         TAILQ_INSERT_TAIL(&hmp->devvpl, e, entry);
1274                 }
1275                 KKASSERT(TAILQ_EMPTY(&devvpl));
1276                 KKASSERT(!TAILQ_EMPTY(&hmp->devvpl));
1277
1278                 /*
1279                  * Really important to get these right or the flush and
1280                  * teardown code will get confused.
1281                  */
1282                 hmp->spmp = hammer2_pfsalloc(NULL, NULL, 0, NULL);
1283                 spmp = hmp->spmp;
1284                 spmp->pfs_hmps[0] = hmp;
1285
1286                 /*
1287                  * Dummy-up vchain and fchain's modify_tid.  mirror_tid
1288                  * is inherited from the volume header.
1289                  */
1290                 xid = 0;
1291                 hmp->vchain.bref.mirror_tid = hmp->voldata.mirror_tid;
1292                 hmp->vchain.bref.modify_tid = hmp->vchain.bref.mirror_tid;
1293                 hmp->vchain.pmp = spmp;
1294                 hmp->fchain.bref.mirror_tid = hmp->voldata.freemap_tid;
1295                 hmp->fchain.bref.modify_tid = hmp->fchain.bref.mirror_tid;
1296                 hmp->fchain.pmp = spmp;
1297
1298                 /*
1299                  * First locate the super-root inode, which is key 0
1300                  * relative to the volume header's blockset.
1301                  *
1302                  * Then locate the root inode by scanning the directory keyspace
1303                  * represented by the label.
1304                  */
1305                 parent = hammer2_chain_lookup_init(&hmp->vchain, 0);
1306                 schain = hammer2_chain_lookup(&parent, &key_dummy,
1307                                       HAMMER2_SROOT_KEY, HAMMER2_SROOT_KEY,
1308                                       &error, 0);
1309                 hammer2_chain_lookup_done(parent);
1310                 if (schain == NULL) {
1311                         kprintf("hammer2_mount: invalid super-root\n");
1312                         hammer2_unmount_helper(mp, NULL, hmp);
1313                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1314                         hammer2_vfs_unmount(mp, MNT_FORCE);
1315                         return EINVAL;
1316                 }
1317                 if (schain->error) {
1318                         kprintf("hammer2_mount: error %s reading super-root\n",
1319                                 hammer2_error_str(schain->error));
1320                         hammer2_chain_unlock(schain);
1321                         hammer2_chain_drop(schain);
1322                         schain = NULL;
1323                         hammer2_unmount_helper(mp, NULL, hmp);
1324                         lockmgr(&hammer2_mntlk, LK_RELEASE);
1325                         hammer2_vfs_unmount(mp, MNT_FORCE);
1326                         return EINVAL;
1327                 }
1328
1329                 /*
1330                  * The super-root always uses an inode_tid of 1 when
1331                  * creating PFSs.
1332                  */
1333                 spmp->inode_tid = 1;
1334                 spmp->modify_tid = schain->bref.modify_tid + 1;
1335
1336                 /*
1337                  * Sanity-check schain's pmp and finish initialization.
1338                  * Any chain belonging to the super-root topology should
1339                  * have a NULL pmp (not even set to spmp).
1340                  */
1341                 ripdata = &schain->data->ipdata;
1342                 KKASSERT(schain->pmp == NULL);
1343                 spmp->pfs_clid = ripdata->meta.pfs_clid;
1344
1345                 /*
1346                  * Replace the dummy spmp->iroot with a real one.  It's
1347                  * easier to just do a wholesale replacement than to try
1348                  * to update the chain and fixup the iroot fields.
1349                  *
1350                  * The returned inode is locked with the supplied cluster.
1351                  */
1352                 hammer2_dummy_xop_from_chain(&xop, schain);
1353                 hammer2_inode_drop(spmp->iroot);
1354                 spmp->iroot = NULL;
1355                 spmp->iroot = hammer2_inode_get(spmp, &xop, -1, -1);
1356                 spmp->spmp_hmp = hmp;
1357                 spmp->pfs_types[0] = ripdata->meta.pfs_type;
1358                 spmp->pfs_hmps[0] = hmp;
1359                 hammer2_inode_ref(spmp->iroot);
1360                 hammer2_inode_unlock(spmp->iroot);
1361                 hammer2_cluster_unlock(&xop.cluster);
1362                 hammer2_chain_drop(schain);
1363                 /* do not call hammer2_cluster_drop() on an embedded cluster */
1364                 schain = NULL;  /* now invalid */
1365                 /* leave spmp->iroot with one ref */
1366
1367                 if (!hmp->ronly) {
1368                         error = hammer2_recovery(hmp);
1369                         if (error == 0)
1370                                 error |= hammer2_fixup_pfses(hmp);
1371                         /* XXX do something with error */
1372                 }
1373                 hammer2_update_pmps(hmp);
1374                 hammer2_iocom_init(hmp);
1375                 hammer2_bulkfree_init(hmp);
1376
1377                 /*
1378                  * Ref the cluster management messaging descriptor.  The mount
1379                  * program deals with the other end of the communications pipe.
1380                  *
1381                  * Root mounts typically do not supply one.
1382                  */
1383                 if (info.cluster_fd >= 0) {
1384                         fp = holdfp(curthread, info.cluster_fd, -1);
1385                         if (fp) {
1386                                 hammer2_cluster_reconnect(hmp, fp);
1387                         } else {
1388                                 kprintf("hammer2_mount: bad cluster_fd!\n");
1389                         }
1390                 }
1391         } else {
1392                 spmp = hmp->spmp;
1393                 if (info.hflags & HMNT2_DEVFLAGS) {
1394                         kprintf("hammer2_mount: Warning: mount flags pertaining "
1395                                 "to the whole device may only be specified "
1396                                 "on the first mount of the device: %08x\n",
1397                                 info.hflags & HMNT2_DEVFLAGS);
1398                 }
1399         }
1400
1401         /*
1402          * Force local mount (disassociate all PFSs from their clusters).
1403          * Used primarily for debugging.
1404          */
1405         force_local = (hmp->hflags & HMNT2_LOCAL) ? hmp : NULL;
1406
1407         /*
1408          * Lookup the mount point under the media-localized super-root.
1409          * Scanning hammer2_pfslist doesn't help us because it represents
1410          * PFS cluster ids which can aggregate several named PFSs together.
1411          *
1412          * cluster->pmp will incorrectly point to spmp and must be fixed
1413          * up later on.
1414          */
1415         hammer2_inode_lock(spmp->iroot, 0);
1416         parent = hammer2_inode_chain(spmp->iroot, 0, HAMMER2_RESOLVE_ALWAYS);
1417         lhc = hammer2_dirhash(label, strlen(label));
1418         chain = hammer2_chain_lookup(&parent, &key_next,
1419                                      lhc, lhc + HAMMER2_DIRHASH_LOMASK,
1420                                      &error, 0);
1421         while (chain) {
1422                 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
1423                     strcmp(label, chain->data->ipdata.filename) == 0) {
1424                         break;
1425                 }
1426                 chain = hammer2_chain_next(&parent, chain, &key_next,
1427                                             key_next,
1428                                             lhc + HAMMER2_DIRHASH_LOMASK,
1429                                             &error, 0);
1430         }
1431         if (parent) {
1432                 hammer2_chain_unlock(parent);
1433                 hammer2_chain_drop(parent);
1434         }
1435         hammer2_inode_unlock(spmp->iroot);
1436
1437         /*
1438          * PFS could not be found?
1439          */
1440         if (chain == NULL) {
1441                 hammer2_unmount_helper(mp, NULL, hmp);
1442                 lockmgr(&hammer2_mntlk, LK_RELEASE);
1443                 hammer2_vfs_unmount(mp, MNT_FORCE);
1444
1445                 if (error) {
1446                         kprintf("hammer2_mount: PFS label I/O error\n");
1447                         return EINVAL;
1448                 } else {
1449                         kprintf("hammer2_mount: PFS label \"%s\" not found\n",
1450                                 label);
1451                         return ENOENT;
1452                 }
1453         }
1454
1455         /*
1456          * Acquire the pmp structure (it should have already been allocated
1457          * via hammer2_update_pmps() so do not pass cluster in to add to
1458          * available chains).
1459          *
1460          * Check if the cluster has already been mounted.  A cluster can
1461          * only be mounted once, use null mounts to mount additional copies.
1462          */
1463         if (chain->error) {
1464                 kprintf("hammer2_mount: PFS label I/O error\n");
1465         } else {
1466                 ripdata = &chain->data->ipdata;
1467                 bref = chain->bref;
1468                 pmp = hammer2_pfsalloc(NULL, ripdata,
1469                                        bref.modify_tid, force_local);
1470         }
1471         hammer2_chain_unlock(chain);
1472         hammer2_chain_drop(chain);
1473
1474         /*
1475          * Finish the mount
1476          */
1477         kprintf("hammer2_mount: hmp=%p pmp=%p\n", hmp, pmp);
1478
1479         if (pmp->mp) {
1480                 kprintf("hammer2_mount: PFS already mounted!\n");
1481                 hammer2_unmount_helper(mp, NULL, hmp);
1482                 lockmgr(&hammer2_mntlk, LK_RELEASE);
1483                 hammer2_vfs_unmount(mp, MNT_FORCE);
1484
1485                 return EBUSY;
1486         }
1487
1488         pmp->hflags = info.hflags;
1489         mp->mnt_flag |= MNT_LOCAL;
1490         mp->mnt_kern_flag |= MNTK_ALL_MPSAFE;   /* all entry pts are SMP */
1491         mp->mnt_kern_flag |= MNTK_THR_SYNC;     /* new vsyncscan semantics */
1492
1493         /*
1494          * required mount structure initializations
1495          */
1496         mp->mnt_stat.f_iosize = HAMMER2_PBUFSIZE;
1497         mp->mnt_stat.f_bsize = HAMMER2_PBUFSIZE;
1498
1499         mp->mnt_vstat.f_frsize = HAMMER2_PBUFSIZE;
1500         mp->mnt_vstat.f_bsize = HAMMER2_PBUFSIZE;
1501
1502         /*
1503          * Optional fields
1504          */
1505         mp->mnt_iosize_max = MAXPHYS;
1506
1507         /*
1508          * Connect up mount pointers.
1509          */
1510         hammer2_mount_helper(mp, pmp);
1511         lockmgr(&hammer2_mntlk, LK_RELEASE);
1512
1513         /*
1514          * Finish setup
1515          */
1516         vfs_getnewfsid(mp);
1517         vfs_add_vnodeops(mp, &hammer2_vnode_vops, &mp->mnt_vn_norm_ops);
1518         vfs_add_vnodeops(mp, &hammer2_spec_vops, &mp->mnt_vn_spec_ops);
1519         vfs_add_vnodeops(mp, &hammer2_fifo_vops, &mp->mnt_vn_fifo_ops);
1520
1521         if (path) {
1522                 copyinstr(info.volume, mp->mnt_stat.f_mntfromname,
1523                           MNAMELEN - 1, &size);
1524                 bzero(mp->mnt_stat.f_mntfromname + size, MNAMELEN - size);
1525         } /* else root mount, already in there */
1526
1527         bzero(mp->mnt_stat.f_mntonname, sizeof(mp->mnt_stat.f_mntonname));
1528         if (path) {
1529                 copyinstr(path, mp->mnt_stat.f_mntonname,
1530                           sizeof(mp->mnt_stat.f_mntonname) - 1,
1531                           &size);
1532         } else {
1533                 /* root mount */
1534                 mp->mnt_stat.f_mntonname[0] = '/';
1535         }
1536
1537         /*
1538          * Initial statfs to prime mnt_stat.
1539          */
1540         hammer2_vfs_statfs(mp, &mp->mnt_stat, cred);
1541
1542         return 0;
1543 }
1544
1545 /*
1546  * Scan PFSs under the super-root and create hammer2_pfs structures.
1547  */
1548 static
1549 void
1550 hammer2_update_pmps(hammer2_dev_t *hmp)
1551 {
1552         const hammer2_inode_data_t *ripdata;
1553         hammer2_chain_t *parent;
1554         hammer2_chain_t *chain;
1555         hammer2_blockref_t bref;
1556         hammer2_dev_t *force_local;
1557         hammer2_pfs_t *spmp;
1558         hammer2_pfs_t *pmp;
1559         hammer2_key_t key_next;
1560         int error;
1561
1562         /*
1563          * Force local mount (disassociate all PFSs from their clusters).
1564          * Used primarily for debugging.
1565          */
1566         force_local = (hmp->hflags & HMNT2_LOCAL) ? hmp : NULL;
1567
1568         /*
1569          * Lookup mount point under the media-localized super-root.
1570          *
1571          * cluster->pmp will incorrectly point to spmp and must be fixed
1572          * up later on.
1573          */
1574         spmp = hmp->spmp;
1575         hammer2_inode_lock(spmp->iroot, 0);
1576         parent = hammer2_inode_chain(spmp->iroot, 0, HAMMER2_RESOLVE_ALWAYS);
1577         chain = hammer2_chain_lookup(&parent, &key_next,
1578                                          HAMMER2_KEY_MIN, HAMMER2_KEY_MAX,
1579                                          &error, 0);
1580         while (chain) {
1581                 if (chain->error) {
1582                         kprintf("I/O error scanning PFS labels\n");
1583                 } else if (chain->bref.type != HAMMER2_BREF_TYPE_INODE) {
1584                         kprintf("Non inode chain type %d under super-root\n",
1585                                 chain->bref.type);
1586                 } else {
1587                         ripdata = &chain->data->ipdata;
1588                         bref = chain->bref;
1589                         pmp = hammer2_pfsalloc(chain, ripdata,
1590                                                bref.modify_tid, force_local);
1591                 }
1592                 chain = hammer2_chain_next(&parent, chain, &key_next,
1593                                            key_next, HAMMER2_KEY_MAX,
1594                                            &error, 0);
1595         }
1596         if (parent) {
1597                 hammer2_chain_unlock(parent);
1598                 hammer2_chain_drop(parent);
1599         }
1600         hammer2_inode_unlock(spmp->iroot);
1601 }
1602
1603 static
1604 int
1605 hammer2_remount(hammer2_dev_t *hmp, struct mount *mp, char *path __unused,
1606                 struct ucred *cred)
1607 {
1608         hammer2_volume_t *vol;
1609         struct vnode *devvp;
1610         int i, error, result = 0;
1611
1612         if (!(hmp->ronly && (mp->mnt_kern_flag & MNTK_WANTRDWR)))
1613                 return 0;
1614
1615         for (i = 0; i < hmp->nvolumes; ++i) {
1616                 vol = &hmp->volumes[i];
1617                 devvp = vol->dev->devvp;
1618                 KKASSERT(devvp);
1619                 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1620                 VOP_OPEN(devvp, FREAD | FWRITE, FSCRED, NULL);
1621                 vn_unlock(devvp);
1622                 error = 0;
1623                 if (vol->id == HAMMER2_ROOT_VOLUME) {
1624                         error = hammer2_recovery(hmp);
1625                         if (error == 0)
1626                                 error |= hammer2_fixup_pfses(hmp);
1627                 }
1628                 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1629                 if (error == 0) {
1630                         VOP_CLOSE(devvp, FREAD, NULL);
1631                 } else {
1632                         VOP_CLOSE(devvp, FREAD | FWRITE, NULL);
1633                 }
1634                 vn_unlock(devvp);
1635                 result |= error;
1636         }
1637         if (result == 0) {
1638                 kprintf("hammer2: enable read/write\n");
1639                 hmp->ronly = 0;
1640         }
1641
1642         return result;
1643 }
1644
1645 static
1646 int
1647 hammer2_vfs_unmount(struct mount *mp, int mntflags)
1648 {
1649         hammer2_pfs_t *pmp;
1650         int flags;
1651         int error = 0;
1652
1653         pmp = MPTOPMP(mp);
1654
1655         if (pmp == NULL)
1656                 return(0);
1657
1658         lockmgr(&hammer2_mntlk, LK_EXCLUSIVE);
1659
1660         /*
1661          * If mount initialization proceeded far enough we must flush
1662          * its vnodes and sync the underlying mount points.  Three syncs
1663          * are required to fully flush the filesystem (freemap updates lag
1664          * by one flush, and one extra for safety).
1665          */
1666         if (mntflags & MNT_FORCE)
1667                 flags = FORCECLOSE;
1668         else
1669                 flags = 0;
1670         if (pmp->iroot) {
1671                 error = vflush(mp, 0, flags);
1672                 if (error)
1673                         goto failed;
1674                 hammer2_vfs_sync(mp, MNT_WAIT);
1675                 hammer2_vfs_sync(mp, MNT_WAIT);
1676                 hammer2_vfs_sync(mp, MNT_WAIT);
1677         }
1678
1679         /*
1680          * Cleanup the frontend support XOPS threads
1681          */
1682         hammer2_xop_helper_cleanup(pmp);
1683
1684         if (pmp->mp)
1685                 hammer2_unmount_helper(mp, pmp, NULL);
1686
1687         error = 0;
1688 failed:
1689         lockmgr(&hammer2_mntlk, LK_RELEASE);
1690
1691         return (error);
1692 }
1693
1694 /*
1695  * Mount helper, hook the system mount into our PFS.
1696  * The mount lock is held.
1697  *
1698  * We must bump the mount_count on related devices for any
1699  * mounted PFSs.
1700  */
1701 static
1702 void
1703 hammer2_mount_helper(struct mount *mp, hammer2_pfs_t *pmp)
1704 {
1705         hammer2_cluster_t *cluster;
1706         hammer2_chain_t *rchain;
1707         int i;
1708
1709         mp->mnt_data = (qaddr_t)pmp;
1710         pmp->mp = mp;
1711
1712         /*
1713          * After pmp->mp is set we have to adjust hmp->mount_count.
1714          */
1715         cluster = &pmp->iroot->cluster;
1716         for (i = 0; i < cluster->nchains; ++i) {
1717                 rchain = cluster->array[i].chain;
1718                 if (rchain == NULL)
1719                         continue;
1720                 ++rchain->hmp->mount_count;
1721         }
1722
1723         /*
1724          * Create missing Xop threads
1725          */
1726         hammer2_xop_helper_create(pmp);
1727 }
1728
1729 /*
1730  * Mount helper, unhook the system mount from our PFS.
1731  * The mount lock is held.
1732  *
1733  * If hmp is supplied a mount responsible for being the first to open
1734  * the block device failed and the block device and all PFSs using the
1735  * block device must be cleaned up.
1736  *
1737  * If pmp is supplied multiple devices might be backing the PFS and each
1738  * must be disconnected.  This might not be the last PFS using some of the
1739  * underlying devices.  Also, we have to adjust our hmp->mount_count
1740  * accounting for the devices backing the pmp which is now undergoing an
1741  * unmount.
1742  */
1743 static
1744 void
1745 hammer2_unmount_helper(struct mount *mp, hammer2_pfs_t *pmp, hammer2_dev_t *hmp)
1746 {
1747         hammer2_cluster_t *cluster;
1748         hammer2_chain_t *rchain;
1749         int dumpcnt;
1750         int i;
1751
1752         /*
1753          * If no device supplied this is a high-level unmount and we have to
1754          * to disconnect the mount, adjust mount_count, and locate devices
1755          * that might now have no mounts.
1756          */
1757         if (pmp) {
1758                 KKASSERT(hmp == NULL);
1759                 KKASSERT((void *)(intptr_t)mp->mnt_data == pmp);
1760                 pmp->mp = NULL;
1761                 mp->mnt_data = NULL;
1762
1763                 /*
1764                  * After pmp->mp is cleared we have to account for
1765                  * mount_count.
1766                  */
1767                 cluster = &pmp->iroot->cluster;
1768                 for (i = 0; i < cluster->nchains; ++i) {
1769                         rchain = cluster->array[i].chain;
1770                         if (rchain == NULL)
1771                                 continue;
1772                         --rchain->hmp->mount_count;
1773                         /* scrapping hmp now may invalidate the pmp */
1774                 }
1775 again:
1776                 TAILQ_FOREACH(hmp, &hammer2_mntlist, mntentry) {
1777                         if (hmp->mount_count == 0) {
1778                                 hammer2_unmount_helper(NULL, NULL, hmp);
1779                                 goto again;
1780                         }
1781                 }
1782                 return;
1783         }
1784
1785         /*
1786          * Try to terminate the block device.  We can't terminate it if
1787          * there are still PFSs referencing it.
1788          */
1789         if (hmp->mount_count)
1790                 return;
1791
1792         /*
1793          * Decomission the network before we start messing with the
1794          * device and PFS.
1795          */
1796         hammer2_iocom_uninit(hmp);
1797
1798         hammer2_bulkfree_uninit(hmp);
1799         hammer2_pfsfree_scan(hmp, 0);
1800
1801         /*
1802          * Cycle the volume data lock as a safety (probably not needed any
1803          * more).  To ensure everything is out we need to flush at least
1804          * three times.  (1) The running of the sideq can dirty the
1805          * filesystem, (2) A normal flush can dirty the freemap, and
1806          * (3) ensure that the freemap is fully synchronized.
1807          *
1808          * The next mount's recovery scan can clean everything up but we want
1809          * to leave the filesystem in a 100% clean state on a normal unmount.
1810          */
1811 #if 0
1812         hammer2_voldata_lock(hmp);
1813         hammer2_voldata_unlock(hmp);
1814 #endif
1815
1816         /*
1817          * Flush whatever is left.  Unmounted but modified PFS's might still
1818          * have some dirty chains on them.
1819          */
1820         hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1821         hammer2_chain_lock(&hmp->fchain, HAMMER2_RESOLVE_ALWAYS);
1822
1823         if (hmp->fchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1824                 hammer2_voldata_modify(hmp);
1825                 hammer2_flush(&hmp->fchain, HAMMER2_FLUSH_TOP |
1826                                             HAMMER2_FLUSH_ALL);
1827         }
1828         hammer2_chain_unlock(&hmp->fchain);
1829
1830         if (hmp->vchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1831                 hammer2_flush(&hmp->vchain, HAMMER2_FLUSH_TOP |
1832                                             HAMMER2_FLUSH_ALL);
1833         }
1834         hammer2_chain_unlock(&hmp->vchain);
1835
1836         if ((hmp->vchain.flags | hmp->fchain.flags) &
1837             HAMMER2_CHAIN_FLUSH_MASK) {
1838                 kprintf("hammer2_unmount: chains left over after final sync\n");
1839                 kprintf("    vchain %08x\n", hmp->vchain.flags);
1840                 kprintf("    fchain %08x\n", hmp->fchain.flags);
1841
1842                 if (hammer2_debug & 0x0010)
1843                         Debugger("entered debugger");
1844         }
1845
1846         hammer2_pfsfree_scan(hmp, 1);
1847
1848         KKASSERT(hmp->spmp == NULL);
1849
1850         /*
1851          * Finish up with the device vnode
1852          */
1853         if (!TAILQ_EMPTY(&hmp->devvpl)) {
1854                 hammer2_close_devvp(&hmp->devvpl, hmp->ronly);
1855                 hammer2_cleanup_devvp(&hmp->devvpl);
1856         }
1857         KKASSERT(TAILQ_EMPTY(&hmp->devvpl));
1858
1859         /*
1860          * Clear vchain/fchain flags that might prevent final cleanup
1861          * of these chains.
1862          */
1863         if (hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED) {
1864                 atomic_add_long(&hammer2_count_modified_chains, -1);
1865                 atomic_clear_int(&hmp->vchain.flags, HAMMER2_CHAIN_MODIFIED);
1866                 hammer2_pfs_memory_wakeup(hmp->vchain.pmp, -1);
1867         }
1868         if (hmp->vchain.flags & HAMMER2_CHAIN_UPDATE) {
1869                 atomic_clear_int(&hmp->vchain.flags, HAMMER2_CHAIN_UPDATE);
1870         }
1871
1872         if (hmp->fchain.flags & HAMMER2_CHAIN_MODIFIED) {
1873                 atomic_add_long(&hammer2_count_modified_chains, -1);
1874                 atomic_clear_int(&hmp->fchain.flags, HAMMER2_CHAIN_MODIFIED);
1875                 hammer2_pfs_memory_wakeup(hmp->fchain.pmp, -1);
1876         }
1877         if (hmp->fchain.flags & HAMMER2_CHAIN_UPDATE) {
1878                 atomic_clear_int(&hmp->fchain.flags, HAMMER2_CHAIN_UPDATE);
1879         }
1880
1881         /*
1882          * Final drop of embedded freemap root chain to
1883          * clean up fchain.core (fchain structure is not
1884          * flagged ALLOCATED so it is cleaned out and then
1885          * left to rot).
1886          */
1887         hammer2_chain_drop(&hmp->fchain);
1888
1889         /*
1890          * Final drop of embedded volume root chain to clean
1891          * up vchain.core (vchain structure is not flagged
1892          * ALLOCATED so it is cleaned out and then left to
1893          * rot).
1894          */
1895         dumpcnt = 50;
1896         hammer2_dump_chain(&hmp->vchain, 0, 0, &dumpcnt, 'v', (u_int)-1);
1897         dumpcnt = 50;
1898         hammer2_dump_chain(&hmp->fchain, 0, 0, &dumpcnt, 'f', (u_int)-1);
1899
1900         hammer2_chain_drop(&hmp->vchain);
1901
1902         hammer2_io_cleanup(hmp, &hmp->iotree);
1903         if (hmp->iofree_count) {
1904                 kprintf("io_cleanup: %d I/O's left hanging\n",
1905                         hmp->iofree_count);
1906         }
1907
1908         TAILQ_REMOVE(&hammer2_mntlist, hmp, mntentry);
1909         kmalloc_destroy_obj(&hmp->mchain);
1910         kmalloc_destroy_obj(&hmp->mio);
1911         kmalloc_destroy(&hmp->mmsg);
1912         kfree(hmp, M_HAMMER2);
1913 }
1914
1915 int
1916 hammer2_vfs_vget(struct mount *mp, struct vnode *dvp,
1917                  ino_t ino, struct vnode **vpp)
1918 {
1919         hammer2_xop_lookup_t *xop;
1920         hammer2_pfs_t *pmp;
1921         hammer2_inode_t *ip;
1922         hammer2_tid_t inum;
1923         int error;
1924
1925         inum = (hammer2_tid_t)ino & HAMMER2_DIRHASH_USERMSK;
1926
1927         error = 0;
1928         pmp = MPTOPMP(mp);
1929
1930         /*
1931          * Easy if we already have it cached
1932          */
1933         ip = hammer2_inode_lookup(pmp, inum);
1934         if (ip) {
1935                 hammer2_inode_lock(ip, HAMMER2_RESOLVE_SHARED);
1936                 *vpp = hammer2_igetv(ip, &error);
1937                 hammer2_inode_unlock(ip);
1938                 hammer2_inode_drop(ip);         /* from lookup */
1939
1940                 return error;
1941         }
1942
1943         /*
1944          * Otherwise we have to find the inode
1945          */
1946         xop = hammer2_xop_alloc(pmp->iroot, 0);
1947         xop->lhc = inum;
1948         hammer2_xop_start(&xop->head, &hammer2_lookup_desc);
1949         error = hammer2_xop_collect(&xop->head, 0);
1950
1951         if (error == 0)
1952                 ip = hammer2_inode_get(pmp, &xop->head, -1, -1);
1953         hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
1954
1955         if (ip) {
1956                 *vpp = hammer2_igetv(ip, &error);
1957                 hammer2_inode_unlock(ip);
1958         } else {
1959                 *vpp = NULL;
1960                 error = ENOENT;
1961         }
1962         return (error);
1963 }
1964
1965 static
1966 int
1967 hammer2_vfs_root(struct mount *mp, struct vnode **vpp)
1968 {
1969         hammer2_pfs_t *pmp;
1970         struct vnode *vp;
1971         int error;
1972
1973         pmp = MPTOPMP(mp);
1974         if (pmp->iroot == NULL) {
1975                 kprintf("hammer2 (%s): no root inode\n",
1976                         mp->mnt_stat.f_mntfromname);
1977                 *vpp = NULL;
1978                 return EINVAL;
1979         }
1980
1981         error = 0;
1982         hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED);
1983
1984         while (pmp->inode_tid == 0) {
1985                 hammer2_xop_ipcluster_t *xop;
1986                 const hammer2_inode_meta_t *meta;
1987
1988                 xop = hammer2_xop_alloc(pmp->iroot, HAMMER2_XOP_MODIFYING);
1989                 hammer2_xop_start(&xop->head, &hammer2_ipcluster_desc);
1990                 error = hammer2_xop_collect(&xop->head, 0);
1991
1992                 if (error == 0) {
1993                         meta = &hammer2_xop_gdata(&xop->head)->ipdata.meta;
1994                         pmp->iroot->meta = *meta;
1995                         pmp->inode_tid = meta->pfs_inum + 1;
1996                         hammer2_xop_pdata(&xop->head);
1997                         /* meta invalid */
1998
1999                         if (pmp->inode_tid < HAMMER2_INODE_START)
2000                                 pmp->inode_tid = HAMMER2_INODE_START;
2001                         pmp->modify_tid =
2002                                 xop->head.cluster.focus->bref.modify_tid + 1;
2003 #if 0
2004                         kprintf("PFS: Starting inode %jd\n",
2005                                 (intmax_t)pmp->inode_tid);
2006                         kprintf("PMP focus good set nextino=%ld mod=%016jx\n",
2007                                 pmp->inode_tid, pmp->modify_tid);
2008 #endif
2009                         wakeup(&pmp->iroot);
2010
2011                         hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
2012
2013                         /*
2014                          * Prime the mount info.
2015                          */
2016                         hammer2_vfs_statfs(mp, &mp->mnt_stat, NULL);
2017                         break;
2018                 }
2019
2020                 /*
2021                  * Loop, try again
2022                  */
2023                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
2024                 hammer2_inode_unlock(pmp->iroot);
2025                 error = tsleep(&pmp->iroot, PCATCH, "h2root", hz);
2026                 hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED);
2027                 if (error == EINTR)
2028                         break;
2029         }
2030
2031         if (error) {
2032                 hammer2_inode_unlock(pmp->iroot);
2033                 *vpp = NULL;
2034         } else {
2035                 vp = hammer2_igetv(pmp->iroot, &error);
2036                 hammer2_inode_unlock(pmp->iroot);
2037                 *vpp = vp;
2038         }
2039
2040         return (error);
2041 }
2042
2043 /*
2044  * Filesystem status
2045  *
2046  * XXX incorporate ipdata->meta.inode_quota and data_quota
2047  */
2048 static
2049 int
2050 hammer2_vfs_statfs(struct mount *mp, struct statfs *sbp, struct ucred *cred)
2051 {
2052         hammer2_pfs_t *pmp;
2053         hammer2_dev_t *hmp;
2054         hammer2_blockref_t bref;
2055         struct statfs tmp;
2056         int i;
2057
2058         /*
2059          * NOTE: iroot might not have validated the cluster yet.
2060          */
2061         pmp = MPTOPMP(mp);
2062
2063         bzero(&tmp, sizeof(tmp));
2064
2065         for (i = 0; i < pmp->iroot->cluster.nchains; ++i) {
2066                 hmp = pmp->pfs_hmps[i];
2067                 if (hmp == NULL)
2068                         continue;
2069                 if (pmp->iroot->cluster.array[i].chain)
2070                         bref = pmp->iroot->cluster.array[i].chain->bref;
2071                 else
2072                         bzero(&bref, sizeof(bref));
2073
2074                 tmp.f_files = bref.embed.stats.inode_count;
2075                 tmp.f_ffree = 0;
2076                 tmp.f_blocks = hmp->voldata.allocator_size /
2077                                mp->mnt_vstat.f_bsize;
2078                 tmp.f_bfree = hmp->voldata.allocator_free /
2079                               mp->mnt_vstat.f_bsize;
2080                 tmp.f_bavail = tmp.f_bfree;
2081
2082                 if (cred && cred->cr_uid != 0) {
2083                         uint64_t adj;
2084
2085                         /* 5% */
2086                         adj = hmp->free_reserved / mp->mnt_vstat.f_bsize;
2087                         tmp.f_blocks -= adj;
2088                         tmp.f_bfree -= adj;
2089                         tmp.f_bavail -= adj;
2090                 }
2091
2092                 mp->mnt_stat.f_blocks = tmp.f_blocks;
2093                 mp->mnt_stat.f_bfree = tmp.f_bfree;
2094                 mp->mnt_stat.f_bavail = tmp.f_bavail;
2095                 mp->mnt_stat.f_files = tmp.f_files;
2096                 mp->mnt_stat.f_ffree = tmp.f_ffree;
2097
2098                 *sbp = mp->mnt_stat;
2099         }
2100         return (0);
2101 }
2102
2103 static
2104 int
2105 hammer2_vfs_statvfs(struct mount *mp, struct statvfs *sbp, struct ucred *cred)
2106 {
2107         hammer2_pfs_t *pmp;
2108         hammer2_dev_t *hmp;
2109         hammer2_blockref_t bref;
2110         struct statvfs tmp;
2111         int i;
2112
2113         /*
2114          * NOTE: iroot might not have validated the cluster yet.
2115          */
2116         pmp = MPTOPMP(mp);
2117         bzero(&tmp, sizeof(tmp));
2118
2119         for (i = 0; i < pmp->iroot->cluster.nchains; ++i) {
2120                 hmp = pmp->pfs_hmps[i];
2121                 if (hmp == NULL)
2122                         continue;
2123                 if (pmp->iroot->cluster.array[i].chain)
2124                         bref = pmp->iroot->cluster.array[i].chain->bref;
2125                 else
2126                         bzero(&bref, sizeof(bref));
2127
2128                 tmp.f_files = bref.embed.stats.inode_count;
2129                 tmp.f_ffree = 0;
2130                 tmp.f_blocks = hmp->voldata.allocator_size /
2131                                mp->mnt_vstat.f_bsize;
2132                 tmp.f_bfree = hmp->voldata.allocator_free /
2133                               mp->mnt_vstat.f_bsize;
2134                 tmp.f_bavail = tmp.f_bfree;
2135
2136                 if (cred && cred->cr_uid != 0) {
2137                         uint64_t adj;
2138
2139                         /* 5% */
2140                         adj = hmp->free_reserved / mp->mnt_vstat.f_bsize;
2141                         tmp.f_blocks -= adj;
2142                         tmp.f_bfree -= adj;
2143                         tmp.f_bavail -= adj;
2144                 }
2145
2146                 mp->mnt_vstat.f_blocks = tmp.f_blocks;
2147                 mp->mnt_vstat.f_bfree = tmp.f_bfree;
2148                 mp->mnt_vstat.f_bavail = tmp.f_bavail;
2149                 mp->mnt_vstat.f_files = tmp.f_files;
2150                 mp->mnt_vstat.f_ffree = tmp.f_ffree;
2151
2152                 *sbp = mp->mnt_vstat;
2153         }
2154         return (0);
2155 }
2156
2157 /*
2158  * Mount-time recovery (RW mounts)
2159  *
2160  * Updates to the free block table are allowed to lag flushes by one
2161  * transaction.  In case of a crash, then on a fresh mount we must do an
2162  * incremental scan of the last committed transaction id and make sure that
2163  * all related blocks have been marked allocated.
2164  */
2165 struct hammer2_recovery_elm {
2166         TAILQ_ENTRY(hammer2_recovery_elm) entry;
2167         hammer2_chain_t *chain;
2168         hammer2_tid_t sync_tid;
2169 };
2170
2171 TAILQ_HEAD(hammer2_recovery_list, hammer2_recovery_elm);
2172
2173 struct hammer2_recovery_info {
2174         struct hammer2_recovery_list list;
2175         hammer2_tid_t   mtid;
2176         int     depth;
2177 };
2178
2179 static int hammer2_recovery_scan(hammer2_dev_t *hmp,
2180                         hammer2_chain_t *parent,
2181                         struct hammer2_recovery_info *info,
2182                         hammer2_tid_t sync_tid);
2183
2184 #define HAMMER2_RECOVERY_MAXDEPTH       10
2185
2186 static
2187 int
2188 hammer2_recovery(hammer2_dev_t *hmp)
2189 {
2190         struct hammer2_recovery_info info;
2191         struct hammer2_recovery_elm *elm;
2192         hammer2_chain_t *parent;
2193         hammer2_tid_t sync_tid;
2194         hammer2_tid_t mirror_tid;
2195         int error;
2196
2197         hammer2_trans_init(hmp->spmp, 0);
2198
2199         sync_tid = hmp->voldata.freemap_tid;
2200         mirror_tid = hmp->voldata.mirror_tid;
2201
2202         kprintf("hammer2_mount: \"%s\": ", hmp->devrepname);
2203         if (sync_tid >= mirror_tid) {
2204                 kprintf("no recovery needed\n");
2205         } else {
2206                 kprintf("freemap recovery %016jx-%016jx\n",
2207                         sync_tid + 1, mirror_tid);
2208         }
2209
2210         TAILQ_INIT(&info.list);
2211         info.depth = 0;
2212         parent = hammer2_chain_lookup_init(&hmp->vchain, 0);
2213         error = hammer2_recovery_scan(hmp, parent, &info, sync_tid);
2214         hammer2_chain_lookup_done(parent);
2215
2216         while ((elm = TAILQ_FIRST(&info.list)) != NULL) {
2217                 TAILQ_REMOVE(&info.list, elm, entry);
2218                 parent = elm->chain;
2219                 sync_tid = elm->sync_tid;
2220                 kfree(elm, M_HAMMER2);
2221
2222                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
2223                 error |= hammer2_recovery_scan(hmp, parent, &info,
2224                                               hmp->voldata.freemap_tid);
2225                 hammer2_chain_unlock(parent);
2226                 hammer2_chain_drop(parent);     /* drop elm->chain ref */
2227         }
2228
2229         hammer2_trans_done(hmp->spmp, 0);
2230
2231         return error;
2232 }
2233
2234 static
2235 int
2236 hammer2_recovery_scan(hammer2_dev_t *hmp, hammer2_chain_t *parent,
2237                       struct hammer2_recovery_info *info,
2238                       hammer2_tid_t sync_tid)
2239 {
2240         const hammer2_inode_data_t *ripdata;
2241         hammer2_chain_t *chain;
2242         hammer2_blockref_t bref;
2243         int tmp_error;
2244         int rup_error;
2245         int error;
2246         int first;
2247
2248         /*
2249          * Adjust freemap to ensure that the block(s) are marked allocated.
2250          */
2251         if (parent->bref.type != HAMMER2_BREF_TYPE_VOLUME) {
2252                 hammer2_freemap_adjust(hmp, &parent->bref,
2253                                        HAMMER2_FREEMAP_DORECOVER);
2254         }
2255
2256         /*
2257          * Check type for recursive scan
2258          */
2259         switch(parent->bref.type) {
2260         case HAMMER2_BREF_TYPE_VOLUME:
2261                 /* data already instantiated */
2262                 break;
2263         case HAMMER2_BREF_TYPE_INODE:
2264                 /*
2265                  * Must instantiate data for DIRECTDATA test and also
2266                  * for recursion.
2267                  */
2268                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
2269                 ripdata = &parent->data->ipdata;
2270                 if (ripdata->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA) {
2271                         /* not applicable to recovery scan */
2272                         hammer2_chain_unlock(parent);
2273                         return 0;
2274                 }
2275                 hammer2_chain_unlock(parent);
2276                 break;
2277         case HAMMER2_BREF_TYPE_INDIRECT:
2278                 /*
2279                  * Must instantiate data for recursion
2280                  */
2281                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
2282                 hammer2_chain_unlock(parent);
2283                 break;
2284         case HAMMER2_BREF_TYPE_DIRENT:
2285         case HAMMER2_BREF_TYPE_DATA:
2286         case HAMMER2_BREF_TYPE_FREEMAP:
2287         case HAMMER2_BREF_TYPE_FREEMAP_NODE:
2288         case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
2289                 /* not applicable to recovery scan */
2290                 return 0;
2291                 break;
2292         default:
2293                 return HAMMER2_ERROR_BADBREF;
2294         }
2295
2296         /*
2297          * Defer operation if depth limit reached.
2298          */
2299         if (info->depth >= HAMMER2_RECOVERY_MAXDEPTH) {
2300                 struct hammer2_recovery_elm *elm;
2301
2302                 elm = kmalloc(sizeof(*elm), M_HAMMER2, M_ZERO | M_WAITOK);
2303                 elm->chain = parent;
2304                 elm->sync_tid = sync_tid;
2305                 hammer2_chain_ref(parent);
2306                 TAILQ_INSERT_TAIL(&info->list, elm, entry);
2307                 /* unlocked by caller */
2308
2309                 return(0);
2310         }
2311
2312
2313         /*
2314          * Recursive scan of the last flushed transaction only.  We are
2315          * doing this without pmp assignments so don't leave the chains
2316          * hanging around after we are done with them.
2317          *
2318          * error        Cumulative error this level only
2319          * rup_error    Cumulative error for recursion
2320          * tmp_error    Specific non-cumulative recursion error
2321          */
2322         chain = NULL;
2323         first = 1;
2324         rup_error = 0;
2325         error = 0;
2326
2327         for (;;) {
2328                 error |= hammer2_chain_scan(parent, &chain, &bref,
2329                                             &first,
2330                                             HAMMER2_LOOKUP_NODATA);
2331
2332                 /*
2333                  * Problem during scan or EOF
2334                  */
2335                 if (error)
2336                         break;
2337
2338                 /*
2339                  * If this is a leaf
2340                  */
2341                 if (chain == NULL) {
2342                         if (bref.mirror_tid > sync_tid) {
2343                                 hammer2_freemap_adjust(hmp, &bref,
2344                                                      HAMMER2_FREEMAP_DORECOVER);
2345                         }
2346                         continue;
2347                 }
2348
2349                 /*
2350                  * This may or may not be a recursive node.
2351                  */
2352                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
2353                 if (bref.mirror_tid > sync_tid) {
2354                         ++info->depth;
2355                         tmp_error = hammer2_recovery_scan(hmp, chain,
2356                                                            info, sync_tid);
2357                         --info->depth;
2358                 } else {
2359                         tmp_error = 0;
2360                 }
2361
2362                 /*
2363                  * Flush the recovery at the PFS boundary to stage it for
2364                  * the final flush of the super-root topology.
2365                  */
2366                 if (tmp_error == 0 &&
2367                     (bref.flags & HAMMER2_BREF_FLAG_PFSROOT) &&
2368                     (chain->flags & HAMMER2_CHAIN_ONFLUSH)) {
2369                         hammer2_flush(chain, HAMMER2_FLUSH_TOP |
2370                                              HAMMER2_FLUSH_ALL);
2371                 }
2372                 rup_error |= tmp_error;
2373         }
2374         return ((error | rup_error) & ~HAMMER2_ERROR_EOF);
2375 }
2376
2377 /*
2378  * This fixes up an error introduced in earlier H2 implementations where
2379  * moving a PFS inode into an indirect block wound up causing the
2380  * HAMMER2_BREF_FLAG_PFSROOT flag in the bref to get cleared.
2381  */
2382 static
2383 int
2384 hammer2_fixup_pfses(hammer2_dev_t *hmp)
2385 {
2386         const hammer2_inode_data_t *ripdata;
2387         hammer2_chain_t *parent;
2388         hammer2_chain_t *chain;
2389         hammer2_key_t key_next;
2390         hammer2_pfs_t *spmp;
2391         int error;
2392
2393         error = 0;
2394
2395         /*
2396          * Lookup mount point under the media-localized super-root.
2397          *
2398          * cluster->pmp will incorrectly point to spmp and must be fixed
2399          * up later on.
2400          */
2401         spmp = hmp->spmp;
2402         hammer2_inode_lock(spmp->iroot, 0);
2403         parent = hammer2_inode_chain(spmp->iroot, 0, HAMMER2_RESOLVE_ALWAYS);
2404         chain = hammer2_chain_lookup(&parent, &key_next,
2405                                          HAMMER2_KEY_MIN, HAMMER2_KEY_MAX,
2406                                          &error, 0);
2407         while (chain) {
2408                 if (chain->bref.type != HAMMER2_BREF_TYPE_INODE)
2409                         continue;
2410                 if (chain->error) {
2411                         kprintf("I/O error scanning PFS labels\n");
2412                         error |= chain->error;
2413                 } else if ((chain->bref.flags &
2414                             HAMMER2_BREF_FLAG_PFSROOT) == 0) {
2415                         int error2;
2416
2417                         ripdata = &chain->data->ipdata;
2418                         hammer2_trans_init(hmp->spmp, 0);
2419                         error2 = hammer2_chain_modify(chain,
2420                                                       chain->bref.modify_tid,
2421                                                       0, 0);
2422                         if (error2 == 0) {
2423                                 kprintf("hammer2: Correct mis-flagged PFS %s\n",
2424                                         ripdata->filename);
2425                                 chain->bref.flags |= HAMMER2_BREF_FLAG_PFSROOT;
2426                         } else {
2427                                 error |= error2;
2428                         }
2429                         hammer2_flush(chain, HAMMER2_FLUSH_TOP |
2430                                              HAMMER2_FLUSH_ALL);
2431                         hammer2_trans_done(hmp->spmp, 0);
2432                 }
2433                 chain = hammer2_chain_next(&parent, chain, &key_next,
2434                                            key_next, HAMMER2_KEY_MAX,
2435                                            &error, 0);
2436         }
2437         if (parent) {
2438                 hammer2_chain_unlock(parent);
2439                 hammer2_chain_drop(parent);
2440         }
2441         hammer2_inode_unlock(spmp->iroot);
2442
2443         return error;
2444 }
2445
2446 /*
2447  * Sync a mount point; this is called periodically on a per-mount basis from
2448  * the filesystem syncer, and whenever a user issues a sync.
2449  */
2450 int
2451 hammer2_vfs_sync(struct mount *mp, int waitfor)
2452 {
2453         int error;
2454
2455         error = hammer2_vfs_sync_pmp(MPTOPMP(mp), waitfor);
2456
2457         return error;
2458 }
2459
2460 /*
2461  * Because frontend operations lock vnodes before we get a chance to
2462  * lock the related inode, we can't just acquire a vnode lock without
2463  * risking a deadlock.  The frontend may be holding a vnode lock while
2464  * also blocked on our SYNCQ flag while trying to get the inode lock.
2465  *
2466  * To deal with this situation we can check the vnode lock situation
2467  * after locking the inode and perform a work-around.
2468  */
2469 int
2470 hammer2_vfs_sync_pmp(hammer2_pfs_t *pmp, int waitfor)
2471 {
2472         struct mount *mp;
2473         /*hammer2_xop_flush_t *xop;*/
2474         /*struct hammer2_sync_info info;*/
2475         hammer2_inode_t *ip;
2476         hammer2_depend_t *depend;
2477         hammer2_depend_t *depend_next;
2478         struct vnode *vp;
2479         uint32_t pass2;
2480         int error;
2481         int wakecount;
2482         int dorestart;
2483
2484         mp = pmp->mp;
2485
2486         /*
2487          * Move all inodes on sideq to syncq.  This will clear sideq.
2488          * This should represent all flushable inodes.  These inodes
2489          * will already have refs due to being on syncq or sideq.  We
2490          * must do this all at once with the spinlock held to ensure that
2491          * all inode dependencies are part of the same flush.
2492          *
2493          * We should be able to do this asynchronously from frontend
2494          * operations because we will be locking the inodes later on
2495          * to actually flush them, and that will partition any frontend
2496          * op using the same inode.  Either it has already locked the
2497          * inode and we will block, or it has not yet locked the inode
2498          * and it will block until we are finished flushing that inode.
2499          *
2500          * When restarting, only move the inodes flagged as PASS2 from
2501          * SIDEQ to SYNCQ.  PASS2 propagation by inode_lock4() and
2502          * inode_depend() are atomic with the spin-lock.
2503          */
2504         hammer2_trans_init(pmp, HAMMER2_TRANS_ISFLUSH);
2505 #ifdef HAMMER2_DEBUG_SYNC
2506         kprintf("FILESYSTEM SYNC BOUNDARY\n");
2507 #endif
2508         dorestart = 0;
2509
2510         /*
2511          * Move inodes from depq to syncq, releasing the related
2512          * depend structures.
2513          */
2514 restart:
2515 #ifdef HAMMER2_DEBUG_SYNC
2516         kprintf("FILESYSTEM SYNC RESTART (%d)\n", dorestart);
2517 #endif
2518         hammer2_trans_setflags(pmp, 0/*HAMMER2_TRANS_COPYQ*/);
2519         hammer2_trans_clearflags(pmp, HAMMER2_TRANS_RESCAN);
2520
2521         /*
2522          * Move inodes from depq to syncq.  When restarting, only depq's
2523          * marked pass2 are moved.
2524          */
2525         hammer2_spin_ex(&pmp->list_spin);
2526         depend_next = TAILQ_FIRST(&pmp->depq);
2527         wakecount = 0;
2528
2529         while ((depend = depend_next) != NULL) {
2530                 depend_next = TAILQ_NEXT(depend, entry);
2531                 if (dorestart && depend->pass2 == 0)
2532                         continue;
2533                 TAILQ_FOREACH(ip, &depend->sideq, entry) {
2534                         KKASSERT(ip->flags & HAMMER2_INODE_SIDEQ);
2535                         atomic_set_int(&ip->flags, HAMMER2_INODE_SYNCQ);
2536                         atomic_clear_int(&ip->flags, HAMMER2_INODE_SIDEQ);
2537                         ip->depend = NULL;
2538                 }
2539
2540                 /*
2541                  * NOTE: pmp->sideq_count includes both sideq and syncq
2542                  */
2543                 TAILQ_CONCAT(&pmp->syncq, &depend->sideq, entry);
2544
2545                 depend->count = 0;
2546                 depend->pass2 = 0;
2547                 TAILQ_REMOVE(&pmp->depq, depend, entry);
2548         }
2549
2550         hammer2_spin_unex(&pmp->list_spin);
2551         hammer2_trans_clearflags(pmp, /*HAMMER2_TRANS_COPYQ |*/
2552                                       HAMMER2_TRANS_WAITING);
2553         dorestart = 0;
2554
2555         /*
2556          * sideq_count may have dropped enough to allow us to unstall
2557          * the frontend.
2558          */
2559         hammer2_pfs_memory_wakeup(pmp, 0);
2560
2561         /*
2562          * Now run through all inodes on syncq.
2563          *
2564          * Flush transactions only interlock with other flush transactions.
2565          * Any conflicting frontend operations will block on the inode, but
2566          * may hold a vnode lock while doing so.
2567          */
2568         hammer2_spin_ex(&pmp->list_spin);
2569         while ((ip = TAILQ_FIRST(&pmp->syncq)) != NULL) {
2570                 /*
2571                  * Remove the inode from the SYNCQ, transfer the syncq ref
2572                  * to us.  We must clear SYNCQ to allow any potential
2573                  * front-end deadlock to proceed.  We must set PASS2 so
2574                  * the dependency code knows what to do.
2575                  */
2576                 pass2 = ip->flags;
2577                 cpu_ccfence();
2578                 if (atomic_cmpset_int(&ip->flags,
2579                               pass2,
2580                               (pass2 & ~(HAMMER2_INODE_SYNCQ |
2581                                          HAMMER2_INODE_SYNCQ_WAKEUP)) |
2582                               HAMMER2_INODE_SYNCQ_PASS2) == 0) {
2583                         continue;
2584                 }
2585                 TAILQ_REMOVE(&pmp->syncq, ip, entry);
2586                 --pmp->sideq_count;
2587                 hammer2_spin_unex(&pmp->list_spin);
2588
2589                 /*
2590                  * Tickle anyone waiting on ip->flags or the hysteresis
2591                  * on the dirty inode count.
2592                  */
2593                 if (pass2 & HAMMER2_INODE_SYNCQ_WAKEUP)
2594                         wakeup(&ip->flags);
2595                 if (++wakecount >= hammer2_limit_dirty_inodes / 20 + 1) {
2596                         wakecount = 0;
2597                         hammer2_pfs_memory_wakeup(pmp, 0);
2598                 }
2599
2600                 /*
2601                  * Relock the inode, and we inherit a ref from the above.
2602                  * We will check for a race after we acquire the vnode.
2603                  */
2604                 hammer2_mtx_ex(&ip->lock);
2605
2606                 /*
2607                  * We need the vp in order to vfsync() dirty buffers, so if
2608                  * one isn't attached we can skip it.
2609                  *
2610                  * Ordering the inode lock and then the vnode lock has the
2611                  * potential to deadlock.  If we had left SYNCQ set that could
2612                  * also deadlock us against the frontend even if we don't hold
2613                  * any locks, but the latter is not a problem now since we
2614                  * cleared it.  igetv will temporarily release the inode lock
2615                  * in a safe manner to work-around the deadlock.
2616                  *
2617                  * Unfortunately it is still possible to deadlock when the
2618                  * frontend obtains multiple inode locks, because all the
2619                  * related vnodes are already locked (nor can the vnode locks
2620                  * be released and reacquired without messing up RECLAIM and
2621                  * INACTIVE sequencing).
2622                  *
2623                  * The solution for now is to move the vp back onto SIDEQ
2624                  * and set dorestart, which will restart the flush after we
2625                  * exhaust the current SYNCQ.  Note that additional
2626                  * dependencies may build up, so we definitely need to move
2627                  * the whole SIDEQ back to SYNCQ when we restart.
2628                  */
2629                 vp = ip->vp;
2630                 if (vp) {
2631                         if (vget(vp, LK_EXCLUSIVE|LK_NOWAIT)) {
2632                                 /*
2633                                  * Failed to get the vnode, requeue the inode
2634                                  * (PASS2 is already set so it will be found
2635                                  * again on the restart).
2636                                  *
2637                                  * Then unlock, possibly sleep, and retry
2638                                  * later.  We sleep if PASS2 was *previously*
2639                                  * set, before we set it again above.
2640                                  */
2641                                 vp = NULL;
2642                                 dorestart = 1;
2643 #ifdef HAMMER2_DEBUG_SYNC
2644                                 kprintf("inum %ld (sync delayed by vnode)\n",
2645                                         (long)ip->meta.inum);
2646 #endif
2647                                 hammer2_inode_delayed_sideq(ip);
2648
2649                                 hammer2_mtx_unlock(&ip->lock);
2650                                 hammer2_inode_drop(ip);
2651
2652                                 if (pass2 & HAMMER2_INODE_SYNCQ_PASS2) {
2653                                         tsleep(&dorestart, 0, "h2syndel", 2);
2654                                 }
2655                                 hammer2_spin_ex(&pmp->list_spin);
2656                                 continue;
2657                         }
2658                 } else {
2659                         vp = NULL;
2660                 }
2661
2662                 /*
2663                  * If the inode wound up on a SIDEQ again it will already be
2664                  * prepped for another PASS2.  In this situation if we flush
2665                  * it now we will just wind up flushing it again in the same
2666                  * syncer run, so we might as well not flush it now.
2667                  */
2668                 if (ip->flags & HAMMER2_INODE_SIDEQ) {
2669                         hammer2_mtx_unlock(&ip->lock);
2670                         hammer2_inode_drop(ip);
2671                         if (vp)
2672                                 vput(vp);
2673                         dorestart = 1;
2674                         hammer2_spin_ex(&pmp->list_spin);
2675                         continue;
2676                 }
2677
2678                 /*
2679                  * Ok we have the inode exclusively locked and if vp is
2680                  * not NULL that will also be exclusively locked.  Do the
2681                  * meat of the flush.
2682                  *
2683                  * vp token needed for v_rbdirty_tree check / vclrisdirty
2684                  * sequencing.  Though we hold the vnode exclusively so
2685                  * we shouldn't need to hold the token also in this case.
2686                  */
2687                 if (vp) {
2688                         vfsync(vp, MNT_WAIT, 1, NULL, NULL);
2689                         bio_track_wait(&vp->v_track_write, 0, 0); /* XXX */
2690                 }
2691
2692                 /*
2693                  * If the inode has not yet been inserted into the tree
2694                  * we must do so.  Then sync and flush it.  The flush should
2695                  * update the parent.
2696                  */
2697                 if (ip->flags & HAMMER2_INODE_DELETING) {
2698 #ifdef HAMMER2_DEBUG_SYNC
2699                         kprintf("inum %ld destroy\n", (long)ip->meta.inum);
2700 #endif
2701                         hammer2_inode_chain_des(ip);
2702                         atomic_add_long(&hammer2_iod_inode_deletes, 1);
2703                 } else if (ip->flags & HAMMER2_INODE_CREATING) {
2704 #ifdef HAMMER2_DEBUG_SYNC
2705                         kprintf("inum %ld insert\n", (long)ip->meta.inum);
2706 #endif
2707                         hammer2_inode_chain_ins(ip);
2708                         atomic_add_long(&hammer2_iod_inode_creates, 1);
2709                 }
2710 #ifdef HAMMER2_DEBUG_SYNC
2711                 kprintf("inum %ld chain-sync\n", (long)ip->meta.inum);
2712 #endif
2713
2714                 /*
2715                  * Because I kinda messed up the design and index the inodes
2716                  * under the root inode, along side the directory entries,
2717                  * we can't flush the inode index under the iroot until the
2718                  * end.  If we do it now we might miss effects created by
2719                  * other inodes on the SYNCQ.
2720                  *
2721                  * Do a normal (non-FSSYNC) flush instead, which allows the
2722                  * vnode code to work the same.  We don't want to force iroot
2723                  * back onto the SIDEQ, and we also don't want the flush code
2724                  * to update pfs_iroot_blocksets until the final flush later.
2725                  *
2726                  * XXX at the moment this will likely result in a double-flush
2727                  * of the iroot chain.
2728                  */
2729                 hammer2_inode_chain_sync(ip);
2730                 if (ip == pmp->iroot) {
2731                         hammer2_inode_chain_flush(ip, HAMMER2_XOP_INODE_STOP);
2732                 } else {
2733                         hammer2_inode_chain_flush(ip, HAMMER2_XOP_INODE_STOP |
2734                                                       HAMMER2_XOP_FSSYNC);
2735                 }
2736                 if (vp) {
2737                         lwkt_gettoken(&vp->v_token);
2738                         if ((ip->flags & (HAMMER2_INODE_MODIFIED |
2739                                           HAMMER2_INODE_RESIZED |
2740                                           HAMMER2_INODE_DIRTYDATA)) == 0 &&
2741                             RB_EMPTY(&vp->v_rbdirty_tree) &&
2742                             !bio_track_active(&vp->v_track_write)) {
2743                                 vclrisdirty(vp);
2744                         } else {
2745                                 hammer2_inode_delayed_sideq(ip);
2746                         }
2747                         lwkt_reltoken(&vp->v_token);
2748                         vput(vp);
2749                         vp = NULL;      /* safety */
2750                 }
2751                 atomic_clear_int(&ip->flags, HAMMER2_INODE_SYNCQ_PASS2);
2752                 hammer2_inode_unlock(ip);       /* unlock+drop */
2753                 /* ip pointer invalid */
2754
2755                 /*
2756                  * If the inode got dirted after we dropped our locks,
2757                  * it will have already been moved back to the SIDEQ.
2758                  */
2759                 hammer2_spin_ex(&pmp->list_spin);
2760         }
2761         hammer2_spin_unex(&pmp->list_spin);
2762         hammer2_pfs_memory_wakeup(pmp, 0);
2763
2764         if (dorestart || (pmp->trans.flags & HAMMER2_TRANS_RESCAN)) {
2765 #ifdef HAMMER2_DEBUG_SYNC
2766                 kprintf("FILESYSTEM SYNC STAGE 1 RESTART\n");
2767                 /*tsleep(&dorestart, 0, "h2STG1-R", hz*20);*/
2768 #endif
2769                 dorestart = 1;
2770                 goto restart;
2771         }
2772 #ifdef HAMMER2_DEBUG_SYNC
2773         kprintf("FILESYSTEM SYNC STAGE 2 BEGIN\n");
2774         /*tsleep(&dorestart, 0, "h2STG2", hz*20);*/
2775 #endif
2776
2777         /*
2778          * We have to flush the PFS root last, even if it does not appear to
2779          * be dirty, because all the inodes in the PFS are indexed under it.
2780          * The normal flushing of iroot above would only occur if directory
2781          * entries under the root were changed.
2782          *
2783          * Specifying VOLHDR will cause an additionl flush of hmp->spmp
2784          * for the media making up the cluster.
2785          */
2786         if ((ip = pmp->iroot) != NULL) {
2787                 hammer2_inode_ref(ip);
2788                 hammer2_mtx_ex(&ip->lock);
2789                 hammer2_inode_chain_sync(ip);
2790                 hammer2_inode_chain_flush(ip, HAMMER2_XOP_INODE_STOP |
2791                                               HAMMER2_XOP_FSSYNC |
2792                                               HAMMER2_XOP_VOLHDR);
2793                 hammer2_inode_unlock(ip);       /* unlock+drop */
2794         }
2795 #ifdef HAMMER2_DEBUG_SYNC
2796         kprintf("FILESYSTEM SYNC STAGE 2 DONE\n");
2797 #endif
2798
2799         /*
2800          * device bioq sync
2801          */
2802         hammer2_bioq_sync(pmp);
2803
2804 #if 0
2805         /*
2806          * Generally speaking we now want to flush the media topology from
2807          * the iroot through to the inodes.  The flush stops at any inode
2808          * boundary, which allows the frontend to continue running concurrent
2809          * modifying operations on inodes (including kernel flushes of
2810          * buffers) without interfering with the main sync.
2811          *
2812          * Use the XOP interface to concurrently flush all nodes to
2813          * synchronize the PFSROOT subtopology to the media.  A standard
2814          * end-of-scan ENOENT error indicates cluster sufficiency.
2815          *
2816          * Note that this flush will not be visible on crash recovery until
2817          * we flush the super-root topology in the next loop.
2818          *
2819          * XXX For now wait for all flushes to complete.
2820          */
2821         if (mp && (ip = pmp->iroot) != NULL) {
2822                 /*
2823                  * If unmounting try to flush everything including any
2824                  * sub-trees under inodes, just in case there is dangling
2825                  * modified data, as a safety.  Otherwise just flush up to
2826                  * the inodes in this stage.
2827                  */
2828                 kprintf("MP & IROOT\n");
2829 #ifdef HAMMER2_DEBUG_SYNC
2830                 kprintf("FILESYSTEM SYNC STAGE 3 IROOT BEGIN\n");
2831 #endif
2832                 if (mp->mnt_kern_flag & MNTK_UNMOUNT) {
2833                         xop = hammer2_xop_alloc(ip, HAMMER2_XOP_MODIFYING |
2834                                                     HAMMER2_XOP_VOLHDR |
2835                                                     HAMMER2_XOP_FSSYNC |
2836                                                     HAMMER2_XOP_INODE_STOP);
2837                 } else {
2838                         xop = hammer2_xop_alloc(ip, HAMMER2_XOP_MODIFYING |
2839                                                     HAMMER2_XOP_INODE_STOP |
2840                                                     HAMMER2_XOP_VOLHDR |
2841                                                     HAMMER2_XOP_FSSYNC |
2842                                                     HAMMER2_XOP_INODE_STOP);
2843                 }
2844                 hammer2_xop_start(&xop->head, &hammer2_inode_flush_desc);
2845                 error = hammer2_xop_collect(&xop->head,
2846                                             HAMMER2_XOP_COLLECT_WAITALL);
2847                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
2848 #ifdef HAMMER2_DEBUG_SYNC
2849                 kprintf("FILESYSTEM SYNC STAGE 3 IROOT END\n");
2850 #endif
2851                 if (error == HAMMER2_ERROR_ENOENT)
2852                         error = 0;
2853                 else
2854                         error = hammer2_error_to_errno(error);
2855         } else {
2856                 error = 0;
2857         }
2858 #endif
2859         error = 0;      /* XXX */
2860         hammer2_trans_done(pmp, HAMMER2_TRANS_ISFLUSH);
2861
2862         return (error);
2863 }
2864
2865 static
2866 int
2867 hammer2_vfs_vptofh(struct vnode *vp, struct fid *fhp)
2868 {
2869         hammer2_inode_t *ip;
2870
2871         KKASSERT(MAXFIDSZ >= 16);
2872         ip = VTOI(vp);
2873         fhp->fid_len = offsetof(struct fid, fid_data[16]);
2874         fhp->fid_ext = 0;
2875         ((hammer2_tid_t *)fhp->fid_data)[0] = ip->meta.inum;
2876         ((hammer2_tid_t *)fhp->fid_data)[1] = 0;
2877
2878         return 0;
2879 }
2880
2881 static
2882 int
2883 hammer2_vfs_fhtovp(struct mount *mp, struct vnode *rootvp,
2884                struct fid *fhp, struct vnode **vpp)
2885 {
2886         hammer2_pfs_t *pmp;
2887         hammer2_tid_t inum;
2888         int error;
2889
2890         pmp = MPTOPMP(mp);
2891         inum = ((hammer2_tid_t *)fhp->fid_data)[0] & HAMMER2_DIRHASH_USERMSK;
2892         if (vpp) {
2893                 if (inum == 1)
2894                         error = hammer2_vfs_root(mp, vpp);
2895                 else
2896                         error = hammer2_vfs_vget(mp, NULL, inum, vpp);
2897         } else {
2898                 error = 0;
2899         }
2900         if (error)
2901                 kprintf("fhtovp: %016jx -> %p, %d\n", inum, *vpp, error);
2902         return error;
2903 }
2904
2905 static
2906 int
2907 hammer2_vfs_checkexp(struct mount *mp, struct sockaddr *nam,
2908                  int *exflagsp, struct ucred **credanonp)
2909 {
2910         hammer2_pfs_t *pmp;
2911         struct netcred *np;
2912         int error;
2913
2914         pmp = MPTOPMP(mp);
2915         np = vfs_export_lookup(mp, &pmp->export, nam);
2916         if (np) {
2917                 *exflagsp = np->netc_exflags;
2918                 *credanonp = &np->netc_anon;
2919                 error = 0;
2920         } else {
2921                 error = EACCES;
2922         }
2923         return error;
2924 }
2925
2926 /*
2927  * This handles hysteresis on regular file flushes.  Because the BIOs are
2928  * routed to a thread it is possible for an excessive number to build up
2929  * and cause long front-end stalls long before the runningbuffspace limit
2930  * is hit, so we implement hammer2_flush_pipe to control the
2931  * hysteresis.
2932  *
2933  * This is a particular problem when compression is used.
2934  */
2935 void
2936 hammer2_lwinprog_ref(hammer2_pfs_t *pmp)
2937 {
2938         atomic_add_int(&pmp->count_lwinprog, 1);
2939 }
2940
2941 void
2942 hammer2_lwinprog_drop(hammer2_pfs_t *pmp)
2943 {
2944         int lwinprog;
2945
2946         lwinprog = atomic_fetchadd_int(&pmp->count_lwinprog, -1);
2947         if ((lwinprog & HAMMER2_LWINPROG_WAITING) &&
2948             (lwinprog & HAMMER2_LWINPROG_MASK) <= hammer2_flush_pipe * 2 / 3) {
2949                 atomic_clear_int(&pmp->count_lwinprog,
2950                                  HAMMER2_LWINPROG_WAITING);
2951                 wakeup(&pmp->count_lwinprog);
2952         }
2953         if ((lwinprog & HAMMER2_LWINPROG_WAITING0) &&
2954             (lwinprog & HAMMER2_LWINPROG_MASK) <= 0) {
2955                 atomic_clear_int(&pmp->count_lwinprog,
2956                                  HAMMER2_LWINPROG_WAITING0);
2957                 wakeup(&pmp->count_lwinprog);
2958         }
2959 }
2960
2961 void
2962 hammer2_lwinprog_wait(hammer2_pfs_t *pmp, int flush_pipe)
2963 {
2964         int lwinprog;
2965         int lwflag = (flush_pipe) ? HAMMER2_LWINPROG_WAITING :
2966                                     HAMMER2_LWINPROG_WAITING0;
2967
2968         for (;;) {
2969                 lwinprog = pmp->count_lwinprog;
2970                 cpu_ccfence();
2971                 if ((lwinprog & HAMMER2_LWINPROG_MASK) <= flush_pipe)
2972                         break;
2973                 tsleep_interlock(&pmp->count_lwinprog, 0);
2974                 atomic_set_int(&pmp->count_lwinprog, lwflag);
2975                 lwinprog = pmp->count_lwinprog;
2976                 if ((lwinprog & HAMMER2_LWINPROG_MASK) <= flush_pipe)
2977                         break;
2978                 tsleep(&pmp->count_lwinprog, PINTERLOCKED, "h2wpipe", hz);
2979         }
2980 }
2981
2982 /*
2983  * It is possible for an excessive number of dirty chains or dirty inodes
2984  * to build up.  When this occurs we start an asynchronous filesystem sync.
2985  * If the level continues to build up, we stall, waiting for it to drop,
2986  * with some hysteresis.
2987  *
2988  * This relies on the kernel calling hammer2_vfs_modifying() prior to
2989  * obtaining any vnode locks before making a modifying VOP call.
2990  */
2991 static int
2992 hammer2_vfs_modifying(struct mount *mp)
2993 {
2994         if (mp->mnt_flag & MNT_RDONLY)
2995                 return EROFS;
2996         hammer2_pfs_memory_wait(MPTOPMP(mp));
2997
2998         return 0;
2999 }
3000
3001 /*
3002  * Initiate an asynchronous filesystem sync and, with hysteresis,
3003  * stall if the internal data structure count becomes too bloated.
3004  */
3005 void
3006 hammer2_pfs_memory_wait(hammer2_pfs_t *pmp)
3007 {
3008         uint32_t waiting;
3009         int pcatch;
3010         int error;
3011
3012         if (pmp == NULL || pmp->mp == NULL)
3013                 return;
3014
3015         for (;;) {
3016                 waiting = pmp->inmem_dirty_chains & HAMMER2_DIRTYCHAIN_MASK;
3017                 cpu_ccfence();
3018
3019                 /*
3020                  * Start the syncer running at 1/2 the limit
3021                  */
3022                 if (waiting > hammer2_limit_dirty_chains / 2 ||
3023                     pmp->sideq_count > hammer2_limit_dirty_inodes / 2) {
3024                         trigger_syncer(pmp->mp);
3025                 }
3026
3027                 /*
3028                  * Stall at the limit waiting for the counts to drop.
3029                  * This code will typically be woken up once the count
3030                  * drops below 3/4 the limit, or in one second.
3031                  */
3032                 if (waiting < hammer2_limit_dirty_chains &&
3033                     pmp->sideq_count < hammer2_limit_dirty_inodes) {
3034                         break;
3035                 }
3036
3037                 pcatch = curthread->td_proc ? PCATCH : 0;
3038
3039                 tsleep_interlock(&pmp->inmem_dirty_chains, pcatch);
3040                 atomic_set_int(&pmp->inmem_dirty_chains,
3041                                HAMMER2_DIRTYCHAIN_WAITING);
3042                 if (waiting < hammer2_limit_dirty_chains &&
3043                     pmp->sideq_count < hammer2_limit_dirty_inodes) {
3044                         break;
3045                 }
3046                 trigger_syncer(pmp->mp);
3047                 error = tsleep(&pmp->inmem_dirty_chains, PINTERLOCKED | pcatch,
3048                                "h2memw", hz);
3049                 if (error == ERESTART)
3050                         break;
3051         }
3052 }
3053
3054 /*
3055  * Wake up any stalled frontend ops waiting, with hysteresis, using
3056  * 2/3 of the limit.
3057  */
3058 void
3059 hammer2_pfs_memory_wakeup(hammer2_pfs_t *pmp, int count)
3060 {
3061         uint32_t waiting;
3062
3063         if (pmp) {
3064                 waiting = atomic_fetchadd_int(&pmp->inmem_dirty_chains, count);
3065                 /* don't need --waiting to test flag */
3066
3067                 if ((waiting & HAMMER2_DIRTYCHAIN_WAITING) &&
3068                     (pmp->inmem_dirty_chains & HAMMER2_DIRTYCHAIN_MASK) <=
3069                     hammer2_limit_dirty_chains * 2 / 3 &&
3070                     pmp->sideq_count <= hammer2_limit_dirty_inodes * 2 / 3) {
3071                         atomic_clear_int(&pmp->inmem_dirty_chains,
3072                                          HAMMER2_DIRTYCHAIN_WAITING);
3073                         wakeup(&pmp->inmem_dirty_chains);
3074                 }
3075         }
3076 }
3077
3078 void
3079 hammer2_pfs_memory_inc(hammer2_pfs_t *pmp)
3080 {
3081         if (pmp) {
3082                 atomic_add_int(&pmp->inmem_dirty_chains, 1);
3083         }
3084 }
3085
3086 /*
3087  * Volume header data locks
3088  */
3089 void
3090 hammer2_voldata_lock(hammer2_dev_t *hmp)
3091 {
3092         lockmgr(&hmp->vollk, LK_EXCLUSIVE);
3093 }
3094
3095 void
3096 hammer2_voldata_unlock(hammer2_dev_t *hmp)
3097 {
3098         lockmgr(&hmp->vollk, LK_RELEASE);
3099 }
3100
3101 void
3102 hammer2_voldata_modify(hammer2_dev_t *hmp)
3103 {
3104         if ((hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED) == 0) {
3105                 atomic_add_long(&hammer2_count_modified_chains, 1);
3106                 atomic_set_int(&hmp->vchain.flags, HAMMER2_CHAIN_MODIFIED);
3107                 hammer2_pfs_memory_inc(hmp->vchain.pmp);
3108         }
3109 }
3110
3111 /*
3112  * Returns 0 if the filesystem has tons of free space
3113  * Returns 1 if the filesystem has less than 10% remaining
3114  * Returns 2 if the filesystem has less than 2%/5% (user/root) remaining.
3115  */
3116 int
3117 hammer2_vfs_enospace(hammer2_inode_t *ip, off_t bytes, struct ucred *cred)
3118 {
3119         hammer2_pfs_t *pmp;
3120         hammer2_dev_t *hmp;
3121         hammer2_off_t free_reserved;
3122         hammer2_off_t free_nominal;
3123         int i;
3124
3125         pmp = ip->pmp;
3126
3127         if (pmp->free_ticks == 0 || pmp->free_ticks != ticks) {
3128                 free_reserved = HAMMER2_SEGSIZE;
3129                 free_nominal = 0x7FFFFFFFFFFFFFFFLLU;
3130                 for (i = 0; i < pmp->iroot->cluster.nchains; ++i) {
3131                         hmp = pmp->pfs_hmps[i];
3132                         if (hmp == NULL)
3133                                 continue;
3134                         if (pmp->pfs_types[i] != HAMMER2_PFSTYPE_MASTER &&
3135                             pmp->pfs_types[i] != HAMMER2_PFSTYPE_SOFT_MASTER)
3136                                 continue;
3137
3138                         if (free_nominal > hmp->voldata.allocator_free)
3139                                 free_nominal = hmp->voldata.allocator_free;
3140                         if (free_reserved < hmp->free_reserved)
3141                                 free_reserved = hmp->free_reserved;
3142                 }
3143
3144                 /*
3145                  * SMP races ok
3146                  */
3147                 pmp->free_reserved = free_reserved;
3148                 pmp->free_nominal = free_nominal;
3149                 pmp->free_ticks = ticks;
3150         } else {
3151                 free_reserved = pmp->free_reserved;
3152                 free_nominal = pmp->free_nominal;
3153         }
3154         if (cred && cred->cr_uid != 0) {
3155                 if ((int64_t)(free_nominal - bytes) <
3156                     (int64_t)free_reserved) {
3157                         return 2;
3158                 }
3159         } else {
3160                 if ((int64_t)(free_nominal - bytes) <
3161                     (int64_t)free_reserved / 2) {
3162                         return 2;
3163                 }
3164         }
3165         if ((int64_t)(free_nominal - bytes) < (int64_t)free_reserved * 2)
3166                 return 1;
3167         return 0;
3168 }
3169
3170 /*
3171  * Debugging
3172  */
3173 void
3174 hammer2_dump_chain(hammer2_chain_t *chain, int tab, int bi, int *countp,
3175                    char pfx, u_int flags)
3176 {
3177         hammer2_chain_t *scan;
3178         hammer2_chain_t *parent;
3179
3180         --*countp;
3181         if (*countp == 0) {
3182                 kprintf("%*.*s...\n", tab, tab, "");
3183                 return;
3184         }
3185         if (*countp < 0)
3186                 return;
3187         kprintf("%*.*s%c-chain %p %s.%-3d %016jx %016jx/%-2d mir=%016jx\n",
3188                 tab, tab, "", pfx, chain,
3189                 hammer2_bref_type_str(chain->bref.type), bi,
3190                 chain->bref.data_off, chain->bref.key, chain->bref.keybits,
3191                 chain->bref.mirror_tid);
3192
3193         kprintf("%*.*s      [%08x] (%s) refs=%d",
3194                 tab, tab, "",
3195                 chain->flags,
3196                 ((chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
3197                 chain->data) ?  (char *)chain->data->ipdata.filename : "?"),
3198                 chain->refs);
3199
3200         parent = chain->parent;
3201         if (parent)
3202                 kprintf("\n%*.*s      p=%p [pflags %08x prefs %d]",
3203                         tab, tab, "",
3204                         parent, parent->flags, parent->refs);
3205         if (RB_EMPTY(&chain->core.rbtree)) {
3206                 kprintf("\n");
3207         } else {
3208                 int bi = 0;
3209                 kprintf(" {\n");
3210                 RB_FOREACH(scan, hammer2_chain_tree, &chain->core.rbtree) {
3211                         if ((scan->flags & flags) || flags == (u_int)-1) {
3212                                 hammer2_dump_chain(scan, tab + 4, bi, countp,
3213                                                    'a', flags);
3214                         }
3215                         bi++;
3216                 }
3217                 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE && chain->data)
3218                         kprintf("%*.*s}(%s)\n", tab, tab, "",
3219                                 chain->data->ipdata.filename);
3220                 else
3221                         kprintf("%*.*s}\n", tab, tab, "");
3222         }
3223 }