sys/vfs/hammer/hammer_vnops.c

   1 /*
   2  * Copyright (c) 2007-2008 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/vfs/hammer/hammer_vnops.c,v 1.102 2008/10/16 17:24:16 dillon Exp $
  35  */
  36
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/kernel.h>
  40 #include <sys/fcntl.h>
  41 #include <sys/namecache.h>
  42 #include <sys/vnode.h>
  43 #include <sys/lockf.h>
  44 #include <sys/event.h>
  45 #include <sys/stat.h>
  46 #include <sys/dirent.h>
  47 #include <sys/file.h>
  48 #include <vm/vm_extern.h>
  49 #include <vfs/fifofs/fifo.h>
  50
  51 #include <sys/mplock2.h>
  52
  53 #include "hammer.h"
  54
  55 /*
  56  * USERFS VNOPS
  57  */
  58 /*static int hammer_vop_vnoperate(struct vop_generic_args *);*/
  59 static int hammer_vop_fsync(struct vop_fsync_args *);
  60 static int hammer_vop_read(struct vop_read_args *);
  61 static int hammer_vop_write(struct vop_write_args *);
  62 static int hammer_vop_access(struct vop_access_args *);
  63 static int hammer_vop_advlock(struct vop_advlock_args *);
  64 static int hammer_vop_close(struct vop_close_args *);
  65 static int hammer_vop_ncreate(struct vop_ncreate_args *);
  66 static int hammer_vop_getattr(struct vop_getattr_args *);
  67 static int hammer_vop_nresolve(struct vop_nresolve_args *);
  68 static int hammer_vop_nlookupdotdot(struct vop_nlookupdotdot_args *);
  69 static int hammer_vop_nlink(struct vop_nlink_args *);
  70 static int hammer_vop_nmkdir(struct vop_nmkdir_args *);
  71 static int hammer_vop_nmknod(struct vop_nmknod_args *);
  72 static int hammer_vop_open(struct vop_open_args *);
  73 static int hammer_vop_print(struct vop_print_args *);
  74 static int hammer_vop_readdir(struct vop_readdir_args *);
  75 static int hammer_vop_readlink(struct vop_readlink_args *);
  76 static int hammer_vop_nremove(struct vop_nremove_args *);
  77 static int hammer_vop_nrename(struct vop_nrename_args *);
  78 static int hammer_vop_nrmdir(struct vop_nrmdir_args *);
  79 static int hammer_vop_markatime(struct vop_markatime_args *);
  80 static int hammer_vop_setattr(struct vop_setattr_args *);
  81 static int hammer_vop_strategy(struct vop_strategy_args *);
  82 static int hammer_vop_bmap(struct vop_bmap_args *ap);
  83 static int hammer_vop_nsymlink(struct vop_nsymlink_args *);
  84 static int hammer_vop_nwhiteout(struct vop_nwhiteout_args *);
  85 static int hammer_vop_ioctl(struct vop_ioctl_args *);
  86 static int hammer_vop_mountctl(struct vop_mountctl_args *);
  87 static int hammer_vop_kqfilter (struct vop_kqfilter_args *);
  88
  89 static int hammer_vop_fifoclose (struct vop_close_args *);
  90 static int hammer_vop_fiforead (struct vop_read_args *);
  91 static int hammer_vop_fifowrite (struct vop_write_args *);
  92 static int hammer_vop_fifokqfilter (struct vop_kqfilter_args *);
  93
  94 struct vop_ops hammer_vnode_vops = {
  95         .vop_default =          vop_defaultop,
  96         .vop_fsync =            hammer_vop_fsync,
  97         .vop_getpages =         vop_stdgetpages,
  98         .vop_putpages =         vop_stdputpages,
  99         .vop_read =             hammer_vop_read,
 100         .vop_write =            hammer_vop_write,
 101         .vop_access =           hammer_vop_access,
 102         .vop_advlock =          hammer_vop_advlock,
 103         .vop_close =            hammer_vop_close,
 104         .vop_ncreate =          hammer_vop_ncreate,
 105         .vop_getattr =          hammer_vop_getattr,
 106         .vop_inactive =         hammer_vop_inactive,
 107         .vop_reclaim =          hammer_vop_reclaim,
 108         .vop_nresolve =         hammer_vop_nresolve,
 109         .vop_nlookupdotdot =    hammer_vop_nlookupdotdot,
 110         .vop_nlink =            hammer_vop_nlink,
 111         .vop_nmkdir =           hammer_vop_nmkdir,
 112         .vop_nmknod =           hammer_vop_nmknod,
 113         .vop_open =             hammer_vop_open,
 114         .vop_pathconf =         vop_stdpathconf,
 115         .vop_print =            hammer_vop_print,
 116         .vop_readdir =          hammer_vop_readdir,
 117         .vop_readlink =         hammer_vop_readlink,
 118         .vop_nremove =          hammer_vop_nremove,
 119         .vop_nrename =          hammer_vop_nrename,
 120         .vop_nrmdir =           hammer_vop_nrmdir,
 121         .vop_markatime =        hammer_vop_markatime,
 122         .vop_setattr =          hammer_vop_setattr,
 123         .vop_bmap =             hammer_vop_bmap,
 124         .vop_strategy =         hammer_vop_strategy,
 125         .vop_nsymlink =         hammer_vop_nsymlink,
 126         .vop_nwhiteout =        hammer_vop_nwhiteout,
 127         .vop_ioctl =            hammer_vop_ioctl,
 128         .vop_mountctl =         hammer_vop_mountctl,
 129         .vop_kqfilter =         hammer_vop_kqfilter
 130 };
 131
 132 struct vop_ops hammer_spec_vops = {
 133         .vop_default =          vop_defaultop,
 134         .vop_fsync =            hammer_vop_fsync,
 135         .vop_read =             vop_stdnoread,
 136         .vop_write =            vop_stdnowrite,
 137         .vop_access =           hammer_vop_access,
 138         .vop_close =            hammer_vop_close,
 139         .vop_markatime =        hammer_vop_markatime,
 140         .vop_getattr =          hammer_vop_getattr,
 141         .vop_inactive =         hammer_vop_inactive,
 142         .vop_reclaim =          hammer_vop_reclaim,
 143         .vop_setattr =          hammer_vop_setattr
 144 };
 145
 146 struct vop_ops hammer_fifo_vops = {
 147         .vop_default =          fifo_vnoperate,
 148         .vop_fsync =            hammer_vop_fsync,
 149         .vop_read =             hammer_vop_fiforead,
 150         .vop_write =            hammer_vop_fifowrite,
 151         .vop_access =           hammer_vop_access,
 152         .vop_close =            hammer_vop_fifoclose,
 153         .vop_markatime =        hammer_vop_markatime,
 154         .vop_getattr =          hammer_vop_getattr,
 155         .vop_inactive =         hammer_vop_inactive,
 156         .vop_reclaim =          hammer_vop_reclaim,
 157         .vop_setattr =          hammer_vop_setattr,
 158         .vop_kqfilter =         hammer_vop_fifokqfilter
 159 };
 160
 161 static __inline
 162 void
 163 hammer_knote(struct vnode *vp, int flags)
 164 {
 165         if (flags)
 166                 KNOTE(&vp->v_pollinfo.vpi_selinfo.si_note, flags);
 167 }
 168
 169 #ifdef DEBUG_TRUNCATE
 170 struct hammer_inode *HammerTruncIp;
 171 #endif
 172
 173 static int hammer_dounlink(hammer_transaction_t trans, struct nchandle *nch,
 174                            struct vnode *dvp, struct ucred *cred,
 175                            int flags, int isdir);
 176 static int hammer_vop_strategy_read(struct vop_strategy_args *ap);
 177 static int hammer_vop_strategy_write(struct vop_strategy_args *ap);
 178
 179 #if 0
 180 static
 181 int
 182 hammer_vop_vnoperate(struct vop_generic_args *)
 183 {
 184         return (VOCALL(&hammer_vnode_vops, ap));
 185 }
 186 #endif
 187
 188 /*
 189  * hammer_vop_fsync { vp, waitfor }
 190  *
 191  * fsync() an inode to disk and wait for it to be completely committed
 192  * such that the information would not be undone if a crash occured after
 193  * return.
 194  *
 195  * NOTE: HAMMER's fsync()'s are going to remain expensive until we implement
 196  *       a REDO log.  A sysctl is provided to relax HAMMER's fsync()
 197  *       operation.
 198  *
 199  *       Ultimately the combination of a REDO log and use of fast storage
 200  *       to front-end cluster caches will make fsync fast, but it aint
 201  *       here yet.  And, in anycase, we need real transactional
 202  *       all-or-nothing features which are not restricted to a single file.
 203  */
 204 static
 205 int
 206 hammer_vop_fsync(struct vop_fsync_args *ap)
 207 {
 208         hammer_inode_t ip = VTOI(ap->a_vp);
 209         hammer_mount_t hmp = ip->hmp;
 210         int waitfor = ap->a_waitfor;
 211         int mode;
 212
 213         /*
 214          * Fsync rule relaxation (default is either full synchronous flush
 215          * or REDO semantics with synchronous flush).
 216          */
 217         if (ap->a_flags & VOP_FSYNC_SYSCALL) {
 218                 switch(hammer_fsync_mode) {
 219                 case 0:
 220 mode0:
 221                         /* no REDO, full synchronous flush */
 222                         goto skip;
 223                 case 1:
 224 mode1:
 225                         /* no REDO, full asynchronous flush */
 226                         if (waitfor == MNT_WAIT)
 227                                 waitfor = MNT_NOWAIT;
 228                         goto skip;
 229                 case 2:
 230                         /* REDO semantics, synchronous flush */
 231                         if (hmp->version < HAMMER_VOL_VERSION_FOUR)
 232                                 goto mode0;
 233                         mode = HAMMER_FLUSH_UNDOS_AUTO;
 234                         break;
 235                 case 3:
 236                         /* REDO semantics, relaxed asynchronous flush */
 237                         if (hmp->version < HAMMER_VOL_VERSION_FOUR)
 238                                 goto mode1;
 239                         mode = HAMMER_FLUSH_UNDOS_RELAXED;
 240                         if (waitfor == MNT_WAIT)
 241                                 waitfor = MNT_NOWAIT;
 242                         break;
 243                 case 4:
 244                         /* ignore the fsync() system call */
 245                         return(0);
 246                 default:
 247                         /* we have to do something */
 248                         mode = HAMMER_FLUSH_UNDOS_RELAXED;
 249                         if (waitfor == MNT_WAIT)
 250                                 waitfor = MNT_NOWAIT;
 251                         break;
 252                 }
 253
 254                 /*
 255                  * Fast fsync only needs to flush the UNDO/REDO fifo if
 256                  * HAMMER_INODE_REDO is non-zero and the only modifications
 257                  * made to the file are write or write-extends.
 258                  */
 259                 if ((ip->flags & HAMMER_INODE_REDO) &&
 260                     (ip->flags & HAMMER_INODE_MODMASK_NOREDO) == 0
 261                 ) {
 262                         ++hammer_count_fsyncs;
 263                         hammer_flusher_flush_undos(hmp, mode);
 264                         ip->redo_count = 0;
 265                         return(0);
 266                 }
 267
 268                 /*
 269                  * REDO is enabled by fsync(), the idea being we really only
 270                  * want to lay down REDO records when programs are using
 271                  * fsync() heavily.  The first fsync() on the file starts
 272                  * the gravy train going and later fsync()s keep it hot by
 273                  * resetting the redo_count.
 274                  *
 275                  * We weren't running REDOs before now so we have to fall
 276                  * through and do a full fsync of what we have.
 277                  */
 278                 if (hmp->version >= HAMMER_VOL_VERSION_FOUR) {
 279                         ip->flags |= HAMMER_INODE_REDO;
 280                         ip->redo_count = 0;
 281                 }
 282         }
 283 skip:
 284
 285         /*
 286          * Do a full flush sequence.
 287          */
 288         ++hammer_count_fsyncs;
 289         vfsync(ap->a_vp, waitfor, 1, NULL, NULL);
 290         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 291         if (waitfor == MNT_WAIT) {
 292                 vn_unlock(ap->a_vp);
 293                 hammer_wait_inode(ip);
 294                 vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
 295         }
 296         return (ip->error);
 297 }
 298
 299 /*
 300  * hammer_vop_read { vp, uio, ioflag, cred }
 301  *
 302  * MPALMOSTSAFE
 303  */
 304 static
 305 int
 306 hammer_vop_read(struct vop_read_args *ap)
 307 {
 308         struct hammer_transaction trans;
 309         hammer_inode_t ip;
 310         off_t offset;
 311         struct buf *bp;
 312         struct uio *uio;
 313         int error;
 314         int n;
 315         int seqcount;
 316         int ioseqcount;
 317         int blksize;
 318         int got_mplock;
 319         int bigread;
 320
 321         if (ap->a_vp->v_type != VREG)
 322                 return (EINVAL);
 323         ip = VTOI(ap->a_vp);
 324         error = 0;
 325         uio = ap->a_uio;
 326
 327         /*
 328          * Allow the UIO's size to override the sequential heuristic.
 329          */
 330         blksize = hammer_blocksize(uio->uio_offset);
 331         seqcount = (uio->uio_resid + (blksize - 1)) / blksize;
 332         ioseqcount = ap->a_ioflag >> 16;
 333         if (seqcount < ioseqcount)
 334                 seqcount = ioseqcount;
 335
 336         /*
 337          * Temporary hack until more of HAMMER can be made MPSAFE.
 338          */
 339 #ifdef SMP
 340         if (curthread->td_mpcount) {
 341                 got_mplock = -1;
 342                 hammer_start_transaction(&trans, ip->hmp);
 343         } else {
 344                 got_mplock = 0;
 345         }
 346 #else
 347         hammer_start_transaction(&trans, ip->hmp);
 348         got_mplock = -1;
 349 #endif
 350
 351         /*
 352          * If reading or writing a huge amount of data we have to break
 353          * atomicy and allow the operation to be interrupted by a signal
 354          * or it can DOS the machine.
 355          */
 356         bigread = (uio->uio_resid > 100 * 1024 * 1024);
 357
 358         /*
 359          * Access the data typically in HAMMER_BUFSIZE blocks via the
 360          * buffer cache, but HAMMER may use a variable block size based
 361          * on the offset.
 362          *
 363          * XXX Temporary hack, delay the start transaction while we remain
 364          *     MPSAFE.  NOTE: ino_data.size cannot change while vnode is
 365          *     locked-shared.
 366          */
 367         while (uio->uio_resid > 0 && uio->uio_offset < ip->ino_data.size) {
 368                 int64_t base_offset;
 369                 int64_t file_limit;
 370
 371                 blksize = hammer_blocksize(uio->uio_offset);
 372                 offset = (int)uio->uio_offset & (blksize - 1);
 373                 base_offset = uio->uio_offset - offset;
 374
 375                 if (bigread && (error = hammer_signal_check(ip->hmp)) != 0)
 376                         break;
 377
 378                 /*
 379                  * MPSAFE
 380                  */
 381                 bp = getcacheblk(ap->a_vp, base_offset);
 382                 if (bp) {
 383                         error = 0;
 384                         goto skip;
 385                 }
 386
 387                 /*
 388                  * MPUNSAFE
 389                  */
 390                 if (got_mplock == 0) {
 391                         got_mplock = 1;
 392                         get_mplock();
 393                         hammer_start_transaction(&trans, ip->hmp);
 394                 }
 395
 396                 if (hammer_cluster_enable) {
 397                         /*
 398                          * Use file_limit to prevent cluster_read() from
 399                          * creating buffers of the wrong block size past
 400                          * the demarc.
 401                          */
 402                         file_limit = ip->ino_data.size;
 403                         if (base_offset < HAMMER_XDEMARC &&
 404                             file_limit > HAMMER_XDEMARC) {
 405                                 file_limit = HAMMER_XDEMARC;
 406                         }
 407                         error = cluster_read(ap->a_vp,
 408                                              file_limit, base_offset,
 409                                              blksize, MAXPHYS,
 410                                              seqcount, &bp);
 411                 } else {
 412                         error = bread(ap->a_vp, base_offset, blksize, &bp);
 413                 }
 414                 if (error) {
 415                         brelse(bp);
 416                         break;
 417                 }
 418 skip:
 419
 420                 /* bp->b_flags |= B_CLUSTEROK; temporarily disabled */
 421                 n = blksize - offset;
 422                 if (n > uio->uio_resid)
 423                         n = uio->uio_resid;
 424                 if (n > ip->ino_data.size - uio->uio_offset)
 425                         n = (int)(ip->ino_data.size - uio->uio_offset);
 426                 error = uiomove((char *)bp->b_data + offset, n, uio);
 427
 428                 /* data has a lower priority then meta-data */
 429                 bp->b_flags |= B_AGE;
 430                 bqrelse(bp);
 431                 if (error)
 432                         break;
 433                 hammer_stats_file_read += n;
 434         }
 435
 436         /*
 437          * XXX only update the atime if we had to get the MP lock.
 438          * XXX hack hack hack, fixme.
 439          */
 440         if (got_mplock) {
 441                 if ((ip->flags & HAMMER_INODE_RO) == 0 &&
 442                     (ip->hmp->mp->mnt_flag & MNT_NOATIME) == 0) {
 443                         ip->ino_data.atime = trans.time;
 444                         hammer_modify_inode(&trans, ip, HAMMER_INODE_ATIME);
 445                 }
 446                 hammer_done_transaction(&trans);
 447                 if (got_mplock > 0)
 448                         rel_mplock();
 449         }
 450         return (error);
 451 }
 452
 453 /*
 454  * hammer_vop_write { vp, uio, ioflag, cred }
 455  */
 456 static
 457 int
 458 hammer_vop_write(struct vop_write_args *ap)
 459 {
 460         struct hammer_transaction trans;
 461         struct hammer_inode *ip;
 462         hammer_mount_t hmp;
 463         struct uio *uio;
 464         int offset;
 465         off_t base_offset;
 466         struct buf *bp;
 467         int kflags;
 468         int error;
 469         int n;
 470         int flags;
 471         int seqcount;
 472         int bigwrite;
 473
 474         if (ap->a_vp->v_type != VREG)
 475                 return (EINVAL);
 476         ip = VTOI(ap->a_vp);
 477         hmp = ip->hmp;
 478         error = 0;
 479         kflags = 0;
 480         seqcount = ap->a_ioflag >> 16;
 481
 482         if (ip->flags & HAMMER_INODE_RO)
 483                 return (EROFS);
 484
 485         /*
 486          * Create a transaction to cover the operations we perform.
 487          */
 488         hammer_start_transaction(&trans, hmp);
 489         uio = ap->a_uio;
 490
 491         /*
 492          * Check append mode
 493          */
 494         if (ap->a_ioflag & IO_APPEND)
 495                 uio->uio_offset = ip->ino_data.size;
 496
 497         /*
 498          * Check for illegal write offsets.  Valid range is 0...2^63-1.
 499          *
 500          * NOTE: the base_off assignment is required to work around what
 501          * I consider to be a GCC-4 optimization bug.
 502          */
 503         if (uio->uio_offset < 0) {
 504                 hammer_done_transaction(&trans);
 505                 return (EFBIG);
 506         }
 507         base_offset = uio->uio_offset + uio->uio_resid; /* work around gcc-4 */
 508         if (uio->uio_resid > 0 && base_offset <= uio->uio_offset) {
 509                 hammer_done_transaction(&trans);
 510                 return (EFBIG);
 511         }
 512
 513         /*
 514          * If reading or writing a huge amount of data we have to break
 515          * atomicy and allow the operation to be interrupted by a signal
 516          * or it can DOS the machine.
 517          *
 518          * Preset redo_count so we stop generating REDOs earlier if the
 519          * limit is exceeded.
 520          */
 521         bigwrite = (uio->uio_resid > 100 * 1024 * 1024);
 522         if ((ip->flags & HAMMER_INODE_REDO) &&
 523             ip->redo_count < hammer_limit_redo) {
 524                 ip->redo_count += uio->uio_resid;
 525         }
 526
 527         /*
 528          * Access the data typically in HAMMER_BUFSIZE blocks via the
 529          * buffer cache, but HAMMER may use a variable block size based
 530          * on the offset.
 531          */
 532         while (uio->uio_resid > 0) {
 533                 int fixsize = 0;
 534                 int blksize;
 535                 int blkmask;
 536                 int trivial;
 537                 int endofblk;
 538                 off_t nsize;
 539
 540                 if ((error = hammer_checkspace(hmp, HAMMER_CHKSPC_WRITE)) != 0)
 541                         break;
 542                 if (bigwrite && (error = hammer_signal_check(hmp)) != 0)
 543                         break;
 544
 545                 blksize = hammer_blocksize(uio->uio_offset);
 546
 547                 /*
 548                  * Do not allow HAMMER to blow out the buffer cache.  Very
 549                  * large UIOs can lockout other processes due to bwillwrite()
 550                  * mechanics.
 551                  *
 552                  * The hammer inode is not locked during these operations.
 553                  * The vnode is locked which can interfere with the pageout
 554                  * daemon for non-UIO_NOCOPY writes but should not interfere
 555                  * with the buffer cache.  Even so, we cannot afford to
 556                  * allow the pageout daemon to build up too many dirty buffer
 557                  * cache buffers.
 558                  *
 559                  * Only call this if we aren't being recursively called from
 560                  * a virtual disk device (vn), else we may deadlock.
 561                  */
 562                 if ((ap->a_ioflag & IO_RECURSE) == 0)
 563                         bwillwrite(blksize);
 564
 565                 /*
 566                  * Control the number of pending records associated with
 567                  * this inode.  If too many have accumulated start a
 568                  * flush.  Try to maintain a pipeline with the flusher.
 569                  */
 570                 if (ip->rsv_recs >= hammer_limit_inode_recs) {
 571                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 572                 }
 573                 if (ip->rsv_recs >= hammer_limit_inode_recs * 2) {
 574                         while (ip->rsv_recs >= hammer_limit_inode_recs) {
 575                                 tsleep(&ip->rsv_recs, 0, "hmrwww", hz);
 576                         }
 577                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 578                 }
 579
 580 #if 0
 581                 /*
 582                  * Do not allow HAMMER to blow out system memory by
 583                  * accumulating too many records.   Records are so well
 584                  * decoupled from the buffer cache that it is possible
 585                  * for userland to push data out to the media via
 586                  * direct-write, but build up the records queued to the
 587                  * backend faster then the backend can flush them out.
 588                  * HAMMER has hit its write limit but the frontend has
 589                  * no pushback to slow it down.
 590                  */
 591                 if (hmp->rsv_recs > hammer_limit_recs / 2) {
 592                         /*
 593                          * Get the inode on the flush list
 594                          */
 595                         if (ip->rsv_recs >= 64)
 596                                 hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 597                         else if (ip->rsv_recs >= 16)
 598                                 hammer_flush_inode(ip, 0);
 599
 600                         /*
 601                          * Keep the flusher going if the system keeps
 602                          * queueing records.
 603                          */
 604                         delta = hmp->count_newrecords -
 605                                 hmp->last_newrecords;
 606                         if (delta < 0 || delta > hammer_limit_recs / 2) {
 607                                 hmp->last_newrecords = hmp->count_newrecords;
 608                                 hammer_sync_hmp(hmp, MNT_NOWAIT);
 609                         }
 610
 611                         /*
 612                          * If we have gotten behind start slowing
 613                          * down the writers.
 614                          */
 615                         delta = (hmp->rsv_recs - hammer_limit_recs) *
 616                                 hz / hammer_limit_recs;
 617                         if (delta > 0)
 618                                 tsleep(&trans, 0, "hmrslo", delta);
 619                 }
 620 #endif
 621
 622                 /*
 623                  * Calculate the blocksize at the current offset and figure
 624                  * out how much we can actually write.
 625                  */
 626                 blkmask = blksize - 1;
 627                 offset = (int)uio->uio_offset & blkmask;
 628                 base_offset = uio->uio_offset & ~(int64_t)blkmask;
 629                 n = blksize - offset;
 630                 if (n > uio->uio_resid) {
 631                         n = uio->uio_resid;
 632                         endofblk = 0;
 633                 } else {
 634                         endofblk = 1;
 635                 }
 636                 nsize = uio->uio_offset + n;
 637                 if (nsize > ip->ino_data.size) {
 638                         if (uio->uio_offset > ip->ino_data.size)
 639                                 trivial = 0;
 640                         else
 641                                 trivial = 1;
 642                         nvextendbuf(ap->a_vp,
 643                                     ip->ino_data.size,
 644                                     nsize,
 645                                     hammer_blocksize(ip->ino_data.size),
 646                                     hammer_blocksize(nsize),
 647                                     hammer_blockoff(ip->ino_data.size),
 648                                     hammer_blockoff(nsize),
 649                                     trivial);
 650                         fixsize = 1;
 651                         kflags |= NOTE_EXTEND;
 652                 }
 653
 654                 if (uio->uio_segflg == UIO_NOCOPY) {
 655                         /*
 656                          * Issuing a write with the same data backing the
 657                          * buffer.  Instantiate the buffer to collect the
 658                          * backing vm pages, then read-in any missing bits.
 659                          *
 660                          * This case is used by vop_stdputpages().
 661                          */
 662                         bp = getblk(ap->a_vp, base_offset,
 663                                     blksize, GETBLK_BHEAVY, 0);
 664                         if ((bp->b_flags & B_CACHE) == 0) {
 665                                 bqrelse(bp);
 666                                 error = bread(ap->a_vp, base_offset,
 667                                               blksize, &bp);
 668                         }
 669                 } else if (offset == 0 && uio->uio_resid >= blksize) {
 670                         /*
 671                          * Even though we are entirely overwriting the buffer
 672                          * we may still have to zero it out to avoid a
 673                          * mmap/write visibility issue.
 674                          */
 675                         bp = getblk(ap->a_vp, base_offset, blksize, GETBLK_BHEAVY, 0);
 676                         if ((bp->b_flags & B_CACHE) == 0)
 677                                 vfs_bio_clrbuf(bp);
 678                 } else if (base_offset >= ip->ino_data.size) {
 679                         /*
 680                          * If the base offset of the buffer is beyond the
 681                          * file EOF, we don't have to issue a read.
 682                          */
 683                         bp = getblk(ap->a_vp, base_offset,
 684                                     blksize, GETBLK_BHEAVY, 0);
 685                         vfs_bio_clrbuf(bp);
 686                 } else {
 687                         /*
 688                          * Partial overwrite, read in any missing bits then
 689                          * replace the portion being written.
 690                          */
 691                         error = bread(ap->a_vp, base_offset, blksize, &bp);
 692                         if (error == 0)
 693                                 bheavy(bp);
 694                 }
 695                 if (error == 0)
 696                         error = uiomove(bp->b_data + offset, n, uio);
 697
 698                 /*
 699                  * Generate REDO records if enabled and redo_count will not
 700                  * exceeded the limit.
 701                  *
 702                  * If redo_count exceeds the limit we stop generating records
 703                  * and clear HAMMER_INODE_REDO.  This will cause the next
 704                  * fsync() to do a full meta-data sync instead of just an
 705                  * UNDO/REDO fifo update.
 706                  *
 707                  * When clearing HAMMER_INODE_REDO any pre-existing REDOs
 708                  * will still be tracked.  The tracks will be terminated
 709                  * when the related meta-data (including possible data
 710                  * modifications which are not tracked via REDO) is
 711                  * flushed.
 712                  */
 713                 if ((ip->flags & HAMMER_INODE_REDO) && error == 0) {
 714                         if (ip->redo_count < hammer_limit_redo) {
 715                                 bp->b_flags |= B_VFSFLAG1;
 716                                 error = hammer_generate_redo(&trans, ip,
 717                                                      base_offset + offset,
 718                                                      HAMMER_REDO_WRITE,
 719                                                      bp->b_data + offset,
 720                                                      (size_t)n);
 721                         } else {
 722                                 ip->flags &= ~HAMMER_INODE_REDO;
 723                         }
 724                 }
 725
 726                 /*
 727                  * If we screwed up we have to undo any VM size changes we
 728                  * made.
 729                  */
 730                 if (error) {
 731                         brelse(bp);
 732                         if (fixsize) {
 733                                 nvtruncbuf(ap->a_vp, ip->ino_data.size,
 734                                           hammer_blocksize(ip->ino_data.size),
 735                                           hammer_blockoff(ip->ino_data.size));
 736                         }
 737                         break;
 738                 }
 739                 kflags |= NOTE_WRITE;
 740                 hammer_stats_file_write += n;
 741                 /* bp->b_flags |= B_CLUSTEROK; temporarily disabled */
 742                 if (ip->ino_data.size < uio->uio_offset) {
 743                         ip->ino_data.size = uio->uio_offset;
 744                         flags = HAMMER_INODE_SDIRTY;
 745                 } else {
 746                         flags = 0;
 747                 }
 748                 ip->ino_data.mtime = trans.time;
 749                 flags |= HAMMER_INODE_MTIME | HAMMER_INODE_BUFS;
 750                 hammer_modify_inode(&trans, ip, flags);
 751
 752                 /*
 753                  * Once we dirty the buffer any cached zone-X offset
 754                  * becomes invalid.  HAMMER NOTE: no-history mode cannot
 755                  * allow overwriting over the same data sector unless
 756                  * we provide UNDOs for the old data, which we don't.
 757                  */
 758                 bp->b_bio2.bio_offset = NOOFFSET;
 759
 760                 /*
 761                  * Final buffer disposition.
 762                  *
 763                  * Because meta-data updates are deferred, HAMMER is
 764                  * especially sensitive to excessive bdwrite()s because
 765                  * the I/O stream is not broken up by disk reads.  So the
 766                  * buffer cache simply cannot keep up.
 767                  *
 768                  * WARNING!  blksize is variable.  cluster_write() is
 769                  *           expected to not blow up if it encounters
 770                  *           buffers that do not match the passed blksize.
 771                  *
 772                  * NOTE!  Hammer shouldn't need to bawrite()/cluster_write().
 773                  *        The ip->rsv_recs check should burst-flush the data.
 774                  *        If we queue it immediately the buf could be left
 775                  *        locked on the device queue for a very long time.
 776                  *
 777                  * NOTE!  To avoid degenerate stalls due to mismatched block
 778                  *        sizes we only honor IO_DIRECT on the write which
 779                  *        abuts the end of the buffer.  However, we must
 780                  *        honor IO_SYNC in case someone is silly enough to
 781                  *        configure a HAMMER file as swap, or when HAMMER
 782                  *        is serving NFS (for commits).  Ick ick.
 783                  */
 784                 bp->b_flags |= B_AGE;
 785                 if (ap->a_ioflag & IO_SYNC) {
 786                         bwrite(bp);
 787                 } else if ((ap->a_ioflag & IO_DIRECT) && endofblk) {
 788                         bawrite(bp);
 789                 } else {
 790 #if 0
 791                 if (offset + n == blksize) {
 792                         if (hammer_cluster_enable == 0 ||
 793                             (ap->a_vp->v_mount->mnt_flag & MNT_NOCLUSTERW)) {
 794                                 bawrite(bp);
 795                         } else {
 796                                 cluster_write(bp, ip->ino_data.size,
 797                                               blksize, seqcount);
 798                         }
 799                 } else {
 800 #endif
 801                         bdwrite(bp);
 802                 }
 803         }
 804         hammer_done_transaction(&trans);
 805         hammer_knote(ap->a_vp, kflags);
 806         return (error);
 807 }
 808
 809 /*
 810  * hammer_vop_access { vp, mode, cred }
 811  */
 812 static
 813 int
 814 hammer_vop_access(struct vop_access_args *ap)
 815 {
 816         struct hammer_inode *ip = VTOI(ap->a_vp);
 817         uid_t uid;
 818         gid_t gid;
 819         int error;
 820
 821         ++hammer_stats_file_iopsr;
 822         uid = hammer_to_unix_xid(&ip->ino_data.uid);
 823         gid = hammer_to_unix_xid(&ip->ino_data.gid);
 824
 825         error = vop_helper_access(ap, uid, gid, ip->ino_data.mode,
 826                                   ip->ino_data.uflags);
 827         return (error);
 828 }
 829
 830 /*
 831  * hammer_vop_advlock { vp, id, op, fl, flags }
 832  */
 833 static
 834 int
 835 hammer_vop_advlock(struct vop_advlock_args *ap)
 836 {
 837         hammer_inode_t ip = VTOI(ap->a_vp);
 838
 839         return (lf_advlock(ap, &ip->advlock, ip->ino_data.size));
 840 }
 841
 842 /*
 843  * hammer_vop_close { vp, fflag }
 844  *
 845  * We can only sync-on-close for normal closes.
 846  */
 847 static
 848 int
 849 hammer_vop_close(struct vop_close_args *ap)
 850 {
 851 #if 0
 852         struct vnode *vp = ap->a_vp;
 853         hammer_inode_t ip = VTOI(vp);
 854         int waitfor;
 855         if (ip->flags & (HAMMER_INODE_CLOSESYNC|HAMMER_INODE_CLOSEASYNC)) {
 856                 if (vn_islocked(vp) == LK_EXCLUSIVE &&
 857                     (vp->v_flag & (VINACTIVE|VRECLAIMED)) == 0) {
 858                         if (ip->flags & HAMMER_INODE_CLOSESYNC)
 859                                 waitfor = MNT_WAIT;
 860                         else
 861                                 waitfor = MNT_NOWAIT;
 862                         ip->flags &= ~(HAMMER_INODE_CLOSESYNC |
 863                                        HAMMER_INODE_CLOSEASYNC);
 864                         VOP_FSYNC(vp, MNT_NOWAIT, waitfor);
 865                 }
 866         }
 867 #endif
 868         return (vop_stdclose(ap));
 869 }
 870
 871 /*
 872  * hammer_vop_ncreate { nch, dvp, vpp, cred, vap }
 873  *
 874  * The operating system has already ensured that the directory entry
 875  * does not exist and done all appropriate namespace locking.
 876  */
 877 static
 878 int
 879 hammer_vop_ncreate(struct vop_ncreate_args *ap)
 880 {
 881         struct hammer_transaction trans;
 882         struct hammer_inode *dip;
 883         struct hammer_inode *nip;
 884         struct nchandle *nch;
 885         int error;
 886
 887         nch = ap->a_nch;
 888         dip = VTOI(ap->a_dvp);
 889
 890         if (dip->flags & HAMMER_INODE_RO)
 891                 return (EROFS);
 892         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
 893                 return (error);
 894
 895         /*
 896          * Create a transaction to cover the operations we perform.
 897          */
 898         hammer_start_transaction(&trans, dip->hmp);
 899         ++hammer_stats_file_iopsw;
 900
 901         /*
 902          * Create a new filesystem object of the requested type.  The
 903          * returned inode will be referenced and shared-locked to prevent
 904          * it from being moved to the flusher.
 905          */
 906         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
 907                                     dip, nch->ncp->nc_name, nch->ncp->nc_nlen,
 908                                     NULL, &nip);
 909         if (error) {
 910                 hkprintf("hammer_create_inode error %d\n", error);
 911                 hammer_done_transaction(&trans);
 912                 *ap->a_vpp = NULL;
 913                 return (error);
 914         }
 915
 916         /*
 917          * Add the new filesystem object to the directory.  This will also
 918          * bump the inode's link count.
 919          */
 920         error = hammer_ip_add_directory(&trans, dip,
 921                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
 922                                         nip);
 923         if (error)
 924                 hkprintf("hammer_ip_add_directory error %d\n", error);
 925
 926         /*
 927          * Finish up.
 928          */
 929         if (error) {
 930                 hammer_rel_inode(nip, 0);
 931                 hammer_done_transaction(&trans);
 932                 *ap->a_vpp = NULL;
 933         } else {
 934                 error = hammer_get_vnode(nip, ap->a_vpp);
 935                 hammer_done_transaction(&trans);
 936                 hammer_rel_inode(nip, 0);
 937                 if (error == 0) {
 938                         cache_setunresolved(ap->a_nch);
 939                         cache_setvp(ap->a_nch, *ap->a_vpp);
 940                 }
 941                 hammer_knote(ap->a_dvp, NOTE_WRITE);
 942         }
 943         return (error);
 944 }
 945
 946 /*
 947  * hammer_vop_getattr { vp, vap }
 948  *
 949  * Retrieve an inode's attribute information.  When accessing inodes
 950  * historically we fake the atime field to ensure consistent results.
 951  * The atime field is stored in the B-Tree element and allowed to be
 952  * updated without cycling the element.
 953  *
 954  * MPSAFE
 955  */
 956 static
 957 int
 958 hammer_vop_getattr(struct vop_getattr_args *ap)
 959 {
 960         struct hammer_inode *ip = VTOI(ap->a_vp);
 961         struct vattr *vap = ap->a_vap;
 962
 963         /*
 964          * We want the fsid to be different when accessing a filesystem
 965          * with different as-of's so programs like diff don't think
 966          * the files are the same.
 967          *
 968          * We also want the fsid to be the same when comparing snapshots,
 969          * or when comparing mirrors (which might be backed by different
 970          * physical devices).  HAMMER fsids are based on the PFS's
 971          * shared_uuid field.
 972          *
 973          * XXX there is a chance of collision here.  The va_fsid reported
 974          * by stat is different from the more involved fsid used in the
 975          * mount structure.
 976          */
 977         ++hammer_stats_file_iopsr;
 978         hammer_lock_sh(&ip->lock);
 979         vap->va_fsid = ip->pfsm->fsid_udev ^ (u_int32_t)ip->obj_asof ^
 980                        (u_int32_t)(ip->obj_asof >> 32);
 981
 982         vap->va_fileid = ip->ino_leaf.base.obj_id;
 983         vap->va_mode = ip->ino_data.mode;
 984         vap->va_nlink = ip->ino_data.nlinks;
 985         vap->va_uid = hammer_to_unix_xid(&ip->ino_data.uid);
 986         vap->va_gid = hammer_to_unix_xid(&ip->ino_data.gid);
 987         vap->va_rmajor = 0;
 988         vap->va_rminor = 0;
 989         vap->va_size = ip->ino_data.size;
 990
 991         /*
 992          * Special case for @@PFS softlinks.  The actual size of the
 993          * expanded softlink is "@@0x%016llx:%05d" == 26 bytes.
 994          * or for MAX_TID is    "@@-1:%05d" == 10 bytes.
 995          */
 996         if (ip->ino_data.obj_type == HAMMER_OBJTYPE_SOFTLINK &&
 997             ip->ino_data.size == 10 &&
 998             ip->obj_asof == HAMMER_MAX_TID &&
 999             ip->obj_localization == 0 &&
1000             strncmp(ip->ino_data.ext.symlink, "@@PFS", 5) == 0) {
1001                     if (ip->pfsm->pfsd.mirror_flags & HAMMER_PFSD_SLAVE)
1002                             vap->va_size = 26;
1003                     else
1004                             vap->va_size = 10;
1005         }
1006
1007         /*
1008          * We must provide a consistent atime and mtime for snapshots
1009          * so people can do a 'tar cf - ... | md5' on them and get
1010          * consistent results.
1011          */
1012         if (ip->flags & HAMMER_INODE_RO) {
1013                 hammer_time_to_timespec(ip->ino_data.ctime, &vap->va_atime);
1014                 hammer_time_to_timespec(ip->ino_data.ctime, &vap->va_mtime);
1015         } else {
1016                 hammer_time_to_timespec(ip->ino_data.atime, &vap->va_atime);
1017                 hammer_time_to_timespec(ip->ino_data.mtime, &vap->va_mtime);
1018         }
1019         hammer_time_to_timespec(ip->ino_data.ctime, &vap->va_ctime);
1020         vap->va_flags = ip->ino_data.uflags;
1021         vap->va_gen = 1;        /* hammer inums are unique for all time */
1022         vap->va_blocksize = HAMMER_BUFSIZE;
1023         if (ip->ino_data.size >= HAMMER_XDEMARC) {
1024                 vap->va_bytes = (ip->ino_data.size + HAMMER_XBUFMASK64) &
1025                                 ~HAMMER_XBUFMASK64;
1026         } else if (ip->ino_data.size > HAMMER_BUFSIZE / 2) {
1027                 vap->va_bytes = (ip->ino_data.size + HAMMER_BUFMASK64) &
1028                                 ~HAMMER_BUFMASK64;
1029         } else {
1030                 vap->va_bytes = (ip->ino_data.size + 15) & ~15;
1031         }
1032
1033         vap->va_type = hammer_get_vnode_type(ip->ino_data.obj_type);
1034         vap->va_filerev = 0;    /* XXX */
1035         vap->va_uid_uuid = ip->ino_data.uid;
1036         vap->va_gid_uuid = ip->ino_data.gid;
1037         vap->va_fsid_uuid = ip->hmp->fsid;
1038         vap->va_vaflags = VA_UID_UUID_VALID | VA_GID_UUID_VALID |
1039                           VA_FSID_UUID_VALID;
1040
1041         switch (ip->ino_data.obj_type) {
1042         case HAMMER_OBJTYPE_CDEV:
1043         case HAMMER_OBJTYPE_BDEV:
1044                 vap->va_rmajor = ip->ino_data.rmajor;
1045                 vap->va_rminor = ip->ino_data.rminor;
1046                 break;
1047         default:
1048                 break;
1049         }
1050         hammer_unlock(&ip->lock);
1051         return(0);
1052 }
1053
1054 /*
1055  * hammer_vop_nresolve { nch, dvp, cred }
1056  *
1057  * Locate the requested directory entry.
1058  */
1059 static
1060 int
1061 hammer_vop_nresolve(struct vop_nresolve_args *ap)
1062 {
1063         struct hammer_transaction trans;
1064         struct namecache *ncp;
1065         hammer_inode_t dip;
1066         hammer_inode_t ip;
1067         hammer_tid_t asof;
1068         struct hammer_cursor cursor;
1069         struct vnode *vp;
1070         int64_t namekey;
1071         int error;
1072         int i;
1073         int nlen;
1074         int flags;
1075         int ispfs;
1076         int64_t obj_id;
1077         u_int32_t localization;
1078         u_int32_t max_iterations;
1079
1080         /*
1081          * Misc initialization, plus handle as-of name extensions.  Look for
1082          * the '@@' extension.  Note that as-of files and directories cannot
1083          * be modified.
1084          */
1085         dip = VTOI(ap->a_dvp);
1086         ncp = ap->a_nch->ncp;
1087         asof = dip->obj_asof;
1088         localization = dip->obj_localization;   /* for code consistency */
1089         nlen = ncp->nc_nlen;
1090         flags = dip->flags & HAMMER_INODE_RO;
1091         ispfs = 0;
1092
1093         hammer_simple_transaction(&trans, dip->hmp);
1094         ++hammer_stats_file_iopsr;
1095
1096         for (i = 0; i < nlen; ++i) {
1097                 if (ncp->nc_name[i] == '@' && ncp->nc_name[i+1] == '@') {
1098                         error = hammer_str_to_tid(ncp->nc_name + i + 2,
1099                                                   &ispfs, &asof, &localization);
1100                         if (error != 0) {
1101                                 i = nlen;
1102                                 break;
1103                         }
1104                         if (asof != HAMMER_MAX_TID)
1105                                 flags |= HAMMER_INODE_RO;
1106                         break;
1107                 }
1108         }
1109         nlen = i;
1110
1111         /*
1112          * If this is a PFS softlink we dive into the PFS
1113          */
1114         if (ispfs && nlen == 0) {
1115                 ip = hammer_get_inode(&trans, dip, HAMMER_OBJID_ROOT,
1116                                       asof, localization,
1117                                       flags, &error);
1118                 if (error == 0) {
1119                         error = hammer_get_vnode(ip, &vp);
1120                         hammer_rel_inode(ip, 0);
1121                 } else {
1122                         vp = NULL;
1123                 }
1124                 if (error == 0) {
1125                         vn_unlock(vp);
1126                         cache_setvp(ap->a_nch, vp);
1127                         vrele(vp);
1128                 }
1129                 goto done;
1130         }
1131
1132         /*
1133          * If there is no path component the time extension is relative to dip.
1134          * e.g. "fubar/@@<snapshot>"
1135          *
1136          * "." is handled by the kernel, but ".@@<snapshot>" is not.
1137          * e.g. "fubar/.@@<snapshot>"
1138          *
1139          * ".." is handled by the kernel.  We do not currently handle
1140          * "..@<snapshot>".
1141          */
1142         if (nlen == 0 || (nlen == 1 && ncp->nc_name[0] == '.')) {
1143                 ip = hammer_get_inode(&trans, dip, dip->obj_id,
1144                                       asof, dip->obj_localization,
1145                                       flags, &error);
1146                 if (error == 0) {
1147                         error = hammer_get_vnode(ip, &vp);
1148                         hammer_rel_inode(ip, 0);
1149                 } else {
1150                         vp = NULL;
1151                 }
1152                 if (error == 0) {
1153                         vn_unlock(vp);
1154                         cache_setvp(ap->a_nch, vp);
1155                         vrele(vp);
1156                 }
1157                 goto done;
1158         }
1159
1160         /*
1161          * Calculate the namekey and setup the key range for the scan.  This
1162          * works kinda like a chained hash table where the lower 32 bits
1163          * of the namekey synthesize the chain.
1164          *
1165          * The key range is inclusive of both key_beg and key_end.
1166          */
1167         namekey = hammer_directory_namekey(dip, ncp->nc_name, nlen,
1168                                            &max_iterations);
1169
1170         error = hammer_init_cursor(&trans, &cursor, &dip->cache[1], dip);
1171         cursor.key_beg.localization = dip->obj_localization +
1172                                       hammer_dir_localization(dip);
1173         cursor.key_beg.obj_id = dip->obj_id;
1174         cursor.key_beg.key = namekey;
1175         cursor.key_beg.create_tid = 0;
1176         cursor.key_beg.delete_tid = 0;
1177         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
1178         cursor.key_beg.obj_type = 0;
1179
1180         cursor.key_end = cursor.key_beg;
1181         cursor.key_end.key += max_iterations;
1182         cursor.asof = asof;
1183         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
1184
1185         /*
1186          * Scan all matching records (the chain), locate the one matching
1187          * the requested path component.
1188          *
1189          * The hammer_ip_*() functions merge in-memory records with on-disk
1190          * records for the purposes of the search.
1191          */
1192         obj_id = 0;
1193         localization = HAMMER_DEF_LOCALIZATION;
1194
1195         if (error == 0) {
1196                 error = hammer_ip_first(&cursor);
1197                 while (error == 0) {
1198                         error = hammer_ip_resolve_data(&cursor);
1199                         if (error)
1200                                 break;
1201                         if (nlen == cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF &&
1202                             bcmp(ncp->nc_name, cursor.data->entry.name, nlen) == 0) {
1203                                 obj_id = cursor.data->entry.obj_id;
1204                                 localization = cursor.data->entry.localization;
1205                                 break;
1206                         }
1207                         error = hammer_ip_next(&cursor);
1208                 }
1209         }
1210         hammer_done_cursor(&cursor);
1211
1212         /*
1213          * Lookup the obj_id.  This should always succeed.  If it does not
1214          * the filesystem may be damaged and we return a dummy inode.
1215          */
1216         if (error == 0) {
1217                 ip = hammer_get_inode(&trans, dip, obj_id,
1218                                       asof, localization,
1219                                       flags, &error);
1220                 if (error == ENOENT) {
1221                         kprintf("HAMMER: WARNING: Missing "
1222                                 "inode for dirent \"%s\"\n"
1223                                 "\tobj_id = %016llx, asof=%016llx, lo=%08x\n",
1224                                 ncp->nc_name,
1225                                 (long long)obj_id, (long long)asof,
1226                                 localization);
1227                         error = 0;
1228                         ip = hammer_get_dummy_inode(&trans, dip, obj_id,
1229                                                     asof, localization,
1230                                                     flags, &error);
1231                 }
1232                 if (error == 0) {
1233                         error = hammer_get_vnode(ip, &vp);
1234                         hammer_rel_inode(ip, 0);
1235                 } else {
1236                         vp = NULL;
1237                 }
1238                 if (error == 0) {
1239                         vn_unlock(vp);
1240                         cache_setvp(ap->a_nch, vp);
1241                         vrele(vp);
1242                 }
1243         } else if (error == ENOENT) {
1244                 cache_setvp(ap->a_nch, NULL);
1245         }
1246 done:
1247         hammer_done_transaction(&trans);
1248         return (error);
1249 }
1250
1251 /*
1252  * hammer_vop_nlookupdotdot { dvp, vpp, cred }
1253  *
1254  * Locate the parent directory of a directory vnode.
1255  *
1256  * dvp is referenced but not locked.  *vpp must be returned referenced and
1257  * locked.  A parent_obj_id of 0 does not necessarily indicate that we are
1258  * at the root, instead it could indicate that the directory we were in was
1259  * removed.
1260  *
1261  * NOTE: as-of sequences are not linked into the directory structure.  If
1262  * we are at the root with a different asof then the mount point, reload
1263  * the same directory with the mount point's asof.   I'm not sure what this
1264  * will do to NFS.  We encode ASOF stamps in NFS file handles so it might not
1265  * get confused, but it hasn't been tested.
1266  */
1267 static
1268 int
1269 hammer_vop_nlookupdotdot(struct vop_nlookupdotdot_args *ap)
1270 {
1271         struct hammer_transaction trans;
1272         struct hammer_inode *dip;
1273         struct hammer_inode *ip;
1274         int64_t parent_obj_id;
1275         u_int32_t parent_obj_localization;
1276         hammer_tid_t asof;
1277         int error;
1278
1279         dip = VTOI(ap->a_dvp);
1280         asof = dip->obj_asof;
1281
1282         /*
1283          * Whos are parent?  This could be the root of a pseudo-filesystem
1284          * whos parent is in another localization domain.
1285          */
1286         parent_obj_id = dip->ino_data.parent_obj_id;
1287         if (dip->obj_id == HAMMER_OBJID_ROOT)
1288                 parent_obj_localization = dip->ino_data.ext.obj.parent_obj_localization;
1289         else
1290                 parent_obj_localization = dip->obj_localization;
1291
1292         if (parent_obj_id == 0) {
1293                 if (dip->obj_id == HAMMER_OBJID_ROOT &&
1294                    asof != dip->hmp->asof) {
1295                         parent_obj_id = dip->obj_id;
1296                         asof = dip->hmp->asof;
1297                         *ap->a_fakename = kmalloc(19, M_TEMP, M_WAITOK);
1298                         ksnprintf(*ap->a_fakename, 19, "0x%016llx",
1299                                   (long long)dip->obj_asof);
1300                 } else {
1301                         *ap->a_vpp = NULL;
1302                         return ENOENT;
1303                 }
1304         }
1305
1306         hammer_simple_transaction(&trans, dip->hmp);
1307         ++hammer_stats_file_iopsr;
1308
1309         ip = hammer_get_inode(&trans, dip, parent_obj_id,
1310                               asof, parent_obj_localization,
1311                               dip->flags, &error);
1312         if (ip) {
1313                 error = hammer_get_vnode(ip, ap->a_vpp);
1314                 hammer_rel_inode(ip, 0);
1315         } else {
1316                 *ap->a_vpp = NULL;
1317         }
1318         hammer_done_transaction(&trans);
1319         return (error);
1320 }
1321
1322 /*
1323  * hammer_vop_nlink { nch, dvp, vp, cred }
1324  */
1325 static
1326 int
1327 hammer_vop_nlink(struct vop_nlink_args *ap)
1328 {
1329         struct hammer_transaction trans;
1330         struct hammer_inode *dip;
1331         struct hammer_inode *ip;
1332         struct nchandle *nch;
1333         int error;
1334
1335         if (ap->a_dvp->v_mount != ap->a_vp->v_mount)
1336                 return(EXDEV);
1337
1338         nch = ap->a_nch;
1339         dip = VTOI(ap->a_dvp);
1340         ip = VTOI(ap->a_vp);
1341
1342         if (dip->obj_localization != ip->obj_localization)
1343                 return(EXDEV);
1344
1345         if (dip->flags & HAMMER_INODE_RO)
1346                 return (EROFS);
1347         if (ip->flags & HAMMER_INODE_RO)
1348                 return (EROFS);
1349         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1350                 return (error);
1351
1352         /*
1353          * Create a transaction to cover the operations we perform.
1354          */
1355         hammer_start_transaction(&trans, dip->hmp);
1356         ++hammer_stats_file_iopsw;
1357
1358         /*
1359          * Add the filesystem object to the directory.  Note that neither
1360          * dip nor ip are referenced or locked, but their vnodes are
1361          * referenced.  This function will bump the inode's link count.
1362          */
1363         error = hammer_ip_add_directory(&trans, dip,
1364                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1365                                         ip);
1366
1367         /*
1368          * Finish up.
1369          */
1370         if (error == 0) {
1371                 cache_setunresolved(nch);
1372                 cache_setvp(nch, ap->a_vp);
1373         }
1374         hammer_done_transaction(&trans);
1375         hammer_knote(ap->a_vp, NOTE_LINK);
1376         hammer_knote(ap->a_dvp, NOTE_WRITE);
1377         return (error);
1378 }
1379
1380 /*
1381  * hammer_vop_nmkdir { nch, dvp, vpp, cred, vap }
1382  *
1383  * The operating system has already ensured that the directory entry
1384  * does not exist and done all appropriate namespace locking.
1385  */
1386 static
1387 int
1388 hammer_vop_nmkdir(struct vop_nmkdir_args *ap)
1389 {
1390         struct hammer_transaction trans;
1391         struct hammer_inode *dip;
1392         struct hammer_inode *nip;
1393         struct nchandle *nch;
1394         int error;
1395
1396         nch = ap->a_nch;
1397         dip = VTOI(ap->a_dvp);
1398
1399         if (dip->flags & HAMMER_INODE_RO)
1400                 return (EROFS);
1401         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1402                 return (error);
1403
1404         /*
1405          * Create a transaction to cover the operations we perform.
1406          */
1407         hammer_start_transaction(&trans, dip->hmp);
1408         ++hammer_stats_file_iopsw;
1409
1410         /*
1411          * Create a new filesystem object of the requested type.  The
1412          * returned inode will be referenced but not locked.
1413          */
1414         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
1415                                     dip, nch->ncp->nc_name, nch->ncp->nc_nlen,
1416                                     NULL, &nip);
1417         if (error) {
1418                 hkprintf("hammer_mkdir error %d\n", error);
1419                 hammer_done_transaction(&trans);
1420                 *ap->a_vpp = NULL;
1421                 return (error);
1422         }
1423         /*
1424          * Add the new filesystem object to the directory.  This will also
1425          * bump the inode's link count.
1426          */
1427         error = hammer_ip_add_directory(&trans, dip,
1428                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1429                                         nip);
1430         if (error)
1431                 hkprintf("hammer_mkdir (add) error %d\n", error);
1432
1433         /*
1434          * Finish up.
1435          */
1436         if (error) {
1437                 hammer_rel_inode(nip, 0);
1438                 *ap->a_vpp = NULL;
1439         } else {
1440                 error = hammer_get_vnode(nip, ap->a_vpp);
1441                 hammer_rel_inode(nip, 0);
1442                 if (error == 0) {
1443                         cache_setunresolved(ap->a_nch);
1444                         cache_setvp(ap->a_nch, *ap->a_vpp);
1445                 }
1446         }
1447         hammer_done_transaction(&trans);
1448         if (error == 0)
1449                 hammer_knote(ap->a_dvp, NOTE_WRITE | NOTE_LINK);
1450         return (error);
1451 }
1452
1453 /*
1454  * hammer_vop_nmknod { nch, dvp, vpp, cred, vap }
1455  *
1456  * The operating system has already ensured that the directory entry
1457  * does not exist and done all appropriate namespace locking.
1458  */
1459 static
1460 int
1461 hammer_vop_nmknod(struct vop_nmknod_args *ap)
1462 {
1463         struct hammer_transaction trans;
1464         struct hammer_inode *dip;
1465         struct hammer_inode *nip;
1466         struct nchandle *nch;
1467         int error;
1468
1469         nch = ap->a_nch;
1470         dip = VTOI(ap->a_dvp);
1471
1472         if (dip->flags & HAMMER_INODE_RO)
1473                 return (EROFS);
1474         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1475                 return (error);
1476
1477         /*
1478          * Create a transaction to cover the operations we perform.
1479          */
1480         hammer_start_transaction(&trans, dip->hmp);
1481         ++hammer_stats_file_iopsw;
1482
1483         /*
1484          * Create a new filesystem object of the requested type.  The
1485          * returned inode will be referenced but not locked.
1486          *
1487          * If mknod specifies a directory a pseudo-fs is created.
1488          */
1489         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
1490                                     dip, nch->ncp->nc_name, nch->ncp->nc_nlen,
1491                                     NULL, &nip);
1492         if (error) {
1493                 hammer_done_transaction(&trans);
1494                 *ap->a_vpp = NULL;
1495                 return (error);
1496         }
1497
1498         /*
1499          * Add the new filesystem object to the directory.  This will also
1500          * bump the inode's link count.
1501          */
1502         error = hammer_ip_add_directory(&trans, dip,
1503                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1504                                         nip);
1505
1506         /*
1507          * Finish up.
1508          */
1509         if (error) {
1510                 hammer_rel_inode(nip, 0);
1511                 *ap->a_vpp = NULL;
1512         } else {
1513                 error = hammer_get_vnode(nip, ap->a_vpp);
1514                 hammer_rel_inode(nip, 0);
1515                 if (error == 0) {
1516                         cache_setunresolved(ap->a_nch);
1517                         cache_setvp(ap->a_nch, *ap->a_vpp);
1518                 }
1519         }
1520         hammer_done_transaction(&trans);
1521         if (error == 0)
1522                 hammer_knote(ap->a_dvp, NOTE_WRITE);
1523         return (error);
1524 }
1525
1526 /*
1527  * hammer_vop_open { vp, mode, cred, fp }
1528  */
1529 static
1530 int
1531 hammer_vop_open(struct vop_open_args *ap)
1532 {
1533         hammer_inode_t ip;
1534
1535         ++hammer_stats_file_iopsr;
1536         ip = VTOI(ap->a_vp);
1537
1538         if ((ap->a_mode & FWRITE) && (ip->flags & HAMMER_INODE_RO))
1539                 return (EROFS);
1540         return(vop_stdopen(ap));
1541 }
1542
1543 /*
1544  * hammer_vop_print { vp }
1545  */
1546 static
1547 int
1548 hammer_vop_print(struct vop_print_args *ap)
1549 {
1550         return EOPNOTSUPP;
1551 }
1552
1553 /*
1554  * hammer_vop_readdir { vp, uio, cred, *eofflag, *ncookies, off_t **cookies }
1555  */
1556 static
1557 int
1558 hammer_vop_readdir(struct vop_readdir_args *ap)
1559 {
1560         struct hammer_transaction trans;
1561         struct hammer_cursor cursor;
1562         struct hammer_inode *ip;
1563         struct uio *uio;
1564         hammer_base_elm_t base;
1565         int error;
1566         int cookie_index;
1567         int ncookies;
1568         off_t *cookies;
1569         off_t saveoff;
1570         int r;
1571         int dtype;
1572
1573         ++hammer_stats_file_iopsr;
1574         ip = VTOI(ap->a_vp);
1575         uio = ap->a_uio;
1576         saveoff = uio->uio_offset;
1577
1578         if (ap->a_ncookies) {
1579                 ncookies = uio->uio_resid / 16 + 1;
1580                 if (ncookies > 1024)
1581                         ncookies = 1024;
1582                 cookies = kmalloc(ncookies * sizeof(off_t), M_TEMP, M_WAITOK);
1583                 cookie_index = 0;
1584         } else {
1585                 ncookies = -1;
1586                 cookies = NULL;
1587                 cookie_index = 0;
1588         }
1589
1590         hammer_simple_transaction(&trans, ip->hmp);
1591
1592         /*
1593          * Handle artificial entries
1594          *
1595          * It should be noted that the minimum value for a directory
1596          * hash key on-media is 0x0000000100000000, so we can use anything
1597          * less then that to represent our 'special' key space.
1598          */
1599         error = 0;
1600         if (saveoff == 0) {
1601                 r = vop_write_dirent(&error, uio, ip->obj_id, DT_DIR, 1, ".");
1602                 if (r)
1603                         goto done;
1604                 if (cookies)
1605                         cookies[cookie_index] = saveoff;
1606                 ++saveoff;
1607                 ++cookie_index;
1608                 if (cookie_index == ncookies)
1609                         goto done;
1610         }
1611         if (saveoff == 1) {
1612                 if (ip->ino_data.parent_obj_id) {
1613                         r = vop_write_dirent(&error, uio,
1614                                              ip->ino_data.parent_obj_id,
1615                                              DT_DIR, 2, "..");
1616                 } else {
1617                         r = vop_write_dirent(&error, uio,
1618                                              ip->obj_id, DT_DIR, 2, "..");
1619                 }
1620                 if (r)
1621                         goto done;
1622                 if (cookies)
1623                         cookies[cookie_index] = saveoff;
1624                 ++saveoff;
1625                 ++cookie_index;
1626                 if (cookie_index == ncookies)
1627                         goto done;
1628         }
1629
1630         /*
1631          * Key range (begin and end inclusive) to scan.  Directory keys
1632          * directly translate to a 64 bit 'seek' position.
1633          */
1634         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
1635         cursor.key_beg.localization = ip->obj_localization +
1636                                       hammer_dir_localization(ip);
1637         cursor.key_beg.obj_id = ip->obj_id;
1638         cursor.key_beg.create_tid = 0;
1639         cursor.key_beg.delete_tid = 0;
1640         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
1641         cursor.key_beg.obj_type = 0;
1642         cursor.key_beg.key = saveoff;
1643
1644         cursor.key_end = cursor.key_beg;
1645         cursor.key_end.key = HAMMER_MAX_KEY;
1646         cursor.asof = ip->obj_asof;
1647         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
1648
1649         error = hammer_ip_first(&cursor);
1650
1651         while (error == 0) {
1652                 error = hammer_ip_resolve_data(&cursor);
1653                 if (error)
1654                         break;
1655                 base = &cursor.leaf->base;
1656                 saveoff = base->key;
1657                 KKASSERT(cursor.leaf->data_len > HAMMER_ENTRY_NAME_OFF);
1658
1659                 if (base->obj_id != ip->obj_id)
1660                         panic("readdir: bad record at %p", cursor.node);
1661
1662                 /*
1663                  * Convert pseudo-filesystems into softlinks
1664                  */
1665                 dtype = hammer_get_dtype(cursor.leaf->base.obj_type);
1666                 r = vop_write_dirent(
1667                              &error, uio, cursor.data->entry.obj_id,
1668                              dtype,
1669                              cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF ,
1670                              (void *)cursor.data->entry.name);
1671                 if (r)
1672                         break;
1673                 ++saveoff;
1674                 if (cookies)
1675                         cookies[cookie_index] = base->key;
1676                 ++cookie_index;
1677                 if (cookie_index == ncookies)
1678                         break;
1679                 error = hammer_ip_next(&cursor);
1680         }
1681         hammer_done_cursor(&cursor);
1682
1683 done:
1684         hammer_done_transaction(&trans);
1685
1686         if (ap->a_eofflag)
1687                 *ap->a_eofflag = (error == ENOENT);
1688         uio->uio_offset = saveoff;
1689         if (error && cookie_index == 0) {
1690                 if (error == ENOENT)
1691                         error = 0;
1692                 if (cookies) {
1693                         kfree(cookies, M_TEMP);
1694                         *ap->a_ncookies = 0;
1695                         *ap->a_cookies = NULL;
1696                 }
1697         } else {
1698                 if (error == ENOENT)
1699                         error = 0;
1700                 if (cookies) {
1701                         *ap->a_ncookies = cookie_index;
1702                         *ap->a_cookies = cookies;
1703                 }
1704         }
1705         return(error);
1706 }
1707
1708 /*
1709  * hammer_vop_readlink { vp, uio, cred }
1710  */
1711 static
1712 int
1713 hammer_vop_readlink(struct vop_readlink_args *ap)
1714 {
1715         struct hammer_transaction trans;
1716         struct hammer_cursor cursor;
1717         struct hammer_inode *ip;
1718         char buf[32];
1719         u_int32_t localization;
1720         hammer_pseudofs_inmem_t pfsm;
1721         int error;
1722
1723         ip = VTOI(ap->a_vp);
1724
1725         /*
1726          * Shortcut if the symlink data was stuffed into ino_data.
1727          *
1728          * Also expand special "@@PFS%05d" softlinks (expansion only
1729          * occurs for non-historical (current) accesses made from the
1730          * primary filesystem).
1731          */
1732         if (ip->ino_data.size <= HAMMER_INODE_BASESYMLEN) {
1733                 char *ptr;
1734                 int bytes;
1735
1736                 ptr = ip->ino_data.ext.symlink;
1737                 bytes = (int)ip->ino_data.size;
1738                 if (bytes == 10 &&
1739                     ip->obj_asof == HAMMER_MAX_TID &&
1740                     ip->obj_localization == 0 &&
1741                     strncmp(ptr, "@@PFS", 5) == 0) {
1742                         hammer_simple_transaction(&trans, ip->hmp);
1743                         bcopy(ptr + 5, buf, 5);
1744                         buf[5] = 0;
1745                         localization = strtoul(buf, NULL, 10) << 16;
1746                         pfsm = hammer_load_pseudofs(&trans, localization,
1747                                                     &error);
1748                         if (error == 0) {
1749                                 if (pfsm->pfsd.mirror_flags &
1750                                     HAMMER_PFSD_SLAVE) {
1751                                         /* vap->va_size == 26 */
1752                                         ksnprintf(buf, sizeof(buf),
1753                                                   "@@0x%016llx:%05d",
1754                                                   (long long)pfsm->pfsd.sync_end_tid,
1755                                                   localization >> 16);
1756                                 } else {
1757                                         /* vap->va_size == 10 */
1758                                         ksnprintf(buf, sizeof(buf),
1759                                                   "@@-1:%05d",
1760                                                   localization >> 16);
1761 #if 0
1762                                         ksnprintf(buf, sizeof(buf),
1763                                                   "@@0x%016llx:%05d",
1764                                                   (long long)HAMMER_MAX_TID,
1765                                                   localization >> 16);
1766 #endif
1767                                 }
1768                                 ptr = buf;
1769                                 bytes = strlen(buf);
1770                         }
1771                         if (pfsm)
1772                                 hammer_rel_pseudofs(trans.hmp, pfsm);
1773                         hammer_done_transaction(&trans);
1774                 }
1775                 error = uiomove(ptr, bytes, ap->a_uio);
1776                 return(error);
1777         }
1778
1779         /*
1780          * Long version
1781          */
1782         hammer_simple_transaction(&trans, ip->hmp);
1783         ++hammer_stats_file_iopsr;
1784         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
1785
1786         /*
1787          * Key range (begin and end inclusive) to scan.  Directory keys
1788          * directly translate to a 64 bit 'seek' position.
1789          */
1790         cursor.key_beg.localization = ip->obj_localization +
1791                                       HAMMER_LOCALIZE_MISC;
1792         cursor.key_beg.obj_id = ip->obj_id;
1793         cursor.key_beg.create_tid = 0;
1794         cursor.key_beg.delete_tid = 0;
1795         cursor.key_beg.rec_type = HAMMER_RECTYPE_FIX;
1796         cursor.key_beg.obj_type = 0;
1797         cursor.key_beg.key = HAMMER_FIXKEY_SYMLINK;
1798         cursor.asof = ip->obj_asof;
1799         cursor.flags |= HAMMER_CURSOR_ASOF;
1800
1801         error = hammer_ip_lookup(&cursor);
1802         if (error == 0) {
1803                 error = hammer_ip_resolve_data(&cursor);
1804                 if (error == 0) {
1805                         KKASSERT(cursor.leaf->data_len >=
1806                                  HAMMER_SYMLINK_NAME_OFF);
1807                         error = uiomove(cursor.data->symlink.name,
1808                                         cursor.leaf->data_len -
1809                                                 HAMMER_SYMLINK_NAME_OFF,
1810                                         ap->a_uio);
1811                 }
1812         }
1813         hammer_done_cursor(&cursor);
1814         hammer_done_transaction(&trans);
1815         return(error);
1816 }
1817
1818 /*
1819  * hammer_vop_nremove { nch, dvp, cred }
1820  */
1821 static
1822 int
1823 hammer_vop_nremove(struct vop_nremove_args *ap)
1824 {
1825         struct hammer_transaction trans;
1826         struct hammer_inode *dip;
1827         int error;
1828
1829         dip = VTOI(ap->a_dvp);
1830
1831         if (hammer_nohistory(dip) == 0 &&
1832             (error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_REMOVE)) != 0) {
1833                 return (error);
1834         }
1835
1836         hammer_start_transaction(&trans, dip->hmp);
1837         ++hammer_stats_file_iopsw;
1838         error = hammer_dounlink(&trans, ap->a_nch, ap->a_dvp, ap->a_cred, 0, 0);
1839         hammer_done_transaction(&trans);
1840         if (error == 0)
1841                 hammer_knote(ap->a_dvp, NOTE_WRITE);
1842         return (error);
1843 }
1844
1845 /*
1846  * hammer_vop_nrename { fnch, tnch, fdvp, tdvp, cred }
1847  */
1848 static
1849 int
1850 hammer_vop_nrename(struct vop_nrename_args *ap)
1851 {
1852         struct hammer_transaction trans;
1853         struct namecache *fncp;
1854         struct namecache *tncp;
1855         struct hammer_inode *fdip;
1856         struct hammer_inode *tdip;
1857         struct hammer_inode *ip;
1858         struct hammer_cursor cursor;
1859         int64_t namekey;
1860         u_int32_t max_iterations;
1861         int nlen, error;
1862
1863         if (ap->a_fdvp->v_mount != ap->a_tdvp->v_mount)
1864                 return(EXDEV);
1865         if (ap->a_fdvp->v_mount != ap->a_fnch->ncp->nc_vp->v_mount)
1866                 return(EXDEV);
1867
1868         fdip = VTOI(ap->a_fdvp);
1869         tdip = VTOI(ap->a_tdvp);
1870         fncp = ap->a_fnch->ncp;
1871         tncp = ap->a_tnch->ncp;
1872         ip = VTOI(fncp->nc_vp);
1873         KKASSERT(ip != NULL);
1874
1875         if (fdip->obj_localization != tdip->obj_localization)
1876                 return(EXDEV);
1877         if (fdip->obj_localization != ip->obj_localization)
1878                 return(EXDEV);
1879
1880         if (fdip->flags & HAMMER_INODE_RO)
1881                 return (EROFS);
1882         if (tdip->flags & HAMMER_INODE_RO)
1883                 return (EROFS);
1884         if (ip->flags & HAMMER_INODE_RO)
1885                 return (EROFS);
1886         if ((error = hammer_checkspace(fdip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1887                 return (error);
1888
1889         hammer_start_transaction(&trans, fdip->hmp);
1890         ++hammer_stats_file_iopsw;
1891
1892         /*
1893          * Remove tncp from the target directory and then link ip as
1894          * tncp. XXX pass trans to dounlink
1895          *
1896          * Force the inode sync-time to match the transaction so it is
1897          * in-sync with the creation of the target directory entry.
1898          */
1899         error = hammer_dounlink(&trans, ap->a_tnch, ap->a_tdvp,
1900                                 ap->a_cred, 0, -1);
1901         if (error == 0 || error == ENOENT) {
1902                 error = hammer_ip_add_directory(&trans, tdip,
1903                                                 tncp->nc_name, tncp->nc_nlen,
1904                                                 ip);
1905                 if (error == 0) {
1906                         ip->ino_data.parent_obj_id = tdip->obj_id;
1907                         ip->ino_data.ctime = trans.time;
1908                         hammer_modify_inode(&trans, ip, HAMMER_INODE_DDIRTY);
1909                 }
1910         }
1911         if (error)
1912                 goto failed; /* XXX */
1913
1914         /*
1915          * Locate the record in the originating directory and remove it.
1916          *
1917          * Calculate the namekey and setup the key range for the scan.  This
1918          * works kinda like a chained hash table where the lower 32 bits
1919          * of the namekey synthesize the chain.
1920          *
1921          * The key range is inclusive of both key_beg and key_end.
1922          */
1923         namekey = hammer_directory_namekey(fdip, fncp->nc_name, fncp->nc_nlen,
1924                                            &max_iterations);
1925 retry:
1926         hammer_init_cursor(&trans, &cursor, &fdip->cache[1], fdip);
1927         cursor.key_beg.localization = fdip->obj_localization +
1928                                       hammer_dir_localization(fdip);
1929         cursor.key_beg.obj_id = fdip->obj_id;
1930         cursor.key_beg.key = namekey;
1931         cursor.key_beg.create_tid = 0;
1932         cursor.key_beg.delete_tid = 0;
1933         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
1934         cursor.key_beg.obj_type = 0;
1935
1936         cursor.key_end = cursor.key_beg;
1937         cursor.key_end.key += max_iterations;
1938         cursor.asof = fdip->obj_asof;
1939         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
1940
1941         /*
1942          * Scan all matching records (the chain), locate the one matching
1943          * the requested path component.
1944          *
1945          * The hammer_ip_*() functions merge in-memory records with on-disk
1946          * records for the purposes of the search.
1947          */
1948         error = hammer_ip_first(&cursor);
1949         while (error == 0) {
1950                 if (hammer_ip_resolve_data(&cursor) != 0)
1951                         break;
1952                 nlen = cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF;
1953                 KKASSERT(nlen > 0);
1954                 if (fncp->nc_nlen == nlen &&
1955                     bcmp(fncp->nc_name, cursor.data->entry.name, nlen) == 0) {
1956                         break;
1957                 }
1958                 error = hammer_ip_next(&cursor);
1959         }
1960
1961         /*
1962          * If all is ok we have to get the inode so we can adjust nlinks.
1963          *
1964          * WARNING: hammer_ip_del_directory() may have to terminate the
1965          * cursor to avoid a recursion.  It's ok to call hammer_done_cursor()
1966          * twice.
1967          */
1968         if (error == 0)
1969                 error = hammer_ip_del_directory(&trans, &cursor, fdip, ip);
1970
1971         /*
1972          * XXX A deadlock here will break rename's atomicy for the purposes
1973          * of crash recovery.
1974          */
1975         if (error == EDEADLK) {
1976                 hammer_done_cursor(&cursor);
1977                 goto retry;
1978         }
1979
1980         /*
1981          * Cleanup and tell the kernel that the rename succeeded.
1982          */
1983         hammer_done_cursor(&cursor);
1984         if (error == 0) {
1985                 cache_rename(ap->a_fnch, ap->a_tnch);
1986                 hammer_knote(ap->a_fdvp, NOTE_WRITE);
1987                 hammer_knote(ap->a_tdvp, NOTE_WRITE);
1988                 if (ip->vp)
1989                         hammer_knote(ip->vp, NOTE_RENAME);
1990         }
1991
1992 failed:
1993         hammer_done_transaction(&trans);
1994         return (error);
1995 }
1996
1997 /*
1998  * hammer_vop_nrmdir { nch, dvp, cred }
1999  */
2000 static
2001 int
2002 hammer_vop_nrmdir(struct vop_nrmdir_args *ap)
2003 {
2004         struct hammer_transaction trans;
2005         struct hammer_inode *dip;
2006         int error;
2007
2008         dip = VTOI(ap->a_dvp);
2009
2010         if (hammer_nohistory(dip) == 0 &&
2011             (error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_REMOVE)) != 0) {
2012                 return (error);
2013         }
2014
2015         hammer_start_transaction(&trans, dip->hmp);
2016         ++hammer_stats_file_iopsw;
2017         error = hammer_dounlink(&trans, ap->a_nch, ap->a_dvp, ap->a_cred, 0, 1);
2018         hammer_done_transaction(&trans);
2019         if (error == 0)
2020                 hammer_knote(ap->a_dvp, NOTE_WRITE | NOTE_LINK);
2021         return (error);
2022 }
2023
2024 /*
2025  * hammer_vop_markatime { vp, cred }
2026  */
2027 static
2028 int
2029 hammer_vop_markatime(struct vop_markatime_args *ap)
2030 {
2031         struct hammer_transaction trans;
2032         struct hammer_inode *ip;
2033
2034         ip = VTOI(ap->a_vp);
2035         if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2036                 return (EROFS);
2037         if (ip->flags & HAMMER_INODE_RO)
2038                 return (EROFS);
2039         if (ip->hmp->mp->mnt_flag & MNT_NOATIME)
2040                 return (0);
2041         hammer_start_transaction(&trans, ip->hmp);
2042         ++hammer_stats_file_iopsw;
2043
2044         ip->ino_data.atime = trans.time;
2045         hammer_modify_inode(&trans, ip, HAMMER_INODE_ATIME);
2046         hammer_done_transaction(&trans);
2047         hammer_knote(ap->a_vp, NOTE_ATTRIB);
2048         return (0);
2049 }
2050
2051 /*
2052  * hammer_vop_setattr { vp, vap, cred }
2053  */
2054 static
2055 int
2056 hammer_vop_setattr(struct vop_setattr_args *ap)
2057 {
2058         struct hammer_transaction trans;
2059         struct vattr *vap;
2060         struct hammer_inode *ip;
2061         int modflags;
2062         int error;
2063         int truncating;
2064         int blksize;
2065         int kflags;
2066 #if 0
2067         int64_t aligned_size;
2068 #endif
2069         u_int32_t flags;
2070
2071         vap = ap->a_vap;
2072         ip = ap->a_vp->v_data;
2073         modflags = 0;
2074         kflags = 0;
2075
2076         if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2077                 return(EROFS);
2078         if (ip->flags & HAMMER_INODE_RO)
2079                 return (EROFS);
2080         if (hammer_nohistory(ip) == 0 &&
2081             (error = hammer_checkspace(ip->hmp, HAMMER_CHKSPC_REMOVE)) != 0) {
2082                 return (error);
2083         }
2084
2085         hammer_start_transaction(&trans, ip->hmp);
2086         ++hammer_stats_file_iopsw;
2087         error = 0;
2088
2089         if (vap->va_flags != VNOVAL) {
2090                 flags = ip->ino_data.uflags;
2091                 error = vop_helper_setattr_flags(&flags, vap->va_flags,
2092                                          hammer_to_unix_xid(&ip->ino_data.uid),
2093                                          ap->a_cred);
2094                 if (error == 0) {
2095                         if (ip->ino_data.uflags != flags) {
2096                                 ip->ino_data.uflags = flags;
2097                                 ip->ino_data.ctime = trans.time;
2098                                 modflags |= HAMMER_INODE_DDIRTY;
2099                                 kflags |= NOTE_ATTRIB;
2100                         }
2101                         if (ip->ino_data.uflags & (IMMUTABLE | APPEND)) {
2102                                 error = 0;
2103                                 goto done;
2104                         }
2105                 }
2106                 goto done;
2107         }
2108         if (ip->ino_data.uflags & (IMMUTABLE | APPEND)) {
2109                 error = EPERM;
2110                 goto done;
2111         }
2112         if (vap->va_uid != (uid_t)VNOVAL || vap->va_gid != (gid_t)VNOVAL) {
2113                 mode_t cur_mode = ip->ino_data.mode;
2114                 uid_t cur_uid = hammer_to_unix_xid(&ip->ino_data.uid);
2115                 gid_t cur_gid = hammer_to_unix_xid(&ip->ino_data.gid);
2116                 uuid_t uuid_uid;
2117                 uuid_t uuid_gid;
2118
2119                 error = vop_helper_chown(ap->a_vp, vap->va_uid, vap->va_gid,
2120                                          ap->a_cred,
2121                                          &cur_uid, &cur_gid, &cur_mode);
2122                 if (error == 0) {
2123                         hammer_guid_to_uuid(&uuid_uid, cur_uid);
2124                         hammer_guid_to_uuid(&uuid_gid, cur_gid);
2125                         if (bcmp(&uuid_uid, &ip->ino_data.uid,
2126                                  sizeof(uuid_uid)) ||
2127                             bcmp(&uuid_gid, &ip->ino_data.gid,
2128                                  sizeof(uuid_gid)) ||
2129                             ip->ino_data.mode != cur_mode
2130                         ) {
2131                                 ip->ino_data.uid = uuid_uid;
2132                                 ip->ino_data.gid = uuid_gid;
2133                                 ip->ino_data.mode = cur_mode;
2134                                 ip->ino_data.ctime = trans.time;
2135                                 modflags |= HAMMER_INODE_DDIRTY;
2136                         }
2137                         kflags |= NOTE_ATTRIB;
2138                 }
2139         }
2140         while (vap->va_size != VNOVAL && ip->ino_data.size != vap->va_size) {
2141                 switch(ap->a_vp->v_type) {
2142                 case VREG:
2143                         if (vap->va_size == ip->ino_data.size)
2144                                 break;
2145
2146                         /*
2147                          * Log the operation if in fast-fsync mode.
2148                          */
2149                         if (ip->flags & HAMMER_INODE_REDO) {
2150                                 error = hammer_generate_redo(&trans, ip,
2151                                                              vap->va_size,
2152                                                              HAMMER_REDO_TRUNC,
2153                                                              NULL, 0);
2154                         }
2155                         blksize = hammer_blocksize(vap->va_size);
2156
2157                         /*
2158                          * XXX break atomicy, we can deadlock the backend
2159                          * if we do not release the lock.  Probably not a
2160                          * big deal here.
2161                          */
2162                         if (vap->va_size < ip->ino_data.size) {
2163                                 nvtruncbuf(ap->a_vp, vap->va_size,
2164                                            blksize,
2165                                            hammer_blockoff(vap->va_size));
2166                                 truncating = 1;
2167                                 kflags |= NOTE_WRITE;
2168                         } else {
2169                                 nvextendbuf(ap->a_vp,
2170                                             ip->ino_data.size,
2171                                             vap->va_size,
2172                                             hammer_blocksize(ip->ino_data.size),
2173                                             hammer_blocksize(vap->va_size),
2174                                             hammer_blockoff(ip->ino_data.size),
2175                                             hammer_blockoff(vap->va_size),
2176                                             0);
2177                                 truncating = 0;
2178                                 kflags |= NOTE_WRITE | NOTE_EXTEND;
2179                         }
2180                         ip->ino_data.size = vap->va_size;
2181                         ip->ino_data.mtime = trans.time;
2182                         /* XXX safe to use SDIRTY instead of DDIRTY here? */
2183                         modflags |= HAMMER_INODE_MTIME | HAMMER_INODE_DDIRTY;
2184
2185                         /*
2186                          * On-media truncation is cached in the inode until
2187                          * the inode is synchronized.  We must immediately
2188                          * handle any frontend records.
2189                          */
2190                         if (truncating) {
2191                                 hammer_ip_frontend_trunc(ip, vap->va_size);
2192 #ifdef DEBUG_TRUNCATE
2193                                 if (HammerTruncIp == NULL)
2194                                         HammerTruncIp = ip;
2195 #endif
2196                                 if ((ip->flags & HAMMER_INODE_TRUNCATED) == 0) {
2197                                         ip->flags |= HAMMER_INODE_TRUNCATED;
2198                                         ip->trunc_off = vap->va_size;
2199 #ifdef DEBUG_TRUNCATE
2200                                         if (ip == HammerTruncIp)
2201                                         kprintf("truncate1 %016llx\n",
2202                                                 (long long)ip->trunc_off);
2203 #endif
2204                                 } else if (ip->trunc_off > vap->va_size) {
2205                                         ip->trunc_off = vap->va_size;
2206 #ifdef DEBUG_TRUNCATE
2207                                         if (ip == HammerTruncIp)
2208                                         kprintf("truncate2 %016llx\n",
2209                                                 (long long)ip->trunc_off);
2210 #endif
2211                                 } else {
2212 #ifdef DEBUG_TRUNCATE
2213                                         if (ip == HammerTruncIp)
2214                                         kprintf("truncate3 %016llx (ignored)\n",
2215                                                 (long long)vap->va_size);
2216 #endif
2217                                 }
2218                         }
2219
2220 #if 0
2221                         /*
2222                          * When truncating, nvtruncbuf() may have cleaned out
2223                          * a portion of the last block on-disk in the buffer
2224                          * cache.  We must clean out any frontend records
2225                          * for blocks beyond the new last block.
2226                          */
2227                         aligned_size = (vap->va_size + (blksize - 1)) &
2228                                        ~(int64_t)(blksize - 1);
2229                         if (truncating && vap->va_size < aligned_size) {
2230                                 aligned_size -= blksize;
2231                                 hammer_ip_frontend_trunc(ip, aligned_size);
2232                         }
2233 #endif
2234                         break;
2235                 case VDATABASE:
2236                         if ((ip->flags & HAMMER_INODE_TRUNCATED) == 0) {
2237                                 ip->flags |= HAMMER_INODE_TRUNCATED;
2238                                 ip->trunc_off = vap->va_size;
2239                         } else if (ip->trunc_off > vap->va_size) {
2240                                 ip->trunc_off = vap->va_size;
2241                         }
2242                         hammer_ip_frontend_trunc(ip, vap->va_size);
2243                         ip->ino_data.size = vap->va_size;
2244                         ip->ino_data.mtime = trans.time;
2245                         modflags |= HAMMER_INODE_MTIME | HAMMER_INODE_DDIRTY;
2246                         kflags |= NOTE_ATTRIB;
2247                         break;
2248                 default:
2249                         error = EINVAL;
2250                         goto done;
2251                 }
2252                 break;
2253         }
2254         if (vap->va_atime.tv_sec != VNOVAL) {
2255                 ip->ino_data.atime = hammer_timespec_to_time(&vap->va_atime);
2256                 modflags |= HAMMER_INODE_ATIME;
2257                 kflags |= NOTE_ATTRIB;
2258         }
2259         if (vap->va_mtime.tv_sec != VNOVAL) {
2260                 ip->ino_data.mtime = hammer_timespec_to_time(&vap->va_mtime);
2261                 modflags |= HAMMER_INODE_MTIME;
2262                 kflags |= NOTE_ATTRIB;
2263         }
2264         if (vap->va_mode != (mode_t)VNOVAL) {
2265                 mode_t   cur_mode = ip->ino_data.mode;
2266                 uid_t cur_uid = hammer_to_unix_xid(&ip->ino_data.uid);
2267                 gid_t cur_gid = hammer_to_unix_xid(&ip->ino_data.gid);
2268
2269                 error = vop_helper_chmod(ap->a_vp, vap->va_mode, ap->a_cred,
2270                                          cur_uid, cur_gid, &cur_mode);
2271                 if (error == 0 && ip->ino_data.mode != cur_mode) {
2272                         ip->ino_data.mode = cur_mode;
2273                         ip->ino_data.ctime = trans.time;
2274                         modflags |= HAMMER_INODE_DDIRTY;
2275                         kflags |= NOTE_ATTRIB;
2276                 }
2277         }
2278 done:
2279         if (error == 0)
2280                 hammer_modify_inode(&trans, ip, modflags);
2281         hammer_done_transaction(&trans);
2282         hammer_knote(ap->a_vp, kflags);
2283         return (error);
2284 }
2285
2286 /*
2287  * hammer_vop_nsymlink { nch, dvp, vpp, cred, vap, target }
2288  */
2289 static
2290 int
2291 hammer_vop_nsymlink(struct vop_nsymlink_args *ap)
2292 {
2293         struct hammer_transaction trans;
2294         struct hammer_inode *dip;
2295         struct hammer_inode *nip;
2296         struct nchandle *nch;
2297         hammer_record_t record;
2298         int error;
2299         int bytes;
2300
2301         ap->a_vap->va_type = VLNK;
2302
2303         nch = ap->a_nch;
2304         dip = VTOI(ap->a_dvp);
2305
2306         if (dip->flags & HAMMER_INODE_RO)
2307                 return (EROFS);
2308         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
2309                 return (error);
2310
2311         /*
2312          * Create a transaction to cover the operations we perform.
2313          */
2314         hammer_start_transaction(&trans, dip->hmp);
2315         ++hammer_stats_file_iopsw;
2316
2317         /*
2318          * Create a new filesystem object of the requested type.  The
2319          * returned inode will be referenced but not locked.
2320          */
2321
2322         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
2323                                     dip, nch->ncp->nc_name, nch->ncp->nc_nlen,
2324                                     NULL, &nip);
2325         if (error) {
2326                 hammer_done_transaction(&trans);
2327                 *ap->a_vpp = NULL;
2328                 return (error);
2329         }
2330
2331         /*
2332          * Add a record representing the symlink.  symlink stores the link
2333          * as pure data, not a string, and is no \0 terminated.
2334          */
2335         if (error == 0) {
2336                 bytes = strlen(ap->a_target);
2337
2338                 if (bytes <= HAMMER_INODE_BASESYMLEN) {
2339                         bcopy(ap->a_target, nip->ino_data.ext.symlink, bytes);
2340                 } else {
2341                         record = hammer_alloc_mem_record(nip, bytes);
2342                         record->type = HAMMER_MEM_RECORD_GENERAL;
2343
2344                         record->leaf.base.localization = nip->obj_localization +
2345                                                          HAMMER_LOCALIZE_MISC;
2346                         record->leaf.base.key = HAMMER_FIXKEY_SYMLINK;
2347                         record->leaf.base.rec_type = HAMMER_RECTYPE_FIX;
2348                         record->leaf.data_len = bytes;
2349                         KKASSERT(HAMMER_SYMLINK_NAME_OFF == 0);
2350                         bcopy(ap->a_target, record->data->symlink.name, bytes);
2351                         error = hammer_ip_add_record(&trans, record);
2352                 }
2353
2354                 /*
2355                  * Set the file size to the length of the link.
2356                  */
2357                 if (error == 0) {
2358                         nip->ino_data.size = bytes;
2359                         hammer_modify_inode(&trans, nip, HAMMER_INODE_DDIRTY);
2360                 }
2361         }
2362         if (error == 0)
2363                 error = hammer_ip_add_directory(&trans, dip, nch->ncp->nc_name,
2364                                                 nch->ncp->nc_nlen, nip);
2365
2366         /*
2367          * Finish up.
2368          */
2369         if (error) {
2370                 hammer_rel_inode(nip, 0);
2371                 *ap->a_vpp = NULL;
2372         } else {
2373                 error = hammer_get_vnode(nip, ap->a_vpp);
2374                 hammer_rel_inode(nip, 0);
2375                 if (error == 0) {
2376                         cache_setunresolved(ap->a_nch);
2377                         cache_setvp(ap->a_nch, *ap->a_vpp);
2378                         hammer_knote(ap->a_dvp, NOTE_WRITE);
2379                 }
2380         }
2381         hammer_done_transaction(&trans);
2382         return (error);
2383 }
2384
2385 /*
2386  * hammer_vop_nwhiteout { nch, dvp, cred, flags }
2387  */
2388 static
2389 int
2390 hammer_vop_nwhiteout(struct vop_nwhiteout_args *ap)
2391 {
2392         struct hammer_transaction trans;
2393         struct hammer_inode *dip;
2394         int error;
2395
2396         dip = VTOI(ap->a_dvp);
2397
2398         if (hammer_nohistory(dip) == 0 &&
2399             (error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0) {
2400                 return (error);
2401         }
2402
2403         hammer_start_transaction(&trans, dip->hmp);
2404         ++hammer_stats_file_iopsw;
2405         error = hammer_dounlink(&trans, ap->a_nch, ap->a_dvp,
2406                                 ap->a_cred, ap->a_flags, -1);
2407         hammer_done_transaction(&trans);
2408
2409         return (error);
2410 }
2411
2412 /*
2413  * hammer_vop_ioctl { vp, command, data, fflag, cred }
2414  */
2415 static
2416 int
2417 hammer_vop_ioctl(struct vop_ioctl_args *ap)
2418 {
2419         struct hammer_inode *ip = ap->a_vp->v_data;
2420
2421         ++hammer_stats_file_iopsr;
2422         return(hammer_ioctl(ip, ap->a_command, ap->a_data,
2423                             ap->a_fflag, ap->a_cred));
2424 }
2425
2426 static
2427 int
2428 hammer_vop_mountctl(struct vop_mountctl_args *ap)
2429 {
2430         static const struct mountctl_opt extraopt[] = {
2431                 { HMNT_NOHISTORY,       "nohistory" },
2432                 { HMNT_MASTERID,        "master" },
2433                 { 0, NULL}
2434
2435         };
2436         struct hammer_mount *hmp;
2437         struct mount *mp;
2438         int usedbytes;
2439         int error;
2440
2441         error = 0;
2442         usedbytes = 0;
2443         mp = ap->a_head.a_ops->head.vv_mount;
2444         KKASSERT(mp->mnt_data != NULL);
2445         hmp = (struct hammer_mount *)mp->mnt_data;
2446
2447         switch(ap->a_op) {
2448
2449         case MOUNTCTL_SET_EXPORT:
2450                 if (ap->a_ctllen != sizeof(struct export_args))
2451                         error = EINVAL;
2452                 else
2453                         error = hammer_vfs_export(mp, ap->a_op,
2454                                       (const struct export_args *)ap->a_ctl);
2455                 break;
2456         case MOUNTCTL_MOUNTFLAGS:
2457         {
2458                 /*
2459                  * Call standard mountctl VOP function
2460                  * so we get user mount flags.
2461                  */
2462                 error = vop_stdmountctl(ap);
2463                 if (error)
2464                         break;
2465
2466                 usedbytes = *ap->a_res;
2467
2468                 if (usedbytes > 0 && usedbytes < ap->a_buflen) {
2469                         usedbytes += vfs_flagstostr(hmp->hflags, extraopt, ap->a_buf,
2470                                                     ap->a_buflen - usedbytes,
2471                                                     &error);
2472                 }
2473
2474                 *ap->a_res += usedbytes;
2475                 break;
2476         }
2477         default:
2478                 error = vop_stdmountctl(ap);
2479                 break;
2480         }
2481         return(error);
2482 }
2483
2484 /*
2485  * hammer_vop_strategy { vp, bio }
2486  *
2487  * Strategy call, used for regular file read & write only.  Note that the
2488  * bp may represent a cluster.
2489  *
2490  * To simplify operation and allow better optimizations in the future,
2491  * this code does not make any assumptions with regards to buffer alignment
2492  * or size.
2493  */
2494 static
2495 int
2496 hammer_vop_strategy(struct vop_strategy_args *ap)
2497 {
2498         struct buf *bp;
2499         int error;
2500
2501         bp = ap->a_bio->bio_buf;
2502
2503         switch(bp->b_cmd) {
2504         case BUF_CMD_READ:
2505                 error = hammer_vop_strategy_read(ap);
2506                 break;
2507         case BUF_CMD_WRITE:
2508                 error = hammer_vop_strategy_write(ap);
2509                 break;
2510         default:
2511                 bp->b_error = error = EINVAL;
2512                 bp->b_flags |= B_ERROR;
2513                 biodone(ap->a_bio);
2514                 break;
2515         }
2516         return (error);
2517 }
2518
2519 /*
2520  * Read from a regular file.  Iterate the related records and fill in the
2521  * BIO/BUF.  Gaps are zero-filled.
2522  *
2523  * The support code in hammer_object.c should be used to deal with mixed
2524  * in-memory and on-disk records.
2525  *
2526  * NOTE: Can be called from the cluster code with an oversized buf.
2527  *
2528  * XXX atime update
2529  */
2530 static
2531 int
2532 hammer_vop_strategy_read(struct vop_strategy_args *ap)
2533 {
2534         struct hammer_transaction trans;
2535         struct hammer_inode *ip;
2536         struct hammer_inode *dip;
2537         struct hammer_cursor cursor;
2538         hammer_base_elm_t base;
2539         hammer_off_t disk_offset;
2540         struct bio *bio;
2541         struct bio *nbio;
2542         struct buf *bp;
2543         int64_t rec_offset;
2544         int64_t ran_end;
2545         int64_t tmp64;
2546         int error;
2547         int boff;
2548         int roff;
2549         int n;
2550
2551         bio = ap->a_bio;
2552         bp = bio->bio_buf;
2553         ip = ap->a_vp->v_data;
2554
2555         /*
2556          * The zone-2 disk offset may have been set by the cluster code via
2557          * a BMAP operation, or else should be NOOFFSET.
2558          *
2559          * Checking the high bits for a match against zone-2 should suffice.
2560          */
2561         nbio = push_bio(bio);
2562         if ((nbio->bio_offset & HAMMER_OFF_ZONE_MASK) ==
2563             HAMMER_ZONE_LARGE_DATA) {
2564                 error = hammer_io_direct_read(ip->hmp, nbio, NULL);
2565                 return (error);
2566         }
2567
2568         /*
2569          * Well, that sucked.  Do it the hard way.  If all the stars are
2570          * aligned we may still be able to issue a direct-read.
2571          */
2572         hammer_simple_transaction(&trans, ip->hmp);
2573         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
2574
2575         /*
2576          * Key range (begin and end inclusive) to scan.  Note that the key's
2577          * stored in the actual records represent BASE+LEN, not BASE.  The
2578          * first record containing bio_offset will have a key > bio_offset.
2579          */
2580         cursor.key_beg.localization = ip->obj_localization +
2581                                       HAMMER_LOCALIZE_MISC;
2582         cursor.key_beg.obj_id = ip->obj_id;
2583         cursor.key_beg.create_tid = 0;
2584         cursor.key_beg.delete_tid = 0;
2585         cursor.key_beg.obj_type = 0;
2586         cursor.key_beg.key = bio->bio_offset + 1;
2587         cursor.asof = ip->obj_asof;
2588         cursor.flags |= HAMMER_CURSOR_ASOF;
2589
2590         cursor.key_end = cursor.key_beg;
2591         KKASSERT(ip->ino_data.obj_type == HAMMER_OBJTYPE_REGFILE);
2592 #if 0
2593         if (ip->ino_data.obj_type == HAMMER_OBJTYPE_DBFILE) {
2594                 cursor.key_beg.rec_type = HAMMER_RECTYPE_DB;
2595                 cursor.key_end.rec_type = HAMMER_RECTYPE_DB;
2596                 cursor.key_end.key = 0x7FFFFFFFFFFFFFFFLL;
2597         } else
2598 #endif
2599         {
2600                 ran_end = bio->bio_offset + bp->b_bufsize;
2601                 cursor.key_beg.rec_type = HAMMER_RECTYPE_DATA;
2602                 cursor.key_end.rec_type = HAMMER_RECTYPE_DATA;
2603                 tmp64 = ran_end + MAXPHYS + 1;  /* work-around GCC-4 bug */
2604                 if (tmp64 < ran_end)
2605                         cursor.key_end.key = 0x7FFFFFFFFFFFFFFFLL;
2606                 else
2607                         cursor.key_end.key = ran_end + MAXPHYS + 1;
2608         }
2609         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE;
2610
2611         error = hammer_ip_first(&cursor);
2612         boff = 0;
2613
2614         while (error == 0) {
2615                 /*
2616                  * Get the base file offset of the record.  The key for
2617                  * data records is (base + bytes) rather then (base).
2618                  */
2619                 base = &cursor.leaf->base;
2620                 rec_offset = base->key - cursor.leaf->data_len;
2621
2622                 /*
2623                  * Calculate the gap, if any, and zero-fill it.
2624                  *
2625                  * n is the offset of the start of the record verses our
2626                  * current seek offset in the bio.
2627                  */
2628                 n = (int)(rec_offset - (bio->bio_offset + boff));
2629                 if (n > 0) {
2630                         if (n > bp->b_bufsize - boff)
2631                                 n = bp->b_bufsize - boff;
2632                         bzero((char *)bp->b_data + boff, n);
2633                         boff += n;
2634                         n = 0;
2635                 }
2636
2637                 /*
2638                  * Calculate the data offset in the record and the number
2639                  * of bytes we can copy.
2640                  *
2641                  * There are two degenerate cases.  First, boff may already
2642                  * be at bp->b_bufsize.  Secondly, the data offset within
2643                  * the record may exceed the record's size.
2644                  */
2645                 roff = -n;
2646                 rec_offset += roff;
2647                 n = cursor.leaf->data_len - roff;
2648                 if (n <= 0) {
2649                         kprintf("strategy_read: bad n=%d roff=%d\n", n, roff);
2650                         n = 0;
2651                 } else if (n > bp->b_bufsize - boff) {
2652                         n = bp->b_bufsize - boff;
2653                 }
2654
2655                 /*
2656                  * Deal with cached truncations.  This cool bit of code
2657                  * allows truncate()/ftruncate() to avoid having to sync
2658                  * the file.
2659                  *
2660                  * If the frontend is truncated then all backend records are
2661                  * subject to the frontend's truncation.
2662                  *
2663                  * If the backend is truncated then backend records on-disk
2664                  * (but not in-memory) are subject to the backend's
2665                  * truncation.  In-memory records owned by the backend
2666                  * represent data written after the truncation point on the
2667                  * backend and must not be truncated.
2668                  *
2669                  * Truncate operations deal with frontend buffer cache
2670                  * buffers and frontend-owned in-memory records synchronously.
2671                  */
2672                 if (ip->flags & HAMMER_INODE_TRUNCATED) {
2673                         if (hammer_cursor_ondisk(&cursor)/* ||
2674                             cursor.iprec->flush_state == HAMMER_FST_FLUSH*/) {
2675                                 if (ip->trunc_off <= rec_offset)
2676                                         n = 0;
2677                                 else if (ip->trunc_off < rec_offset + n)
2678                                         n = (int)(ip->trunc_off - rec_offset);
2679                         }
2680                 }
2681                 if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
2682                         if (hammer_cursor_ondisk(&cursor)) {
2683                                 if (ip->sync_trunc_off <= rec_offset)
2684                                         n = 0;
2685                                 else if (ip->sync_trunc_off < rec_offset + n)
2686                                         n = (int)(ip->sync_trunc_off - rec_offset);
2687                         }
2688                 }
2689
2690                 /*
2691                  * Try to issue a direct read into our bio if possible,
2692                  * otherwise resolve the element data into a hammer_buffer
2693                  * and copy.
2694                  *
2695                  * The buffer on-disk should be zerod past any real
2696                  * truncation point, but may not be for any synthesized
2697                  * truncation point from above.
2698                  */
2699                 disk_offset = cursor.leaf->data_offset + roff;
2700                 if (boff == 0 && n == bp->b_bufsize &&
2701                     hammer_cursor_ondisk(&cursor) &&
2702                     (disk_offset & HAMMER_BUFMASK) == 0) {
2703                         KKASSERT((disk_offset & HAMMER_OFF_ZONE_MASK) ==
2704                                  HAMMER_ZONE_LARGE_DATA);
2705                         nbio->bio_offset = disk_offset;
2706                         error = hammer_io_direct_read(trans.hmp, nbio,
2707                                                       cursor.leaf);
2708                         goto done;
2709                 } else if (n) {
2710                         error = hammer_ip_resolve_data(&cursor);
2711                         if (error == 0) {
2712                                 bcopy((char *)cursor.data + roff,
2713                                       (char *)bp->b_data + boff, n);
2714                         }
2715                 }
2716                 if (error)
2717                         break;
2718
2719                 /*
2720                  * Iterate until we have filled the request.
2721                  */
2722                 boff += n;
2723                 if (boff == bp->b_bufsize)
2724                         break;
2725                 error = hammer_ip_next(&cursor);
2726         }
2727
2728         /*
2729          * There may have been a gap after the last record
2730          */
2731         if (error == ENOENT)
2732                 error = 0;
2733         if (error == 0 && boff != bp->b_bufsize) {
2734                 KKASSERT(boff < bp->b_bufsize);
2735                 bzero((char *)bp->b_data + boff, bp->b_bufsize - boff);
2736                 /* boff = bp->b_bufsize; */
2737         }
2738         bp->b_resid = 0;
2739         bp->b_error = error;
2740         if (error)
2741                 bp->b_flags |= B_ERROR;
2742         biodone(ap->a_bio);
2743
2744 done:
2745         /*
2746          * Cache the b-tree node for the last data read in cache[1].
2747          *
2748          * If we hit the file EOF then also cache the node in the
2749          * governing director's cache[3], it will be used to initialize
2750          * the inode's cache[1] for any inodes looked up via the directory.
2751          *
2752          * This doesn't reduce disk accesses since the B-Tree chain is
2753          * likely cached, but it does reduce cpu overhead when looking
2754          * up file offsets for cpdup/tar/cpio style iterations.
2755          */
2756         if (cursor.node)
2757                 hammer_cache_node(&ip->cache[1], cursor.node);
2758         if (ran_end >= ip->ino_data.size) {
2759                 dip = hammer_find_inode(&trans, ip->ino_data.parent_obj_id,
2760                                         ip->obj_asof, ip->obj_localization);
2761                 if (dip) {
2762                         hammer_cache_node(&dip->cache[3], cursor.node);
2763                         hammer_rel_inode(dip, 0);
2764                 }
2765         }
2766         hammer_done_cursor(&cursor);
2767         hammer_done_transaction(&trans);
2768         return(error);
2769 }
2770
2771 /*
2772  * BMAP operation - used to support cluster_read() only.
2773  *
2774  * (struct vnode *vp, off_t loffset, off_t *doffsetp, int *runp, int *runb)
2775  *
2776  * This routine may return EOPNOTSUPP if the opration is not supported for
2777  * the specified offset.  The contents of the pointer arguments do not
2778  * need to be initialized in that case.
2779  *
2780  * If a disk address is available and properly aligned return 0 with
2781  * *doffsetp set to the zone-2 address, and *runp / *runb set appropriately
2782  * to the run-length relative to that offset.  Callers may assume that
2783  * *doffsetp is valid if 0 is returned, even if *runp is not sufficiently
2784  * large, so return EOPNOTSUPP if it is not sufficiently large.
2785  */
2786 static
2787 int
2788 hammer_vop_bmap(struct vop_bmap_args *ap)
2789 {
2790         struct hammer_transaction trans;
2791         struct hammer_inode *ip;
2792         struct hammer_cursor cursor;
2793         hammer_base_elm_t base;
2794         int64_t rec_offset;
2795         int64_t ran_end;
2796         int64_t tmp64;
2797         int64_t base_offset;
2798         int64_t base_disk_offset;
2799         int64_t last_offset;
2800         hammer_off_t last_disk_offset;
2801         hammer_off_t disk_offset;
2802         int     rec_len;
2803         int     error;
2804         int     blksize;
2805
2806         ++hammer_stats_file_iopsr;
2807         ip = ap->a_vp->v_data;
2808
2809         /*
2810          * We can only BMAP regular files.  We can't BMAP database files,
2811          * directories, etc.
2812          */
2813         if (ip->ino_data.obj_type != HAMMER_OBJTYPE_REGFILE)
2814                 return(EOPNOTSUPP);
2815
2816         /*
2817          * bmap is typically called with runp/runb both NULL when used
2818          * for writing.  We do not support BMAP for writing atm.
2819          */
2820         if (ap->a_cmd != BUF_CMD_READ)
2821                 return(EOPNOTSUPP);
2822
2823         /*
2824          * Scan the B-Tree to acquire blockmap addresses, then translate
2825          * to raw addresses.
2826          */
2827         hammer_simple_transaction(&trans, ip->hmp);
2828 #if 0
2829         kprintf("bmap_beg %016llx ip->cache %p\n",
2830                 (long long)ap->a_loffset, ip->cache[1]);
2831 #endif
2832         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
2833
2834         /*
2835          * Key range (begin and end inclusive) to scan.  Note that the key's
2836          * stored in the actual records represent BASE+LEN, not BASE.  The
2837          * first record containing bio_offset will have a key > bio_offset.
2838          */
2839         cursor.key_beg.localization = ip->obj_localization +
2840                                       HAMMER_LOCALIZE_MISC;
2841         cursor.key_beg.obj_id = ip->obj_id;
2842         cursor.key_beg.create_tid = 0;
2843         cursor.key_beg.delete_tid = 0;
2844         cursor.key_beg.obj_type = 0;
2845         if (ap->a_runb)
2846                 cursor.key_beg.key = ap->a_loffset - MAXPHYS + 1;
2847         else
2848                 cursor.key_beg.key = ap->a_loffset + 1;
2849         if (cursor.key_beg.key < 0)
2850                 cursor.key_beg.key = 0;
2851         cursor.asof = ip->obj_asof;
2852         cursor.flags |= HAMMER_CURSOR_ASOF;
2853
2854         cursor.key_end = cursor.key_beg;
2855         KKASSERT(ip->ino_data.obj_type == HAMMER_OBJTYPE_REGFILE);
2856
2857         ran_end = ap->a_loffset + MAXPHYS;
2858         cursor.key_beg.rec_type = HAMMER_RECTYPE_DATA;
2859         cursor.key_end.rec_type = HAMMER_RECTYPE_DATA;
2860         tmp64 = ran_end + MAXPHYS + 1;  /* work-around GCC-4 bug */
2861         if (tmp64 < ran_end)
2862                 cursor.key_end.key = 0x7FFFFFFFFFFFFFFFLL;
2863         else
2864                 cursor.key_end.key = ran_end + MAXPHYS + 1;
2865
2866         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE;
2867
2868         error = hammer_ip_first(&cursor);
2869         base_offset = last_offset = 0;
2870         base_disk_offset = last_disk_offset = 0;
2871
2872         while (error == 0) {
2873                 /*
2874                  * Get the base file offset of the record.  The key for
2875                  * data records is (base + bytes) rather then (base).
2876                  *
2877                  * NOTE: rec_offset + rec_len may exceed the end-of-file.
2878                  * The extra bytes should be zero on-disk and the BMAP op
2879                  * should still be ok.
2880                  */
2881                 base = &cursor.leaf->base;
2882                 rec_offset = base->key - cursor.leaf->data_len;
2883                 rec_len    = cursor.leaf->data_len;
2884
2885                 /*
2886                  * Incorporate any cached truncation.
2887                  *
2888                  * NOTE: Modifications to rec_len based on synthesized
2889                  * truncation points remove the guarantee that any extended
2890                  * data on disk is zero (since the truncations may not have
2891                  * taken place on-media yet).
2892                  */
2893                 if (ip->flags & HAMMER_INODE_TRUNCATED) {
2894                         if (hammer_cursor_ondisk(&cursor) ||
2895                             cursor.iprec->flush_state == HAMMER_FST_FLUSH) {
2896                                 if (ip->trunc_off <= rec_offset)
2897                                         rec_len = 0;
2898                                 else if (ip->trunc_off < rec_offset + rec_len)
2899                                         rec_len = (int)(ip->trunc_off - rec_offset);
2900                         }
2901                 }
2902                 if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
2903                         if (hammer_cursor_ondisk(&cursor)) {
2904                                 if (ip->sync_trunc_off <= rec_offset)
2905                                         rec_len = 0;
2906                                 else if (ip->sync_trunc_off < rec_offset + rec_len)
2907                                         rec_len = (int)(ip->sync_trunc_off - rec_offset);
2908                         }
2909                 }
2910
2911                 /*
2912                  * Accumulate information.  If we have hit a discontiguous
2913                  * block reset base_offset unless we are already beyond the
2914                  * requested offset.  If we are, that's it, we stop.
2915                  */
2916                 if (error)
2917                         break;
2918                 if (hammer_cursor_ondisk(&cursor)) {
2919                         disk_offset = cursor.leaf->data_offset;
2920                         if (rec_offset != last_offset ||
2921                             disk_offset != last_disk_offset) {
2922                                 if (rec_offset > ap->a_loffset)
2923                                         break;
2924                                 base_offset = rec_offset;
2925                                 base_disk_offset = disk_offset;
2926                         }
2927                         last_offset = rec_offset + rec_len;
2928                         last_disk_offset = disk_offset + rec_len;
2929                 }
2930                 error = hammer_ip_next(&cursor);
2931         }
2932
2933 #if 0
2934         kprintf("BMAP %016llx:  %016llx - %016llx\n",
2935                 (long long)ap->a_loffset,
2936                 (long long)base_offset,
2937                 (long long)last_offset);
2938         kprintf("BMAP %16s:  %016llx - %016llx\n", "",
2939                 (long long)base_disk_offset,
2940                 (long long)last_disk_offset);
2941 #endif
2942
2943         if (cursor.node) {
2944                 hammer_cache_node(&ip->cache[1], cursor.node);
2945 #if 0
2946                 kprintf("bmap_end2 %016llx ip->cache %p\n",
2947                         (long long)ap->a_loffset, ip->cache[1]);
2948 #endif
2949         }
2950         hammer_done_cursor(&cursor);
2951         hammer_done_transaction(&trans);
2952
2953         /*
2954          * If we couldn't find any records or the records we did find were
2955          * all behind the requested offset, return failure.  A forward
2956          * truncation can leave a hole w/ no on-disk records.
2957          */
2958         if (last_offset == 0 || last_offset < ap->a_loffset)
2959                 return (EOPNOTSUPP);
2960
2961         /*
2962          * Figure out the block size at the requested offset and adjust
2963          * our limits so the cluster_read() does not create inappropriately
2964          * sized buffer cache buffers.
2965          */
2966         blksize = hammer_blocksize(ap->a_loffset);
2967         if (hammer_blocksize(base_offset) != blksize) {
2968                 base_offset = hammer_blockdemarc(base_offset, ap->a_loffset);
2969         }
2970         if (last_offset != ap->a_loffset &&
2971             hammer_blocksize(last_offset - 1) != blksize) {
2972                 last_offset = hammer_blockdemarc(ap->a_loffset,
2973                                                  last_offset - 1);
2974         }
2975
2976         /*
2977          * Returning EOPNOTSUPP simply prevents the direct-IO optimization
2978          * from occuring.
2979          */
2980         disk_offset = base_disk_offset + (ap->a_loffset - base_offset);
2981
2982         if ((disk_offset & HAMMER_OFF_ZONE_MASK) != HAMMER_ZONE_LARGE_DATA) {
2983                 /*
2984                  * Only large-data zones can be direct-IOd
2985                  */
2986                 error = EOPNOTSUPP;
2987         } else if ((disk_offset & HAMMER_BUFMASK) ||
2988                    (last_offset - ap->a_loffset) < blksize) {
2989                 /*
2990                  * doffsetp is not aligned or the forward run size does
2991                  * not cover a whole buffer, disallow the direct I/O.
2992                  */
2993                 error = EOPNOTSUPP;
2994         } else {
2995                 /*
2996                  * We're good.
2997                  */
2998                 *ap->a_doffsetp = disk_offset;
2999                 if (ap->a_runb) {
3000                         *ap->a_runb = ap->a_loffset - base_offset;
3001                         KKASSERT(*ap->a_runb >= 0);
3002                 }
3003                 if (ap->a_runp) {
3004                         *ap->a_runp = last_offset - ap->a_loffset;
3005                         KKASSERT(*ap->a_runp >= 0);
3006                 }
3007                 error = 0;
3008         }
3009         return(error);
3010 }
3011
3012 /*
3013  * Write to a regular file.   Because this is a strategy call the OS is
3014  * trying to actually get data onto the media.
3015  */
3016 static
3017 int
3018 hammer_vop_strategy_write(struct vop_strategy_args *ap)
3019 {
3020         hammer_record_t record;
3021         hammer_mount_t hmp;
3022         hammer_inode_t ip;
3023         struct bio *bio;
3024         struct buf *bp;
3025         int blksize;
3026         int bytes;
3027         int error;
3028
3029         bio = ap->a_bio;
3030         bp = bio->bio_buf;
3031         ip = ap->a_vp->v_data;
3032         hmp = ip->hmp;
3033
3034         blksize = hammer_blocksize(bio->bio_offset);
3035         KKASSERT(bp->b_bufsize == blksize);
3036
3037         if (ip->flags & HAMMER_INODE_RO) {
3038                 bp->b_error = EROFS;
3039                 bp->b_flags |= B_ERROR;
3040                 biodone(ap->a_bio);
3041                 return(EROFS);
3042         }
3043
3044         /*
3045          * Interlock with inode destruction (no in-kernel or directory
3046          * topology visibility).  If we queue new IO while trying to
3047          * destroy the inode we can deadlock the vtrunc call in
3048          * hammer_inode_unloadable_check().
3049          *
3050          * Besides, there's no point flushing a bp associated with an
3051          * inode that is being destroyed on-media and has no kernel
3052          * references.
3053          */
3054         if ((ip->flags | ip->sync_flags) &
3055             (HAMMER_INODE_DELETING|HAMMER_INODE_DELETED)) {
3056                 bp->b_resid = 0;
3057                 biodone(ap->a_bio);
3058                 return(0);
3059         }
3060
3061         /*
3062          * Reserve space and issue a direct-write from the front-end.
3063          * NOTE: The direct_io code will hammer_bread/bcopy smaller
3064          * allocations.
3065          *
3066          * An in-memory record will be installed to reference the storage
3067          * until the flusher can get to it.
3068          *
3069          * Since we own the high level bio the front-end will not try to
3070          * do a direct-read until the write completes.
3071          *
3072          * NOTE: The only time we do not reserve a full-sized buffers
3073          * worth of data is if the file is small.  We do not try to
3074          * allocate a fragment (from the small-data zone) at the end of
3075          * an otherwise large file as this can lead to wildly separated
3076          * data.
3077          */
3078         KKASSERT((bio->bio_offset & HAMMER_BUFMASK) == 0);
3079         KKASSERT(bio->bio_offset < ip->ino_data.size);
3080         if (bio->bio_offset || ip->ino_data.size > HAMMER_BUFSIZE / 2)
3081                 bytes = bp->b_bufsize;
3082         else
3083                 bytes = ((int)ip->ino_data.size + 15) & ~15;
3084
3085         record = hammer_ip_add_bulk(ip, bio->bio_offset, bp->b_data,
3086                                     bytes, &error);
3087
3088         /*
3089          * B_VFSFLAG1 indicates that a REDO_WRITE entry was generated
3090          * in hammer_vop_write().  We must flag the record so the proper
3091          * REDO_TERM_WRITE entry is generated during the flush.
3092          */
3093         if (record) {
3094                 if (bp->b_flags & B_VFSFLAG1) {
3095                         record->flags |= HAMMER_RECF_REDO;
3096                         bp->b_flags &= ~B_VFSFLAG1;
3097                 }
3098                 hammer_io_direct_write(hmp, bio, record);
3099                 if (ip->rsv_recs > 1 && hmp->rsv_recs > hammer_limit_recs)
3100                         hammer_flush_inode(ip, 0);
3101         } else {
3102                 bp->b_bio2.bio_offset = NOOFFSET;
3103                 bp->b_error = error;
3104                 bp->b_flags |= B_ERROR;
3105                 biodone(ap->a_bio);
3106         }
3107         return(error);
3108 }
3109
3110 /*
3111  * dounlink - disconnect a directory entry
3112  *
3113  * XXX whiteout support not really in yet
3114  */
3115 static int
3116 hammer_dounlink(hammer_transaction_t trans, struct nchandle *nch,
3117                 struct vnode *dvp, struct ucred *cred,
3118                 int flags, int isdir)
3119 {
3120         struct namecache *ncp;
3121         hammer_inode_t dip;
3122         hammer_inode_t ip;
3123         struct hammer_cursor cursor;
3124         int64_t namekey;
3125         u_int32_t max_iterations;
3126         int nlen, error;
3127
3128         /*
3129          * Calculate the namekey and setup the key range for the scan.  This
3130          * works kinda like a chained hash table where the lower 32 bits
3131          * of the namekey synthesize the chain.
3132          *
3133          * The key range is inclusive of both key_beg and key_end.
3134          */
3135         dip = VTOI(dvp);
3136         ncp = nch->ncp;
3137
3138         if (dip->flags & HAMMER_INODE_RO)
3139                 return (EROFS);
3140
3141         namekey = hammer_directory_namekey(dip, ncp->nc_name, ncp->nc_nlen,
3142                                            &max_iterations);
3143 retry:
3144         hammer_init_cursor(trans, &cursor, &dip->cache[1], dip);
3145         cursor.key_beg.localization = dip->obj_localization +
3146                                       hammer_dir_localization(dip);
3147         cursor.key_beg.obj_id = dip->obj_id;
3148         cursor.key_beg.key = namekey;
3149         cursor.key_beg.create_tid = 0;
3150         cursor.key_beg.delete_tid = 0;
3151         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
3152         cursor.key_beg.obj_type = 0;
3153
3154         cursor.key_end = cursor.key_beg;
3155         cursor.key_end.key += max_iterations;
3156         cursor.asof = dip->obj_asof;
3157         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
3158
3159         /*
3160          * Scan all matching records (the chain), locate the one matching
3161          * the requested path component.  info->last_error contains the
3162          * error code on search termination and could be 0, ENOENT, or
3163          * something else.
3164          *
3165          * The hammer_ip_*() functions merge in-memory records with on-disk
3166          * records for the purposes of the search.
3167          */
3168         error = hammer_ip_first(&cursor);
3169
3170         while (error == 0) {
3171                 error = hammer_ip_resolve_data(&cursor);
3172                 if (error)
3173                         break;
3174                 nlen = cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF;
3175                 KKASSERT(nlen > 0);
3176                 if (ncp->nc_nlen == nlen &&
3177                     bcmp(ncp->nc_name, cursor.data->entry.name, nlen) == 0) {
3178                         break;
3179                 }
3180                 error = hammer_ip_next(&cursor);
3181         }
3182
3183         /*
3184          * If all is ok we have to get the inode so we can adjust nlinks.
3185          * To avoid a deadlock with the flusher we must release the inode
3186          * lock on the directory when acquiring the inode for the entry.
3187          *
3188          * If the target is a directory, it must be empty.
3189          */
3190         if (error == 0) {
3191                 hammer_unlock(&cursor.ip->lock);
3192                 ip = hammer_get_inode(trans, dip, cursor.data->entry.obj_id,
3193                                       dip->hmp->asof,
3194                                       cursor.data->entry.localization,
3195                                       0, &error);
3196                 hammer_lock_sh(&cursor.ip->lock);
3197                 if (error == ENOENT) {
3198                         kprintf("HAMMER: WARNING: Removing "
3199                                 "dirent w/missing inode \"%s\"\n"
3200                                 "\tobj_id = %016llx\n",
3201                                 ncp->nc_name,
3202                                 (long long)cursor.data->entry.obj_id);
3203                         error = 0;
3204                 }
3205
3206                 /*
3207                  * If isdir >= 0 we validate that the entry is or is not a
3208                  * directory.  If isdir < 0 we don't care.
3209                  */
3210                 if (error == 0 && isdir >= 0 && ip) {
3211                         if (isdir &&
3212                             ip->ino_data.obj_type != HAMMER_OBJTYPE_DIRECTORY) {
3213                                 error = ENOTDIR;
3214                         } else if (isdir == 0 &&
3215                             ip->ino_data.obj_type == HAMMER_OBJTYPE_DIRECTORY) {
3216                                 error = EISDIR;
3217                         }
3218                 }
3219
3220                 /*
3221                  * If we are trying to remove a directory the directory must
3222                  * be empty.
3223                  *
3224                  * The check directory code can loop and deadlock/retry.  Our
3225                  * own cursor's node locks must be released to avoid a 3-way
3226                  * deadlock with the flusher if the check directory code
3227                  * blocks.
3228                  *
3229                  * If any changes whatsoever have been made to the cursor
3230                  * set EDEADLK and retry.
3231                  *
3232                  * WARNING: See warnings in hammer_unlock_cursor()
3233                  *          function.
3234                  */
3235                 if (error == 0 && ip && ip->ino_data.obj_type ==
3236                                         HAMMER_OBJTYPE_DIRECTORY) {
3237                         hammer_unlock_cursor(&cursor);
3238                         error = hammer_ip_check_directory_empty(trans, ip);
3239                         hammer_lock_cursor(&cursor);
3240                         if (cursor.flags & HAMMER_CURSOR_RETEST) {
3241                                 kprintf("HAMMER: Warning: avoided deadlock "
3242                                         "on rmdir '%s'\n",
3243                                         ncp->nc_name);
3244                                 error = EDEADLK;
3245                         }
3246                 }
3247
3248                 /*
3249                  * Delete the directory entry.
3250                  *
3251                  * WARNING: hammer_ip_del_directory() may have to terminate
3252                  * the cursor to avoid a deadlock.  It is ok to call
3253                  * hammer_done_cursor() twice.
3254                  */
3255                 if (error == 0) {
3256                         error = hammer_ip_del_directory(trans, &cursor,
3257                                                         dip, ip);
3258                 }
3259                 hammer_done_cursor(&cursor);
3260                 if (error == 0) {
3261                         cache_setunresolved(nch);
3262                         cache_setvp(nch, NULL);
3263
3264                         /*
3265                          * XXX locking.  Note: ip->vp might get ripped out
3266                          * when we setunresolved() the nch since we had
3267                          * no other reference to it.  In that case ip->vp
3268                          * will be NULL.
3269                          */
3270                         if (ip && ip->vp) {
3271                                 hammer_knote(ip->vp, NOTE_DELETE);
3272                                 cache_inval_vp(ip->vp, CINV_DESTROY);
3273                         }
3274                 }
3275                 if (ip)
3276                         hammer_rel_inode(ip, 0);
3277         } else {
3278                 hammer_done_cursor(&cursor);
3279         }
3280         if (error == EDEADLK)
3281                 goto retry;
3282
3283         return (error);
3284 }
3285
3286 /************************************************************************
3287  *                          FIFO AND SPECFS OPS                         *
3288  ************************************************************************
3289  *
3290  */
3291
3292 static int
3293 hammer_vop_fifoclose (struct vop_close_args *ap)
3294 {
3295         /* XXX update itimes */
3296         return (VOCALL(&fifo_vnode_vops, &ap->a_head));
3297 }
3298
3299 static int
3300 hammer_vop_fiforead (struct vop_read_args *ap)
3301 {
3302         int error;
3303
3304         error = VOCALL(&fifo_vnode_vops, &ap->a_head);
3305         /* XXX update access time */
3306         return (error);
3307 }
3308
3309 static int
3310 hammer_vop_fifowrite (struct vop_write_args *ap)
3311 {
3312         int error;
3313
3314         error = VOCALL(&fifo_vnode_vops, &ap->a_head);
3315         /* XXX update access time */
3316         return (error);
3317 }
3318
3319 static
3320 int
3321 hammer_vop_fifokqfilter(struct vop_kqfilter_args *ap)
3322 {
3323         int error;
3324
3325         error = VOCALL(&fifo_vnode_vops, &ap->a_head);
3326         if (error)
3327                 error = hammer_vop_kqfilter(ap);
3328         return(error);
3329 }
3330
3331 /************************************************************************
3332  *                          KQFILTER OPS                                *
3333  ************************************************************************
3334  *
3335  */
3336 static void filt_hammerdetach(struct knote *kn);
3337 static int filt_hammerread(struct knote *kn, long hint);
3338 static int filt_hammerwrite(struct knote *kn, long hint);
3339 static int filt_hammervnode(struct knote *kn, long hint);
3340
3341 static struct filterops hammerread_filtops =
3342         { 1, NULL, filt_hammerdetach, filt_hammerread };
3343 static struct filterops hammerwrite_filtops =
3344         { 1, NULL, filt_hammerdetach, filt_hammerwrite };
3345 static struct filterops hammervnode_filtops =
3346         { 1, NULL, filt_hammerdetach, filt_hammervnode };
3347
3348 static
3349 int
3350 hammer_vop_kqfilter(struct vop_kqfilter_args *ap)
3351 {
3352         struct vnode *vp = ap->a_vp;
3353         struct knote *kn = ap->a_kn;
3354         lwkt_tokref vlock;
3355
3356         switch (kn->kn_filter) {
3357         case EVFILT_READ:
3358                 kn->kn_fop = &hammerread_filtops;
3359                 break;
3360         case EVFILT_WRITE:
3361                 kn->kn_fop = &hammerwrite_filtops;
3362                 break;
3363         case EVFILT_VNODE:
3364                 kn->kn_fop = &hammervnode_filtops;
3365                 break;
3366         default:
3367                 return (1);
3368         }
3369
3370         kn->kn_hook = (caddr_t)vp;
3371
3372         lwkt_gettoken(&vlock, &vp->v_token);
3373         SLIST_INSERT_HEAD(&vp->v_pollinfo.vpi_selinfo.si_note, kn, kn_selnext);
3374         lwkt_reltoken(&vlock);
3375
3376         return(0);
3377 }
3378
3379 static void
3380 filt_hammerdetach(struct knote *kn)
3381 {
3382         struct vnode *vp = (void *)kn->kn_hook;
3383         lwkt_tokref vlock;
3384
3385         lwkt_gettoken(&vlock, &vp->v_token);
3386         SLIST_REMOVE(&vp->v_pollinfo.vpi_selinfo.si_note,
3387                      kn, knote, kn_selnext);
3388         lwkt_reltoken(&vlock);
3389 }
3390
3391 static int
3392 filt_hammerread(struct knote *kn, long hint)
3393 {
3394         struct vnode *vp = (void *)kn->kn_hook;
3395         hammer_inode_t ip = VTOI(vp);
3396
3397         if (hint == NOTE_REVOKE) {
3398                 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
3399                 return(1);
3400         }
3401         kn->kn_data = ip->ino_data.size - kn->kn_fp->f_offset;
3402         return (kn->kn_data != 0);
3403 }
3404
3405 static int
3406 filt_hammerwrite(struct knote *kn, long hint)
3407 {
3408         if (hint == NOTE_REVOKE)
3409                 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
3410         kn->kn_data = 0;
3411         return (1);
3412 }
3413
3414 static int
3415 filt_hammervnode(struct knote *kn, long hint)
3416 {
3417         if (kn->kn_sfflags & hint)
3418                 kn->kn_fflags |= hint;
3419         if (hint == NOTE_REVOKE) {
3420                 kn->kn_flags |= EV_EOF;
3421                 return (1);
3422         }
3423         return (kn->kn_fflags != 0);
3424 }
3425