sys/vfs/hammer/hammer_vnops.c

   1 /*
   2  * Copyright (c) 2007-2008 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/kernel.h>
  38 #include <sys/fcntl.h>
  39 #include <sys/namecache.h>
  40 #include <sys/vnode.h>
  41 #include <sys/lockf.h>
  42 #include <sys/event.h>
  43 #include <sys/stat.h>
  44 #include <sys/dirent.h>
  45 #include <sys/file.h>
  46 #include <vm/vm_extern.h>
  47 #include <vm/swap_pager.h>
  48 #include <vfs/fifofs/fifo.h>
  49
  50 #include "hammer.h"
  51
  52 /*
  53  * USERFS VNOPS
  54  */
  55 /*static int hammer_vop_vnoperate(struct vop_generic_args *);*/
  56 static int hammer_vop_fsync(struct vop_fsync_args *);
  57 static int hammer_vop_read(struct vop_read_args *);
  58 static int hammer_vop_write(struct vop_write_args *);
  59 static int hammer_vop_access(struct vop_access_args *);
  60 static int hammer_vop_advlock(struct vop_advlock_args *);
  61 static int hammer_vop_close(struct vop_close_args *);
  62 static int hammer_vop_ncreate(struct vop_ncreate_args *);
  63 static int hammer_vop_getattr(struct vop_getattr_args *);
  64 static int hammer_vop_nresolve(struct vop_nresolve_args *);
  65 static int hammer_vop_nlookupdotdot(struct vop_nlookupdotdot_args *);
  66 static int hammer_vop_nlink(struct vop_nlink_args *);
  67 static int hammer_vop_nmkdir(struct vop_nmkdir_args *);
  68 static int hammer_vop_nmknod(struct vop_nmknod_args *);
  69 static int hammer_vop_open(struct vop_open_args *);
  70 static int hammer_vop_print(struct vop_print_args *);
  71 static int hammer_vop_readdir(struct vop_readdir_args *);
  72 static int hammer_vop_readlink(struct vop_readlink_args *);
  73 static int hammer_vop_nremove(struct vop_nremove_args *);
  74 static int hammer_vop_nrename(struct vop_nrename_args *);
  75 static int hammer_vop_nrmdir(struct vop_nrmdir_args *);
  76 static int hammer_vop_markatime(struct vop_markatime_args *);
  77 static int hammer_vop_setattr(struct vop_setattr_args *);
  78 static int hammer_vop_strategy(struct vop_strategy_args *);
  79 static int hammer_vop_bmap(struct vop_bmap_args *ap);
  80 static int hammer_vop_nsymlink(struct vop_nsymlink_args *);
  81 static int hammer_vop_nwhiteout(struct vop_nwhiteout_args *);
  82 static int hammer_vop_ioctl(struct vop_ioctl_args *);
  83 static int hammer_vop_mountctl(struct vop_mountctl_args *);
  84 static int hammer_vop_kqfilter (struct vop_kqfilter_args *);
  85
  86 static int hammer_vop_fifoclose (struct vop_close_args *);
  87 static int hammer_vop_fiforead (struct vop_read_args *);
  88 static int hammer_vop_fifowrite (struct vop_write_args *);
  89 static int hammer_vop_fifokqfilter (struct vop_kqfilter_args *);
  90
  91 struct vop_ops hammer_vnode_vops = {
  92         .vop_default =          vop_defaultop,
  93         .vop_fsync =            hammer_vop_fsync,
  94         .vop_getpages =         vop_stdgetpages,
  95         .vop_putpages =         vop_stdputpages,
  96         .vop_read =             hammer_vop_read,
  97         .vop_write =            hammer_vop_write,
  98         .vop_access =           hammer_vop_access,
  99         .vop_advlock =          hammer_vop_advlock,
 100         .vop_close =            hammer_vop_close,
 101         .vop_ncreate =          hammer_vop_ncreate,
 102         .vop_getattr =          hammer_vop_getattr,
 103         .vop_inactive =         hammer_vop_inactive,
 104         .vop_reclaim =          hammer_vop_reclaim,
 105         .vop_nresolve =         hammer_vop_nresolve,
 106         .vop_nlookupdotdot =    hammer_vop_nlookupdotdot,
 107         .vop_nlink =            hammer_vop_nlink,
 108         .vop_nmkdir =           hammer_vop_nmkdir,
 109         .vop_nmknod =           hammer_vop_nmknod,
 110         .vop_open =             hammer_vop_open,
 111         .vop_pathconf =         vop_stdpathconf,
 112         .vop_print =            hammer_vop_print,
 113         .vop_readdir =          hammer_vop_readdir,
 114         .vop_readlink =         hammer_vop_readlink,
 115         .vop_nremove =          hammer_vop_nremove,
 116         .vop_nrename =          hammer_vop_nrename,
 117         .vop_nrmdir =           hammer_vop_nrmdir,
 118         .vop_markatime =        hammer_vop_markatime,
 119         .vop_setattr =          hammer_vop_setattr,
 120         .vop_bmap =             hammer_vop_bmap,
 121         .vop_strategy =         hammer_vop_strategy,
 122         .vop_nsymlink =         hammer_vop_nsymlink,
 123         .vop_nwhiteout =        hammer_vop_nwhiteout,
 124         .vop_ioctl =            hammer_vop_ioctl,
 125         .vop_mountctl =         hammer_vop_mountctl,
 126         .vop_kqfilter =         hammer_vop_kqfilter
 127 };
 128
 129 struct vop_ops hammer_spec_vops = {
 130         .vop_default =          vop_defaultop,
 131         .vop_fsync =            hammer_vop_fsync,
 132         .vop_read =             vop_stdnoread,
 133         .vop_write =            vop_stdnowrite,
 134         .vop_access =           hammer_vop_access,
 135         .vop_close =            hammer_vop_close,
 136         .vop_markatime =        hammer_vop_markatime,
 137         .vop_getattr =          hammer_vop_getattr,
 138         .vop_inactive =         hammer_vop_inactive,
 139         .vop_reclaim =          hammer_vop_reclaim,
 140         .vop_setattr =          hammer_vop_setattr
 141 };
 142
 143 struct vop_ops hammer_fifo_vops = {
 144         .vop_default =          fifo_vnoperate,
 145         .vop_fsync =            hammer_vop_fsync,
 146         .vop_read =             hammer_vop_fiforead,
 147         .vop_write =            hammer_vop_fifowrite,
 148         .vop_access =           hammer_vop_access,
 149         .vop_close =            hammer_vop_fifoclose,
 150         .vop_markatime =        hammer_vop_markatime,
 151         .vop_getattr =          hammer_vop_getattr,
 152         .vop_inactive =         hammer_vop_inactive,
 153         .vop_reclaim =          hammer_vop_reclaim,
 154         .vop_setattr =          hammer_vop_setattr,
 155         .vop_kqfilter =         hammer_vop_fifokqfilter
 156 };
 157
 158 static __inline
 159 void
 160 hammer_knote(struct vnode *vp, int flags)
 161 {
 162         if (flags)
 163                 KNOTE(&vp->v_pollinfo.vpi_kqinfo.ki_note, flags);
 164 }
 165
 166 #ifdef DEBUG_TRUNCATE
 167 struct hammer_inode *HammerTruncIp;
 168 #endif
 169
 170 static int hammer_dounlink(hammer_transaction_t trans, struct nchandle *nch,
 171                            struct vnode *dvp, struct ucred *cred,
 172                            int flags, int isdir);
 173 static int hammer_vop_strategy_read(struct vop_strategy_args *ap);
 174 static int hammer_vop_strategy_write(struct vop_strategy_args *ap);
 175
 176 #if 0
 177 static
 178 int
 179 hammer_vop_vnoperate(struct vop_generic_args *)
 180 {
 181         return (VOCALL(&hammer_vnode_vops, ap));
 182 }
 183 #endif
 184
 185 /*
 186  * hammer_vop_fsync { vp, waitfor }
 187  *
 188  * fsync() an inode to disk and wait for it to be completely committed
 189  * such that the information would not be undone if a crash occured after
 190  * return.
 191  *
 192  * NOTE: HAMMER's fsync()'s are going to remain expensive until we implement
 193  *       a REDO log.  A sysctl is provided to relax HAMMER's fsync()
 194  *       operation.
 195  *
 196  *       Ultimately the combination of a REDO log and use of fast storage
 197  *       to front-end cluster caches will make fsync fast, but it aint
 198  *       here yet.  And, in anycase, we need real transactional
 199  *       all-or-nothing features which are not restricted to a single file.
 200  */
 201 static
 202 int
 203 hammer_vop_fsync(struct vop_fsync_args *ap)
 204 {
 205         hammer_inode_t ip = VTOI(ap->a_vp);
 206         hammer_mount_t hmp = ip->hmp;
 207         int waitfor = ap->a_waitfor;
 208         int mode;
 209
 210         lwkt_gettoken(&hmp->fs_token);
 211
 212         /*
 213          * Fsync rule relaxation (default is either full synchronous flush
 214          * or REDO semantics with synchronous flush).
 215          */
 216         if (ap->a_flags & VOP_FSYNC_SYSCALL) {
 217                 switch(hammer_fsync_mode) {
 218                 case 0:
 219 mode0:
 220                         /* no REDO, full synchronous flush */
 221                         goto skip;
 222                 case 1:
 223 mode1:
 224                         /* no REDO, full asynchronous flush */
 225                         if (waitfor == MNT_WAIT)
 226                                 waitfor = MNT_NOWAIT;
 227                         goto skip;
 228                 case 2:
 229                         /* REDO semantics, synchronous flush */
 230                         if (hmp->version < HAMMER_VOL_VERSION_FOUR)
 231                                 goto mode0;
 232                         mode = HAMMER_FLUSH_UNDOS_AUTO;
 233                         break;
 234                 case 3:
 235                         /* REDO semantics, relaxed asynchronous flush */
 236                         if (hmp->version < HAMMER_VOL_VERSION_FOUR)
 237                                 goto mode1;
 238                         mode = HAMMER_FLUSH_UNDOS_RELAXED;
 239                         if (waitfor == MNT_WAIT)
 240                                 waitfor = MNT_NOWAIT;
 241                         break;
 242                 case 4:
 243                         /* ignore the fsync() system call */
 244                         lwkt_reltoken(&hmp->fs_token);
 245                         return(0);
 246                 default:
 247                         /* we have to do something */
 248                         mode = HAMMER_FLUSH_UNDOS_RELAXED;
 249                         if (waitfor == MNT_WAIT)
 250                                 waitfor = MNT_NOWAIT;
 251                         break;
 252                 }
 253
 254                 /*
 255                  * Fast fsync only needs to flush the UNDO/REDO fifo if
 256                  * HAMMER_INODE_REDO is non-zero and the only modifications
 257                  * made to the file are write or write-extends.
 258                  */
 259                 if ((ip->flags & HAMMER_INODE_REDO) &&
 260                     (ip->flags & HAMMER_INODE_MODMASK_NOREDO) == 0
 261                 ) {
 262                         ++hammer_count_fsyncs;
 263                         hammer_flusher_flush_undos(hmp, mode);
 264                         ip->redo_count = 0;
 265                         if (ip->vp && (ip->flags & HAMMER_INODE_MODMASK) == 0)
 266                                 vclrisdirty(ip->vp);
 267                         lwkt_reltoken(&hmp->fs_token);
 268                         return(0);
 269                 }
 270
 271                 /*
 272                  * REDO is enabled by fsync(), the idea being we really only
 273                  * want to lay down REDO records when programs are using
 274                  * fsync() heavily.  The first fsync() on the file starts
 275                  * the gravy train going and later fsync()s keep it hot by
 276                  * resetting the redo_count.
 277                  *
 278                  * We weren't running REDOs before now so we have to fall
 279                  * through and do a full fsync of what we have.
 280                  */
 281                 if (hmp->version >= HAMMER_VOL_VERSION_FOUR &&
 282                     (hmp->flags & HAMMER_MOUNT_REDO_RECOVERY_RUN) == 0) {
 283                         ip->flags |= HAMMER_INODE_REDO;
 284                         ip->redo_count = 0;
 285                 }
 286         }
 287 skip:
 288
 289         /*
 290          * Do a full flush sequence.
 291          *
 292          * Attempt to release the vnode while waiting for the inode to
 293          * finish flushing.  This can really mess up inactive->reclaim
 294          * sequences so only do it if the vnode is active.
 295          *
 296          * WARNING! The VX lock functions must be used.  vn_lock() will
 297          *          fail when this is part of a VOP_RECLAIM sequence.
 298          */
 299         ++hammer_count_fsyncs;
 300         vfsync(ap->a_vp, waitfor, 1, NULL, NULL);
 301         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 302         if (waitfor == MNT_WAIT) {
 303                 int dorelock;
 304
 305                 if ((ap->a_vp->v_flag & VRECLAIMED) == 0) {
 306                         vx_unlock(ap->a_vp);
 307                         dorelock = 1;
 308                 } else {
 309                         dorelock = 0;
 310                 }
 311                 hammer_wait_inode(ip);
 312                 if (dorelock)
 313                         vx_lock(ap->a_vp);
 314         }
 315         if (ip->vp && (ip->flags & HAMMER_INODE_MODMASK) == 0)
 316                 vclrisdirty(ip->vp);
 317         lwkt_reltoken(&hmp->fs_token);
 318         return (ip->error);
 319 }
 320
 321 /*
 322  * hammer_vop_read { vp, uio, ioflag, cred }
 323  *
 324  * MPSAFE (for the cache safe does not require fs_token)
 325  */
 326 static
 327 int
 328 hammer_vop_read(struct vop_read_args *ap)
 329 {
 330         struct hammer_transaction trans;
 331         hammer_inode_t ip;
 332         hammer_mount_t hmp;
 333         off_t offset;
 334         struct buf *bp;
 335         struct uio *uio;
 336         int error;
 337         int n;
 338         int seqcount;
 339         int ioseqcount;
 340         int blksize;
 341         int bigread;
 342         int got_fstoken;
 343         size_t resid;
 344
 345         if (ap->a_vp->v_type != VREG)
 346                 return (EINVAL);
 347         ip = VTOI(ap->a_vp);
 348         hmp = ip->hmp;
 349         error = 0;
 350         got_fstoken = 0;
 351         uio = ap->a_uio;
 352
 353         /*
 354          * Attempt to shortcut directly to the VM object using lwbufs.
 355          * This is much faster than instantiating buffer cache buffers.
 356          */
 357         resid = uio->uio_resid;
 358         error = vop_helper_read_shortcut(ap);
 359         hammer_stats_file_read += resid - uio->uio_resid;
 360         if (error)
 361                 return (error);
 362         if (uio->uio_resid == 0)
 363                 goto finished;
 364
 365         /*
 366          * Allow the UIO's size to override the sequential heuristic.
 367          */
 368         blksize = hammer_blocksize(uio->uio_offset);
 369         seqcount = (uio->uio_resid + (BKVASIZE - 1)) / BKVASIZE;
 370         ioseqcount = (ap->a_ioflag >> 16);
 371         if (seqcount < ioseqcount)
 372                 seqcount = ioseqcount;
 373
 374         /*
 375          * If reading or writing a huge amount of data we have to break
 376          * atomicy and allow the operation to be interrupted by a signal
 377          * or it can DOS the machine.
 378          */
 379         bigread = (uio->uio_resid > 100 * 1024 * 1024);
 380
 381         /*
 382          * Access the data typically in HAMMER_BUFSIZE blocks via the
 383          * buffer cache, but HAMMER may use a variable block size based
 384          * on the offset.
 385          *
 386          * XXX Temporary hack, delay the start transaction while we remain
 387          *     MPSAFE.  NOTE: ino_data.size cannot change while vnode is
 388          *     locked-shared.
 389          */
 390         while (uio->uio_resid > 0 && uio->uio_offset < ip->ino_data.size) {
 391                 int64_t base_offset;
 392                 int64_t file_limit;
 393
 394                 blksize = hammer_blocksize(uio->uio_offset);
 395                 offset = (int)uio->uio_offset & (blksize - 1);
 396                 base_offset = uio->uio_offset - offset;
 397
 398                 if (bigread && (error = hammer_signal_check(ip->hmp)) != 0)
 399                         break;
 400
 401                 /*
 402                  * MPSAFE
 403                  */
 404                 bp = getblk(ap->a_vp, base_offset, blksize, 0, 0);
 405                 if ((bp->b_flags & (B_INVAL | B_CACHE | B_RAM)) == B_CACHE) {
 406                         bp->b_flags &= ~B_AGE;
 407                         error = 0;
 408                         goto skip;
 409                 }
 410                 if (ap->a_ioflag & IO_NRDELAY) {
 411                         bqrelse(bp);
 412                         return (EWOULDBLOCK);
 413                 }
 414
 415                 /*
 416                  * MPUNSAFE
 417                  */
 418                 if (got_fstoken == 0) {
 419                         lwkt_gettoken(&hmp->fs_token);
 420                         got_fstoken = 1;
 421                         hammer_start_transaction(&trans, ip->hmp);
 422                 }
 423
 424                 /*
 425                  * NOTE: A valid bp has already been acquired, but was not
 426                  *       B_CACHE.
 427                  */
 428                 if (hammer_cluster_enable) {
 429                         /*
 430                          * Use file_limit to prevent cluster_read() from
 431                          * creating buffers of the wrong block size past
 432                          * the demarc.
 433                          */
 434                         file_limit = ip->ino_data.size;
 435                         if (base_offset < HAMMER_XDEMARC &&
 436                             file_limit > HAMMER_XDEMARC) {
 437                                 file_limit = HAMMER_XDEMARC;
 438                         }
 439                         error = cluster_readx(ap->a_vp,
 440                                              file_limit, base_offset,
 441                                              blksize, uio->uio_resid,
 442                                              seqcount * BKVASIZE, &bp);
 443                 } else {
 444                         error = breadnx(ap->a_vp, base_offset, blksize,
 445                                         NULL, NULL, 0, &bp);
 446                 }
 447                 if (error) {
 448                         brelse(bp);
 449                         break;
 450                 }
 451 skip:
 452                 if ((hammer_debug_io & 0x0001) && (bp->b_flags & B_IODEBUG)) {
 453                         kprintf("doff %016jx read file %016jx@%016jx\n",
 454                                 (intmax_t)bp->b_bio2.bio_offset,
 455                                 (intmax_t)ip->obj_id,
 456                                 (intmax_t)bp->b_loffset);
 457                 }
 458                 bp->b_flags &= ~B_IODEBUG;
 459                 if (blksize == HAMMER_XBUFSIZE)
 460                         bp->b_flags |= B_CLUSTEROK;
 461
 462                 n = blksize - offset;
 463                 if (n > uio->uio_resid)
 464                         n = uio->uio_resid;
 465                 if (n > ip->ino_data.size - uio->uio_offset)
 466                         n = (int)(ip->ino_data.size - uio->uio_offset);
 467                 if (got_fstoken)
 468                         lwkt_reltoken(&hmp->fs_token);
 469
 470                 /*
 471                  * Set B_AGE, data has a lower priority than meta-data.
 472                  *
 473                  * Use a hold/unlock/drop sequence to run the uiomove
 474                  * with the buffer unlocked, avoiding deadlocks against
 475                  * read()s on mmap()'d spaces.
 476                  */
 477                 bp->b_flags |= B_AGE;
 478                 error = uiomovebp(bp, (char *)bp->b_data + offset, n, uio);
 479                 bqrelse(bp);
 480
 481                 if (got_fstoken)
 482                         lwkt_gettoken(&hmp->fs_token);
 483
 484                 if (error)
 485                         break;
 486                 hammer_stats_file_read += n;
 487         }
 488
 489 finished:
 490
 491         /*
 492          * Try to update the atime with just the inode lock for maximum
 493          * concurrency.  If we can't shortcut it we have to get the full
 494          * blown transaction.
 495          */
 496         if (got_fstoken == 0 && hammer_update_atime_quick(ip) < 0) {
 497                 lwkt_gettoken(&hmp->fs_token);
 498                 got_fstoken = 1;
 499                 hammer_start_transaction(&trans, ip->hmp);
 500         }
 501
 502         if (got_fstoken) {
 503                 if ((ip->flags & HAMMER_INODE_RO) == 0 &&
 504                     (ip->hmp->mp->mnt_flag & MNT_NOATIME) == 0) {
 505                         ip->ino_data.atime = trans.time;
 506                         hammer_modify_inode(&trans, ip, HAMMER_INODE_ATIME);
 507                 }
 508                 hammer_done_transaction(&trans);
 509                 lwkt_reltoken(&hmp->fs_token);
 510         }
 511         return (error);
 512 }
 513
 514 /*
 515  * hammer_vop_write { vp, uio, ioflag, cred }
 516  */
 517 static
 518 int
 519 hammer_vop_write(struct vop_write_args *ap)
 520 {
 521         struct hammer_transaction trans;
 522         struct hammer_inode *ip;
 523         hammer_mount_t hmp;
 524         thread_t td;
 525         struct uio *uio;
 526         int offset;
 527         off_t base_offset;
 528         int64_t cluster_eof;
 529         struct buf *bp;
 530         int kflags;
 531         int error;
 532         int n;
 533         int flags;
 534         int seqcount;
 535         int bigwrite;
 536
 537         if (ap->a_vp->v_type != VREG)
 538                 return (EINVAL);
 539         ip = VTOI(ap->a_vp);
 540         hmp = ip->hmp;
 541         error = 0;
 542         kflags = 0;
 543         seqcount = ap->a_ioflag >> 16;
 544
 545         if (ip->flags & HAMMER_INODE_RO)
 546                 return (EROFS);
 547
 548         /*
 549          * Create a transaction to cover the operations we perform.
 550          */
 551         lwkt_gettoken(&hmp->fs_token);
 552         hammer_start_transaction(&trans, hmp);
 553         uio = ap->a_uio;
 554
 555         /*
 556          * Check append mode
 557          */
 558         if (ap->a_ioflag & IO_APPEND)
 559                 uio->uio_offset = ip->ino_data.size;
 560
 561         /*
 562          * Check for illegal write offsets.  Valid range is 0...2^63-1.
 563          *
 564          * NOTE: the base_off assignment is required to work around what
 565          * I consider to be a GCC-4 optimization bug.
 566          */
 567         if (uio->uio_offset < 0) {
 568                 hammer_done_transaction(&trans);
 569                 lwkt_reltoken(&hmp->fs_token);
 570                 return (EFBIG);
 571         }
 572         base_offset = uio->uio_offset + uio->uio_resid; /* work around gcc-4 */
 573         if (uio->uio_resid > 0 && base_offset <= uio->uio_offset) {
 574                 hammer_done_transaction(&trans);
 575                 lwkt_reltoken(&hmp->fs_token);
 576                 return (EFBIG);
 577         }
 578
 579         if (uio->uio_resid > 0 && (td = uio->uio_td) != NULL && td->td_proc &&
 580             base_offset > td->td_proc->p_rlimit[RLIMIT_FSIZE].rlim_cur) {
 581                 hammer_done_transaction(&trans);
 582                 lwkt_reltoken(&hmp->fs_token);
 583                 lwpsignal(td->td_proc, td->td_lwp, SIGXFSZ);
 584                 return (EFBIG);
 585         }
 586
 587         /*
 588          * If reading or writing a huge amount of data we have to break
 589          * atomicy and allow the operation to be interrupted by a signal
 590          * or it can DOS the machine.
 591          *
 592          * Preset redo_count so we stop generating REDOs earlier if the
 593          * limit is exceeded.
 594          */
 595         bigwrite = (uio->uio_resid > 100 * 1024 * 1024);
 596         if ((ip->flags & HAMMER_INODE_REDO) &&
 597             ip->redo_count < hammer_limit_redo) {
 598                 ip->redo_count += uio->uio_resid;
 599         }
 600
 601         /*
 602          * Access the data typically in HAMMER_BUFSIZE blocks via the
 603          * buffer cache, but HAMMER may use a variable block size based
 604          * on the offset.
 605          */
 606         while (uio->uio_resid > 0) {
 607                 int fixsize = 0;
 608                 int blksize;
 609                 int blkmask;
 610                 int trivial;
 611                 int endofblk;
 612                 off_t nsize;
 613
 614                 if ((error = hammer_checkspace(hmp, HAMMER_CHKSPC_WRITE)) != 0)
 615                         break;
 616                 if (bigwrite && (error = hammer_signal_check(hmp)) != 0)
 617                         break;
 618
 619                 blksize = hammer_blocksize(uio->uio_offset);
 620
 621                 /*
 622                  * Do not allow HAMMER to blow out the buffer cache.  Very
 623                  * large UIOs can lockout other processes due to bwillwrite()
 624                  * mechanics.
 625                  *
 626                  * The hammer inode is not locked during these operations.
 627                  * The vnode is locked which can interfere with the pageout
 628                  * daemon for non-UIO_NOCOPY writes but should not interfere
 629                  * with the buffer cache.  Even so, we cannot afford to
 630                  * allow the pageout daemon to build up too many dirty buffer
 631                  * cache buffers.
 632                  *
 633                  * Only call this if we aren't being recursively called from
 634                  * a virtual disk device (vn), else we may deadlock.
 635                  */
 636                 if ((ap->a_ioflag & IO_RECURSE) == 0)
 637                         bwillwrite(blksize);
 638
 639                 /*
 640                  * Control the number of pending records associated with
 641                  * this inode.  If too many have accumulated start a
 642                  * flush.  Try to maintain a pipeline with the flusher.
 643                  *
 644                  * NOTE: It is possible for other sources to grow the
 645                  *       records but not necessarily issue another flush,
 646                  *       so use a timeout and ensure that a re-flush occurs.
 647                  */
 648                 if (ip->rsv_recs >= hammer_limit_inode_recs) {
 649                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 650                         while (ip->rsv_recs >= hammer_limit_inode_recs * 2) {
 651                                 ip->flags |= HAMMER_INODE_RECSW;
 652                                 tsleep(&ip->rsv_recs, 0, "hmrwww", hz);
 653                                 hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 654                         }
 655                 }
 656
 657 #if 0
 658                 /*
 659                  * Do not allow HAMMER to blow out system memory by
 660                  * accumulating too many records.   Records are so well
 661                  * decoupled from the buffer cache that it is possible
 662                  * for userland to push data out to the media via
 663                  * direct-write, but build up the records queued to the
 664                  * backend faster then the backend can flush them out.
 665                  * HAMMER has hit its write limit but the frontend has
 666                  * no pushback to slow it down.
 667                  */
 668                 if (hmp->rsv_recs > hammer_limit_recs / 2) {
 669                         /*
 670                          * Get the inode on the flush list
 671                          */
 672                         if (ip->rsv_recs >= 64)
 673                                 hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 674                         else if (ip->rsv_recs >= 16)
 675                                 hammer_flush_inode(ip, 0);
 676
 677                         /*
 678                          * Keep the flusher going if the system keeps
 679                          * queueing records.
 680                          */
 681                         delta = hmp->count_newrecords -
 682                                 hmp->last_newrecords;
 683                         if (delta < 0 || delta > hammer_limit_recs / 2) {
 684                                 hmp->last_newrecords = hmp->count_newrecords;
 685                                 hammer_sync_hmp(hmp, MNT_NOWAIT);
 686                         }
 687
 688                         /*
 689                          * If we have gotten behind start slowing
 690                          * down the writers.
 691                          */
 692                         delta = (hmp->rsv_recs - hammer_limit_recs) *
 693                                 hz / hammer_limit_recs;
 694                         if (delta > 0)
 695                                 tsleep(&trans, 0, "hmrslo", delta);
 696                 }
 697 #endif
 698
 699                 /*
 700                  * Calculate the blocksize at the current offset and figure
 701                  * out how much we can actually write.
 702                  */
 703                 blkmask = blksize - 1;
 704                 offset = (int)uio->uio_offset & blkmask;
 705                 base_offset = uio->uio_offset & ~(int64_t)blkmask;
 706                 n = blksize - offset;
 707                 if (n > uio->uio_resid) {
 708                         n = uio->uio_resid;
 709                         endofblk = 0;
 710                 } else {
 711                         endofblk = 1;
 712                 }
 713                 nsize = uio->uio_offset + n;
 714                 if (nsize > ip->ino_data.size) {
 715                         if (uio->uio_offset > ip->ino_data.size)
 716                                 trivial = 0;
 717                         else
 718                                 trivial = 1;
 719                         nvextendbuf(ap->a_vp,
 720                                     ip->ino_data.size,
 721                                     nsize,
 722                                     hammer_blocksize(ip->ino_data.size),
 723                                     hammer_blocksize(nsize),
 724                                     hammer_blockoff(ip->ino_data.size),
 725                                     hammer_blockoff(nsize),
 726                                     trivial);
 727                         fixsize = 1;
 728                         kflags |= NOTE_EXTEND;
 729                 }
 730
 731                 if (uio->uio_segflg == UIO_NOCOPY) {
 732                         /*
 733                          * Issuing a write with the same data backing the
 734                          * buffer.  Instantiate the buffer to collect the
 735                          * backing vm pages, then read-in any missing bits.
 736                          *
 737                          * This case is used by vop_stdputpages().
 738                          */
 739                         bp = getblk(ap->a_vp, base_offset,
 740                                     blksize, GETBLK_BHEAVY, 0);
 741                         if ((bp->b_flags & B_CACHE) == 0) {
 742                                 bqrelse(bp);
 743                                 error = bread(ap->a_vp, base_offset,
 744                                               blksize, &bp);
 745                         }
 746                 } else if (offset == 0 && uio->uio_resid >= blksize) {
 747                         /*
 748                          * Even though we are entirely overwriting the buffer
 749                          * we may still have to zero it out to avoid a
 750                          * mmap/write visibility issue.
 751                          */
 752                         bp = getblk(ap->a_vp, base_offset, blksize, GETBLK_BHEAVY, 0);
 753                         if ((bp->b_flags & B_CACHE) == 0)
 754                                 vfs_bio_clrbuf(bp);
 755                 } else if (base_offset >= ip->ino_data.size) {
 756                         /*
 757                          * If the base offset of the buffer is beyond the
 758                          * file EOF, we don't have to issue a read.
 759                          */
 760                         bp = getblk(ap->a_vp, base_offset,
 761                                     blksize, GETBLK_BHEAVY, 0);
 762                         vfs_bio_clrbuf(bp);
 763                 } else {
 764                         /*
 765                          * Partial overwrite, read in any missing bits then
 766                          * replace the portion being written.
 767                          */
 768                         error = bread(ap->a_vp, base_offset, blksize, &bp);
 769                         if (error == 0)
 770                                 bheavy(bp);
 771                 }
 772                 if (error == 0) {
 773                         lwkt_reltoken(&hmp->fs_token);
 774                         error = uiomovebp(bp, bp->b_data + offset, n, uio);
 775                         lwkt_gettoken(&hmp->fs_token);
 776                 }
 777
 778                 /*
 779                  * Generate REDO records if enabled and redo_count will not
 780                  * exceeded the limit.
 781                  *
 782                  * If redo_count exceeds the limit we stop generating records
 783                  * and clear HAMMER_INODE_REDO.  This will cause the next
 784                  * fsync() to do a full meta-data sync instead of just an
 785                  * UNDO/REDO fifo update.
 786                  *
 787                  * When clearing HAMMER_INODE_REDO any pre-existing REDOs
 788                  * will still be tracked.  The tracks will be terminated
 789                  * when the related meta-data (including possible data
 790                  * modifications which are not tracked via REDO) is
 791                  * flushed.
 792                  */
 793                 if ((ip->flags & HAMMER_INODE_REDO) && error == 0) {
 794                         if (ip->redo_count < hammer_limit_redo) {
 795                                 bp->b_flags |= B_VFSFLAG1;
 796                                 error = hammer_generate_redo(&trans, ip,
 797                                                      base_offset + offset,
 798                                                      HAMMER_REDO_WRITE,
 799                                                      bp->b_data + offset,
 800                                                      (size_t)n);
 801                         } else {
 802                                 ip->flags &= ~HAMMER_INODE_REDO;
 803                         }
 804                 }
 805
 806                 /*
 807                  * If we screwed up we have to undo any VM size changes we
 808                  * made.
 809                  */
 810                 if (error) {
 811                         brelse(bp);
 812                         if (fixsize) {
 813                                 nvtruncbuf(ap->a_vp, ip->ino_data.size,
 814                                           hammer_blocksize(ip->ino_data.size),
 815                                           hammer_blockoff(ip->ino_data.size),
 816                                           0);
 817                         }
 818                         break;
 819                 }
 820                 kflags |= NOTE_WRITE;
 821                 hammer_stats_file_write += n;
 822                 if (blksize == HAMMER_XBUFSIZE)
 823                         bp->b_flags |= B_CLUSTEROK;
 824                 if (ip->ino_data.size < uio->uio_offset) {
 825                         ip->ino_data.size = uio->uio_offset;
 826                         flags = HAMMER_INODE_SDIRTY;
 827                 } else {
 828                         flags = 0;
 829                 }
 830                 ip->ino_data.mtime = trans.time;
 831                 flags |= HAMMER_INODE_MTIME | HAMMER_INODE_BUFS;
 832                 hammer_modify_inode(&trans, ip, flags);
 833
 834                 /*
 835                  * Once we dirty the buffer any cached zone-X offset
 836                  * becomes invalid.  HAMMER NOTE: no-history mode cannot
 837                  * allow overwriting over the same data sector unless
 838                  * we provide UNDOs for the old data, which we don't.
 839                  */
 840                 bp->b_bio2.bio_offset = NOOFFSET;
 841
 842                 /*
 843                  * Final buffer disposition.
 844                  *
 845                  * Because meta-data updates are deferred, HAMMER is
 846                  * especially sensitive to excessive bdwrite()s because
 847                  * the I/O stream is not broken up by disk reads.  So the
 848                  * buffer cache simply cannot keep up.
 849                  *
 850                  * WARNING!  blksize is variable.  cluster_write() is
 851                  *           expected to not blow up if it encounters
 852                  *           buffers that do not match the passed blksize.
 853                  *
 854                  * NOTE!  Hammer shouldn't need to bawrite()/cluster_write().
 855                  *        The ip->rsv_recs check should burst-flush the data.
 856                  *        If we queue it immediately the buf could be left
 857                  *        locked on the device queue for a very long time.
 858                  *
 859                  *        However, failing to flush a dirty buffer out when
 860                  *        issued from the pageout daemon can result in a low
 861                  *        memory deadlock against bio_page_alloc(), so we
 862                  *        have to bawrite() on IO_ASYNC as well.
 863                  *
 864                  * NOTE!  To avoid degenerate stalls due to mismatched block
 865                  *        sizes we only honor IO_DIRECT on the write which
 866                  *        abuts the end of the buffer.  However, we must
 867                  *        honor IO_SYNC in case someone is silly enough to
 868                  *        configure a HAMMER file as swap, or when HAMMER
 869                  *        is serving NFS (for commits).  Ick ick.
 870                  */
 871                 bp->b_flags |= B_AGE;
 872                 if (blksize == HAMMER_XBUFSIZE)
 873                         bp->b_flags |= B_CLUSTEROK;
 874
 875                 if (ap->a_ioflag & IO_SYNC) {
 876                         bwrite(bp);
 877                 } else if ((ap->a_ioflag & IO_DIRECT) && endofblk) {
 878                         bawrite(bp);
 879                 } else if (ap->a_ioflag & IO_ASYNC) {
 880                         bawrite(bp);
 881                 } else if (hammer_cluster_enable &&
 882                            !(ap->a_vp->v_mount->mnt_flag & MNT_NOCLUSTERW)) {
 883                         if (base_offset < HAMMER_XDEMARC)
 884                                 cluster_eof = hammer_blockdemarc(base_offset,
 885                                                          ip->ino_data.size);
 886                         else
 887                                 cluster_eof = ip->ino_data.size;
 888                         cluster_write(bp, cluster_eof, blksize, seqcount);
 889                 } else {
 890                         bdwrite(bp);
 891                 }
 892         }
 893         hammer_done_transaction(&trans);
 894         hammer_knote(ap->a_vp, kflags);
 895         lwkt_reltoken(&hmp->fs_token);
 896         return (error);
 897 }
 898
 899 /*
 900  * hammer_vop_access { vp, mode, cred }
 901  *
 902  * MPSAFE - does not require fs_token
 903  */
 904 static
 905 int
 906 hammer_vop_access(struct vop_access_args *ap)
 907 {
 908         struct hammer_inode *ip = VTOI(ap->a_vp);
 909         uid_t uid;
 910         gid_t gid;
 911         int error;
 912
 913         ++hammer_stats_file_iopsr;
 914         uid = hammer_to_unix_xid(&ip->ino_data.uid);
 915         gid = hammer_to_unix_xid(&ip->ino_data.gid);
 916
 917         error = vop_helper_access(ap, uid, gid, ip->ino_data.mode,
 918                                   ip->ino_data.uflags);
 919         return (error);
 920 }
 921
 922 /*
 923  * hammer_vop_advlock { vp, id, op, fl, flags }
 924  *
 925  * MPSAFE - does not require fs_token
 926  */
 927 static
 928 int
 929 hammer_vop_advlock(struct vop_advlock_args *ap)
 930 {
 931         hammer_inode_t ip = VTOI(ap->a_vp);
 932
 933         return (lf_advlock(ap, &ip->advlock, ip->ino_data.size));
 934 }
 935
 936 /*
 937  * hammer_vop_close { vp, fflag }
 938  *
 939  * We can only sync-on-close for normal closes.  XXX disabled for now.
 940  */
 941 static
 942 int
 943 hammer_vop_close(struct vop_close_args *ap)
 944 {
 945 #if 0
 946         struct vnode *vp = ap->a_vp;
 947         hammer_inode_t ip = VTOI(vp);
 948         int waitfor;
 949         if (ip->flags & (HAMMER_INODE_CLOSESYNC|HAMMER_INODE_CLOSEASYNC)) {
 950                 if (vn_islocked(vp) == LK_EXCLUSIVE &&
 951                     (vp->v_flag & (VINACTIVE|VRECLAIMED)) == 0) {
 952                         if (ip->flags & HAMMER_INODE_CLOSESYNC)
 953                                 waitfor = MNT_WAIT;
 954                         else
 955                                 waitfor = MNT_NOWAIT;
 956                         ip->flags &= ~(HAMMER_INODE_CLOSESYNC |
 957                                        HAMMER_INODE_CLOSEASYNC);
 958                         VOP_FSYNC(vp, MNT_NOWAIT, waitfor);
 959                 }
 960         }
 961 #endif
 962         return (vop_stdclose(ap));
 963 }
 964
 965 /*
 966  * hammer_vop_ncreate { nch, dvp, vpp, cred, vap }
 967  *
 968  * The operating system has already ensured that the directory entry
 969  * does not exist and done all appropriate namespace locking.
 970  */
 971 static
 972 int
 973 hammer_vop_ncreate(struct vop_ncreate_args *ap)
 974 {
 975         struct hammer_transaction trans;
 976         struct hammer_inode *dip;
 977         struct hammer_inode *nip;
 978         struct nchandle *nch;
 979         hammer_mount_t hmp;
 980         int error;
 981
 982         nch = ap->a_nch;
 983         dip = VTOI(ap->a_dvp);
 984         hmp = dip->hmp;
 985
 986         if (dip->flags & HAMMER_INODE_RO)
 987                 return (EROFS);
 988         if ((error = hammer_checkspace(hmp, HAMMER_CHKSPC_CREATE)) != 0)
 989                 return (error);
 990
 991         /*
 992          * Create a transaction to cover the operations we perform.
 993          */
 994         lwkt_gettoken(&hmp->fs_token);
 995         hammer_start_transaction(&trans, hmp);
 996         ++hammer_stats_file_iopsw;
 997
 998         /*
 999          * Create a new filesystem object of the requested type.  The
1000          * returned inode will be referenced and shared-locked to prevent
1001          * it from being moved to the flusher.
1002          */
1003         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
1004                                     dip, nch->ncp->nc_name, nch->ncp->nc_nlen,
1005                                     NULL, &nip);
1006         if (error) {
1007                 hkprintf("hammer_create_inode error %d\n", error);
1008                 hammer_done_transaction(&trans);
1009                 *ap->a_vpp = NULL;
1010                 lwkt_reltoken(&hmp->fs_token);
1011                 return (error);
1012         }
1013
1014         /*
1015          * Add the new filesystem object to the directory.  This will also
1016          * bump the inode's link count.
1017          */
1018         error = hammer_ip_add_directory(&trans, dip,
1019                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1020                                         nip);
1021         if (error)
1022                 hkprintf("hammer_ip_add_directory error %d\n", error);
1023
1024         /*
1025          * Finish up.
1026          */
1027         if (error) {
1028                 hammer_rel_inode(nip, 0);
1029                 hammer_done_transaction(&trans);
1030                 *ap->a_vpp = NULL;
1031         } else {
1032                 error = hammer_get_vnode(nip, ap->a_vpp);
1033                 hammer_done_transaction(&trans);
1034                 hammer_rel_inode(nip, 0);
1035                 if (error == 0) {
1036                         cache_setunresolved(ap->a_nch);
1037                         cache_setvp(ap->a_nch, *ap->a_vpp);
1038                 }
1039                 hammer_knote(ap->a_dvp, NOTE_WRITE);
1040         }
1041         lwkt_reltoken(&hmp->fs_token);
1042         return (error);
1043 }
1044
1045 /*
1046  * hammer_vop_getattr { vp, vap }
1047  *
1048  * Retrieve an inode's attribute information.  When accessing inodes
1049  * historically we fake the atime field to ensure consistent results.
1050  * The atime field is stored in the B-Tree element and allowed to be
1051  * updated without cycling the element.
1052  *
1053  * MPSAFE - does not require fs_token
1054  */
1055 static
1056 int
1057 hammer_vop_getattr(struct vop_getattr_args *ap)
1058 {
1059         struct hammer_inode *ip = VTOI(ap->a_vp);
1060         struct vattr *vap = ap->a_vap;
1061
1062         /*
1063          * We want the fsid to be different when accessing a filesystem
1064          * with different as-of's so programs like diff don't think
1065          * the files are the same.
1066          *
1067          * We also want the fsid to be the same when comparing snapshots,
1068          * or when comparing mirrors (which might be backed by different
1069          * physical devices).  HAMMER fsids are based on the PFS's
1070          * shared_uuid field.
1071          *
1072          * XXX there is a chance of collision here.  The va_fsid reported
1073          * by stat is different from the more involved fsid used in the
1074          * mount structure.
1075          */
1076         ++hammer_stats_file_iopsr;
1077         hammer_lock_sh(&ip->lock);
1078         vap->va_fsid = ip->pfsm->fsid_udev ^ (u_int32_t)ip->obj_asof ^
1079                        (u_int32_t)(ip->obj_asof >> 32);
1080
1081         vap->va_fileid = ip->ino_leaf.base.obj_id;
1082         vap->va_mode = ip->ino_data.mode;
1083         vap->va_nlink = ip->ino_data.nlinks;
1084         vap->va_uid = hammer_to_unix_xid(&ip->ino_data.uid);
1085         vap->va_gid = hammer_to_unix_xid(&ip->ino_data.gid);
1086         vap->va_rmajor = 0;
1087         vap->va_rminor = 0;
1088         vap->va_size = ip->ino_data.size;
1089
1090         /*
1091          * Special case for @@PFS softlinks.  The actual size of the
1092          * expanded softlink is "@@0x%016llx:%05d" == 26 bytes.
1093          * or for MAX_TID is    "@@-1:%05d" == 10 bytes.
1094          */
1095         if (ip->ino_data.obj_type == HAMMER_OBJTYPE_SOFTLINK &&
1096             ip->ino_data.size == 10 &&
1097             ip->obj_asof == HAMMER_MAX_TID &&
1098             ip->obj_localization == 0 &&
1099             strncmp(ip->ino_data.ext.symlink, "@@PFS", 5) == 0) {
1100                     if (ip->pfsm->pfsd.mirror_flags & HAMMER_PFSD_SLAVE)
1101                             vap->va_size = 26;
1102                     else
1103                             vap->va_size = 10;
1104         }
1105
1106         /*
1107          * We must provide a consistent atime and mtime for snapshots
1108          * so people can do a 'tar cf - ... | md5' on them and get
1109          * consistent results.
1110          */
1111         if (ip->flags & HAMMER_INODE_RO) {
1112                 hammer_time_to_timespec(ip->ino_data.ctime, &vap->va_atime);
1113                 hammer_time_to_timespec(ip->ino_data.ctime, &vap->va_mtime);
1114         } else {
1115                 hammer_time_to_timespec(ip->ino_data.atime, &vap->va_atime);
1116                 hammer_time_to_timespec(ip->ino_data.mtime, &vap->va_mtime);
1117         }
1118         hammer_time_to_timespec(ip->ino_data.ctime, &vap->va_ctime);
1119         vap->va_flags = ip->ino_data.uflags;
1120         vap->va_gen = 1;        /* hammer inums are unique for all time */
1121         vap->va_blocksize = HAMMER_BUFSIZE;
1122         if (ip->ino_data.size >= HAMMER_XDEMARC) {
1123                 vap->va_bytes = (ip->ino_data.size + HAMMER_XBUFMASK64) &
1124                                 ~HAMMER_XBUFMASK64;
1125         } else if (ip->ino_data.size > HAMMER_BUFSIZE / 2) {
1126                 vap->va_bytes = (ip->ino_data.size + HAMMER_BUFMASK64) &
1127                                 ~HAMMER_BUFMASK64;
1128         } else {
1129                 vap->va_bytes = (ip->ino_data.size + 15) & ~15;
1130         }
1131
1132         vap->va_type = hammer_get_vnode_type(ip->ino_data.obj_type);
1133         vap->va_filerev = 0;    /* XXX */
1134         vap->va_uid_uuid = ip->ino_data.uid;
1135         vap->va_gid_uuid = ip->ino_data.gid;
1136         vap->va_fsid_uuid = ip->hmp->fsid;
1137         vap->va_vaflags = VA_UID_UUID_VALID | VA_GID_UUID_VALID |
1138                           VA_FSID_UUID_VALID;
1139
1140         switch (ip->ino_data.obj_type) {
1141         case HAMMER_OBJTYPE_CDEV:
1142         case HAMMER_OBJTYPE_BDEV:
1143                 vap->va_rmajor = ip->ino_data.rmajor;
1144                 vap->va_rminor = ip->ino_data.rminor;
1145                 break;
1146         default:
1147                 break;
1148         }
1149         hammer_unlock(&ip->lock);
1150         return(0);
1151 }
1152
1153 /*
1154  * hammer_vop_nresolve { nch, dvp, cred }
1155  *
1156  * Locate the requested directory entry.
1157  */
1158 static
1159 int
1160 hammer_vop_nresolve(struct vop_nresolve_args *ap)
1161 {
1162         struct hammer_transaction trans;
1163         struct namecache *ncp;
1164         hammer_mount_t hmp;
1165         hammer_inode_t dip;
1166         hammer_inode_t ip;
1167         hammer_tid_t asof;
1168         struct hammer_cursor cursor;
1169         struct vnode *vp;
1170         int64_t namekey;
1171         int error;
1172         int i;
1173         int nlen;
1174         int flags;
1175         int ispfs;
1176         int64_t obj_id;
1177         u_int32_t localization;
1178         u_int32_t max_iterations;
1179
1180         /*
1181          * Misc initialization, plus handle as-of name extensions.  Look for
1182          * the '@@' extension.  Note that as-of files and directories cannot
1183          * be modified.
1184          */
1185         dip = VTOI(ap->a_dvp);
1186         ncp = ap->a_nch->ncp;
1187         asof = dip->obj_asof;
1188         localization = dip->obj_localization;   /* for code consistency */
1189         nlen = ncp->nc_nlen;
1190         flags = dip->flags & HAMMER_INODE_RO;
1191         ispfs = 0;
1192         hmp = dip->hmp;
1193
1194         lwkt_gettoken(&hmp->fs_token);
1195         hammer_simple_transaction(&trans, hmp);
1196         ++hammer_stats_file_iopsr;
1197
1198         for (i = 0; i < nlen; ++i) {
1199                 if (ncp->nc_name[i] == '@' && ncp->nc_name[i+1] == '@') {
1200                         error = hammer_str_to_tid(ncp->nc_name + i + 2,
1201                                                   &ispfs, &asof, &localization);
1202                         if (error != 0) {
1203                                 i = nlen;
1204                                 break;
1205                         }
1206                         if (asof != HAMMER_MAX_TID)
1207                                 flags |= HAMMER_INODE_RO;
1208                         break;
1209                 }
1210         }
1211         nlen = i;
1212
1213         /*
1214          * If this is a PFS softlink we dive into the PFS
1215          */
1216         if (ispfs && nlen == 0) {
1217                 ip = hammer_get_inode(&trans, dip, HAMMER_OBJID_ROOT,
1218                                       asof, localization,
1219                                       flags, &error);
1220                 if (error == 0) {
1221                         error = hammer_get_vnode(ip, &vp);
1222                         hammer_rel_inode(ip, 0);
1223                 } else {
1224                         vp = NULL;
1225                 }
1226                 if (error == 0) {
1227                         vn_unlock(vp);
1228                         cache_setvp(ap->a_nch, vp);
1229                         vrele(vp);
1230                 }
1231                 goto done;
1232         }
1233
1234         /*
1235          * If there is no path component the time extension is relative to dip.
1236          * e.g. "fubar/@@<snapshot>"
1237          *
1238          * "." is handled by the kernel, but ".@@<snapshot>" is not.
1239          * e.g. "fubar/.@@<snapshot>"
1240          *
1241          * ".." is handled by the kernel.  We do not currently handle
1242          * "..@<snapshot>".
1243          */
1244         if (nlen == 0 || (nlen == 1 && ncp->nc_name[0] == '.')) {
1245                 ip = hammer_get_inode(&trans, dip, dip->obj_id,
1246                                       asof, dip->obj_localization,
1247                                       flags, &error);
1248                 if (error == 0) {
1249                         error = hammer_get_vnode(ip, &vp);
1250                         hammer_rel_inode(ip, 0);
1251                 } else {
1252                         vp = NULL;
1253                 }
1254                 if (error == 0) {
1255                         vn_unlock(vp);
1256                         cache_setvp(ap->a_nch, vp);
1257                         vrele(vp);
1258                 }
1259                 goto done;
1260         }
1261
1262         /*
1263          * Calculate the namekey and setup the key range for the scan.  This
1264          * works kinda like a chained hash table where the lower 32 bits
1265          * of the namekey synthesize the chain.
1266          *
1267          * The key range is inclusive of both key_beg and key_end.
1268          */
1269         namekey = hammer_directory_namekey(dip, ncp->nc_name, nlen,
1270                                            &max_iterations);
1271
1272         error = hammer_init_cursor(&trans, &cursor, &dip->cache[1], dip);
1273         cursor.key_beg.localization = dip->obj_localization +
1274                                       hammer_dir_localization(dip);
1275         cursor.key_beg.obj_id = dip->obj_id;
1276         cursor.key_beg.key = namekey;
1277         cursor.key_beg.create_tid = 0;
1278         cursor.key_beg.delete_tid = 0;
1279         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
1280         cursor.key_beg.obj_type = 0;
1281
1282         cursor.key_end = cursor.key_beg;
1283         cursor.key_end.key += max_iterations;
1284         cursor.asof = asof;
1285         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
1286
1287         /*
1288          * Scan all matching records (the chain), locate the one matching
1289          * the requested path component.
1290          *
1291          * The hammer_ip_*() functions merge in-memory records with on-disk
1292          * records for the purposes of the search.
1293          */
1294         obj_id = 0;
1295         localization = HAMMER_DEF_LOCALIZATION;
1296
1297         if (error == 0) {
1298                 error = hammer_ip_first(&cursor);
1299                 while (error == 0) {
1300                         error = hammer_ip_resolve_data(&cursor);
1301                         if (error)
1302                                 break;
1303                         if (nlen == cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF &&
1304                             bcmp(ncp->nc_name, cursor.data->entry.name, nlen) == 0) {
1305                                 obj_id = cursor.data->entry.obj_id;
1306                                 localization = cursor.data->entry.localization;
1307                                 break;
1308                         }
1309                         error = hammer_ip_next(&cursor);
1310                 }
1311         }
1312         hammer_done_cursor(&cursor);
1313
1314         /*
1315          * Lookup the obj_id.  This should always succeed.  If it does not
1316          * the filesystem may be damaged and we return a dummy inode.
1317          */
1318         if (error == 0) {
1319                 ip = hammer_get_inode(&trans, dip, obj_id,
1320                                       asof, localization,
1321                                       flags, &error);
1322                 if (error == ENOENT) {
1323                         kprintf("HAMMER: WARNING: Missing "
1324                                 "inode for dirent \"%s\"\n"
1325                                 "\tobj_id = %016llx, asof=%016llx, lo=%08x\n",
1326                                 ncp->nc_name,
1327                                 (long long)obj_id, (long long)asof,
1328                                 localization);
1329                         error = 0;
1330                         ip = hammer_get_dummy_inode(&trans, dip, obj_id,
1331                                                     asof, localization,
1332                                                     flags, &error);
1333                 }
1334                 if (error == 0) {
1335                         error = hammer_get_vnode(ip, &vp);
1336                         hammer_rel_inode(ip, 0);
1337                 } else {
1338                         vp = NULL;
1339                 }
1340                 if (error == 0) {
1341                         vn_unlock(vp);
1342                         cache_setvp(ap->a_nch, vp);
1343                         vrele(vp);
1344                 }
1345         } else if (error == ENOENT) {
1346                 cache_setvp(ap->a_nch, NULL);
1347         }
1348 done:
1349         hammer_done_transaction(&trans);
1350         lwkt_reltoken(&hmp->fs_token);
1351         return (error);
1352 }
1353
1354 /*
1355  * hammer_vop_nlookupdotdot { dvp, vpp, cred }
1356  *
1357  * Locate the parent directory of a directory vnode.
1358  *
1359  * dvp is referenced but not locked.  *vpp must be returned referenced and
1360  * locked.  A parent_obj_id of 0 does not necessarily indicate that we are
1361  * at the root, instead it could indicate that the directory we were in was
1362  * removed.
1363  *
1364  * NOTE: as-of sequences are not linked into the directory structure.  If
1365  * we are at the root with a different asof then the mount point, reload
1366  * the same directory with the mount point's asof.   I'm not sure what this
1367  * will do to NFS.  We encode ASOF stamps in NFS file handles so it might not
1368  * get confused, but it hasn't been tested.
1369  */
1370 static
1371 int
1372 hammer_vop_nlookupdotdot(struct vop_nlookupdotdot_args *ap)
1373 {
1374         struct hammer_transaction trans;
1375         struct hammer_inode *dip;
1376         struct hammer_inode *ip;
1377         hammer_mount_t hmp;
1378         int64_t parent_obj_id;
1379         u_int32_t parent_obj_localization;
1380         hammer_tid_t asof;
1381         int error;
1382
1383         dip = VTOI(ap->a_dvp);
1384         asof = dip->obj_asof;
1385         hmp = dip->hmp;
1386
1387         /*
1388          * Whos are parent?  This could be the root of a pseudo-filesystem
1389          * whos parent is in another localization domain.
1390          */
1391         lwkt_gettoken(&hmp->fs_token);
1392         parent_obj_id = dip->ino_data.parent_obj_id;
1393         if (dip->obj_id == HAMMER_OBJID_ROOT)
1394                 parent_obj_localization = dip->ino_data.ext.obj.parent_obj_localization;
1395         else
1396                 parent_obj_localization = dip->obj_localization;
1397
1398         if (parent_obj_id == 0) {
1399                 if (dip->obj_id == HAMMER_OBJID_ROOT &&
1400                    asof != hmp->asof) {
1401                         parent_obj_id = dip->obj_id;
1402                         asof = hmp->asof;
1403                         *ap->a_fakename = kmalloc(19, M_TEMP, M_WAITOK);
1404                         ksnprintf(*ap->a_fakename, 19, "0x%016llx",
1405                                   (long long)dip->obj_asof);
1406                 } else {
1407                         *ap->a_vpp = NULL;
1408                         lwkt_reltoken(&hmp->fs_token);
1409                         return ENOENT;
1410                 }
1411         }
1412
1413         hammer_simple_transaction(&trans, hmp);
1414         ++hammer_stats_file_iopsr;
1415
1416         ip = hammer_get_inode(&trans, dip, parent_obj_id,
1417                               asof, parent_obj_localization,
1418                               dip->flags, &error);
1419         if (ip) {
1420                 error = hammer_get_vnode(ip, ap->a_vpp);
1421                 hammer_rel_inode(ip, 0);
1422         } else {
1423                 *ap->a_vpp = NULL;
1424         }
1425         hammer_done_transaction(&trans);
1426         lwkt_reltoken(&hmp->fs_token);
1427         return (error);
1428 }
1429
1430 /*
1431  * hammer_vop_nlink { nch, dvp, vp, cred }
1432  */
1433 static
1434 int
1435 hammer_vop_nlink(struct vop_nlink_args *ap)
1436 {
1437         struct hammer_transaction trans;
1438         struct hammer_inode *dip;
1439         struct hammer_inode *ip;
1440         struct nchandle *nch;
1441         hammer_mount_t hmp;
1442         int error;
1443
1444         if (ap->a_dvp->v_mount != ap->a_vp->v_mount)
1445                 return(EXDEV);
1446
1447         nch = ap->a_nch;
1448         dip = VTOI(ap->a_dvp);
1449         ip = VTOI(ap->a_vp);
1450         hmp = dip->hmp;
1451
1452         if (dip->obj_localization != ip->obj_localization)
1453                 return(EXDEV);
1454
1455         if (dip->flags & HAMMER_INODE_RO)
1456                 return (EROFS);
1457         if (ip->flags & HAMMER_INODE_RO)
1458                 return (EROFS);
1459         if ((error = hammer_checkspace(hmp, HAMMER_CHKSPC_CREATE)) != 0)
1460                 return (error);
1461
1462         /*
1463          * Create a transaction to cover the operations we perform.
1464          */
1465         lwkt_gettoken(&hmp->fs_token);
1466         hammer_start_transaction(&trans, hmp);
1467         ++hammer_stats_file_iopsw;
1468
1469         /*
1470          * Add the filesystem object to the directory.  Note that neither
1471          * dip nor ip are referenced or locked, but their vnodes are
1472          * referenced.  This function will bump the inode's link count.
1473          */
1474         error = hammer_ip_add_directory(&trans, dip,
1475                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1476                                         ip);
1477
1478         /*
1479          * Finish up.
1480          */
1481         if (error == 0) {
1482                 cache_setunresolved(nch);
1483                 cache_setvp(nch, ap->a_vp);
1484         }
1485         hammer_done_transaction(&trans);
1486         hammer_knote(ap->a_vp, NOTE_LINK);
1487         hammer_knote(ap->a_dvp, NOTE_WRITE);
1488         lwkt_reltoken(&hmp->fs_token);
1489         return (error);
1490 }
1491
1492 /*
1493  * hammer_vop_nmkdir { nch, dvp, vpp, cred, vap }
1494  *
1495  * The operating system has already ensured that the directory entry
1496  * does not exist and done all appropriate namespace locking.
1497  */
1498 static
1499 int
1500 hammer_vop_nmkdir(struct vop_nmkdir_args *ap)
1501 {
1502         struct hammer_transaction trans;
1503         struct hammer_inode *dip;
1504         struct hammer_inode *nip;
1505         struct nchandle *nch;
1506         hammer_mount_t hmp;
1507         int error;
1508
1509         nch = ap->a_nch;
1510         dip = VTOI(ap->a_dvp);
1511         hmp = dip->hmp;
1512
1513         if (dip->flags & HAMMER_INODE_RO)
1514                 return (EROFS);
1515         if ((error = hammer_checkspace(hmp, HAMMER_CHKSPC_CREATE)) != 0)
1516                 return (error);
1517
1518         /*
1519          * Create a transaction to cover the operations we perform.
1520          */
1521         lwkt_gettoken(&hmp->fs_token);
1522         hammer_start_transaction(&trans, hmp);
1523         ++hammer_stats_file_iopsw;
1524
1525         /*
1526          * Create a new filesystem object of the requested type.  The
1527          * returned inode will be referenced but not locked.
1528          */
1529         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
1530                                     dip, nch->ncp->nc_name, nch->ncp->nc_nlen,
1531                                     NULL, &nip);
1532         if (error) {
1533                 hkprintf("hammer_mkdir error %d\n", error);
1534                 hammer_done_transaction(&trans);
1535                 *ap->a_vpp = NULL;
1536                 lwkt_reltoken(&hmp->fs_token);
1537                 return (error);
1538         }
1539         /*
1540          * Add the new filesystem object to the directory.  This will also
1541          * bump the inode's link count.
1542          */
1543         error = hammer_ip_add_directory(&trans, dip,
1544                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1545                                         nip);
1546         if (error)
1547                 hkprintf("hammer_mkdir (add) error %d\n", error);
1548
1549         /*
1550          * Finish up.
1551          */
1552         if (error) {
1553                 hammer_rel_inode(nip, 0);
1554                 *ap->a_vpp = NULL;
1555         } else {
1556                 error = hammer_get_vnode(nip, ap->a_vpp);
1557                 hammer_rel_inode(nip, 0);
1558                 if (error == 0) {
1559                         cache_setunresolved(ap->a_nch);
1560                         cache_setvp(ap->a_nch, *ap->a_vpp);
1561                 }
1562         }
1563         hammer_done_transaction(&trans);
1564         if (error == 0)
1565                 hammer_knote(ap->a_dvp, NOTE_WRITE | NOTE_LINK);
1566         lwkt_reltoken(&hmp->fs_token);
1567         return (error);
1568 }
1569
1570 /*
1571  * hammer_vop_nmknod { nch, dvp, vpp, cred, vap }
1572  *
1573  * The operating system has already ensured that the directory entry
1574  * does not exist and done all appropriate namespace locking.
1575  */
1576 static
1577 int
1578 hammer_vop_nmknod(struct vop_nmknod_args *ap)
1579 {
1580         struct hammer_transaction trans;
1581         struct hammer_inode *dip;
1582         struct hammer_inode *nip;
1583         struct nchandle *nch;
1584         hammer_mount_t hmp;
1585         int error;
1586
1587         nch = ap->a_nch;
1588         dip = VTOI(ap->a_dvp);
1589         hmp = dip->hmp;
1590
1591         if (dip->flags & HAMMER_INODE_RO)
1592                 return (EROFS);
1593         if ((error = hammer_checkspace(hmp, HAMMER_CHKSPC_CREATE)) != 0)
1594                 return (error);
1595
1596         /*
1597          * Create a transaction to cover the operations we perform.
1598          */
1599         lwkt_gettoken(&hmp->fs_token);
1600         hammer_start_transaction(&trans, hmp);
1601         ++hammer_stats_file_iopsw;
1602
1603         /*
1604          * Create a new filesystem object of the requested type.  The
1605          * returned inode will be referenced but not locked.
1606          *
1607          * If mknod specifies a directory a pseudo-fs is created.
1608          */
1609         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
1610                                     dip, nch->ncp->nc_name, nch->ncp->nc_nlen,
1611                                     NULL, &nip);
1612         if (error) {
1613                 hammer_done_transaction(&trans);
1614                 *ap->a_vpp = NULL;
1615                 lwkt_reltoken(&hmp->fs_token);
1616                 return (error);
1617         }
1618
1619         /*
1620          * Add the new filesystem object to the directory.  This will also
1621          * bump the inode's link count.
1622          */
1623         error = hammer_ip_add_directory(&trans, dip,
1624                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1625                                         nip);
1626
1627         /*
1628          * Finish up.
1629          */
1630         if (error) {
1631                 hammer_rel_inode(nip, 0);
1632                 *ap->a_vpp = NULL;
1633         } else {
1634                 error = hammer_get_vnode(nip, ap->a_vpp);
1635                 hammer_rel_inode(nip, 0);
1636                 if (error == 0) {
1637                         cache_setunresolved(ap->a_nch);
1638                         cache_setvp(ap->a_nch, *ap->a_vpp);
1639                 }
1640         }
1641         hammer_done_transaction(&trans);
1642         if (error == 0)
1643                 hammer_knote(ap->a_dvp, NOTE_WRITE);
1644         lwkt_reltoken(&hmp->fs_token);
1645         return (error);
1646 }
1647
1648 /*
1649  * hammer_vop_open { vp, mode, cred, fp }
1650  *
1651  * MPSAFE (does not require fs_token)
1652  */
1653 static
1654 int
1655 hammer_vop_open(struct vop_open_args *ap)
1656 {
1657         hammer_inode_t ip;
1658
1659         ++hammer_stats_file_iopsr;
1660         ip = VTOI(ap->a_vp);
1661
1662         if ((ap->a_mode & FWRITE) && (ip->flags & HAMMER_INODE_RO))
1663                 return (EROFS);
1664         return(vop_stdopen(ap));
1665 }
1666
1667 /*
1668  * hammer_vop_print { vp }
1669  */
1670 static
1671 int
1672 hammer_vop_print(struct vop_print_args *ap)
1673 {
1674         return EOPNOTSUPP;
1675 }
1676
1677 /*
1678  * hammer_vop_readdir { vp, uio, cred, *eofflag, *ncookies, off_t **cookies }
1679  */
1680 static
1681 int
1682 hammer_vop_readdir(struct vop_readdir_args *ap)
1683 {
1684         struct hammer_transaction trans;
1685         struct hammer_cursor cursor;
1686         struct hammer_inode *ip;
1687         hammer_mount_t hmp;
1688         struct uio *uio;
1689         hammer_base_elm_t base;
1690         int error;
1691         int cookie_index;
1692         int ncookies;
1693         off_t *cookies;
1694         off_t saveoff;
1695         int r;
1696         int dtype;
1697
1698         ++hammer_stats_file_iopsr;
1699         ip = VTOI(ap->a_vp);
1700         uio = ap->a_uio;
1701         saveoff = uio->uio_offset;
1702         hmp = ip->hmp;
1703
1704         if (ap->a_ncookies) {
1705                 ncookies = uio->uio_resid / 16 + 1;
1706                 if (ncookies > 1024)
1707                         ncookies = 1024;
1708                 cookies = kmalloc(ncookies * sizeof(off_t), M_TEMP, M_WAITOK);
1709                 cookie_index = 0;
1710         } else {
1711                 ncookies = -1;
1712                 cookies = NULL;
1713                 cookie_index = 0;
1714         }
1715
1716         lwkt_gettoken(&hmp->fs_token);
1717         hammer_simple_transaction(&trans, hmp);
1718
1719         /*
1720          * Handle artificial entries
1721          *
1722          * It should be noted that the minimum value for a directory
1723          * hash key on-media is 0x0000000100000000, so we can use anything
1724          * less then that to represent our 'special' key space.
1725          */
1726         error = 0;
1727         if (saveoff == 0) {
1728                 r = vop_write_dirent(&error, uio, ip->obj_id, DT_DIR, 1, ".");
1729                 if (r)
1730                         goto done;
1731                 if (cookies)
1732                         cookies[cookie_index] = saveoff;
1733                 ++saveoff;
1734                 ++cookie_index;
1735                 if (cookie_index == ncookies)
1736                         goto done;
1737         }
1738         if (saveoff == 1) {
1739                 if (ip->ino_data.parent_obj_id) {
1740                         r = vop_write_dirent(&error, uio,
1741                                              ip->ino_data.parent_obj_id,
1742                                              DT_DIR, 2, "..");
1743                 } else {
1744                         r = vop_write_dirent(&error, uio,
1745                                              ip->obj_id, DT_DIR, 2, "..");
1746                 }
1747                 if (r)
1748                         goto done;
1749                 if (cookies)
1750                         cookies[cookie_index] = saveoff;
1751                 ++saveoff;
1752                 ++cookie_index;
1753                 if (cookie_index == ncookies)
1754                         goto done;
1755         }
1756
1757         /*
1758          * Key range (begin and end inclusive) to scan.  Directory keys
1759          * directly translate to a 64 bit 'seek' position.
1760          */
1761         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
1762         cursor.key_beg.localization = ip->obj_localization +
1763                                       hammer_dir_localization(ip);
1764         cursor.key_beg.obj_id = ip->obj_id;
1765         cursor.key_beg.create_tid = 0;
1766         cursor.key_beg.delete_tid = 0;
1767         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
1768         cursor.key_beg.obj_type = 0;
1769         cursor.key_beg.key = saveoff;
1770
1771         cursor.key_end = cursor.key_beg;
1772         cursor.key_end.key = HAMMER_MAX_KEY;
1773         cursor.asof = ip->obj_asof;
1774         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
1775
1776         error = hammer_ip_first(&cursor);
1777
1778         while (error == 0) {
1779                 error = hammer_ip_resolve_data(&cursor);
1780                 if (error)
1781                         break;
1782                 base = &cursor.leaf->base;
1783                 saveoff = base->key;
1784                 KKASSERT(cursor.leaf->data_len > HAMMER_ENTRY_NAME_OFF);
1785
1786                 if (base->obj_id != ip->obj_id)
1787                         panic("readdir: bad record at %p", cursor.node);
1788
1789                 /*
1790                  * Convert pseudo-filesystems into softlinks
1791                  */
1792                 dtype = hammer_get_dtype(cursor.leaf->base.obj_type);
1793                 r = vop_write_dirent(
1794                              &error, uio, cursor.data->entry.obj_id,
1795                              dtype,
1796                              cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF ,
1797                              (void *)cursor.data->entry.name);
1798                 if (r)
1799                         break;
1800                 ++saveoff;
1801                 if (cookies)
1802                         cookies[cookie_index] = base->key;
1803                 ++cookie_index;
1804                 if (cookie_index == ncookies)
1805                         break;
1806                 error = hammer_ip_next(&cursor);
1807         }
1808         hammer_done_cursor(&cursor);
1809
1810 done:
1811         hammer_done_transaction(&trans);
1812
1813         if (ap->a_eofflag)
1814                 *ap->a_eofflag = (error == ENOENT);
1815         uio->uio_offset = saveoff;
1816         if (error && cookie_index == 0) {
1817                 if (error == ENOENT)
1818                         error = 0;
1819                 if (cookies) {
1820                         kfree(cookies, M_TEMP);
1821                         *ap->a_ncookies = 0;
1822                         *ap->a_cookies = NULL;
1823                 }
1824         } else {
1825                 if (error == ENOENT)
1826                         error = 0;
1827                 if (cookies) {
1828                         *ap->a_ncookies = cookie_index;
1829                         *ap->a_cookies = cookies;
1830                 }
1831         }
1832         lwkt_reltoken(&hmp->fs_token);
1833         return(error);
1834 }
1835
1836 /*
1837  * hammer_vop_readlink { vp, uio, cred }
1838  */
1839 static
1840 int
1841 hammer_vop_readlink(struct vop_readlink_args *ap)
1842 {
1843         struct hammer_transaction trans;
1844         struct hammer_cursor cursor;
1845         struct hammer_inode *ip;
1846         hammer_mount_t hmp;
1847         char buf[32];
1848         u_int32_t localization;
1849         hammer_pseudofs_inmem_t pfsm;
1850         int error;
1851
1852         ip = VTOI(ap->a_vp);
1853         hmp = ip->hmp;
1854
1855         lwkt_gettoken(&hmp->fs_token);
1856
1857         /*
1858          * Shortcut if the symlink data was stuffed into ino_data.
1859          *
1860          * Also expand special "@@PFS%05d" softlinks (expansion only
1861          * occurs for non-historical (current) accesses made from the
1862          * primary filesystem).
1863          */
1864         if (ip->ino_data.size <= HAMMER_INODE_BASESYMLEN) {
1865                 char *ptr;
1866                 int bytes;
1867
1868                 ptr = ip->ino_data.ext.symlink;
1869                 bytes = (int)ip->ino_data.size;
1870                 if (bytes == 10 &&
1871                     ip->obj_asof == HAMMER_MAX_TID &&
1872                     ip->obj_localization == 0 &&
1873                     strncmp(ptr, "@@PFS", 5) == 0) {
1874                         hammer_simple_transaction(&trans, hmp);
1875                         bcopy(ptr + 5, buf, 5);
1876                         buf[5] = 0;
1877                         localization = strtoul(buf, NULL, 10) << 16;
1878                         pfsm = hammer_load_pseudofs(&trans, localization,
1879                                                     &error);
1880                         if (error == 0) {
1881                                 if (pfsm->pfsd.mirror_flags &
1882                                     HAMMER_PFSD_SLAVE) {
1883                                         /* vap->va_size == 26 */
1884                                         ksnprintf(buf, sizeof(buf),
1885                                                   "@@0x%016llx:%05d",
1886                                                   (long long)pfsm->pfsd.sync_end_tid,
1887                                                   localization >> 16);
1888                                 } else {
1889                                         /* vap->va_size == 10 */
1890                                         ksnprintf(buf, sizeof(buf),
1891                                                   "@@-1:%05d",
1892                                                   localization >> 16);
1893 #if 0
1894                                         ksnprintf(buf, sizeof(buf),
1895                                                   "@@0x%016llx:%05d",
1896                                                   (long long)HAMMER_MAX_TID,
1897                                                   localization >> 16);
1898 #endif
1899                                 }
1900                                 ptr = buf;
1901                                 bytes = strlen(buf);
1902                         }
1903                         if (pfsm)
1904                                 hammer_rel_pseudofs(hmp, pfsm);
1905                         hammer_done_transaction(&trans);
1906                 }
1907                 error = uiomove(ptr, bytes, ap->a_uio);
1908                 lwkt_reltoken(&hmp->fs_token);
1909                 return(error);
1910         }
1911
1912         /*
1913          * Long version
1914          */
1915         hammer_simple_transaction(&trans, hmp);
1916         ++hammer_stats_file_iopsr;
1917         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
1918
1919         /*
1920          * Key range (begin and end inclusive) to scan.  Directory keys
1921          * directly translate to a 64 bit 'seek' position.
1922          */
1923         cursor.key_beg.localization = ip->obj_localization +
1924                                       HAMMER_LOCALIZE_MISC;
1925         cursor.key_beg.obj_id = ip->obj_id;
1926         cursor.key_beg.create_tid = 0;
1927         cursor.key_beg.delete_tid = 0;
1928         cursor.key_beg.rec_type = HAMMER_RECTYPE_FIX;
1929         cursor.key_beg.obj_type = 0;
1930         cursor.key_beg.key = HAMMER_FIXKEY_SYMLINK;
1931         cursor.asof = ip->obj_asof;
1932         cursor.flags |= HAMMER_CURSOR_ASOF;
1933
1934         error = hammer_ip_lookup(&cursor);
1935         if (error == 0) {
1936                 error = hammer_ip_resolve_data(&cursor);
1937                 if (error == 0) {
1938                         KKASSERT(cursor.leaf->data_len >=
1939                                  HAMMER_SYMLINK_NAME_OFF);
1940                         error = uiomove(cursor.data->symlink.name,
1941                                         cursor.leaf->data_len -
1942                                                 HAMMER_SYMLINK_NAME_OFF,
1943                                         ap->a_uio);
1944                 }
1945         }
1946         hammer_done_cursor(&cursor);
1947         hammer_done_transaction(&trans);
1948         lwkt_reltoken(&hmp->fs_token);
1949         return(error);
1950 }
1951
1952 /*
1953  * hammer_vop_nremove { nch, dvp, cred }
1954  */
1955 static
1956 int
1957 hammer_vop_nremove(struct vop_nremove_args *ap)
1958 {
1959         struct hammer_transaction trans;
1960         struct hammer_inode *dip;
1961         hammer_mount_t hmp;
1962         int error;
1963
1964         dip = VTOI(ap->a_dvp);
1965         hmp = dip->hmp;
1966
1967         if (hammer_nohistory(dip) == 0 &&
1968             (error = hammer_checkspace(hmp, HAMMER_CHKSPC_REMOVE)) != 0) {
1969                 return (error);
1970         }
1971
1972         lwkt_gettoken(&hmp->fs_token);
1973         hammer_start_transaction(&trans, hmp);
1974         ++hammer_stats_file_iopsw;
1975         error = hammer_dounlink(&trans, ap->a_nch, ap->a_dvp, ap->a_cred, 0, 0);
1976         hammer_done_transaction(&trans);
1977         if (error == 0)
1978                 hammer_knote(ap->a_dvp, NOTE_WRITE);
1979         lwkt_reltoken(&hmp->fs_token);
1980         return (error);
1981 }
1982
1983 /*
1984  * hammer_vop_nrename { fnch, tnch, fdvp, tdvp, cred }
1985  */
1986 static
1987 int
1988 hammer_vop_nrename(struct vop_nrename_args *ap)
1989 {
1990         struct hammer_transaction trans;
1991         struct namecache *fncp;
1992         struct namecache *tncp;
1993         struct hammer_inode *fdip;
1994         struct hammer_inode *tdip;
1995         struct hammer_inode *ip;
1996         hammer_mount_t hmp;
1997         struct hammer_cursor cursor;
1998         int64_t namekey;
1999         u_int32_t max_iterations;
2000         int nlen, error;
2001
2002         if (ap->a_fdvp->v_mount != ap->a_tdvp->v_mount)
2003                 return(EXDEV);
2004         if (ap->a_fdvp->v_mount != ap->a_fnch->ncp->nc_vp->v_mount)
2005                 return(EXDEV);
2006
2007         fdip = VTOI(ap->a_fdvp);
2008         tdip = VTOI(ap->a_tdvp);
2009         fncp = ap->a_fnch->ncp;
2010         tncp = ap->a_tnch->ncp;
2011         ip = VTOI(fncp->nc_vp);
2012         KKASSERT(ip != NULL);
2013
2014         hmp = ip->hmp;
2015
2016         if (fdip->obj_localization != tdip->obj_localization)
2017                 return(EXDEV);
2018         if (fdip->obj_localization != ip->obj_localization)
2019                 return(EXDEV);
2020
2021         if (fdip->flags & HAMMER_INODE_RO)
2022                 return (EROFS);
2023         if (tdip->flags & HAMMER_INODE_RO)
2024                 return (EROFS);
2025         if (ip->flags & HAMMER_INODE_RO)
2026                 return (EROFS);
2027         if ((error = hammer_checkspace(hmp, HAMMER_CHKSPC_CREATE)) != 0)
2028                 return (error);
2029
2030         lwkt_gettoken(&hmp->fs_token);
2031         hammer_start_transaction(&trans, hmp);
2032         ++hammer_stats_file_iopsw;
2033
2034         /*
2035          * Remove tncp from the target directory and then link ip as
2036          * tncp. XXX pass trans to dounlink
2037          *
2038          * Force the inode sync-time to match the transaction so it is
2039          * in-sync with the creation of the target directory entry.
2040          */
2041         error = hammer_dounlink(&trans, ap->a_tnch, ap->a_tdvp,
2042                                 ap->a_cred, 0, -1);
2043         if (error == 0 || error == ENOENT) {
2044                 error = hammer_ip_add_directory(&trans, tdip,
2045                                                 tncp->nc_name, tncp->nc_nlen,
2046                                                 ip);
2047                 if (error == 0) {
2048                         ip->ino_data.parent_obj_id = tdip->obj_id;
2049                         ip->ino_data.ctime = trans.time;
2050                         hammer_modify_inode(&trans, ip, HAMMER_INODE_DDIRTY);
2051                 }
2052         }
2053         if (error)
2054                 goto failed; /* XXX */
2055
2056         /*
2057          * Locate the record in the originating directory and remove it.
2058          *
2059          * Calculate the namekey and setup the key range for the scan.  This
2060          * works kinda like a chained hash table where the lower 32 bits
2061          * of the namekey synthesize the chain.
2062          *
2063          * The key range is inclusive of both key_beg and key_end.
2064          */
2065         namekey = hammer_directory_namekey(fdip, fncp->nc_name, fncp->nc_nlen,
2066                                            &max_iterations);
2067 retry:
2068         hammer_init_cursor(&trans, &cursor, &fdip->cache[1], fdip);
2069         cursor.key_beg.localization = fdip->obj_localization +
2070                                       hammer_dir_localization(fdip);
2071         cursor.key_beg.obj_id = fdip->obj_id;
2072         cursor.key_beg.key = namekey;
2073         cursor.key_beg.create_tid = 0;
2074         cursor.key_beg.delete_tid = 0;
2075         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
2076         cursor.key_beg.obj_type = 0;
2077
2078         cursor.key_end = cursor.key_beg;
2079         cursor.key_end.key += max_iterations;
2080         cursor.asof = fdip->obj_asof;
2081         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
2082
2083         /*
2084          * Scan all matching records (the chain), locate the one matching
2085          * the requested path component.
2086          *
2087          * The hammer_ip_*() functions merge in-memory records with on-disk
2088          * records for the purposes of the search.
2089          */
2090         error = hammer_ip_first(&cursor);
2091         while (error == 0) {
2092                 if (hammer_ip_resolve_data(&cursor) != 0)
2093                         break;
2094                 nlen = cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF;
2095                 KKASSERT(nlen > 0);
2096                 if (fncp->nc_nlen == nlen &&
2097                     bcmp(fncp->nc_name, cursor.data->entry.name, nlen) == 0) {
2098                         break;
2099                 }
2100                 error = hammer_ip_next(&cursor);
2101         }
2102
2103         /*
2104          * If all is ok we have to get the inode so we can adjust nlinks.
2105          *
2106          * WARNING: hammer_ip_del_directory() may have to terminate the
2107          * cursor to avoid a recursion.  It's ok to call hammer_done_cursor()
2108          * twice.
2109          */
2110         if (error == 0)
2111                 error = hammer_ip_del_directory(&trans, &cursor, fdip, ip);
2112
2113         /*
2114          * XXX A deadlock here will break rename's atomicy for the purposes
2115          * of crash recovery.
2116          */
2117         if (error == EDEADLK) {
2118                 hammer_done_cursor(&cursor);
2119                 goto retry;
2120         }
2121
2122         /*
2123          * Cleanup and tell the kernel that the rename succeeded.
2124          *
2125          * NOTE: ip->vp, if non-NULL, cannot be directly referenced
2126          *       without formally acquiring the vp since the vp might
2127          *       have zero refs on it, or in the middle of a reclaim,
2128          *       etc.
2129          */
2130         hammer_done_cursor(&cursor);
2131         if (error == 0) {
2132                 cache_rename(ap->a_fnch, ap->a_tnch);
2133                 hammer_knote(ap->a_fdvp, NOTE_WRITE);
2134                 hammer_knote(ap->a_tdvp, NOTE_WRITE);
2135                 while (ip->vp) {
2136                         struct vnode *vp;
2137
2138                         error = hammer_get_vnode(ip, &vp);
2139                         if (error == 0 && vp) {
2140                                 vn_unlock(vp);
2141                                 hammer_knote(ip->vp, NOTE_RENAME);
2142                                 vrele(vp);
2143                                 break;
2144                         }
2145                         kprintf("Debug: HAMMER ip/vp race2 avoided\n");
2146                 }
2147         }
2148
2149 failed:
2150         hammer_done_transaction(&trans);
2151         lwkt_reltoken(&hmp->fs_token);
2152         return (error);
2153 }
2154
2155 /*
2156  * hammer_vop_nrmdir { nch, dvp, cred }
2157  */
2158 static
2159 int
2160 hammer_vop_nrmdir(struct vop_nrmdir_args *ap)
2161 {
2162         struct hammer_transaction trans;
2163         struct hammer_inode *dip;
2164         hammer_mount_t hmp;
2165         int error;
2166
2167         dip = VTOI(ap->a_dvp);
2168         hmp = dip->hmp;
2169
2170         if (hammer_nohistory(dip) == 0 &&
2171             (error = hammer_checkspace(hmp, HAMMER_CHKSPC_REMOVE)) != 0) {
2172                 return (error);
2173         }
2174
2175         lwkt_gettoken(&hmp->fs_token);
2176         hammer_start_transaction(&trans, hmp);
2177         ++hammer_stats_file_iopsw;
2178         error = hammer_dounlink(&trans, ap->a_nch, ap->a_dvp, ap->a_cred, 0, 1);
2179         hammer_done_transaction(&trans);
2180         if (error == 0)
2181                 hammer_knote(ap->a_dvp, NOTE_WRITE | NOTE_LINK);
2182         lwkt_reltoken(&hmp->fs_token);
2183         return (error);
2184 }
2185
2186 /*
2187  * hammer_vop_markatime { vp, cred }
2188  */
2189 static
2190 int
2191 hammer_vop_markatime(struct vop_markatime_args *ap)
2192 {
2193         struct hammer_transaction trans;
2194         struct hammer_inode *ip;
2195         hammer_mount_t hmp;
2196
2197         ip = VTOI(ap->a_vp);
2198         if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2199                 return (EROFS);
2200         if (ip->flags & HAMMER_INODE_RO)
2201                 return (EROFS);
2202         hmp = ip->hmp;
2203         if (hmp->mp->mnt_flag & MNT_NOATIME)
2204                 return (0);
2205         lwkt_gettoken(&hmp->fs_token);
2206         hammer_start_transaction(&trans, hmp);
2207         ++hammer_stats_file_iopsw;
2208
2209         ip->ino_data.atime = trans.time;
2210         hammer_modify_inode(&trans, ip, HAMMER_INODE_ATIME);
2211         hammer_done_transaction(&trans);
2212         hammer_knote(ap->a_vp, NOTE_ATTRIB);
2213         lwkt_reltoken(&hmp->fs_token);
2214         return (0);
2215 }
2216
2217 /*
2218  * hammer_vop_setattr { vp, vap, cred }
2219  */
2220 static
2221 int
2222 hammer_vop_setattr(struct vop_setattr_args *ap)
2223 {
2224         struct hammer_transaction trans;
2225         struct hammer_inode *ip;
2226         struct vattr *vap;
2227         hammer_mount_t hmp;
2228         int modflags;
2229         int error;
2230         int truncating;
2231         int blksize;
2232         int kflags;
2233 #if 0
2234         int64_t aligned_size;
2235 #endif
2236         u_int32_t flags;
2237
2238         vap = ap->a_vap;
2239         ip = ap->a_vp->v_data;
2240         modflags = 0;
2241         kflags = 0;
2242         hmp = ip->hmp;
2243
2244         if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2245                 return(EROFS);
2246         if (ip->flags & HAMMER_INODE_RO)
2247                 return (EROFS);
2248         if (hammer_nohistory(ip) == 0 &&
2249             (error = hammer_checkspace(hmp, HAMMER_CHKSPC_REMOVE)) != 0) {
2250                 return (error);
2251         }
2252
2253         lwkt_gettoken(&hmp->fs_token);
2254         hammer_start_transaction(&trans, hmp);
2255         ++hammer_stats_file_iopsw;
2256         error = 0;
2257
2258         if (vap->va_flags != VNOVAL) {
2259                 flags = ip->ino_data.uflags;
2260                 error = vop_helper_setattr_flags(&flags, vap->va_flags,
2261                                          hammer_to_unix_xid(&ip->ino_data.uid),
2262                                          ap->a_cred);
2263                 if (error == 0) {
2264                         if (ip->ino_data.uflags != flags) {
2265                                 ip->ino_data.uflags = flags;
2266                                 ip->ino_data.ctime = trans.time;
2267                                 modflags |= HAMMER_INODE_DDIRTY;
2268                                 kflags |= NOTE_ATTRIB;
2269                         }
2270                         if (ip->ino_data.uflags & (IMMUTABLE | APPEND)) {
2271                                 error = 0;
2272                                 goto done;
2273                         }
2274                 }
2275                 goto done;
2276         }
2277         if (ip->ino_data.uflags & (IMMUTABLE | APPEND)) {
2278                 error = EPERM;
2279                 goto done;
2280         }
2281         if (vap->va_uid != (uid_t)VNOVAL || vap->va_gid != (gid_t)VNOVAL) {
2282                 mode_t cur_mode = ip->ino_data.mode;
2283                 uid_t cur_uid = hammer_to_unix_xid(&ip->ino_data.uid);
2284                 gid_t cur_gid = hammer_to_unix_xid(&ip->ino_data.gid);
2285                 uuid_t uuid_uid;
2286                 uuid_t uuid_gid;
2287
2288                 error = vop_helper_chown(ap->a_vp, vap->va_uid, vap->va_gid,
2289                                          ap->a_cred,
2290                                          &cur_uid, &cur_gid, &cur_mode);
2291                 if (error == 0) {
2292                         hammer_guid_to_uuid(&uuid_uid, cur_uid);
2293                         hammer_guid_to_uuid(&uuid_gid, cur_gid);
2294                         if (bcmp(&uuid_uid, &ip->ino_data.uid,
2295                                  sizeof(uuid_uid)) ||
2296                             bcmp(&uuid_gid, &ip->ino_data.gid,
2297                                  sizeof(uuid_gid)) ||
2298                             ip->ino_data.mode != cur_mode
2299                         ) {
2300                                 ip->ino_data.uid = uuid_uid;
2301                                 ip->ino_data.gid = uuid_gid;
2302                                 ip->ino_data.mode = cur_mode;
2303                                 ip->ino_data.ctime = trans.time;
2304                                 modflags |= HAMMER_INODE_DDIRTY;
2305                         }
2306                         kflags |= NOTE_ATTRIB;
2307                 }
2308         }
2309         while (vap->va_size != VNOVAL && ip->ino_data.size != vap->va_size) {
2310                 switch(ap->a_vp->v_type) {
2311                 case VREG:
2312                         if (vap->va_size == ip->ino_data.size)
2313                                 break;
2314
2315                         /*
2316                          * Log the operation if in fast-fsync mode or if
2317                          * there are unterminated redo write records present.
2318                          *
2319                          * The second check is needed so the recovery code
2320                          * properly truncates write redos even if nominal
2321                          * REDO operations is turned off due to excessive
2322                          * writes, because the related records might be
2323                          * destroyed and never lay down a TERM_WRITE.
2324                          */
2325                         if ((ip->flags & HAMMER_INODE_REDO) ||
2326                             (ip->flags & HAMMER_INODE_RDIRTY)) {
2327                                 error = hammer_generate_redo(&trans, ip,
2328                                                              vap->va_size,
2329                                                              HAMMER_REDO_TRUNC,
2330                                                              NULL, 0);
2331                         }
2332                         blksize = hammer_blocksize(vap->va_size);
2333
2334                         /*
2335                          * XXX break atomicy, we can deadlock the backend
2336                          * if we do not release the lock.  Probably not a
2337                          * big deal here.
2338                          */
2339                         if (vap->va_size < ip->ino_data.size) {
2340                                 nvtruncbuf(ap->a_vp, vap->va_size,
2341                                            blksize,
2342                                            hammer_blockoff(vap->va_size),
2343                                            0);
2344                                 truncating = 1;
2345                                 kflags |= NOTE_WRITE;
2346                         } else {
2347                                 nvextendbuf(ap->a_vp,
2348                                             ip->ino_data.size,
2349                                             vap->va_size,
2350                                             hammer_blocksize(ip->ino_data.size),
2351                                             hammer_blocksize(vap->va_size),
2352                                             hammer_blockoff(ip->ino_data.size),
2353                                             hammer_blockoff(vap->va_size),
2354                                             0);
2355                                 truncating = 0;
2356                                 kflags |= NOTE_WRITE | NOTE_EXTEND;
2357                         }
2358                         ip->ino_data.size = vap->va_size;
2359                         ip->ino_data.mtime = trans.time;
2360                         /* XXX safe to use SDIRTY instead of DDIRTY here? */
2361                         modflags |= HAMMER_INODE_MTIME | HAMMER_INODE_DDIRTY;
2362
2363                         /*
2364                          * On-media truncation is cached in the inode until
2365                          * the inode is synchronized.  We must immediately
2366                          * handle any frontend records.
2367                          */
2368                         if (truncating) {
2369                                 hammer_ip_frontend_trunc(ip, vap->va_size);
2370 #ifdef DEBUG_TRUNCATE
2371                                 if (HammerTruncIp == NULL)
2372                                         HammerTruncIp = ip;
2373 #endif
2374                                 if ((ip->flags & HAMMER_INODE_TRUNCATED) == 0) {
2375                                         ip->flags |= HAMMER_INODE_TRUNCATED;
2376                                         ip->trunc_off = vap->va_size;
2377                                         hammer_inode_dirty(ip);
2378 #ifdef DEBUG_TRUNCATE
2379                                         if (ip == HammerTruncIp)
2380                                         kprintf("truncate1 %016llx\n",
2381                                                 (long long)ip->trunc_off);
2382 #endif
2383                                 } else if (ip->trunc_off > vap->va_size) {
2384                                         ip->trunc_off = vap->va_size;
2385 #ifdef DEBUG_TRUNCATE
2386                                         if (ip == HammerTruncIp)
2387                                         kprintf("truncate2 %016llx\n",
2388                                                 (long long)ip->trunc_off);
2389 #endif
2390                                 } else {
2391 #ifdef DEBUG_TRUNCATE
2392                                         if (ip == HammerTruncIp)
2393                                         kprintf("truncate3 %016llx (ignored)\n",
2394                                                 (long long)vap->va_size);
2395 #endif
2396                                 }
2397                         }
2398
2399 #if 0
2400                         /*
2401                          * When truncating, nvtruncbuf() may have cleaned out
2402                          * a portion of the last block on-disk in the buffer
2403                          * cache.  We must clean out any frontend records
2404                          * for blocks beyond the new last block.
2405                          */
2406                         aligned_size = (vap->va_size + (blksize - 1)) &
2407                                        ~(int64_t)(blksize - 1);
2408                         if (truncating && vap->va_size < aligned_size) {
2409                                 aligned_size -= blksize;
2410                                 hammer_ip_frontend_trunc(ip, aligned_size);
2411                         }
2412 #endif
2413                         break;
2414                 case VDATABASE:
2415                         if ((ip->flags & HAMMER_INODE_TRUNCATED) == 0) {
2416                                 ip->flags |= HAMMER_INODE_TRUNCATED;
2417                                 ip->trunc_off = vap->va_size;
2418                                 hammer_inode_dirty(ip);
2419                         } else if (ip->trunc_off > vap->va_size) {
2420                                 ip->trunc_off = vap->va_size;
2421                         }
2422                         hammer_ip_frontend_trunc(ip, vap->va_size);
2423                         ip->ino_data.size = vap->va_size;
2424                         ip->ino_data.mtime = trans.time;
2425                         modflags |= HAMMER_INODE_MTIME | HAMMER_INODE_DDIRTY;
2426                         kflags |= NOTE_ATTRIB;
2427                         break;
2428                 default:
2429                         error = EINVAL;
2430                         goto done;
2431                 }
2432                 break;
2433         }
2434         if (vap->va_atime.tv_sec != VNOVAL) {
2435                 ip->ino_data.atime = hammer_timespec_to_time(&vap->va_atime);
2436                 modflags |= HAMMER_INODE_ATIME;
2437                 kflags |= NOTE_ATTRIB;
2438         }
2439         if (vap->va_mtime.tv_sec != VNOVAL) {
2440                 ip->ino_data.mtime = hammer_timespec_to_time(&vap->va_mtime);
2441                 modflags |= HAMMER_INODE_MTIME;
2442                 kflags |= NOTE_ATTRIB;
2443         }
2444         if (vap->va_mode != (mode_t)VNOVAL) {
2445                 mode_t   cur_mode = ip->ino_data.mode;
2446                 uid_t cur_uid = hammer_to_unix_xid(&ip->ino_data.uid);
2447                 gid_t cur_gid = hammer_to_unix_xid(&ip->ino_data.gid);
2448
2449                 error = vop_helper_chmod(ap->a_vp, vap->va_mode, ap->a_cred,
2450                                          cur_uid, cur_gid, &cur_mode);
2451                 if (error == 0 && ip->ino_data.mode != cur_mode) {
2452                         ip->ino_data.mode = cur_mode;
2453                         ip->ino_data.ctime = trans.time;
2454                         modflags |= HAMMER_INODE_DDIRTY;
2455                         kflags |= NOTE_ATTRIB;
2456                 }
2457         }
2458 done:
2459         if (error == 0)
2460                 hammer_modify_inode(&trans, ip, modflags);
2461         hammer_done_transaction(&trans);
2462         hammer_knote(ap->a_vp, kflags);
2463         lwkt_reltoken(&hmp->fs_token);
2464         return (error);
2465 }
2466
2467 /*
2468  * hammer_vop_nsymlink { nch, dvp, vpp, cred, vap, target }
2469  */
2470 static
2471 int
2472 hammer_vop_nsymlink(struct vop_nsymlink_args *ap)
2473 {
2474         struct hammer_transaction trans;
2475         struct hammer_inode *dip;
2476         struct hammer_inode *nip;
2477         hammer_record_t record;
2478         struct nchandle *nch;
2479         hammer_mount_t hmp;
2480         int error;
2481         int bytes;
2482
2483         ap->a_vap->va_type = VLNK;
2484
2485         nch = ap->a_nch;
2486         dip = VTOI(ap->a_dvp);
2487         hmp = dip->hmp;
2488
2489         if (dip->flags & HAMMER_INODE_RO)
2490                 return (EROFS);
2491         if ((error = hammer_checkspace(hmp, HAMMER_CHKSPC_CREATE)) != 0)
2492                 return (error);
2493
2494         /*
2495          * Create a transaction to cover the operations we perform.
2496          */
2497         lwkt_gettoken(&hmp->fs_token);
2498         hammer_start_transaction(&trans, hmp);
2499         ++hammer_stats_file_iopsw;
2500
2501         /*
2502          * Create a new filesystem object of the requested type.  The
2503          * returned inode will be referenced but not locked.
2504          */
2505
2506         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
2507                                     dip, nch->ncp->nc_name, nch->ncp->nc_nlen,
2508                                     NULL, &nip);
2509         if (error) {
2510                 hammer_done_transaction(&trans);
2511                 *ap->a_vpp = NULL;
2512                 lwkt_reltoken(&hmp->fs_token);
2513                 return (error);
2514         }
2515
2516         /*
2517          * Add a record representing the symlink.  symlink stores the link
2518          * as pure data, not a string, and is no \0 terminated.
2519          */
2520         if (error == 0) {
2521                 bytes = strlen(ap->a_target);
2522
2523                 if (bytes <= HAMMER_INODE_BASESYMLEN) {
2524                         bcopy(ap->a_target, nip->ino_data.ext.symlink, bytes);
2525                 } else {
2526                         record = hammer_alloc_mem_record(nip, bytes);
2527                         record->type = HAMMER_MEM_RECORD_GENERAL;
2528
2529                         record->leaf.base.localization = nip->obj_localization +
2530                                                          HAMMER_LOCALIZE_MISC;
2531                         record->leaf.base.key = HAMMER_FIXKEY_SYMLINK;
2532                         record->leaf.base.rec_type = HAMMER_RECTYPE_FIX;
2533                         record->leaf.data_len = bytes;
2534                         KKASSERT(HAMMER_SYMLINK_NAME_OFF == 0);
2535                         bcopy(ap->a_target, record->data->symlink.name, bytes);
2536                         error = hammer_ip_add_record(&trans, record);
2537                 }
2538
2539                 /*
2540                  * Set the file size to the length of the link.
2541                  */
2542                 if (error == 0) {
2543                         nip->ino_data.size = bytes;
2544                         hammer_modify_inode(&trans, nip, HAMMER_INODE_DDIRTY);
2545                 }
2546         }
2547         if (error == 0)
2548                 error = hammer_ip_add_directory(&trans, dip, nch->ncp->nc_name,
2549                                                 nch->ncp->nc_nlen, nip);
2550
2551         /*
2552          * Finish up.
2553          */
2554         if (error) {
2555                 hammer_rel_inode(nip, 0);
2556                 *ap->a_vpp = NULL;
2557         } else {
2558                 error = hammer_get_vnode(nip, ap->a_vpp);
2559                 hammer_rel_inode(nip, 0);
2560                 if (error == 0) {
2561                         cache_setunresolved(ap->a_nch);
2562                         cache_setvp(ap->a_nch, *ap->a_vpp);
2563                         hammer_knote(ap->a_dvp, NOTE_WRITE);
2564                 }
2565         }
2566         hammer_done_transaction(&trans);
2567         lwkt_reltoken(&hmp->fs_token);
2568         return (error);
2569 }
2570
2571 /*
2572  * hammer_vop_nwhiteout { nch, dvp, cred, flags }
2573  */
2574 static
2575 int
2576 hammer_vop_nwhiteout(struct vop_nwhiteout_args *ap)
2577 {
2578         struct hammer_transaction trans;
2579         struct hammer_inode *dip;
2580         hammer_mount_t hmp;
2581         int error;
2582
2583         dip = VTOI(ap->a_dvp);
2584         hmp = dip->hmp;
2585
2586         if (hammer_nohistory(dip) == 0 &&
2587             (error = hammer_checkspace(hmp, HAMMER_CHKSPC_CREATE)) != 0) {
2588                 return (error);
2589         }
2590
2591         lwkt_gettoken(&hmp->fs_token);
2592         hammer_start_transaction(&trans, hmp);
2593         ++hammer_stats_file_iopsw;
2594         error = hammer_dounlink(&trans, ap->a_nch, ap->a_dvp,
2595                                 ap->a_cred, ap->a_flags, -1);
2596         hammer_done_transaction(&trans);
2597         lwkt_reltoken(&hmp->fs_token);
2598
2599         return (error);
2600 }
2601
2602 /*
2603  * hammer_vop_ioctl { vp, command, data, fflag, cred }
2604  */
2605 static
2606 int
2607 hammer_vop_ioctl(struct vop_ioctl_args *ap)
2608 {
2609         struct hammer_inode *ip = ap->a_vp->v_data;
2610         hammer_mount_t hmp = ip->hmp;
2611         int error;
2612
2613         ++hammer_stats_file_iopsr;
2614         lwkt_gettoken(&hmp->fs_token);
2615         error = hammer_ioctl(ip, ap->a_command, ap->a_data,
2616                              ap->a_fflag, ap->a_cred);
2617         lwkt_reltoken(&hmp->fs_token);
2618         return (error);
2619 }
2620
2621 static
2622 int
2623 hammer_vop_mountctl(struct vop_mountctl_args *ap)
2624 {
2625         static const struct mountctl_opt extraopt[] = {
2626                 { HMNT_NOHISTORY,       "nohistory" },
2627                 { HMNT_MASTERID,        "master" },
2628                 { 0, NULL}
2629
2630         };
2631         struct hammer_mount *hmp;
2632         struct mount *mp;
2633         int usedbytes;
2634         int error;
2635
2636         error = 0;
2637         usedbytes = 0;
2638         mp = ap->a_head.a_ops->head.vv_mount;
2639         KKASSERT(mp->mnt_data != NULL);
2640         hmp = (struct hammer_mount *)mp->mnt_data;
2641
2642         lwkt_gettoken(&hmp->fs_token);
2643
2644         switch(ap->a_op) {
2645         case MOUNTCTL_SET_EXPORT:
2646                 if (ap->a_ctllen != sizeof(struct export_args))
2647                         error = EINVAL;
2648                 else
2649                         error = hammer_vfs_export(mp, ap->a_op,
2650                                       (const struct export_args *)ap->a_ctl);
2651                 break;
2652         case MOUNTCTL_MOUNTFLAGS:
2653         {
2654                 /*
2655                  * Call standard mountctl VOP function
2656                  * so we get user mount flags.
2657                  */
2658                 error = vop_stdmountctl(ap);
2659                 if (error)
2660                         break;
2661
2662                 usedbytes = *ap->a_res;
2663
2664                 if (usedbytes > 0 && usedbytes < ap->a_buflen) {
2665                         usedbytes += vfs_flagstostr(hmp->hflags, extraopt,
2666                                                     ap->a_buf,
2667                                                     ap->a_buflen - usedbytes,
2668                                                     &error);
2669                 }
2670
2671                 *ap->a_res += usedbytes;
2672                 break;
2673         }
2674         default:
2675                 error = vop_stdmountctl(ap);
2676                 break;
2677         }
2678         lwkt_reltoken(&hmp->fs_token);
2679         return(error);
2680 }
2681
2682 /*
2683  * hammer_vop_strategy { vp, bio }
2684  *
2685  * Strategy call, used for regular file read & write only.  Note that the
2686  * bp may represent a cluster.
2687  *
2688  * To simplify operation and allow better optimizations in the future,
2689  * this code does not make any assumptions with regards to buffer alignment
2690  * or size.
2691  */
2692 static
2693 int
2694 hammer_vop_strategy(struct vop_strategy_args *ap)
2695 {
2696         struct buf *bp;
2697         int error;
2698
2699         bp = ap->a_bio->bio_buf;
2700
2701         switch(bp->b_cmd) {
2702         case BUF_CMD_READ:
2703                 error = hammer_vop_strategy_read(ap);
2704                 break;
2705         case BUF_CMD_WRITE:
2706                 error = hammer_vop_strategy_write(ap);
2707                 break;
2708         default:
2709                 bp->b_error = error = EINVAL;
2710                 bp->b_flags |= B_ERROR;
2711                 biodone(ap->a_bio);
2712                 break;
2713         }
2714
2715         /* hammer_dump_dedup_cache(((hammer_inode_t)ap->a_vp->v_data)->hmp); */
2716
2717         return (error);
2718 }
2719
2720 /*
2721  * Read from a regular file.  Iterate the related records and fill in the
2722  * BIO/BUF.  Gaps are zero-filled.
2723  *
2724  * The support code in hammer_object.c should be used to deal with mixed
2725  * in-memory and on-disk records.
2726  *
2727  * NOTE: Can be called from the cluster code with an oversized buf.
2728  *
2729  * XXX atime update
2730  */
2731 static
2732 int
2733 hammer_vop_strategy_read(struct vop_strategy_args *ap)
2734 {
2735         struct hammer_transaction trans;
2736         struct hammer_inode *ip;
2737         struct hammer_inode *dip;
2738         hammer_mount_t hmp;
2739         struct hammer_cursor cursor;
2740         hammer_base_elm_t base;
2741         hammer_off_t disk_offset;
2742         struct bio *bio;
2743         struct bio *nbio;
2744         struct buf *bp;
2745         int64_t rec_offset;
2746         int64_t ran_end;
2747         int64_t tmp64;
2748         int error;
2749         int boff;
2750         int roff;
2751         int n;
2752         int isdedupable;
2753
2754         bio = ap->a_bio;
2755         bp = bio->bio_buf;
2756         ip = ap->a_vp->v_data;
2757         hmp = ip->hmp;
2758
2759         /*
2760          * The zone-2 disk offset may have been set by the cluster code via
2761          * a BMAP operation, or else should be NOOFFSET.
2762          *
2763          * Checking the high bits for a match against zone-2 should suffice.
2764          *
2765          * In cases where a lot of data duplication is present it may be
2766          * more beneficial to drop through and doubule-buffer through the
2767          * device.
2768          */
2769         nbio = push_bio(bio);
2770         if ((nbio->bio_offset & HAMMER_OFF_ZONE_MASK) ==
2771             HAMMER_ZONE_LARGE_DATA) {
2772                 if (hammer_double_buffer == 0) {
2773                         lwkt_gettoken(&hmp->fs_token);
2774                         error = hammer_io_direct_read(hmp, nbio, NULL);
2775                         lwkt_reltoken(&hmp->fs_token);
2776                         return (error);
2777                 }
2778
2779                 /*
2780                  * Try to shortcut requests for double_buffer mode too.
2781                  * Since this mode runs through the device buffer cache
2782                  * only compatible buffer sizes (meaning those generated
2783                  * by normal filesystem buffers) are legal.
2784                  */
2785                 if (hammer_live_dedup == 0 && (bp->b_flags & B_PAGING) == 0) {
2786                         error = hammer_io_indirect_read(hmp, nbio, NULL);
2787                         return (error);
2788                 }
2789         }
2790
2791         /*
2792          * Well, that sucked.  Do it the hard way.  If all the stars are
2793          * aligned we may still be able to issue a direct-read.
2794          */
2795         lwkt_gettoken(&hmp->fs_token);
2796         hammer_simple_transaction(&trans, hmp);
2797         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
2798
2799         /*
2800          * Key range (begin and end inclusive) to scan.  Note that the key's
2801          * stored in the actual records represent BASE+LEN, not BASE.  The
2802          * first record containing bio_offset will have a key > bio_offset.
2803          */
2804         cursor.key_beg.localization = ip->obj_localization +
2805                                       HAMMER_LOCALIZE_MISC;
2806         cursor.key_beg.obj_id = ip->obj_id;
2807         cursor.key_beg.create_tid = 0;
2808         cursor.key_beg.delete_tid = 0;
2809         cursor.key_beg.obj_type = 0;
2810         cursor.key_beg.key = bio->bio_offset + 1;
2811         cursor.asof = ip->obj_asof;
2812         cursor.flags |= HAMMER_CURSOR_ASOF;
2813
2814         cursor.key_end = cursor.key_beg;
2815         KKASSERT(ip->ino_data.obj_type == HAMMER_OBJTYPE_REGFILE);
2816 #if 0
2817         if (ip->ino_data.obj_type == HAMMER_OBJTYPE_DBFILE) {
2818                 cursor.key_beg.rec_type = HAMMER_RECTYPE_DB;
2819                 cursor.key_end.rec_type = HAMMER_RECTYPE_DB;
2820                 cursor.key_end.key = 0x7FFFFFFFFFFFFFFFLL;
2821         } else
2822 #endif
2823         {
2824                 ran_end = bio->bio_offset + bp->b_bufsize;
2825                 cursor.key_beg.rec_type = HAMMER_RECTYPE_DATA;
2826                 cursor.key_end.rec_type = HAMMER_RECTYPE_DATA;
2827                 tmp64 = ran_end + MAXPHYS + 1;  /* work-around GCC-4 bug */
2828                 if (tmp64 < ran_end)
2829                         cursor.key_end.key = 0x7FFFFFFFFFFFFFFFLL;
2830                 else
2831                         cursor.key_end.key = ran_end + MAXPHYS + 1;
2832         }
2833         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE;
2834
2835         /*
2836          * Set NOSWAPCACHE for cursor data extraction if double buffering
2837          * is disabled or (if the file is not marked cacheable via chflags
2838          * and vm.swapcache_use_chflags is enabled).
2839          */
2840         if (hammer_double_buffer == 0 ||
2841             ((ap->a_vp->v_flag & VSWAPCACHE) == 0 &&
2842              vm_swapcache_use_chflags)) {
2843                 cursor.flags |= HAMMER_CURSOR_NOSWAPCACHE;
2844         }
2845
2846         error = hammer_ip_first(&cursor);
2847         boff = 0;
2848
2849         while (error == 0) {
2850                 /*
2851                  * Get the base file offset of the record.  The key for
2852                  * data records is (base + bytes) rather then (base).
2853                  */
2854                 base = &cursor.leaf->base;
2855                 rec_offset = base->key - cursor.leaf->data_len;
2856
2857                 /*
2858                  * Calculate the gap, if any, and zero-fill it.
2859                  *
2860                  * n is the offset of the start of the record verses our
2861                  * current seek offset in the bio.
2862                  */
2863                 n = (int)(rec_offset - (bio->bio_offset + boff));
2864                 if (n > 0) {
2865                         if (n > bp->b_bufsize - boff)
2866                                 n = bp->b_bufsize - boff;
2867                         bzero((char *)bp->b_data + boff, n);
2868                         boff += n;
2869                         n = 0;
2870                 }
2871
2872                 /*
2873                  * Calculate the data offset in the record and the number
2874                  * of bytes we can copy.
2875                  *
2876                  * There are two degenerate cases.  First, boff may already
2877                  * be at bp->b_bufsize.  Secondly, the data offset within
2878                  * the record may exceed the record's size.
2879                  */
2880                 roff = -n;
2881                 rec_offset += roff;
2882                 n = cursor.leaf->data_len - roff;
2883                 if (n <= 0) {
2884                         kprintf("strategy_read: bad n=%d roff=%d\n", n, roff);
2885                         n = 0;
2886                 } else if (n > bp->b_bufsize - boff) {
2887                         n = bp->b_bufsize - boff;
2888                 }
2889
2890                 /*
2891                  * Deal with cached truncations.  This cool bit of code
2892                  * allows truncate()/ftruncate() to avoid having to sync
2893                  * the file.
2894                  *
2895                  * If the frontend is truncated then all backend records are
2896                  * subject to the frontend's truncation.
2897                  *
2898                  * If the backend is truncated then backend records on-disk
2899                  * (but not in-memory) are subject to the backend's
2900                  * truncation.  In-memory records owned by the backend
2901                  * represent data written after the truncation point on the
2902                  * backend and must not be truncated.
2903                  *
2904                  * Truncate operations deal with frontend buffer cache
2905                  * buffers and frontend-owned in-memory records synchronously.
2906                  */
2907                 if (ip->flags & HAMMER_INODE_TRUNCATED) {
2908                         if (hammer_cursor_ondisk(&cursor)/* ||
2909                             cursor.iprec->flush_state == HAMMER_FST_FLUSH*/) {
2910                                 if (ip->trunc_off <= rec_offset)
2911                                         n = 0;
2912                                 else if (ip->trunc_off < rec_offset + n)
2913                                         n = (int)(ip->trunc_off - rec_offset);
2914                         }
2915                 }
2916                 if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
2917                         if (hammer_cursor_ondisk(&cursor)) {
2918                                 if (ip->sync_trunc_off <= rec_offset)
2919                                         n = 0;
2920                                 else if (ip->sync_trunc_off < rec_offset + n)
2921                                         n = (int)(ip->sync_trunc_off - rec_offset);
2922                         }
2923                 }
2924
2925                 /*
2926                  * Try to issue a direct read into our bio if possible,
2927                  * otherwise resolve the element data into a hammer_buffer
2928                  * and copy.
2929                  *
2930                  * The buffer on-disk should be zerod past any real
2931                  * truncation point, but may not be for any synthesized
2932                  * truncation point from above.
2933                  *
2934                  * NOTE: disk_offset is only valid if the cursor data is
2935                  *       on-disk.
2936                  */
2937                 disk_offset = cursor.leaf->data_offset + roff;
2938                 isdedupable = (boff == 0 && n == bp->b_bufsize &&
2939                                hammer_cursor_ondisk(&cursor) &&
2940                                ((int)disk_offset & HAMMER_BUFMASK) == 0);
2941
2942                 if (isdedupable && hammer_double_buffer == 0) {
2943                         /*
2944                          * Direct read case
2945                          */
2946                         KKASSERT((disk_offset & HAMMER_OFF_ZONE_MASK) ==
2947                                  HAMMER_ZONE_LARGE_DATA);
2948                         nbio->bio_offset = disk_offset;
2949                         error = hammer_io_direct_read(hmp, nbio, cursor.leaf);
2950                         if (hammer_live_dedup && error == 0)
2951                                 hammer_dedup_cache_add(ip, cursor.leaf);
2952                         goto done;
2953                 } else if (isdedupable) {
2954                         /*
2955                          * Async I/O case for reading from backing store
2956                          * and copying the data to the filesystem buffer.
2957                          * live-dedup has to verify the data anyway if it
2958                          * gets a hit later so we can just add the entry
2959                          * now.
2960                          */
2961                         KKASSERT((disk_offset & HAMMER_OFF_ZONE_MASK) ==
2962                                  HAMMER_ZONE_LARGE_DATA);
2963                         nbio->bio_offset = disk_offset;
2964                         if (hammer_live_dedup)
2965                                 hammer_dedup_cache_add(ip, cursor.leaf);
2966                         error = hammer_io_indirect_read(hmp, nbio, cursor.leaf);
2967                         goto done;
2968                 } else if (n) {
2969                         error = hammer_ip_resolve_data(&cursor);
2970                         if (error == 0) {
2971                                 if (hammer_live_dedup && isdedupable)
2972                                         hammer_dedup_cache_add(ip, cursor.leaf);
2973                                 bcopy((char *)cursor.data + roff,
2974                                       (char *)bp->b_data + boff, n);
2975                         }
2976                 }
2977                 if (error)
2978                         break;
2979
2980                 /*
2981                  * We have to be sure that the only elements added to the
2982                  * dedup cache are those which are already on-media.
2983                  */
2984                 if (hammer_live_dedup && hammer_cursor_ondisk(&cursor))
2985                         hammer_dedup_cache_add(ip, cursor.leaf);
2986
2987                 /*
2988                  * Iterate until we have filled the request.
2989                  */
2990                 boff += n;
2991                 if (boff == bp->b_bufsize)
2992                         break;
2993                 error = hammer_ip_next(&cursor);
2994         }
2995
2996         /*
2997          * There may have been a gap after the last record
2998          */
2999         if (error == ENOENT)
3000                 error = 0;
3001         if (error == 0 && boff != bp->b_bufsize) {
3002                 KKASSERT(boff < bp->b_bufsize);
3003                 bzero((char *)bp->b_data + boff, bp->b_bufsize - boff);
3004                 /* boff = bp->b_bufsize; */
3005         }
3006
3007         /*
3008          * Disallow swapcache operation on the vnode buffer if double
3009          * buffering is enabled, the swapcache will get the data via
3010          * the block device buffer.
3011          */
3012         if (hammer_double_buffer)
3013                 bp->b_flags |= B_NOTMETA;
3014
3015         /*
3016          * Cleanup
3017          */
3018         bp->b_resid = 0;
3019         bp->b_error = error;
3020         if (error)
3021                 bp->b_flags |= B_ERROR;
3022         biodone(ap->a_bio);
3023
3024 done:
3025         /*
3026          * Cache the b-tree node for the last data read in cache[1].
3027          *
3028          * If we hit the file EOF then also cache the node in the
3029          * governing director's cache[3], it will be used to initialize
3030          * the inode's cache[1] for any inodes looked up via the directory.
3031          *
3032          * This doesn't reduce disk accesses since the B-Tree chain is
3033          * likely cached, but it does reduce cpu overhead when looking
3034          * up file offsets for cpdup/tar/cpio style iterations.
3035          */
3036         if (cursor.node)
3037                 hammer_cache_node(&ip->cache[1], cursor.node);
3038         if (ran_end >= ip->ino_data.size) {
3039                 dip = hammer_find_inode(&trans, ip->ino_data.parent_obj_id,
3040                                         ip->obj_asof, ip->obj_localization);
3041                 if (dip) {
3042                         hammer_cache_node(&dip->cache[3], cursor.node);
3043                         hammer_rel_inode(dip, 0);
3044                 }
3045         }
3046         hammer_done_cursor(&cursor);
3047         hammer_done_transaction(&trans);
3048         lwkt_reltoken(&hmp->fs_token);
3049         return(error);
3050 }
3051
3052 /*
3053  * BMAP operation - used to support cluster_read() only.
3054  *
3055  * (struct vnode *vp, off_t loffset, off_t *doffsetp, int *runp, int *runb)
3056  *
3057  * This routine may return EOPNOTSUPP if the opration is not supported for
3058  * the specified offset.  The contents of the pointer arguments do not
3059  * need to be initialized in that case.
3060  *
3061  * If a disk address is available and properly aligned return 0 with
3062  * *doffsetp set to the zone-2 address, and *runp / *runb set appropriately
3063  * to the run-length relative to that offset.  Callers may assume that
3064  * *doffsetp is valid if 0 is returned, even if *runp is not sufficiently
3065  * large, so return EOPNOTSUPP if it is not sufficiently large.
3066  */
3067 static
3068 int
3069 hammer_vop_bmap(struct vop_bmap_args *ap)
3070 {
3071         struct hammer_transaction trans;
3072         struct hammer_inode *ip;
3073         hammer_mount_t hmp;
3074         struct hammer_cursor cursor;
3075         hammer_base_elm_t base;
3076         int64_t rec_offset;
3077         int64_t ran_end;
3078         int64_t tmp64;
3079         int64_t base_offset;
3080         int64_t base_disk_offset;
3081         int64_t last_offset;
3082         hammer_off_t last_disk_offset;
3083         hammer_off_t disk_offset;
3084         int     rec_len;
3085         int     error;
3086         int     blksize;
3087
3088         ++hammer_stats_file_iopsr;
3089         ip = ap->a_vp->v_data;
3090         hmp = ip->hmp;
3091
3092         /*
3093          * We can only BMAP regular files.  We can't BMAP database files,
3094          * directories, etc.
3095          */
3096         if (ip->ino_data.obj_type != HAMMER_OBJTYPE_REGFILE)
3097                 return(EOPNOTSUPP);
3098
3099         /*
3100          * bmap is typically called with runp/runb both NULL when used
3101          * for writing.  We do not support BMAP for writing atm.
3102          */
3103         if (ap->a_cmd != BUF_CMD_READ)
3104                 return(EOPNOTSUPP);
3105
3106         /*
3107          * Scan the B-Tree to acquire blockmap addresses, then translate
3108          * to raw addresses.
3109          */
3110         lwkt_gettoken(&hmp->fs_token);
3111         hammer_simple_transaction(&trans, hmp);
3112 #if 0
3113         kprintf("bmap_beg %016llx ip->cache %p\n",
3114                 (long long)ap->a_loffset, ip->cache[1]);
3115 #endif
3116         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
3117
3118         /*
3119          * Key range (begin and end inclusive) to scan.  Note that the key's
3120          * stored in the actual records represent BASE+LEN, not BASE.  The
3121          * first record containing bio_offset will have a key > bio_offset.
3122          */
3123         cursor.key_beg.localization = ip->obj_localization +
3124                                       HAMMER_LOCALIZE_MISC;
3125         cursor.key_beg.obj_id = ip->obj_id;
3126         cursor.key_beg.create_tid = 0;
3127         cursor.key_beg.delete_tid = 0;
3128         cursor.key_beg.obj_type = 0;
3129         if (ap->a_runb)
3130                 cursor.key_beg.key = ap->a_loffset - MAXPHYS + 1;
3131         else
3132                 cursor.key_beg.key = ap->a_loffset + 1;
3133         if (cursor.key_beg.key < 0)
3134                 cursor.key_beg.key = 0;
3135         cursor.asof = ip->obj_asof;
3136         cursor.flags |= HAMMER_CURSOR_ASOF;
3137
3138         cursor.key_end = cursor.key_beg;
3139         KKASSERT(ip->ino_data.obj_type == HAMMER_OBJTYPE_REGFILE);
3140
3141         ran_end = ap->a_loffset + MAXPHYS;
3142         cursor.key_beg.rec_type = HAMMER_RECTYPE_DATA;
3143         cursor.key_end.rec_type = HAMMER_RECTYPE_DATA;
3144         tmp64 = ran_end + MAXPHYS + 1;  /* work-around GCC-4 bug */
3145         if (tmp64 < ran_end)
3146                 cursor.key_end.key = 0x7FFFFFFFFFFFFFFFLL;
3147         else
3148                 cursor.key_end.key = ran_end + MAXPHYS + 1;
3149
3150         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE;
3151
3152         error = hammer_ip_first(&cursor);
3153         base_offset = last_offset = 0;
3154         base_disk_offset = last_disk_offset = 0;
3155
3156         while (error == 0) {
3157                 /*
3158                  * Get the base file offset of the record.  The key for
3159                  * data records is (base + bytes) rather then (base).
3160                  *
3161                  * NOTE: rec_offset + rec_len may exceed the end-of-file.
3162                  * The extra bytes should be zero on-disk and the BMAP op
3163                  * should still be ok.
3164                  */
3165                 base = &cursor.leaf->base;
3166                 rec_offset = base->key - cursor.leaf->data_len;
3167                 rec_len    = cursor.leaf->data_len;
3168
3169                 /*
3170                  * Incorporate any cached truncation.
3171                  *
3172                  * NOTE: Modifications to rec_len based on synthesized
3173                  * truncation points remove the guarantee that any extended
3174                  * data on disk is zero (since the truncations may not have
3175                  * taken place on-media yet).
3176                  */
3177                 if (ip->flags & HAMMER_INODE_TRUNCATED) {
3178                         if (hammer_cursor_ondisk(&cursor) ||
3179                             cursor.iprec->flush_state == HAMMER_FST_FLUSH) {
3180                                 if (ip->trunc_off <= rec_offset)
3181                                         rec_len = 0;
3182                                 else if (ip->trunc_off < rec_offset + rec_len)
3183                                         rec_len = (int)(ip->trunc_off - rec_offset);
3184                         }
3185                 }
3186                 if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
3187                         if (hammer_cursor_ondisk(&cursor)) {
3188                                 if (ip->sync_trunc_off <= rec_offset)
3189                                         rec_len = 0;
3190                                 else if (ip->sync_trunc_off < rec_offset + rec_len)
3191                                         rec_len = (int)(ip->sync_trunc_off - rec_offset);
3192                         }
3193                 }
3194
3195                 /*
3196                  * Accumulate information.  If we have hit a discontiguous
3197                  * block reset base_offset unless we are already beyond the
3198                  * requested offset.  If we are, that's it, we stop.
3199                  */
3200                 if (error)
3201                         break;
3202                 if (hammer_cursor_ondisk(&cursor)) {
3203                         disk_offset = cursor.leaf->data_offset;
3204                         if (rec_offset != last_offset ||
3205                             disk_offset != last_disk_offset) {
3206                                 if (rec_offset > ap->a_loffset)
3207                                         break;
3208                                 base_offset = rec_offset;
3209                                 base_disk_offset = disk_offset;
3210                         }
3211                         last_offset = rec_offset + rec_len;
3212                         last_disk_offset = disk_offset + rec_len;
3213
3214                         if (hammer_live_dedup)
3215                                 hammer_dedup_cache_add(ip, cursor.leaf);
3216                 }
3217
3218                 error = hammer_ip_next(&cursor);
3219         }
3220
3221 #if 0
3222         kprintf("BMAP %016llx:  %016llx - %016llx\n",
3223                 (long long)ap->a_loffset,
3224                 (long long)base_offset,
3225                 (long long)last_offset);
3226         kprintf("BMAP %16s:  %016llx - %016llx\n", "",
3227                 (long long)base_disk_offset,
3228                 (long long)last_disk_offset);
3229 #endif
3230
3231         if (cursor.node) {
3232                 hammer_cache_node(&ip->cache[1], cursor.node);
3233 #if 0
3234                 kprintf("bmap_end2 %016llx ip->cache %p\n",
3235                         (long long)ap->a_loffset, ip->cache[1]);
3236 #endif
3237         }
3238         hammer_done_cursor(&cursor);
3239         hammer_done_transaction(&trans);
3240         lwkt_reltoken(&hmp->fs_token);
3241
3242         /*
3243          * If we couldn't find any records or the records we did find were
3244          * all behind the requested offset, return failure.  A forward
3245          * truncation can leave a hole w/ no on-disk records.
3246          */
3247         if (last_offset == 0 || last_offset < ap->a_loffset)
3248                 return (EOPNOTSUPP);
3249
3250         /*
3251          * Figure out the block size at the requested offset and adjust
3252          * our limits so the cluster_read() does not create inappropriately
3253          * sized buffer cache buffers.
3254          */
3255         blksize = hammer_blocksize(ap->a_loffset);
3256         if (hammer_blocksize(base_offset) != blksize) {
3257                 base_offset = hammer_blockdemarc(base_offset, ap->a_loffset);
3258         }
3259         if (last_offset != ap->a_loffset &&
3260             hammer_blocksize(last_offset - 1) != blksize) {
3261                 last_offset = hammer_blockdemarc(ap->a_loffset,
3262                                                  last_offset - 1);
3263         }
3264
3265         /*
3266          * Returning EOPNOTSUPP simply prevents the direct-IO optimization
3267          * from occuring.
3268          */
3269         disk_offset = base_disk_offset + (ap->a_loffset - base_offset);
3270
3271         if ((disk_offset & HAMMER_OFF_ZONE_MASK) != HAMMER_ZONE_LARGE_DATA) {
3272                 /*
3273                  * Only large-data zones can be direct-IOd
3274                  */
3275                 error = EOPNOTSUPP;
3276         } else if ((disk_offset & HAMMER_BUFMASK) ||
3277                    (last_offset - ap->a_loffset) < blksize) {
3278                 /*
3279                  * doffsetp is not aligned or the forward run size does
3280                  * not cover a whole buffer, disallow the direct I/O.
3281                  */
3282                 error = EOPNOTSUPP;
3283         } else {
3284                 /*
3285                  * We're good.
3286                  */
3287                 *ap->a_doffsetp = disk_offset;
3288                 if (ap->a_runb) {
3289                         *ap->a_runb = ap->a_loffset - base_offset;
3290                         KKASSERT(*ap->a_runb >= 0);
3291                 }
3292                 if (ap->a_runp) {
3293                         *ap->a_runp = last_offset - ap->a_loffset;
3294                         KKASSERT(*ap->a_runp >= 0);
3295                 }
3296                 error = 0;
3297         }
3298         return(error);
3299 }
3300
3301 /*
3302  * Write to a regular file.   Because this is a strategy call the OS is
3303  * trying to actually get data onto the media.
3304  */
3305 static
3306 int
3307 hammer_vop_strategy_write(struct vop_strategy_args *ap)
3308 {
3309         hammer_record_t record;
3310         hammer_mount_t hmp;
3311         hammer_inode_t ip;
3312         struct bio *bio;
3313         struct buf *bp;
3314         int blksize __debugvar;
3315         int bytes;
3316         int error;
3317
3318         bio = ap->a_bio;
3319         bp = bio->bio_buf;
3320         ip = ap->a_vp->v_data;
3321         hmp = ip->hmp;
3322
3323         blksize = hammer_blocksize(bio->bio_offset);
3324         KKASSERT(bp->b_bufsize == blksize);
3325
3326         if (ip->flags & HAMMER_INODE_RO) {
3327                 bp->b_error = EROFS;
3328                 bp->b_flags |= B_ERROR;
3329                 biodone(ap->a_bio);
3330                 return(EROFS);
3331         }
3332
3333         lwkt_gettoken(&hmp->fs_token);
3334
3335         /*
3336          * Disallow swapcache operation on the vnode buffer if double
3337          * buffering is enabled, the swapcache will get the data via
3338          * the block device buffer.
3339          */
3340         if (hammer_double_buffer)
3341                 bp->b_flags |= B_NOTMETA;
3342
3343         /*
3344          * Interlock with inode destruction (no in-kernel or directory
3345          * topology visibility).  If we queue new IO while trying to
3346          * destroy the inode we can deadlock the vtrunc call in
3347          * hammer_inode_unloadable_check().
3348          *
3349          * Besides, there's no point flushing a bp associated with an
3350          * inode that is being destroyed on-media and has no kernel
3351          * references.
3352          */
3353         if ((ip->flags | ip->sync_flags) &
3354             (HAMMER_INODE_DELETING|HAMMER_INODE_DELETED)) {
3355                 bp->b_resid = 0;
3356                 biodone(ap->a_bio);
3357                 lwkt_reltoken(&hmp->fs_token);
3358                 return(0);
3359         }
3360
3361         /*
3362          * Reserve space and issue a direct-write from the front-end.
3363          * NOTE: The direct_io code will hammer_bread/bcopy smaller
3364          * allocations.
3365          *
3366          * An in-memory record will be installed to reference the storage
3367          * until the flusher can get to it.
3368          *
3369          * Since we own the high level bio the front-end will not try to
3370          * do a direct-read until the write completes.
3371          *
3372          * NOTE: The only time we do not reserve a full-sized buffers
3373          * worth of data is if the file is small.  We do not try to
3374          * allocate a fragment (from the small-data zone) at the end of
3375          * an otherwise large file as this can lead to wildly separated
3376          * data.
3377          */
3378         KKASSERT((bio->bio_offset & HAMMER_BUFMASK) == 0);
3379         KKASSERT(bio->bio_offset < ip->ino_data.size);
3380         if (bio->bio_offset || ip->ino_data.size > HAMMER_BUFSIZE / 2)
3381                 bytes = bp->b_bufsize;
3382         else
3383                 bytes = ((int)ip->ino_data.size + 15) & ~15;
3384
3385         record = hammer_ip_add_bulk(ip, bio->bio_offset, bp->b_data,
3386                                     bytes, &error);
3387
3388         /*
3389          * B_VFSFLAG1 indicates that a REDO_WRITE entry was generated
3390          * in hammer_vop_write().  We must flag the record so the proper
3391          * REDO_TERM_WRITE entry is generated during the flush.
3392          */
3393         if (record) {
3394                 if (bp->b_flags & B_VFSFLAG1) {
3395                         record->flags |= HAMMER_RECF_REDO;
3396                         bp->b_flags &= ~B_VFSFLAG1;
3397                 }
3398                 if (record->flags & HAMMER_RECF_DEDUPED) {
3399                         bp->b_resid = 0;
3400                         hammer_ip_replace_bulk(hmp, record);
3401                         biodone(ap->a_bio);
3402                 } else {
3403                         hammer_io_direct_write(hmp, bio, record);
3404                 }
3405                 if (ip->rsv_recs > 1 && hmp->rsv_recs > hammer_limit_recs)
3406                         hammer_flush_inode(ip, 0);
3407         } else {
3408                 bp->b_bio2.bio_offset = NOOFFSET;
3409                 bp->b_error = error;
3410                 bp->b_flags |= B_ERROR;
3411                 biodone(ap->a_bio);
3412         }
3413         lwkt_reltoken(&hmp->fs_token);
3414         return(error);
3415 }
3416
3417 /*
3418  * dounlink - disconnect a directory entry
3419  *
3420  * XXX whiteout support not really in yet
3421  */
3422 static int
3423 hammer_dounlink(hammer_transaction_t trans, struct nchandle *nch,
3424                 struct vnode *dvp, struct ucred *cred,
3425                 int flags, int isdir)
3426 {
3427         struct namecache *ncp;
3428         hammer_inode_t dip;
3429         hammer_inode_t ip;
3430         hammer_mount_t hmp;
3431         struct hammer_cursor cursor;
3432         int64_t namekey;
3433         u_int32_t max_iterations;
3434         int nlen, error;
3435
3436         /*
3437          * Calculate the namekey and setup the key range for the scan.  This
3438          * works kinda like a chained hash table where the lower 32 bits
3439          * of the namekey synthesize the chain.
3440          *
3441          * The key range is inclusive of both key_beg and key_end.
3442          */
3443         dip = VTOI(dvp);
3444         ncp = nch->ncp;
3445         hmp = dip->hmp;
3446
3447         if (dip->flags & HAMMER_INODE_RO)
3448                 return (EROFS);
3449
3450         namekey = hammer_directory_namekey(dip, ncp->nc_name, ncp->nc_nlen,
3451                                            &max_iterations);
3452 retry:
3453         hammer_init_cursor(trans, &cursor, &dip->cache[1], dip);
3454         cursor.key_beg.localization = dip->obj_localization +
3455                                       hammer_dir_localization(dip);
3456         cursor.key_beg.obj_id = dip->obj_id;
3457         cursor.key_beg.key = namekey;
3458         cursor.key_beg.create_tid = 0;
3459         cursor.key_beg.delete_tid = 0;
3460         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
3461         cursor.key_beg.obj_type = 0;
3462
3463         cursor.key_end = cursor.key_beg;
3464         cursor.key_end.key += max_iterations;
3465         cursor.asof = dip->obj_asof;
3466         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
3467
3468         /*
3469          * Scan all matching records (the chain), locate the one matching
3470          * the requested path component.  info->last_error contains the
3471          * error code on search termination and could be 0, ENOENT, or
3472          * something else.
3473          *
3474          * The hammer_ip_*() functions merge in-memory records with on-disk
3475          * records for the purposes of the search.
3476          */
3477         error = hammer_ip_first(&cursor);
3478
3479         while (error == 0) {
3480                 error = hammer_ip_resolve_data(&cursor);
3481                 if (error)
3482                         break;
3483                 nlen = cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF;
3484                 KKASSERT(nlen > 0);
3485                 if (ncp->nc_nlen == nlen &&
3486                     bcmp(ncp->nc_name, cursor.data->entry.name, nlen) == 0) {
3487                         break;
3488                 }
3489                 error = hammer_ip_next(&cursor);
3490         }
3491
3492         /*
3493          * If all is ok we have to get the inode so we can adjust nlinks.
3494          * To avoid a deadlock with the flusher we must release the inode
3495          * lock on the directory when acquiring the inode for the entry.
3496          *
3497          * If the target is a directory, it must be empty.
3498          */
3499         if (error == 0) {
3500                 hammer_unlock(&cursor.ip->lock);
3501                 ip = hammer_get_inode(trans, dip, cursor.data->entry.obj_id,
3502                                       hmp->asof,
3503                                       cursor.data->entry.localization,
3504                                       0, &error);
3505                 hammer_lock_sh(&cursor.ip->lock);
3506                 if (error == ENOENT) {
3507                         kprintf("HAMMER: WARNING: Removing "
3508                                 "dirent w/missing inode \"%s\"\n"
3509                                 "\tobj_id = %016llx\n",
3510                                 ncp->nc_name,
3511                                 (long long)cursor.data->entry.obj_id);
3512                         error = 0;
3513                 }
3514
3515                 /*
3516                  * If isdir >= 0 we validate that the entry is or is not a
3517                  * directory.  If isdir < 0 we don't care.
3518                  */
3519                 if (error == 0 && isdir >= 0 && ip) {
3520                         if (isdir &&
3521                             ip->ino_data.obj_type != HAMMER_OBJTYPE_DIRECTORY) {
3522                                 error = ENOTDIR;
3523                         } else if (isdir == 0 &&
3524                             ip->ino_data.obj_type == HAMMER_OBJTYPE_DIRECTORY) {
3525                                 error = EISDIR;
3526                         }
3527                 }
3528
3529                 /*
3530                  * If we are trying to remove a directory the directory must
3531                  * be empty.
3532                  *
3533                  * The check directory code can loop and deadlock/retry.  Our
3534                  * own cursor's node locks must be released to avoid a 3-way
3535                  * deadlock with the flusher if the check directory code
3536                  * blocks.
3537                  *
3538                  * If any changes whatsoever have been made to the cursor
3539                  * set EDEADLK and retry.
3540                  *
3541                  * WARNING: See warnings in hammer_unlock_cursor()
3542                  *          function.
3543                  */
3544                 if (error == 0 && ip && ip->ino_data.obj_type ==
3545                                         HAMMER_OBJTYPE_DIRECTORY) {
3546                         hammer_unlock_cursor(&cursor);
3547                         error = hammer_ip_check_directory_empty(trans, ip);
3548                         hammer_lock_cursor(&cursor);
3549                         if (cursor.flags & HAMMER_CURSOR_RETEST) {
3550                                 kprintf("HAMMER: Warning: avoided deadlock "
3551                                         "on rmdir '%s'\n",
3552                                         ncp->nc_name);
3553                                 error = EDEADLK;
3554                         }
3555                 }
3556
3557                 /*
3558                  * Delete the directory entry.
3559                  *
3560                  * WARNING: hammer_ip_del_directory() may have to terminate
3561                  * the cursor to avoid a deadlock.  It is ok to call
3562                  * hammer_done_cursor() twice.
3563                  */
3564                 if (error == 0) {
3565                         error = hammer_ip_del_directory(trans, &cursor,
3566                                                         dip, ip);
3567                 }
3568                 hammer_done_cursor(&cursor);
3569                 if (error == 0) {
3570                         /*
3571                          * Tell the namecache that we are now unlinked.
3572                          */
3573                         cache_unlink(nch);
3574
3575                         /*
3576                          * NOTE: ip->vp, if non-NULL, cannot be directly
3577                          *       referenced without formally acquiring the
3578                          *       vp since the vp might have zero refs on it,
3579                          *       or in the middle of a reclaim, etc.
3580                          *
3581                          * NOTE: The cache_setunresolved() can rip the vp
3582                          *       out from under us since the vp may not have
3583                          *       any refs, in which case ip->vp will be NULL
3584                          *       from the outset.
3585                          */
3586                         while (ip && ip->vp) {
3587                                 struct vnode *vp;
3588
3589                                 error = hammer_get_vnode(ip, &vp);
3590                                 if (error == 0 && vp) {
3591                                         vn_unlock(vp);
3592                                         hammer_knote(ip->vp, NOTE_DELETE);
3593                                         cache_inval_vp(ip->vp, CINV_DESTROY);
3594                                         vrele(vp);
3595                                         break;
3596                                 }
3597                                 kprintf("Debug: HAMMER ip/vp race1 avoided\n");
3598                         }
3599                 }
3600                 if (ip)
3601                         hammer_rel_inode(ip, 0);
3602         } else {
3603                 hammer_done_cursor(&cursor);
3604         }
3605         if (error == EDEADLK)
3606                 goto retry;
3607
3608         return (error);
3609 }
3610
3611 /************************************************************************
3612  *                          FIFO AND SPECFS OPS                         *
3613  ************************************************************************
3614  *
3615  */
3616 static int
3617 hammer_vop_fifoclose (struct vop_close_args *ap)
3618 {
3619         /* XXX update itimes */
3620         return (VOCALL(&fifo_vnode_vops, &ap->a_head));
3621 }
3622
3623 static int
3624 hammer_vop_fiforead (struct vop_read_args *ap)
3625 {
3626         int error;
3627
3628         error = VOCALL(&fifo_vnode_vops, &ap->a_head);
3629         /* XXX update access time */
3630         return (error);
3631 }
3632
3633 static int
3634 hammer_vop_fifowrite (struct vop_write_args *ap)
3635 {
3636         int error;
3637
3638         error = VOCALL(&fifo_vnode_vops, &ap->a_head);
3639         /* XXX update access time */
3640         return (error);
3641 }
3642
3643 static
3644 int
3645 hammer_vop_fifokqfilter(struct vop_kqfilter_args *ap)
3646 {
3647         int error;
3648
3649         error = VOCALL(&fifo_vnode_vops, &ap->a_head);
3650         if (error)
3651                 error = hammer_vop_kqfilter(ap);
3652         return(error);
3653 }
3654
3655 /************************************************************************
3656  *                          KQFILTER OPS                                *
3657  ************************************************************************
3658  *
3659  */
3660 static void filt_hammerdetach(struct knote *kn);
3661 static int filt_hammerread(struct knote *kn, long hint);
3662 static int filt_hammerwrite(struct knote *kn, long hint);
3663 static int filt_hammervnode(struct knote *kn, long hint);
3664
3665 static struct filterops hammerread_filtops =
3666         { FILTEROP_ISFD | FILTEROP_MPSAFE,
3667           NULL, filt_hammerdetach, filt_hammerread };
3668 static struct filterops hammerwrite_filtops =
3669         { FILTEROP_ISFD | FILTEROP_MPSAFE,
3670           NULL, filt_hammerdetach, filt_hammerwrite };
3671 static struct filterops hammervnode_filtops =
3672         { FILTEROP_ISFD | FILTEROP_MPSAFE,
3673           NULL, filt_hammerdetach, filt_hammervnode };
3674
3675 static
3676 int
3677 hammer_vop_kqfilter(struct vop_kqfilter_args *ap)
3678 {
3679         struct vnode *vp = ap->a_vp;
3680         struct knote *kn = ap->a_kn;
3681
3682         switch (kn->kn_filter) {
3683         case EVFILT_READ:
3684                 kn->kn_fop = &hammerread_filtops;
3685                 break;
3686         case EVFILT_WRITE:
3687                 kn->kn_fop = &hammerwrite_filtops;
3688                 break;
3689         case EVFILT_VNODE:
3690                 kn->kn_fop = &hammervnode_filtops;
3691                 break;
3692         default:
3693                 return (EOPNOTSUPP);
3694         }
3695
3696         kn->kn_hook = (caddr_t)vp;
3697
3698         knote_insert(&vp->v_pollinfo.vpi_kqinfo.ki_note, kn);
3699
3700         return(0);
3701 }
3702
3703 static void
3704 filt_hammerdetach(struct knote *kn)
3705 {
3706         struct vnode *vp = (void *)kn->kn_hook;
3707
3708         knote_remove(&vp->v_pollinfo.vpi_kqinfo.ki_note, kn);
3709 }
3710
3711 static int
3712 filt_hammerread(struct knote *kn, long hint)
3713 {
3714         struct vnode *vp = (void *)kn->kn_hook;
3715         hammer_inode_t ip = VTOI(vp);
3716         hammer_mount_t hmp = ip->hmp;
3717         off_t off;
3718
3719         if (hint == NOTE_REVOKE) {
3720                 kn->kn_flags |= (EV_EOF | EV_NODATA | EV_ONESHOT);
3721                 return(1);
3722         }
3723         lwkt_gettoken(&hmp->fs_token);  /* XXX use per-ip-token */
3724         off = ip->ino_data.size - kn->kn_fp->f_offset;
3725         kn->kn_data = (off < INTPTR_MAX) ? off : INTPTR_MAX;
3726         lwkt_reltoken(&hmp->fs_token);
3727         if (kn->kn_sfflags & NOTE_OLDAPI)
3728                 return(1);
3729         return (kn->kn_data != 0);
3730 }
3731
3732 static int
3733 filt_hammerwrite(struct knote *kn, long hint)
3734 {
3735         if (hint == NOTE_REVOKE)
3736                 kn->kn_flags |= (EV_EOF | EV_NODATA | EV_ONESHOT);
3737         kn->kn_data = 0;
3738         return (1);
3739 }
3740
3741 static int
3742 filt_hammervnode(struct knote *kn, long hint)
3743 {
3744         if (kn->kn_sfflags & hint)
3745                 kn->kn_fflags |= hint;
3746         if (hint == NOTE_REVOKE) {
3747                 kn->kn_flags |= (EV_EOF | EV_NODATA);
3748                 return (1);
3749         }
3750         return (kn->kn_fflags != 0);
3751 }
3752