sys/vfs/hammer/hammer_vnops.c

   1 /*
   2  * Copyright (c) 2007-2008 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/vfs/hammer/hammer_vnops.c,v 1.102 2008/10/16 17:24:16 dillon Exp $
  35  */
  36
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/kernel.h>
  40 #include <sys/fcntl.h>
  41 #include <sys/namecache.h>
  42 #include <sys/vnode.h>
  43 #include <sys/lockf.h>
  44 #include <sys/event.h>
  45 #include <sys/stat.h>
  46 #include <sys/dirent.h>
  47 #include <sys/file.h>
  48 #include <vm/vm_extern.h>
  49 #include <vfs/fifofs/fifo.h>
  50 #include "hammer.h"
  51
  52 /*
  53  * USERFS VNOPS
  54  */
  55 /*static int hammer_vop_vnoperate(struct vop_generic_args *);*/
  56 static int hammer_vop_fsync(struct vop_fsync_args *);
  57 static int hammer_vop_read(struct vop_read_args *);
  58 static int hammer_vop_write(struct vop_write_args *);
  59 static int hammer_vop_access(struct vop_access_args *);
  60 static int hammer_vop_advlock(struct vop_advlock_args *);
  61 static int hammer_vop_close(struct vop_close_args *);
  62 static int hammer_vop_ncreate(struct vop_ncreate_args *);
  63 static int hammer_vop_getattr(struct vop_getattr_args *);
  64 static int hammer_vop_nresolve(struct vop_nresolve_args *);
  65 static int hammer_vop_nlookupdotdot(struct vop_nlookupdotdot_args *);
  66 static int hammer_vop_nlink(struct vop_nlink_args *);
  67 static int hammer_vop_nmkdir(struct vop_nmkdir_args *);
  68 static int hammer_vop_nmknod(struct vop_nmknod_args *);
  69 static int hammer_vop_open(struct vop_open_args *);
  70 static int hammer_vop_print(struct vop_print_args *);
  71 static int hammer_vop_readdir(struct vop_readdir_args *);
  72 static int hammer_vop_readlink(struct vop_readlink_args *);
  73 static int hammer_vop_nremove(struct vop_nremove_args *);
  74 static int hammer_vop_nrename(struct vop_nrename_args *);
  75 static int hammer_vop_nrmdir(struct vop_nrmdir_args *);
  76 static int hammer_vop_markatime(struct vop_markatime_args *);
  77 static int hammer_vop_setattr(struct vop_setattr_args *);
  78 static int hammer_vop_strategy(struct vop_strategy_args *);
  79 static int hammer_vop_bmap(struct vop_bmap_args *ap);
  80 static int hammer_vop_nsymlink(struct vop_nsymlink_args *);
  81 static int hammer_vop_nwhiteout(struct vop_nwhiteout_args *);
  82 static int hammer_vop_ioctl(struct vop_ioctl_args *);
  83 static int hammer_vop_mountctl(struct vop_mountctl_args *);
  84 static int hammer_vop_kqfilter (struct vop_kqfilter_args *);
  85
  86 static int hammer_vop_fifoclose (struct vop_close_args *);
  87 static int hammer_vop_fiforead (struct vop_read_args *);
  88 static int hammer_vop_fifowrite (struct vop_write_args *);
  89 static int hammer_vop_fifokqfilter (struct vop_kqfilter_args *);
  90
  91 struct vop_ops hammer_vnode_vops = {
  92         .vop_default =          vop_defaultop,
  93         .vop_fsync =            hammer_vop_fsync,
  94         .vop_getpages =         vop_stdgetpages,
  95         .vop_putpages =         vop_stdputpages,
  96         .vop_read =             hammer_vop_read,
  97         .vop_write =            hammer_vop_write,
  98         .vop_access =           hammer_vop_access,
  99         .vop_advlock =          hammer_vop_advlock,
 100         .vop_close =            hammer_vop_close,
 101         .vop_ncreate =          hammer_vop_ncreate,
 102         .vop_getattr =          hammer_vop_getattr,
 103         .vop_inactive =         hammer_vop_inactive,
 104         .vop_reclaim =          hammer_vop_reclaim,
 105         .vop_nresolve =         hammer_vop_nresolve,
 106         .vop_nlookupdotdot =    hammer_vop_nlookupdotdot,
 107         .vop_nlink =            hammer_vop_nlink,
 108         .vop_nmkdir =           hammer_vop_nmkdir,
 109         .vop_nmknod =           hammer_vop_nmknod,
 110         .vop_open =             hammer_vop_open,
 111         .vop_pathconf =         vop_stdpathconf,
 112         .vop_print =            hammer_vop_print,
 113         .vop_readdir =          hammer_vop_readdir,
 114         .vop_readlink =         hammer_vop_readlink,
 115         .vop_nremove =          hammer_vop_nremove,
 116         .vop_nrename =          hammer_vop_nrename,
 117         .vop_nrmdir =           hammer_vop_nrmdir,
 118         .vop_markatime =        hammer_vop_markatime,
 119         .vop_setattr =          hammer_vop_setattr,
 120         .vop_bmap =             hammer_vop_bmap,
 121         .vop_strategy =         hammer_vop_strategy,
 122         .vop_nsymlink =         hammer_vop_nsymlink,
 123         .vop_nwhiteout =        hammer_vop_nwhiteout,
 124         .vop_ioctl =            hammer_vop_ioctl,
 125         .vop_mountctl =         hammer_vop_mountctl,
 126         .vop_kqfilter =         hammer_vop_kqfilter
 127 };
 128
 129 struct vop_ops hammer_spec_vops = {
 130         .vop_default =          vop_defaultop,
 131         .vop_fsync =            hammer_vop_fsync,
 132         .vop_read =             vop_stdnoread,
 133         .vop_write =            vop_stdnowrite,
 134         .vop_access =           hammer_vop_access,
 135         .vop_close =            hammer_vop_close,
 136         .vop_markatime =        hammer_vop_markatime,
 137         .vop_getattr =          hammer_vop_getattr,
 138         .vop_inactive =         hammer_vop_inactive,
 139         .vop_reclaim =          hammer_vop_reclaim,
 140         .vop_setattr =          hammer_vop_setattr
 141 };
 142
 143 struct vop_ops hammer_fifo_vops = {
 144         .vop_default =          fifo_vnoperate,
 145         .vop_fsync =            hammer_vop_fsync,
 146         .vop_read =             hammer_vop_fiforead,
 147         .vop_write =            hammer_vop_fifowrite,
 148         .vop_access =           hammer_vop_access,
 149         .vop_close =            hammer_vop_fifoclose,
 150         .vop_markatime =        hammer_vop_markatime,
 151         .vop_getattr =          hammer_vop_getattr,
 152         .vop_inactive =         hammer_vop_inactive,
 153         .vop_reclaim =          hammer_vop_reclaim,
 154         .vop_setattr =          hammer_vop_setattr,
 155         .vop_kqfilter =         hammer_vop_fifokqfilter
 156 };
 157
 158 static __inline
 159 void
 160 hammer_knote(struct vnode *vp, int flags)
 161 {
 162         if (flags)
 163                 KNOTE(&vp->v_pollinfo.vpi_selinfo.si_note, flags);
 164 }
 165
 166 #ifdef DEBUG_TRUNCATE
 167 struct hammer_inode *HammerTruncIp;
 168 #endif
 169
 170 static int hammer_dounlink(hammer_transaction_t trans, struct nchandle *nch,
 171                            struct vnode *dvp, struct ucred *cred,
 172                            int flags, int isdir);
 173 static int hammer_vop_strategy_read(struct vop_strategy_args *ap);
 174 static int hammer_vop_strategy_write(struct vop_strategy_args *ap);
 175
 176 #if 0
 177 static
 178 int
 179 hammer_vop_vnoperate(struct vop_generic_args *)
 180 {
 181         return (VOCALL(&hammer_vnode_vops, ap));
 182 }
 183 #endif
 184
 185 /*
 186  * hammer_vop_fsync { vp, waitfor }
 187  *
 188  * fsync() an inode to disk and wait for it to be completely committed
 189  * such that the information would not be undone if a crash occured after
 190  * return.
 191  */
 192 static
 193 int
 194 hammer_vop_fsync(struct vop_fsync_args *ap)
 195 {
 196         hammer_inode_t ip = VTOI(ap->a_vp);
 197
 198         ++hammer_count_fsyncs;
 199         vfsync(ap->a_vp, ap->a_waitfor, 1, NULL, NULL);
 200         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 201         if (ap->a_waitfor == MNT_WAIT) {
 202                 vn_unlock(ap->a_vp);
 203                 hammer_wait_inode(ip);
 204                 vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
 205         }
 206         return (ip->error);
 207 }
 208
 209 /*
 210  * hammer_vop_read { vp, uio, ioflag, cred }
 211  *
 212  * MPALMOSTSAFE
 213  */
 214 static
 215 int
 216 hammer_vop_read(struct vop_read_args *ap)
 217 {
 218         struct hammer_transaction trans;
 219         hammer_inode_t ip;
 220         off_t offset;
 221         struct buf *bp;
 222         struct uio *uio;
 223         int error;
 224         int n;
 225         int seqcount;
 226         int ioseqcount;
 227         int blksize;
 228         int got_mplock;
 229         int bigread;
 230
 231         if (ap->a_vp->v_type != VREG)
 232                 return (EINVAL);
 233         ip = VTOI(ap->a_vp);
 234         error = 0;
 235         uio = ap->a_uio;
 236
 237         /*
 238          * Allow the UIO's size to override the sequential heuristic.
 239          */
 240         blksize = hammer_blocksize(uio->uio_offset);
 241         seqcount = (uio->uio_resid + (blksize - 1)) / blksize;
 242         ioseqcount = ap->a_ioflag >> 16;
 243         if (seqcount < ioseqcount)
 244                 seqcount = ioseqcount;
 245
 246         /*
 247          * Temporary hack until more of HAMMER can be made MPSAFE.
 248          */
 249 #ifdef SMP
 250         if (curthread->td_mpcount) {
 251                 got_mplock = -1;
 252                 hammer_start_transaction(&trans, ip->hmp);
 253         } else {
 254                 got_mplock = 0;
 255         }
 256 #else
 257         hammer_start_transaction(&trans, ip->hmp);
 258         got_mplock = -1;
 259 #endif
 260
 261         /*
 262          * If reading or writing a huge amount of data we have to break
 263          * atomicy and allow the operation to be interrupted by a signal
 264          * or it can DOS the machine.
 265          */
 266         bigread = (uio->uio_resid > 100 * 1024 * 1024);
 267
 268         /*
 269          * Access the data typically in HAMMER_BUFSIZE blocks via the
 270          * buffer cache, but HAMMER may use a variable block size based
 271          * on the offset.
 272          *
 273          * XXX Temporary hack, delay the start transaction while we remain
 274          *     MPSAFE.  NOTE: ino_data.size cannot change while vnode is
 275          *     locked-shared.
 276          */
 277         while (uio->uio_resid > 0 && uio->uio_offset < ip->ino_data.size) {
 278                 int64_t base_offset;
 279                 int64_t file_limit;
 280
 281                 blksize = hammer_blocksize(uio->uio_offset);
 282                 offset = (int)uio->uio_offset & (blksize - 1);
 283                 base_offset = uio->uio_offset - offset;
 284
 285                 if (bigread && (error = hammer_signal_check(ip->hmp)) != 0)
 286                         break;
 287
 288                 /*
 289                  * MPSAFE
 290                  */
 291                 bp = getcacheblk(ap->a_vp, base_offset);
 292                 if (bp) {
 293                         error = 0;
 294                         goto skip;
 295                 }
 296
 297                 /*
 298                  * MPUNSAFE
 299                  */
 300                 if (got_mplock == 0) {
 301                         got_mplock = 1;
 302                         get_mplock();
 303                         hammer_start_transaction(&trans, ip->hmp);
 304                 }
 305
 306                 if (hammer_cluster_enable) {
 307                         /*
 308                          * Use file_limit to prevent cluster_read() from
 309                          * creating buffers of the wrong block size past
 310                          * the demarc.
 311                          */
 312                         file_limit = ip->ino_data.size;
 313                         if (base_offset < HAMMER_XDEMARC &&
 314                             file_limit > HAMMER_XDEMARC) {
 315                                 file_limit = HAMMER_XDEMARC;
 316                         }
 317                         error = cluster_read(ap->a_vp,
 318                                              file_limit, base_offset,
 319                                              blksize, MAXPHYS,
 320                                              seqcount, &bp);
 321                 } else {
 322                         error = bread(ap->a_vp, base_offset, blksize, &bp);
 323                 }
 324                 if (error) {
 325                         kprintf("error %d\n", error);
 326                         brelse(bp);
 327                         break;
 328                 }
 329 skip:
 330
 331                 /* bp->b_flags |= B_CLUSTEROK; temporarily disabled */
 332                 n = blksize - offset;
 333                 if (n > uio->uio_resid)
 334                         n = uio->uio_resid;
 335                 if (n > ip->ino_data.size - uio->uio_offset)
 336                         n = (int)(ip->ino_data.size - uio->uio_offset);
 337                 error = uiomove((char *)bp->b_data + offset, n, uio);
 338
 339                 /* data has a lower priority then meta-data */
 340                 bp->b_flags |= B_AGE;
 341                 bqrelse(bp);
 342                 if (error)
 343                         break;
 344                 hammer_stats_file_read += n;
 345         }
 346
 347         /*
 348          * XXX only update the atime if we had to get the MP lock.
 349          * XXX hack hack hack, fixme.
 350          */
 351         if (got_mplock) {
 352                 if ((ip->flags & HAMMER_INODE_RO) == 0 &&
 353                     (ip->hmp->mp->mnt_flag & MNT_NOATIME) == 0) {
 354                         ip->ino_data.atime = trans.time;
 355                         hammer_modify_inode(ip, HAMMER_INODE_ATIME);
 356                 }
 357                 hammer_done_transaction(&trans);
 358                 if (got_mplock > 0)
 359                         rel_mplock();
 360         }
 361         return (error);
 362 }
 363
 364 /*
 365  * hammer_vop_write { vp, uio, ioflag, cred }
 366  */
 367 static
 368 int
 369 hammer_vop_write(struct vop_write_args *ap)
 370 {
 371         struct hammer_transaction trans;
 372         struct hammer_inode *ip;
 373         hammer_mount_t hmp;
 374         struct uio *uio;
 375         int offset;
 376         off_t base_offset;
 377         struct buf *bp;
 378         int kflags;
 379         int error;
 380         int n;
 381         int flags;
 382         int seqcount;
 383         int bigwrite;
 384
 385         if (ap->a_vp->v_type != VREG)
 386                 return (EINVAL);
 387         ip = VTOI(ap->a_vp);
 388         hmp = ip->hmp;
 389         error = 0;
 390         kflags = 0;
 391         seqcount = ap->a_ioflag >> 16;
 392
 393         if (ip->flags & HAMMER_INODE_RO)
 394                 return (EROFS);
 395
 396         /*
 397          * Create a transaction to cover the operations we perform.
 398          */
 399         hammer_start_transaction(&trans, hmp);
 400         uio = ap->a_uio;
 401
 402         /*
 403          * Check append mode
 404          */
 405         if (ap->a_ioflag & IO_APPEND)
 406                 uio->uio_offset = ip->ino_data.size;
 407
 408         /*
 409          * Check for illegal write offsets.  Valid range is 0...2^63-1.
 410          *
 411          * NOTE: the base_off assignment is required to work around what
 412          * I consider to be a GCC-4 optimization bug.
 413          */
 414         if (uio->uio_offset < 0) {
 415                 hammer_done_transaction(&trans);
 416                 return (EFBIG);
 417         }
 418         base_offset = uio->uio_offset + uio->uio_resid; /* work around gcc-4 */
 419         if (uio->uio_resid > 0 && base_offset <= uio->uio_offset) {
 420                 hammer_done_transaction(&trans);
 421                 return (EFBIG);
 422         }
 423
 424         /*
 425          * If reading or writing a huge amount of data we have to break
 426          * atomicy and allow the operation to be interrupted by a signal
 427          * or it can DOS the machine.
 428          */
 429         bigwrite = (uio->uio_resid > 100 * 1024 * 1024);
 430
 431         /*
 432          * Access the data typically in HAMMER_BUFSIZE blocks via the
 433          * buffer cache, but HAMMER may use a variable block size based
 434          * on the offset.
 435          */
 436         while (uio->uio_resid > 0) {
 437                 int fixsize = 0;
 438                 int blksize;
 439                 int blkmask;
 440
 441                 if ((error = hammer_checkspace(hmp, HAMMER_CHKSPC_WRITE)) != 0)
 442                         break;
 443                 if (bigwrite && (error = hammer_signal_check(hmp)) != 0)
 444                         break;
 445
 446                 blksize = hammer_blocksize(uio->uio_offset);
 447
 448                 /*
 449                  * Do not allow HAMMER to blow out the buffer cache.  Very
 450                  * large UIOs can lockout other processes due to bwillwrite()
 451                  * mechanics.
 452                  *
 453                  * The hammer inode is not locked during these operations.
 454                  * The vnode is locked which can interfere with the pageout
 455                  * daemon for non-UIO_NOCOPY writes but should not interfere
 456                  * with the buffer cache.  Even so, we cannot afford to
 457                  * allow the pageout daemon to build up too many dirty buffer
 458                  * cache buffers.
 459                  *
 460                  * Only call this if we aren't being recursively called from
 461                  * a virtual disk device (vn), else we may deadlock.
 462                  */
 463                 if ((ap->a_ioflag & IO_RECURSE) == 0)
 464                         bwillwrite(blksize);
 465
 466                 /*
 467                  * Control the number of pending records associated with
 468                  * this inode.  If too many have accumulated start a
 469                  * flush.  Try to maintain a pipeline with the flusher.
 470                  */
 471                 if (ip->rsv_recs >= hammer_limit_inode_recs) {
 472                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 473                 }
 474                 if (ip->rsv_recs >= hammer_limit_inode_recs * 2) {
 475                         while (ip->rsv_recs >= hammer_limit_inode_recs) {
 476                                 tsleep(&ip->rsv_recs, 0, "hmrwww", hz);
 477                         }
 478                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 479                 }
 480
 481 #if 0
 482                 /*
 483                  * Do not allow HAMMER to blow out system memory by
 484                  * accumulating too many records.   Records are so well
 485                  * decoupled from the buffer cache that it is possible
 486                  * for userland to push data out to the media via
 487                  * direct-write, but build up the records queued to the
 488                  * backend faster then the backend can flush them out.
 489                  * HAMMER has hit its write limit but the frontend has
 490                  * no pushback to slow it down.
 491                  */
 492                 if (hmp->rsv_recs > hammer_limit_recs / 2) {
 493                         /*
 494                          * Get the inode on the flush list
 495                          */
 496                         if (ip->rsv_recs >= 64)
 497                                 hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 498                         else if (ip->rsv_recs >= 16)
 499                                 hammer_flush_inode(ip, 0);
 500
 501                         /*
 502                          * Keep the flusher going if the system keeps
 503                          * queueing records.
 504                          */
 505                         delta = hmp->count_newrecords -
 506                                 hmp->last_newrecords;
 507                         if (delta < 0 || delta > hammer_limit_recs / 2) {
 508                                 hmp->last_newrecords = hmp->count_newrecords;
 509                                 hammer_sync_hmp(hmp, MNT_NOWAIT);
 510                         }
 511
 512                         /*
 513                          * If we have gotten behind start slowing
 514                          * down the writers.
 515                          */
 516                         delta = (hmp->rsv_recs - hammer_limit_recs) *
 517                                 hz / hammer_limit_recs;
 518                         if (delta > 0)
 519                                 tsleep(&trans, 0, "hmrslo", delta);
 520                 }
 521 #endif
 522
 523                 /*
 524                  * Calculate the blocksize at the current offset and figure
 525                  * out how much we can actually write.
 526                  */
 527                 blkmask = blksize - 1;
 528                 offset = (int)uio->uio_offset & blkmask;
 529                 base_offset = uio->uio_offset & ~(int64_t)blkmask;
 530                 n = blksize - offset;
 531                 if (n > uio->uio_resid)
 532                         n = uio->uio_resid;
 533                 if (uio->uio_offset + n > ip->ino_data.size) {
 534                         vnode_pager_setsize(ap->a_vp, uio->uio_offset + n);
 535                         fixsize = 1;
 536                         kflags |= NOTE_EXTEND;
 537                 }
 538
 539                 if (uio->uio_segflg == UIO_NOCOPY) {
 540                         /*
 541                          * Issuing a write with the same data backing the
 542                          * buffer.  Instantiate the buffer to collect the
 543                          * backing vm pages, then read-in any missing bits.
 544                          *
 545                          * This case is used by vop_stdputpages().
 546                          */
 547                         bp = getblk(ap->a_vp, base_offset,
 548                                     blksize, GETBLK_BHEAVY, 0);
 549                         if ((bp->b_flags & B_CACHE) == 0) {
 550                                 bqrelse(bp);
 551                                 error = bread(ap->a_vp, base_offset,
 552                                               blksize, &bp);
 553                         }
 554                 } else if (offset == 0 && uio->uio_resid >= blksize) {
 555                         /*
 556                          * Even though we are entirely overwriting the buffer
 557                          * we may still have to zero it out to avoid a
 558                          * mmap/write visibility issue.
 559                          */
 560                         bp = getblk(ap->a_vp, base_offset, blksize, GETBLK_BHEAVY, 0);
 561                         if ((bp->b_flags & B_CACHE) == 0)
 562                                 vfs_bio_clrbuf(bp);
 563                 } else if (base_offset >= ip->ino_data.size) {
 564                         /*
 565                          * If the base offset of the buffer is beyond the
 566                          * file EOF, we don't have to issue a read.
 567                          */
 568                         bp = getblk(ap->a_vp, base_offset,
 569                                     blksize, GETBLK_BHEAVY, 0);
 570                         vfs_bio_clrbuf(bp);
 571                 } else {
 572                         /*
 573                          * Partial overwrite, read in any missing bits then
 574                          * replace the portion being written.
 575                          */
 576                         error = bread(ap->a_vp, base_offset, blksize, &bp);
 577                         if (error == 0)
 578                                 bheavy(bp);
 579                 }
 580                 if (error == 0) {
 581                         error = uiomove((char *)bp->b_data + offset,
 582                                         n, uio);
 583                 }
 584
 585                 /*
 586                  * If we screwed up we have to undo any VM size changes we
 587                  * made.
 588                  */
 589                 if (error) {
 590                         brelse(bp);
 591                         if (fixsize) {
 592                                 vtruncbuf(ap->a_vp, ip->ino_data.size,
 593                                           hammer_blocksize(ip->ino_data.size));
 594                         }
 595                         break;
 596                 }
 597                 kflags |= NOTE_WRITE;
 598                 hammer_stats_file_write += n;
 599                 /* bp->b_flags |= B_CLUSTEROK; temporarily disabled */
 600                 if (ip->ino_data.size < uio->uio_offset) {
 601                         ip->ino_data.size = uio->uio_offset;
 602                         flags = HAMMER_INODE_DDIRTY;
 603                         vnode_pager_setsize(ap->a_vp, ip->ino_data.size);
 604                 } else {
 605                         flags = 0;
 606                 }
 607                 ip->ino_data.mtime = trans.time;
 608                 flags |= HAMMER_INODE_MTIME | HAMMER_INODE_BUFS;
 609                 hammer_modify_inode(ip, flags);
 610
 611                 /*
 612                  * Once we dirty the buffer any cached zone-X offset
 613                  * becomes invalid.  HAMMER NOTE: no-history mode cannot
 614                  * allow overwriting over the same data sector unless
 615                  * we provide UNDOs for the old data, which we don't.
 616                  */
 617                 bp->b_bio2.bio_offset = NOOFFSET;
 618
 619                 /*
 620                  * Final buffer disposition.
 621                  *
 622                  * Because meta-data updates are deferred, HAMMER is
 623                  * especially sensitive to excessive bdwrite()s because
 624                  * the I/O stream is not broken up by disk reads.  So the
 625                  * buffer cache simply cannot keep up.
 626                  *
 627                  * WARNING!  blksize is variable.  cluster_write() is
 628                  * expected to not blow up if it encounters buffers that
 629                  * do not match the passed blksize.
 630                  *
 631                  * NOTE!  Hammer shouldn't need to bawrite()/cluster_write().
 632                  *        The ip->rsv_recs check should burst-flush the data.
 633                  *        If we queue it immediately the buf could be left
 634                  *        locked on the device queue for a very long time.
 635                  */
 636                 bp->b_flags |= B_AGE;
 637                 if (ap->a_ioflag & IO_SYNC) {
 638                         bwrite(bp);
 639                 } else if (ap->a_ioflag & IO_DIRECT) {
 640                         bawrite(bp);
 641                 } else {
 642 #if 0
 643                 if (offset + n == blksize) {
 644                         if (hammer_cluster_enable == 0 ||
 645                             (ap->a_vp->v_mount->mnt_flag & MNT_NOCLUSTERW)) {
 646                                 bawrite(bp);
 647                         } else {
 648                                 cluster_write(bp, ip->ino_data.size,
 649                                               blksize, seqcount);
 650                         }
 651                 } else {
 652 #endif
 653                         bdwrite(bp);
 654                 }
 655         }
 656         hammer_done_transaction(&trans);
 657         hammer_knote(ap->a_vp, kflags);
 658         return (error);
 659 }
 660
 661 /*
 662  * hammer_vop_access { vp, mode, cred }
 663  */
 664 static
 665 int
 666 hammer_vop_access(struct vop_access_args *ap)
 667 {
 668         struct hammer_inode *ip = VTOI(ap->a_vp);
 669         uid_t uid;
 670         gid_t gid;
 671         int error;
 672
 673         ++hammer_stats_file_iopsr;
 674         uid = hammer_to_unix_xid(&ip->ino_data.uid);
 675         gid = hammer_to_unix_xid(&ip->ino_data.gid);
 676
 677         error = vop_helper_access(ap, uid, gid, ip->ino_data.mode,
 678                                   ip->ino_data.uflags);
 679         return (error);
 680 }
 681
 682 /*
 683  * hammer_vop_advlock { vp, id, op, fl, flags }
 684  */
 685 static
 686 int
 687 hammer_vop_advlock(struct vop_advlock_args *ap)
 688 {
 689         hammer_inode_t ip = VTOI(ap->a_vp);
 690
 691         return (lf_advlock(ap, &ip->advlock, ip->ino_data.size));
 692 }
 693
 694 /*
 695  * hammer_vop_close { vp, fflag }
 696  */
 697 static
 698 int
 699 hammer_vop_close(struct vop_close_args *ap)
 700 {
 701         /*hammer_inode_t ip = VTOI(ap->a_vp);*/
 702         return (vop_stdclose(ap));
 703 }
 704
 705 /*
 706  * hammer_vop_ncreate { nch, dvp, vpp, cred, vap }
 707  *
 708  * The operating system has already ensured that the directory entry
 709  * does not exist and done all appropriate namespace locking.
 710  */
 711 static
 712 int
 713 hammer_vop_ncreate(struct vop_ncreate_args *ap)
 714 {
 715         struct hammer_transaction trans;
 716         struct hammer_inode *dip;
 717         struct hammer_inode *nip;
 718         struct nchandle *nch;
 719         int error;
 720
 721         nch = ap->a_nch;
 722         dip = VTOI(ap->a_dvp);
 723
 724         if (dip->flags & HAMMER_INODE_RO)
 725                 return (EROFS);
 726         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
 727                 return (error);
 728
 729         /*
 730          * Create a transaction to cover the operations we perform.
 731          */
 732         hammer_start_transaction(&trans, dip->hmp);
 733         ++hammer_stats_file_iopsw;
 734
 735         /*
 736          * Create a new filesystem object of the requested type.  The
 737          * returned inode will be referenced and shared-locked to prevent
 738          * it from being moved to the flusher.
 739          */
 740         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
 741                                     dip, nch->ncp->nc_name, nch->ncp->nc_nlen,
 742                                     NULL, &nip);
 743         if (error) {
 744                 hkprintf("hammer_create_inode error %d\n", error);
 745                 hammer_done_transaction(&trans);
 746                 *ap->a_vpp = NULL;
 747                 return (error);
 748         }
 749
 750         /*
 751          * Add the new filesystem object to the directory.  This will also
 752          * bump the inode's link count.
 753          */
 754         error = hammer_ip_add_directory(&trans, dip,
 755                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
 756                                         nip);
 757         if (error)
 758                 hkprintf("hammer_ip_add_directory error %d\n", error);
 759
 760         /*
 761          * Finish up.
 762          */
 763         if (error) {
 764                 hammer_rel_inode(nip, 0);
 765                 hammer_done_transaction(&trans);
 766                 *ap->a_vpp = NULL;
 767         } else {
 768                 error = hammer_get_vnode(nip, ap->a_vpp);
 769                 hammer_done_transaction(&trans);
 770                 hammer_rel_inode(nip, 0);
 771                 if (error == 0) {
 772                         cache_setunresolved(ap->a_nch);
 773                         cache_setvp(ap->a_nch, *ap->a_vpp);
 774                 }
 775                 hammer_knote(ap->a_dvp, NOTE_WRITE);
 776         }
 777         return (error);
 778 }
 779
 780 /*
 781  * hammer_vop_getattr { vp, vap }
 782  *
 783  * Retrieve an inode's attribute information.  When accessing inodes
 784  * historically we fake the atime field to ensure consistent results.
 785  * The atime field is stored in the B-Tree element and allowed to be
 786  * updated without cycling the element.
 787  *
 788  * MPSAFE
 789  */
 790 static
 791 int
 792 hammer_vop_getattr(struct vop_getattr_args *ap)
 793 {
 794         struct hammer_inode *ip = VTOI(ap->a_vp);
 795         struct vattr *vap = ap->a_vap;
 796
 797         /*
 798          * We want the fsid to be different when accessing a filesystem
 799          * with different as-of's so programs like diff don't think
 800          * the files are the same.
 801          *
 802          * We also want the fsid to be the same when comparing snapshots,
 803          * or when comparing mirrors (which might be backed by different
 804          * physical devices).  HAMMER fsids are based on the PFS's
 805          * shared_uuid field.
 806          *
 807          * XXX there is a chance of collision here.  The va_fsid reported
 808          * by stat is different from the more involved fsid used in the
 809          * mount structure.
 810          */
 811         ++hammer_stats_file_iopsr;
 812         hammer_lock_sh(&ip->lock);
 813         vap->va_fsid = ip->pfsm->fsid_udev ^ (u_int32_t)ip->obj_asof ^
 814                        (u_int32_t)(ip->obj_asof >> 32);
 815
 816         vap->va_fileid = ip->ino_leaf.base.obj_id;
 817         vap->va_mode = ip->ino_data.mode;
 818         vap->va_nlink = ip->ino_data.nlinks;
 819         vap->va_uid = hammer_to_unix_xid(&ip->ino_data.uid);
 820         vap->va_gid = hammer_to_unix_xid(&ip->ino_data.gid);
 821         vap->va_rmajor = 0;
 822         vap->va_rminor = 0;
 823         vap->va_size = ip->ino_data.size;
 824
 825         /*
 826          * Special case for @@PFS softlinks.  The actual size of the
 827          * expanded softlink is "@@0x%016llx:%05d" == 26 bytes.
 828          * or for MAX_TID is    "@@-1:%05d" == 10 bytes.
 829          */
 830         if (ip->ino_data.obj_type == HAMMER_OBJTYPE_SOFTLINK &&
 831             ip->ino_data.size == 10 &&
 832             ip->obj_asof == HAMMER_MAX_TID &&
 833             ip->obj_localization == 0 &&
 834             strncmp(ip->ino_data.ext.symlink, "@@PFS", 5) == 0) {
 835                     if (ip->pfsm->pfsd.mirror_flags & HAMMER_PFSD_SLAVE)
 836                             vap->va_size = 26;
 837                     else
 838                             vap->va_size = 10;
 839         }
 840
 841         /*
 842          * We must provide a consistent atime and mtime for snapshots
 843          * so people can do a 'tar cf - ... | md5' on them and get
 844          * consistent results.
 845          */
 846         if (ip->flags & HAMMER_INODE_RO) {
 847                 hammer_time_to_timespec(ip->ino_data.ctime, &vap->va_atime);
 848                 hammer_time_to_timespec(ip->ino_data.ctime, &vap->va_mtime);
 849         } else {
 850                 hammer_time_to_timespec(ip->ino_data.atime, &vap->va_atime);
 851                 hammer_time_to_timespec(ip->ino_data.mtime, &vap->va_mtime);
 852         }
 853         hammer_time_to_timespec(ip->ino_data.ctime, &vap->va_ctime);
 854         vap->va_flags = ip->ino_data.uflags;
 855         vap->va_gen = 1;        /* hammer inums are unique for all time */
 856         vap->va_blocksize = HAMMER_BUFSIZE;
 857         if (ip->ino_data.size >= HAMMER_XDEMARC) {
 858                 vap->va_bytes = (ip->ino_data.size + HAMMER_XBUFMASK64) &
 859                                 ~HAMMER_XBUFMASK64;
 860         } else if (ip->ino_data.size > HAMMER_BUFSIZE / 2) {
 861                 vap->va_bytes = (ip->ino_data.size + HAMMER_BUFMASK64) &
 862                                 ~HAMMER_BUFMASK64;
 863         } else {
 864                 vap->va_bytes = (ip->ino_data.size + 15) & ~15;
 865         }
 866
 867         vap->va_type = hammer_get_vnode_type(ip->ino_data.obj_type);
 868         vap->va_filerev = 0;    /* XXX */
 869         /* mtime uniquely identifies any adjustments made to the file XXX */
 870         vap->va_fsmid = ip->ino_data.mtime;
 871         vap->va_uid_uuid = ip->ino_data.uid;
 872         vap->va_gid_uuid = ip->ino_data.gid;
 873         vap->va_fsid_uuid = ip->hmp->fsid;
 874         vap->va_vaflags = VA_UID_UUID_VALID | VA_GID_UUID_VALID |
 875                           VA_FSID_UUID_VALID;
 876
 877         switch (ip->ino_data.obj_type) {
 878         case HAMMER_OBJTYPE_CDEV:
 879         case HAMMER_OBJTYPE_BDEV:
 880                 vap->va_rmajor = ip->ino_data.rmajor;
 881                 vap->va_rminor = ip->ino_data.rminor;
 882                 break;
 883         default:
 884                 break;
 885         }
 886         hammer_unlock(&ip->lock);
 887         return(0);
 888 }
 889
 890 /*
 891  * hammer_vop_nresolve { nch, dvp, cred }
 892  *
 893  * Locate the requested directory entry.
 894  */
 895 static
 896 int
 897 hammer_vop_nresolve(struct vop_nresolve_args *ap)
 898 {
 899         struct hammer_transaction trans;
 900         struct namecache *ncp;
 901         hammer_inode_t dip;
 902         hammer_inode_t ip;
 903         hammer_tid_t asof;
 904         struct hammer_cursor cursor;
 905         struct vnode *vp;
 906         int64_t namekey;
 907         int error;
 908         int i;
 909         int nlen;
 910         int flags;
 911         int ispfs;
 912         int64_t obj_id;
 913         u_int32_t localization;
 914         u_int32_t max_iterations;
 915
 916         /*
 917          * Misc initialization, plus handle as-of name extensions.  Look for
 918          * the '@@' extension.  Note that as-of files and directories cannot
 919          * be modified.
 920          */
 921         dip = VTOI(ap->a_dvp);
 922         ncp = ap->a_nch->ncp;
 923         asof = dip->obj_asof;
 924         localization = dip->obj_localization;   /* for code consistency */
 925         nlen = ncp->nc_nlen;
 926         flags = dip->flags & HAMMER_INODE_RO;
 927         ispfs = 0;
 928
 929         hammer_simple_transaction(&trans, dip->hmp);
 930         ++hammer_stats_file_iopsr;
 931
 932         for (i = 0; i < nlen; ++i) {
 933                 if (ncp->nc_name[i] == '@' && ncp->nc_name[i+1] == '@') {
 934                         error = hammer_str_to_tid(ncp->nc_name + i + 2,
 935                                                   &ispfs, &asof, &localization);
 936                         if (error != 0) {
 937                                 i = nlen;
 938                                 break;
 939                         }
 940                         if (asof != HAMMER_MAX_TID)
 941                                 flags |= HAMMER_INODE_RO;
 942                         break;
 943                 }
 944         }
 945         nlen = i;
 946
 947         /*
 948          * If this is a PFS softlink we dive into the PFS
 949          */
 950         if (ispfs && nlen == 0) {
 951                 ip = hammer_get_inode(&trans, dip, HAMMER_OBJID_ROOT,
 952                                       asof, localization,
 953                                       flags, &error);
 954                 if (error == 0) {
 955                         error = hammer_get_vnode(ip, &vp);
 956                         hammer_rel_inode(ip, 0);
 957                 } else {
 958                         vp = NULL;
 959                 }
 960                 if (error == 0) {
 961                         vn_unlock(vp);
 962                         cache_setvp(ap->a_nch, vp);
 963                         vrele(vp);
 964                 }
 965                 goto done;
 966         }
 967
 968         /*
 969          * If there is no path component the time extension is relative to dip.
 970          * e.g. "fubar/@@<snapshot>"
 971          *
 972          * "." is handled by the kernel, but ".@@<snapshot>" is not.
 973          * e.g. "fubar/.@@<snapshot>"
 974          *
 975          * ".." is handled by the kernel.  We do not currently handle
 976          * "..@<snapshot>".
 977          */
 978         if (nlen == 0 || (nlen == 1 && ncp->nc_name[0] == '.')) {
 979                 ip = hammer_get_inode(&trans, dip, dip->obj_id,
 980                                       asof, dip->obj_localization,
 981                                       flags, &error);
 982                 if (error == 0) {
 983                         error = hammer_get_vnode(ip, &vp);
 984                         hammer_rel_inode(ip, 0);
 985                 } else {
 986                         vp = NULL;
 987                 }
 988                 if (error == 0) {
 989                         vn_unlock(vp);
 990                         cache_setvp(ap->a_nch, vp);
 991                         vrele(vp);
 992                 }
 993                 goto done;
 994         }
 995
 996         /*
 997          * Calculate the namekey and setup the key range for the scan.  This
 998          * works kinda like a chained hash table where the lower 32 bits
 999          * of the namekey synthesize the chain.
1000          *
1001          * The key range is inclusive of both key_beg and key_end.
1002          */
1003         namekey = hammer_directory_namekey(dip, ncp->nc_name, nlen,
1004                                            &max_iterations);
1005
1006         error = hammer_init_cursor(&trans, &cursor, &dip->cache[1], dip);
1007         cursor.key_beg.localization = dip->obj_localization +
1008                                       hammer_dir_localization(dip);
1009         cursor.key_beg.obj_id = dip->obj_id;
1010         cursor.key_beg.key = namekey;
1011         cursor.key_beg.create_tid = 0;
1012         cursor.key_beg.delete_tid = 0;
1013         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
1014         cursor.key_beg.obj_type = 0;
1015
1016         cursor.key_end = cursor.key_beg;
1017         cursor.key_end.key += max_iterations;
1018         cursor.asof = asof;
1019         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
1020
1021         /*
1022          * Scan all matching records (the chain), locate the one matching
1023          * the requested path component.
1024          *
1025          * The hammer_ip_*() functions merge in-memory records with on-disk
1026          * records for the purposes of the search.
1027          */
1028         obj_id = 0;
1029         localization = HAMMER_DEF_LOCALIZATION;
1030
1031         if (error == 0) {
1032                 error = hammer_ip_first(&cursor);
1033                 while (error == 0) {
1034                         error = hammer_ip_resolve_data(&cursor);
1035                         if (error)
1036                                 break;
1037                         if (nlen == cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF &&
1038                             bcmp(ncp->nc_name, cursor.data->entry.name, nlen) == 0) {
1039                                 obj_id = cursor.data->entry.obj_id;
1040                                 localization = cursor.data->entry.localization;
1041                                 break;
1042                         }
1043                         error = hammer_ip_next(&cursor);
1044                 }
1045         }
1046         hammer_done_cursor(&cursor);
1047
1048         /*
1049          * Lookup the obj_id.  This should always succeed.  If it does not
1050          * the filesystem may be damaged and we return a dummy inode.
1051          */
1052         if (error == 0) {
1053                 ip = hammer_get_inode(&trans, dip, obj_id,
1054                                       asof, localization,
1055                                       flags, &error);
1056                 if (error == ENOENT) {
1057                         kprintf("HAMMER: WARNING: Missing "
1058                                 "inode for dirent \"%s\"\n"
1059                                 "\tobj_id = %016llx, asof=%016llx, lo=%08x\n",
1060                                 ncp->nc_name,
1061                                 (long long)obj_id, (long long)asof,
1062                                 localization);
1063                         error = 0;
1064                         ip = hammer_get_dummy_inode(&trans, dip, obj_id,
1065                                                     asof, localization,
1066                                                     flags, &error);
1067                 }
1068                 if (error == 0) {
1069                         error = hammer_get_vnode(ip, &vp);
1070                         hammer_rel_inode(ip, 0);
1071                 } else {
1072                         vp = NULL;
1073                 }
1074                 if (error == 0) {
1075                         vn_unlock(vp);
1076                         cache_setvp(ap->a_nch, vp);
1077                         vrele(vp);
1078                 }
1079         } else if (error == ENOENT) {
1080                 cache_setvp(ap->a_nch, NULL);
1081         }
1082 done:
1083         hammer_done_transaction(&trans);
1084         return (error);
1085 }
1086
1087 /*
1088  * hammer_vop_nlookupdotdot { dvp, vpp, cred }
1089  *
1090  * Locate the parent directory of a directory vnode.
1091  *
1092  * dvp is referenced but not locked.  *vpp must be returned referenced and
1093  * locked.  A parent_obj_id of 0 does not necessarily indicate that we are
1094  * at the root, instead it could indicate that the directory we were in was
1095  * removed.
1096  *
1097  * NOTE: as-of sequences are not linked into the directory structure.  If
1098  * we are at the root with a different asof then the mount point, reload
1099  * the same directory with the mount point's asof.   I'm not sure what this
1100  * will do to NFS.  We encode ASOF stamps in NFS file handles so it might not
1101  * get confused, but it hasn't been tested.
1102  */
1103 static
1104 int
1105 hammer_vop_nlookupdotdot(struct vop_nlookupdotdot_args *ap)
1106 {
1107         struct hammer_transaction trans;
1108         struct hammer_inode *dip;
1109         struct hammer_inode *ip;
1110         int64_t parent_obj_id;
1111         u_int32_t parent_obj_localization;
1112         hammer_tid_t asof;
1113         int error;
1114
1115         dip = VTOI(ap->a_dvp);
1116         asof = dip->obj_asof;
1117
1118         /*
1119          * Whos are parent?  This could be the root of a pseudo-filesystem
1120          * whos parent is in another localization domain.
1121          */
1122         parent_obj_id = dip->ino_data.parent_obj_id;
1123         if (dip->obj_id == HAMMER_OBJID_ROOT)
1124                 parent_obj_localization = dip->ino_data.ext.obj.parent_obj_localization;
1125         else
1126                 parent_obj_localization = dip->obj_localization;
1127
1128         if (parent_obj_id == 0) {
1129                 if (dip->obj_id == HAMMER_OBJID_ROOT &&
1130                    asof != dip->hmp->asof) {
1131                         parent_obj_id = dip->obj_id;
1132                         asof = dip->hmp->asof;
1133                         *ap->a_fakename = kmalloc(19, M_TEMP, M_WAITOK);
1134                         ksnprintf(*ap->a_fakename, 19, "0x%016llx",
1135                                   (long long)dip->obj_asof);
1136                 } else {
1137                         *ap->a_vpp = NULL;
1138                         return ENOENT;
1139                 }
1140         }
1141
1142         hammer_simple_transaction(&trans, dip->hmp);
1143         ++hammer_stats_file_iopsr;
1144
1145         ip = hammer_get_inode(&trans, dip, parent_obj_id,
1146                               asof, parent_obj_localization,
1147                               dip->flags, &error);
1148         if (ip) {
1149                 error = hammer_get_vnode(ip, ap->a_vpp);
1150                 hammer_rel_inode(ip, 0);
1151         } else {
1152                 *ap->a_vpp = NULL;
1153         }
1154         hammer_done_transaction(&trans);
1155         return (error);
1156 }
1157
1158 /*
1159  * hammer_vop_nlink { nch, dvp, vp, cred }
1160  */
1161 static
1162 int
1163 hammer_vop_nlink(struct vop_nlink_args *ap)
1164 {
1165         struct hammer_transaction trans;
1166         struct hammer_inode *dip;
1167         struct hammer_inode *ip;
1168         struct nchandle *nch;
1169         int error;
1170
1171         if (ap->a_dvp->v_mount != ap->a_vp->v_mount)
1172                 return(EXDEV);
1173
1174         nch = ap->a_nch;
1175         dip = VTOI(ap->a_dvp);
1176         ip = VTOI(ap->a_vp);
1177
1178         if (dip->obj_localization != ip->obj_localization)
1179                 return(EXDEV);
1180
1181         if (dip->flags & HAMMER_INODE_RO)
1182                 return (EROFS);
1183         if (ip->flags & HAMMER_INODE_RO)
1184                 return (EROFS);
1185         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1186                 return (error);
1187
1188         /*
1189          * Create a transaction to cover the operations we perform.
1190          */
1191         hammer_start_transaction(&trans, dip->hmp);
1192         ++hammer_stats_file_iopsw;
1193
1194         /*
1195          * Add the filesystem object to the directory.  Note that neither
1196          * dip nor ip are referenced or locked, but their vnodes are
1197          * referenced.  This function will bump the inode's link count.
1198          */
1199         error = hammer_ip_add_directory(&trans, dip,
1200                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1201                                         ip);
1202
1203         /*
1204          * Finish up.
1205          */
1206         if (error == 0) {
1207                 cache_setunresolved(nch);
1208                 cache_setvp(nch, ap->a_vp);
1209         }
1210         hammer_done_transaction(&trans);
1211         hammer_knote(ap->a_vp, NOTE_LINK);
1212         hammer_knote(ap->a_dvp, NOTE_WRITE);
1213         return (error);
1214 }
1215
1216 /*
1217  * hammer_vop_nmkdir { nch, dvp, vpp, cred, vap }
1218  *
1219  * The operating system has already ensured that the directory entry
1220  * does not exist and done all appropriate namespace locking.
1221  */
1222 static
1223 int
1224 hammer_vop_nmkdir(struct vop_nmkdir_args *ap)
1225 {
1226         struct hammer_transaction trans;
1227         struct hammer_inode *dip;
1228         struct hammer_inode *nip;
1229         struct nchandle *nch;
1230         int error;
1231
1232         nch = ap->a_nch;
1233         dip = VTOI(ap->a_dvp);
1234
1235         if (dip->flags & HAMMER_INODE_RO)
1236                 return (EROFS);
1237         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1238                 return (error);
1239
1240         /*
1241          * Create a transaction to cover the operations we perform.
1242          */
1243         hammer_start_transaction(&trans, dip->hmp);
1244         ++hammer_stats_file_iopsw;
1245
1246         /*
1247          * Create a new filesystem object of the requested type.  The
1248          * returned inode will be referenced but not locked.
1249          */
1250         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
1251                                     dip, nch->ncp->nc_name, nch->ncp->nc_nlen,
1252                                     NULL, &nip);
1253         if (error) {
1254                 hkprintf("hammer_mkdir error %d\n", error);
1255                 hammer_done_transaction(&trans);
1256                 *ap->a_vpp = NULL;
1257                 return (error);
1258         }
1259         /*
1260          * Add the new filesystem object to the directory.  This will also
1261          * bump the inode's link count.
1262          */
1263         error = hammer_ip_add_directory(&trans, dip,
1264                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1265                                         nip);
1266         if (error)
1267                 hkprintf("hammer_mkdir (add) error %d\n", error);
1268
1269         /*
1270          * Finish up.
1271          */
1272         if (error) {
1273                 hammer_rel_inode(nip, 0);
1274                 *ap->a_vpp = NULL;
1275         } else {
1276                 error = hammer_get_vnode(nip, ap->a_vpp);
1277                 hammer_rel_inode(nip, 0);
1278                 if (error == 0) {
1279                         cache_setunresolved(ap->a_nch);
1280                         cache_setvp(ap->a_nch, *ap->a_vpp);
1281                 }
1282         }
1283         hammer_done_transaction(&trans);
1284         if (error == 0)
1285                 hammer_knote(ap->a_dvp, NOTE_WRITE | NOTE_LINK);
1286         return (error);
1287 }
1288
1289 /*
1290  * hammer_vop_nmknod { nch, dvp, vpp, cred, vap }
1291  *
1292  * The operating system has already ensured that the directory entry
1293  * does not exist and done all appropriate namespace locking.
1294  */
1295 static
1296 int
1297 hammer_vop_nmknod(struct vop_nmknod_args *ap)
1298 {
1299         struct hammer_transaction trans;
1300         struct hammer_inode *dip;
1301         struct hammer_inode *nip;
1302         struct nchandle *nch;
1303         int error;
1304
1305         nch = ap->a_nch;
1306         dip = VTOI(ap->a_dvp);
1307
1308         if (dip->flags & HAMMER_INODE_RO)
1309                 return (EROFS);
1310         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1311                 return (error);
1312
1313         /*
1314          * Create a transaction to cover the operations we perform.
1315          */
1316         hammer_start_transaction(&trans, dip->hmp);
1317         ++hammer_stats_file_iopsw;
1318
1319         /*
1320          * Create a new filesystem object of the requested type.  The
1321          * returned inode will be referenced but not locked.
1322          *
1323          * If mknod specifies a directory a pseudo-fs is created.
1324          */
1325         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
1326                                     dip, nch->ncp->nc_name, nch->ncp->nc_nlen,
1327                                     NULL, &nip);
1328         if (error) {
1329                 hammer_done_transaction(&trans);
1330                 *ap->a_vpp = NULL;
1331                 return (error);
1332         }
1333
1334         /*
1335          * Add the new filesystem object to the directory.  This will also
1336          * bump the inode's link count.
1337          */
1338         error = hammer_ip_add_directory(&trans, dip,
1339                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1340                                         nip);
1341
1342         /*
1343          * Finish up.
1344          */
1345         if (error) {
1346                 hammer_rel_inode(nip, 0);
1347                 *ap->a_vpp = NULL;
1348         } else {
1349                 error = hammer_get_vnode(nip, ap->a_vpp);
1350                 hammer_rel_inode(nip, 0);
1351                 if (error == 0) {
1352                         cache_setunresolved(ap->a_nch);
1353                         cache_setvp(ap->a_nch, *ap->a_vpp);
1354                 }
1355         }
1356         hammer_done_transaction(&trans);
1357         if (error == 0)
1358                 hammer_knote(ap->a_dvp, NOTE_WRITE);
1359         return (error);
1360 }
1361
1362 /*
1363  * hammer_vop_open { vp, mode, cred, fp }
1364  */
1365 static
1366 int
1367 hammer_vop_open(struct vop_open_args *ap)
1368 {
1369         hammer_inode_t ip;
1370
1371         ++hammer_stats_file_iopsr;
1372         ip = VTOI(ap->a_vp);
1373
1374         if ((ap->a_mode & FWRITE) && (ip->flags & HAMMER_INODE_RO))
1375                 return (EROFS);
1376         return(vop_stdopen(ap));
1377 }
1378
1379 /*
1380  * hammer_vop_print { vp }
1381  */
1382 static
1383 int
1384 hammer_vop_print(struct vop_print_args *ap)
1385 {
1386         return EOPNOTSUPP;
1387 }
1388
1389 /*
1390  * hammer_vop_readdir { vp, uio, cred, *eofflag, *ncookies, off_t **cookies }
1391  */
1392 static
1393 int
1394 hammer_vop_readdir(struct vop_readdir_args *ap)
1395 {
1396         struct hammer_transaction trans;
1397         struct hammer_cursor cursor;
1398         struct hammer_inode *ip;
1399         struct uio *uio;
1400         hammer_base_elm_t base;
1401         int error;
1402         int cookie_index;
1403         int ncookies;
1404         off_t *cookies;
1405         off_t saveoff;
1406         int r;
1407         int dtype;
1408
1409         ++hammer_stats_file_iopsr;
1410         ip = VTOI(ap->a_vp);
1411         uio = ap->a_uio;
1412         saveoff = uio->uio_offset;
1413
1414         if (ap->a_ncookies) {
1415                 ncookies = uio->uio_resid / 16 + 1;
1416                 if (ncookies > 1024)
1417                         ncookies = 1024;
1418                 cookies = kmalloc(ncookies * sizeof(off_t), M_TEMP, M_WAITOK);
1419                 cookie_index = 0;
1420         } else {
1421                 ncookies = -1;
1422                 cookies = NULL;
1423                 cookie_index = 0;
1424         }
1425
1426         hammer_simple_transaction(&trans, ip->hmp);
1427
1428         /*
1429          * Handle artificial entries
1430          *
1431          * It should be noted that the minimum value for a directory
1432          * hash key on-media is 0x0000000100000000, so we can use anything
1433          * less then that to represent our 'special' key space.
1434          */
1435         error = 0;
1436         if (saveoff == 0) {
1437                 r = vop_write_dirent(&error, uio, ip->obj_id, DT_DIR, 1, ".");
1438                 if (r)
1439                         goto done;
1440                 if (cookies)
1441                         cookies[cookie_index] = saveoff;
1442                 ++saveoff;
1443                 ++cookie_index;
1444                 if (cookie_index == ncookies)
1445                         goto done;
1446         }
1447         if (saveoff == 1) {
1448                 if (ip->ino_data.parent_obj_id) {
1449                         r = vop_write_dirent(&error, uio,
1450                                              ip->ino_data.parent_obj_id,
1451                                              DT_DIR, 2, "..");
1452                 } else {
1453                         r = vop_write_dirent(&error, uio,
1454                                              ip->obj_id, DT_DIR, 2, "..");
1455                 }
1456                 if (r)
1457                         goto done;
1458                 if (cookies)
1459                         cookies[cookie_index] = saveoff;
1460                 ++saveoff;
1461                 ++cookie_index;
1462                 if (cookie_index == ncookies)
1463                         goto done;
1464         }
1465
1466         /*
1467          * Key range (begin and end inclusive) to scan.  Directory keys
1468          * directly translate to a 64 bit 'seek' position.
1469          */
1470         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
1471         cursor.key_beg.localization = ip->obj_localization +
1472                                       hammer_dir_localization(ip);
1473         cursor.key_beg.obj_id = ip->obj_id;
1474         cursor.key_beg.create_tid = 0;
1475         cursor.key_beg.delete_tid = 0;
1476         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
1477         cursor.key_beg.obj_type = 0;
1478         cursor.key_beg.key = saveoff;
1479
1480         cursor.key_end = cursor.key_beg;
1481         cursor.key_end.key = HAMMER_MAX_KEY;
1482         cursor.asof = ip->obj_asof;
1483         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
1484
1485         error = hammer_ip_first(&cursor);
1486
1487         while (error == 0) {
1488                 error = hammer_ip_resolve_data(&cursor);
1489                 if (error)
1490                         break;
1491                 base = &cursor.leaf->base;
1492                 saveoff = base->key;
1493                 KKASSERT(cursor.leaf->data_len > HAMMER_ENTRY_NAME_OFF);
1494
1495                 if (base->obj_id != ip->obj_id)
1496                         panic("readdir: bad record at %p", cursor.node);
1497
1498                 /*
1499                  * Convert pseudo-filesystems into softlinks
1500                  */
1501                 dtype = hammer_get_dtype(cursor.leaf->base.obj_type);
1502                 r = vop_write_dirent(
1503                              &error, uio, cursor.data->entry.obj_id,
1504                              dtype,
1505                              cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF ,
1506                              (void *)cursor.data->entry.name);
1507                 if (r)
1508                         break;
1509                 ++saveoff;
1510                 if (cookies)
1511                         cookies[cookie_index] = base->key;
1512                 ++cookie_index;
1513                 if (cookie_index == ncookies)
1514                         break;
1515                 error = hammer_ip_next(&cursor);
1516         }
1517         hammer_done_cursor(&cursor);
1518
1519 done:
1520         hammer_done_transaction(&trans);
1521
1522         if (ap->a_eofflag)
1523                 *ap->a_eofflag = (error == ENOENT);
1524         uio->uio_offset = saveoff;
1525         if (error && cookie_index == 0) {
1526                 if (error == ENOENT)
1527                         error = 0;
1528                 if (cookies) {
1529                         kfree(cookies, M_TEMP);
1530                         *ap->a_ncookies = 0;
1531                         *ap->a_cookies = NULL;
1532                 }
1533         } else {
1534                 if (error == ENOENT)
1535                         error = 0;
1536                 if (cookies) {
1537                         *ap->a_ncookies = cookie_index;
1538                         *ap->a_cookies = cookies;
1539                 }
1540         }
1541         return(error);
1542 }
1543
1544 /*
1545  * hammer_vop_readlink { vp, uio, cred }
1546  */
1547 static
1548 int
1549 hammer_vop_readlink(struct vop_readlink_args *ap)
1550 {
1551         struct hammer_transaction trans;
1552         struct hammer_cursor cursor;
1553         struct hammer_inode *ip;
1554         char buf[32];
1555         u_int32_t localization;
1556         hammer_pseudofs_inmem_t pfsm;
1557         int error;
1558
1559         ip = VTOI(ap->a_vp);
1560
1561         /*
1562          * Shortcut if the symlink data was stuffed into ino_data.
1563          *
1564          * Also expand special "@@PFS%05d" softlinks (expansion only
1565          * occurs for non-historical (current) accesses made from the
1566          * primary filesystem).
1567          */
1568         if (ip->ino_data.size <= HAMMER_INODE_BASESYMLEN) {
1569                 char *ptr;
1570                 int bytes;
1571
1572                 ptr = ip->ino_data.ext.symlink;
1573                 bytes = (int)ip->ino_data.size;
1574                 if (bytes == 10 &&
1575                     ip->obj_asof == HAMMER_MAX_TID &&
1576                     ip->obj_localization == 0 &&
1577                     strncmp(ptr, "@@PFS", 5) == 0) {
1578                         hammer_simple_transaction(&trans, ip->hmp);
1579                         bcopy(ptr + 5, buf, 5);
1580                         buf[5] = 0;
1581                         localization = strtoul(buf, NULL, 10) << 16;
1582                         pfsm = hammer_load_pseudofs(&trans, localization,
1583                                                     &error);
1584                         if (error == 0) {
1585                                 if (pfsm->pfsd.mirror_flags &
1586                                     HAMMER_PFSD_SLAVE) {
1587                                         /* vap->va_size == 26 */
1588                                         ksnprintf(buf, sizeof(buf),
1589                                                   "@@0x%016llx:%05d",
1590                                                   (long long)pfsm->pfsd.sync_end_tid,
1591                                                   localization >> 16);
1592                                 } else {
1593                                         /* vap->va_size == 10 */
1594                                         ksnprintf(buf, sizeof(buf),
1595                                                   "@@-1:%05d",
1596                                                   localization >> 16);
1597 #if 0
1598                                         ksnprintf(buf, sizeof(buf),
1599                                                   "@@0x%016llx:%05d",
1600                                                   (long long)HAMMER_MAX_TID,
1601                                                   localization >> 16);
1602 #endif
1603                                 }
1604                                 ptr = buf;
1605                                 bytes = strlen(buf);
1606                         }
1607                         if (pfsm)
1608                                 hammer_rel_pseudofs(trans.hmp, pfsm);
1609                         hammer_done_transaction(&trans);
1610                 }
1611                 error = uiomove(ptr, bytes, ap->a_uio);
1612                 return(error);
1613         }
1614
1615         /*
1616          * Long version
1617          */
1618         hammer_simple_transaction(&trans, ip->hmp);
1619         ++hammer_stats_file_iopsr;
1620         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
1621
1622         /*
1623          * Key range (begin and end inclusive) to scan.  Directory keys
1624          * directly translate to a 64 bit 'seek' position.
1625          */
1626         cursor.key_beg.localization = ip->obj_localization +
1627                                       HAMMER_LOCALIZE_MISC;
1628         cursor.key_beg.obj_id = ip->obj_id;
1629         cursor.key_beg.create_tid = 0;
1630         cursor.key_beg.delete_tid = 0;
1631         cursor.key_beg.rec_type = HAMMER_RECTYPE_FIX;
1632         cursor.key_beg.obj_type = 0;
1633         cursor.key_beg.key = HAMMER_FIXKEY_SYMLINK;
1634         cursor.asof = ip->obj_asof;
1635         cursor.flags |= HAMMER_CURSOR_ASOF;
1636
1637         error = hammer_ip_lookup(&cursor);
1638         if (error == 0) {
1639                 error = hammer_ip_resolve_data(&cursor);
1640                 if (error == 0) {
1641                         KKASSERT(cursor.leaf->data_len >=
1642                                  HAMMER_SYMLINK_NAME_OFF);
1643                         error = uiomove(cursor.data->symlink.name,
1644                                         cursor.leaf->data_len -
1645                                                 HAMMER_SYMLINK_NAME_OFF,
1646                                         ap->a_uio);
1647                 }
1648         }
1649         hammer_done_cursor(&cursor);
1650         hammer_done_transaction(&trans);
1651         return(error);
1652 }
1653
1654 /*
1655  * hammer_vop_nremove { nch, dvp, cred }
1656  */
1657 static
1658 int
1659 hammer_vop_nremove(struct vop_nremove_args *ap)
1660 {
1661         struct hammer_transaction trans;
1662         struct hammer_inode *dip;
1663         int error;
1664
1665         dip = VTOI(ap->a_dvp);
1666
1667         if (hammer_nohistory(dip) == 0 &&
1668             (error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_REMOVE)) != 0) {
1669                 return (error);
1670         }
1671
1672         hammer_start_transaction(&trans, dip->hmp);
1673         ++hammer_stats_file_iopsw;
1674         error = hammer_dounlink(&trans, ap->a_nch, ap->a_dvp, ap->a_cred, 0, 0);
1675         hammer_done_transaction(&trans);
1676         if (error == 0)
1677                 hammer_knote(ap->a_dvp, NOTE_WRITE);
1678         return (error);
1679 }
1680
1681 /*
1682  * hammer_vop_nrename { fnch, tnch, fdvp, tdvp, cred }
1683  */
1684 static
1685 int
1686 hammer_vop_nrename(struct vop_nrename_args *ap)
1687 {
1688         struct hammer_transaction trans;
1689         struct namecache *fncp;
1690         struct namecache *tncp;
1691         struct hammer_inode *fdip;
1692         struct hammer_inode *tdip;
1693         struct hammer_inode *ip;
1694         struct hammer_cursor cursor;
1695         int64_t namekey;
1696         u_int32_t max_iterations;
1697         int nlen, error;
1698
1699         if (ap->a_fdvp->v_mount != ap->a_tdvp->v_mount)
1700                 return(EXDEV);
1701         if (ap->a_fdvp->v_mount != ap->a_fnch->ncp->nc_vp->v_mount)
1702                 return(EXDEV);
1703
1704         fdip = VTOI(ap->a_fdvp);
1705         tdip = VTOI(ap->a_tdvp);
1706         fncp = ap->a_fnch->ncp;
1707         tncp = ap->a_tnch->ncp;
1708         ip = VTOI(fncp->nc_vp);
1709         KKASSERT(ip != NULL);
1710
1711         if (fdip->obj_localization != tdip->obj_localization)
1712                 return(EXDEV);
1713         if (fdip->obj_localization != ip->obj_localization)
1714                 return(EXDEV);
1715
1716         if (fdip->flags & HAMMER_INODE_RO)
1717                 return (EROFS);
1718         if (tdip->flags & HAMMER_INODE_RO)
1719                 return (EROFS);
1720         if (ip->flags & HAMMER_INODE_RO)
1721                 return (EROFS);
1722         if ((error = hammer_checkspace(fdip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1723                 return (error);
1724
1725         hammer_start_transaction(&trans, fdip->hmp);
1726         ++hammer_stats_file_iopsw;
1727
1728         /*
1729          * Remove tncp from the target directory and then link ip as
1730          * tncp. XXX pass trans to dounlink
1731          *
1732          * Force the inode sync-time to match the transaction so it is
1733          * in-sync with the creation of the target directory entry.
1734          */
1735         error = hammer_dounlink(&trans, ap->a_tnch, ap->a_tdvp,
1736                                 ap->a_cred, 0, -1);
1737         if (error == 0 || error == ENOENT) {
1738                 error = hammer_ip_add_directory(&trans, tdip,
1739                                                 tncp->nc_name, tncp->nc_nlen,
1740                                                 ip);
1741                 if (error == 0) {
1742                         ip->ino_data.parent_obj_id = tdip->obj_id;
1743                         ip->ino_data.ctime = trans.time;
1744                         hammer_modify_inode(ip, HAMMER_INODE_DDIRTY);
1745                 }
1746         }
1747         if (error)
1748                 goto failed; /* XXX */
1749
1750         /*
1751          * Locate the record in the originating directory and remove it.
1752          *
1753          * Calculate the namekey and setup the key range for the scan.  This
1754          * works kinda like a chained hash table where the lower 32 bits
1755          * of the namekey synthesize the chain.
1756          *
1757          * The key range is inclusive of both key_beg and key_end.
1758          */
1759         namekey = hammer_directory_namekey(fdip, fncp->nc_name, fncp->nc_nlen,
1760                                            &max_iterations);
1761 retry:
1762         hammer_init_cursor(&trans, &cursor, &fdip->cache[1], fdip);
1763         cursor.key_beg.localization = fdip->obj_localization +
1764                                       hammer_dir_localization(fdip);
1765         cursor.key_beg.obj_id = fdip->obj_id;
1766         cursor.key_beg.key = namekey;
1767         cursor.key_beg.create_tid = 0;
1768         cursor.key_beg.delete_tid = 0;
1769         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
1770         cursor.key_beg.obj_type = 0;
1771
1772         cursor.key_end = cursor.key_beg;
1773         cursor.key_end.key += max_iterations;
1774         cursor.asof = fdip->obj_asof;
1775         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
1776
1777         /*
1778          * Scan all matching records (the chain), locate the one matching
1779          * the requested path component.
1780          *
1781          * The hammer_ip_*() functions merge in-memory records with on-disk
1782          * records for the purposes of the search.
1783          */
1784         error = hammer_ip_first(&cursor);
1785         while (error == 0) {
1786                 if (hammer_ip_resolve_data(&cursor) != 0)
1787                         break;
1788                 nlen = cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF;
1789                 KKASSERT(nlen > 0);
1790                 if (fncp->nc_nlen == nlen &&
1791                     bcmp(fncp->nc_name, cursor.data->entry.name, nlen) == 0) {
1792                         break;
1793                 }
1794                 error = hammer_ip_next(&cursor);
1795         }
1796
1797         /*
1798          * If all is ok we have to get the inode so we can adjust nlinks.
1799          *
1800          * WARNING: hammer_ip_del_directory() may have to terminate the
1801          * cursor to avoid a recursion.  It's ok to call hammer_done_cursor()
1802          * twice.
1803          */
1804         if (error == 0)
1805                 error = hammer_ip_del_directory(&trans, &cursor, fdip, ip);
1806
1807         /*
1808          * XXX A deadlock here will break rename's atomicy for the purposes
1809          * of crash recovery.
1810          */
1811         if (error == EDEADLK) {
1812                 hammer_done_cursor(&cursor);
1813                 goto retry;
1814         }
1815
1816         /*
1817          * Cleanup and tell the kernel that the rename succeeded.
1818          */
1819         hammer_done_cursor(&cursor);
1820         if (error == 0) {
1821                 cache_rename(ap->a_fnch, ap->a_tnch);
1822                 hammer_knote(ap->a_fdvp, NOTE_WRITE);
1823                 hammer_knote(ap->a_tdvp, NOTE_WRITE);
1824                 if (ip->vp)
1825                         hammer_knote(ip->vp, NOTE_RENAME);
1826         }
1827
1828 failed:
1829         hammer_done_transaction(&trans);
1830         return (error);
1831 }
1832
1833 /*
1834  * hammer_vop_nrmdir { nch, dvp, cred }
1835  */
1836 static
1837 int
1838 hammer_vop_nrmdir(struct vop_nrmdir_args *ap)
1839 {
1840         struct hammer_transaction trans;
1841         struct hammer_inode *dip;
1842         int error;
1843
1844         dip = VTOI(ap->a_dvp);
1845
1846         if (hammer_nohistory(dip) == 0 &&
1847             (error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_REMOVE)) != 0) {
1848                 return (error);
1849         }
1850
1851         hammer_start_transaction(&trans, dip->hmp);
1852         ++hammer_stats_file_iopsw;
1853         error = hammer_dounlink(&trans, ap->a_nch, ap->a_dvp, ap->a_cred, 0, 1);
1854         hammer_done_transaction(&trans);
1855         if (error == 0)
1856                 hammer_knote(ap->a_dvp, NOTE_WRITE | NOTE_LINK);
1857         return (error);
1858 }
1859
1860 /*
1861  * hammer_vop_markatime { vp, cred }
1862  */
1863 static
1864 int
1865 hammer_vop_markatime(struct vop_markatime_args *ap)
1866 {
1867         struct hammer_transaction trans;
1868         struct hammer_inode *ip;
1869
1870         ip = VTOI(ap->a_vp);
1871         if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
1872                 return (EROFS);
1873         if (ip->flags & HAMMER_INODE_RO)
1874                 return (EROFS);
1875         if (ip->hmp->mp->mnt_flag & MNT_NOATIME)
1876                 return (0);
1877         hammer_start_transaction(&trans, ip->hmp);
1878         ++hammer_stats_file_iopsw;
1879
1880         ip->ino_data.atime = trans.time;
1881         hammer_modify_inode(ip, HAMMER_INODE_ATIME);
1882         hammer_done_transaction(&trans);
1883         hammer_knote(ap->a_vp, NOTE_ATTRIB);
1884         return (0);
1885 }
1886
1887 /*
1888  * hammer_vop_setattr { vp, vap, cred }
1889  */
1890 static
1891 int
1892 hammer_vop_setattr(struct vop_setattr_args *ap)
1893 {
1894         struct hammer_transaction trans;
1895         struct vattr *vap;
1896         struct hammer_inode *ip;
1897         int modflags;
1898         int error;
1899         int truncating;
1900         int blksize;
1901         int kflags;
1902         int64_t aligned_size;
1903         u_int32_t flags;
1904
1905         vap = ap->a_vap;
1906         ip = ap->a_vp->v_data;
1907         modflags = 0;
1908         kflags = 0;
1909
1910         if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
1911                 return(EROFS);
1912         if (ip->flags & HAMMER_INODE_RO)
1913                 return (EROFS);
1914         if (hammer_nohistory(ip) == 0 &&
1915             (error = hammer_checkspace(ip->hmp, HAMMER_CHKSPC_REMOVE)) != 0) {
1916                 return (error);
1917         }
1918
1919         hammer_start_transaction(&trans, ip->hmp);
1920         ++hammer_stats_file_iopsw;
1921         error = 0;
1922
1923         if (vap->va_flags != VNOVAL) {
1924                 flags = ip->ino_data.uflags;
1925                 error = vop_helper_setattr_flags(&flags, vap->va_flags,
1926                                          hammer_to_unix_xid(&ip->ino_data.uid),
1927                                          ap->a_cred);
1928                 if (error == 0) {
1929                         if (ip->ino_data.uflags != flags) {
1930                                 ip->ino_data.uflags = flags;
1931                                 ip->ino_data.ctime = trans.time;
1932                                 modflags |= HAMMER_INODE_DDIRTY;
1933                                 kflags |= NOTE_ATTRIB;
1934                         }
1935                         if (ip->ino_data.uflags & (IMMUTABLE | APPEND)) {
1936                                 error = 0;
1937                                 goto done;
1938                         }
1939                 }
1940                 goto done;
1941         }
1942         if (ip->ino_data.uflags & (IMMUTABLE | APPEND)) {
1943                 error = EPERM;
1944                 goto done;
1945         }
1946         if (vap->va_uid != (uid_t)VNOVAL || vap->va_gid != (gid_t)VNOVAL) {
1947                 mode_t cur_mode = ip->ino_data.mode;
1948                 uid_t cur_uid = hammer_to_unix_xid(&ip->ino_data.uid);
1949                 gid_t cur_gid = hammer_to_unix_xid(&ip->ino_data.gid);
1950                 uuid_t uuid_uid;
1951                 uuid_t uuid_gid;
1952
1953                 error = vop_helper_chown(ap->a_vp, vap->va_uid, vap->va_gid,
1954                                          ap->a_cred,
1955                                          &cur_uid, &cur_gid, &cur_mode);
1956                 if (error == 0) {
1957                         hammer_guid_to_uuid(&uuid_uid, cur_uid);
1958                         hammer_guid_to_uuid(&uuid_gid, cur_gid);
1959                         if (bcmp(&uuid_uid, &ip->ino_data.uid,
1960                                  sizeof(uuid_uid)) ||
1961                             bcmp(&uuid_gid, &ip->ino_data.gid,
1962                                  sizeof(uuid_gid)) ||
1963                             ip->ino_data.mode != cur_mode
1964                         ) {
1965                                 ip->ino_data.uid = uuid_uid;
1966                                 ip->ino_data.gid = uuid_gid;
1967                                 ip->ino_data.mode = cur_mode;
1968                                 ip->ino_data.ctime = trans.time;
1969                                 modflags |= HAMMER_INODE_DDIRTY;
1970                         }
1971                         kflags |= NOTE_ATTRIB;
1972                 }
1973         }
1974         while (vap->va_size != VNOVAL && ip->ino_data.size != vap->va_size) {
1975                 switch(ap->a_vp->v_type) {
1976                 case VREG:
1977                         if (vap->va_size == ip->ino_data.size)
1978                                 break;
1979                         /*
1980                          * XXX break atomicy, we can deadlock the backend
1981                          * if we do not release the lock.  Probably not a
1982                          * big deal here.
1983                          */
1984                         blksize = hammer_blocksize(vap->va_size);
1985                         if (vap->va_size < ip->ino_data.size) {
1986                                 vtruncbuf(ap->a_vp, vap->va_size, blksize);
1987                                 truncating = 1;
1988                                 kflags |= NOTE_WRITE;
1989                         } else {
1990                                 vnode_pager_setsize(ap->a_vp, vap->va_size);
1991                                 truncating = 0;
1992                                 kflags |= NOTE_WRITE | NOTE_EXTEND;
1993                         }
1994                         ip->ino_data.size = vap->va_size;
1995                         ip->ino_data.mtime = trans.time;
1996                         modflags |= HAMMER_INODE_MTIME | HAMMER_INODE_DDIRTY;
1997
1998                         /*
1999                          * on-media truncation is cached in the inode until
2000                          * the inode is synchronized.
2001                          */
2002                         if (truncating) {
2003                                 hammer_ip_frontend_trunc(ip, vap->va_size);
2004 #ifdef DEBUG_TRUNCATE
2005                                 if (HammerTruncIp == NULL)
2006                                         HammerTruncIp = ip;
2007 #endif
2008                                 if ((ip->flags & HAMMER_INODE_TRUNCATED) == 0) {
2009                                         ip->flags |= HAMMER_INODE_TRUNCATED;
2010                                         ip->trunc_off = vap->va_size;
2011 #ifdef DEBUG_TRUNCATE
2012                                         if (ip == HammerTruncIp)
2013                                         kprintf("truncate1 %016llx\n",
2014                                                 (long long)ip->trunc_off);
2015 #endif
2016                                 } else if (ip->trunc_off > vap->va_size) {
2017                                         ip->trunc_off = vap->va_size;
2018 #ifdef DEBUG_TRUNCATE
2019                                         if (ip == HammerTruncIp)
2020                                         kprintf("truncate2 %016llx\n",
2021                                                 (long long)ip->trunc_off);
2022 #endif
2023                                 } else {
2024 #ifdef DEBUG_TRUNCATE
2025                                         if (ip == HammerTruncIp)
2026                                         kprintf("truncate3 %016llx (ignored)\n",
2027                                                 (long long)vap->va_size);
2028 #endif
2029                                 }
2030                         }
2031
2032                         /*
2033                          * If truncating we have to clean out a portion of
2034                          * the last block on-disk.  We do this in the
2035                          * front-end buffer cache.
2036                          */
2037                         aligned_size = (vap->va_size + (blksize - 1)) &
2038                                        ~(int64_t)(blksize - 1);
2039                         if (truncating && vap->va_size < aligned_size) {
2040                                 struct buf *bp;
2041                                 int offset;
2042
2043                                 aligned_size -= blksize;
2044
2045                                 offset = (int)vap->va_size & (blksize - 1);
2046                                 error = bread(ap->a_vp, aligned_size,
2047                                               blksize, &bp);
2048                                 hammer_ip_frontend_trunc(ip, aligned_size);
2049                                 if (error == 0) {
2050                                         bzero(bp->b_data + offset,
2051                                               blksize - offset);
2052                                         /* must de-cache direct-io offset */
2053                                         bp->b_bio2.bio_offset = NOOFFSET;
2054                                         bdwrite(bp);
2055                                 } else {
2056                                         kprintf("ERROR %d\n", error);
2057                                         brelse(bp);
2058                                 }
2059                         }
2060                         break;
2061                 case VDATABASE:
2062                         if ((ip->flags & HAMMER_INODE_TRUNCATED) == 0) {
2063                                 ip->flags |= HAMMER_INODE_TRUNCATED;
2064                                 ip->trunc_off = vap->va_size;
2065                         } else if (ip->trunc_off > vap->va_size) {
2066                                 ip->trunc_off = vap->va_size;
2067                         }
2068                         hammer_ip_frontend_trunc(ip, vap->va_size);
2069                         ip->ino_data.size = vap->va_size;
2070                         ip->ino_data.mtime = trans.time;
2071                         modflags |= HAMMER_INODE_MTIME | HAMMER_INODE_DDIRTY;
2072                         kflags |= NOTE_ATTRIB;
2073                         break;
2074                 default:
2075                         error = EINVAL;
2076                         goto done;
2077                 }
2078                 break;
2079         }
2080         if (vap->va_atime.tv_sec != VNOVAL) {
2081                 ip->ino_data.atime = hammer_timespec_to_time(&vap->va_atime);
2082                 modflags |= HAMMER_INODE_ATIME;
2083                 kflags |= NOTE_ATTRIB;
2084         }
2085         if (vap->va_mtime.tv_sec != VNOVAL) {
2086                 ip->ino_data.mtime = hammer_timespec_to_time(&vap->va_mtime);
2087                 modflags |= HAMMER_INODE_MTIME;
2088                 kflags |= NOTE_ATTRIB;
2089         }
2090         if (vap->va_mode != (mode_t)VNOVAL) {
2091                 mode_t   cur_mode = ip->ino_data.mode;
2092                 uid_t cur_uid = hammer_to_unix_xid(&ip->ino_data.uid);
2093                 gid_t cur_gid = hammer_to_unix_xid(&ip->ino_data.gid);
2094
2095                 error = vop_helper_chmod(ap->a_vp, vap->va_mode, ap->a_cred,
2096                                          cur_uid, cur_gid, &cur_mode);
2097                 if (error == 0 && ip->ino_data.mode != cur_mode) {
2098                         ip->ino_data.mode = cur_mode;
2099                         ip->ino_data.ctime = trans.time;
2100                         modflags |= HAMMER_INODE_DDIRTY;
2101                         kflags |= NOTE_ATTRIB;
2102                 }
2103         }
2104 done:
2105         if (error == 0)
2106                 hammer_modify_inode(ip, modflags);
2107         hammer_done_transaction(&trans);
2108         hammer_knote(ap->a_vp, kflags);
2109         return (error);
2110 }
2111
2112 /*
2113  * hammer_vop_nsymlink { nch, dvp, vpp, cred, vap, target }
2114  */
2115 static
2116 int
2117 hammer_vop_nsymlink(struct vop_nsymlink_args *ap)
2118 {
2119         struct hammer_transaction trans;
2120         struct hammer_inode *dip;
2121         struct hammer_inode *nip;
2122         struct nchandle *nch;
2123         hammer_record_t record;
2124         int error;
2125         int bytes;
2126
2127         ap->a_vap->va_type = VLNK;
2128
2129         nch = ap->a_nch;
2130         dip = VTOI(ap->a_dvp);
2131
2132         if (dip->flags & HAMMER_INODE_RO)
2133                 return (EROFS);
2134         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
2135                 return (error);
2136
2137         /*
2138          * Create a transaction to cover the operations we perform.
2139          */
2140         hammer_start_transaction(&trans, dip->hmp);
2141         ++hammer_stats_file_iopsw;
2142
2143         /*
2144          * Create a new filesystem object of the requested type.  The
2145          * returned inode will be referenced but not locked.
2146          */
2147
2148         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
2149                                     dip, nch->ncp->nc_name, nch->ncp->nc_nlen,
2150                                     NULL, &nip);
2151         if (error) {
2152                 hammer_done_transaction(&trans);
2153                 *ap->a_vpp = NULL;
2154                 return (error);
2155         }
2156
2157         /*
2158          * Add a record representing the symlink.  symlink stores the link
2159          * as pure data, not a string, and is no \0 terminated.
2160          */
2161         if (error == 0) {
2162                 bytes = strlen(ap->a_target);
2163
2164                 if (bytes <= HAMMER_INODE_BASESYMLEN) {
2165                         bcopy(ap->a_target, nip->ino_data.ext.symlink, bytes);
2166                 } else {
2167                         record = hammer_alloc_mem_record(nip, bytes);
2168                         record->type = HAMMER_MEM_RECORD_GENERAL;
2169
2170                         record->leaf.base.localization = nip->obj_localization +
2171                                                          HAMMER_LOCALIZE_MISC;
2172                         record->leaf.base.key = HAMMER_FIXKEY_SYMLINK;
2173                         record->leaf.base.rec_type = HAMMER_RECTYPE_FIX;
2174                         record->leaf.data_len = bytes;
2175                         KKASSERT(HAMMER_SYMLINK_NAME_OFF == 0);
2176                         bcopy(ap->a_target, record->data->symlink.name, bytes);
2177                         error = hammer_ip_add_record(&trans, record);
2178                 }
2179
2180                 /*
2181                  * Set the file size to the length of the link.
2182                  */
2183                 if (error == 0) {
2184                         nip->ino_data.size = bytes;
2185                         hammer_modify_inode(nip, HAMMER_INODE_DDIRTY);
2186                 }
2187         }
2188         if (error == 0)
2189                 error = hammer_ip_add_directory(&trans, dip, nch->ncp->nc_name,
2190                                                 nch->ncp->nc_nlen, nip);
2191
2192         /*
2193          * Finish up.
2194          */
2195         if (error) {
2196                 hammer_rel_inode(nip, 0);
2197                 *ap->a_vpp = NULL;
2198         } else {
2199                 error = hammer_get_vnode(nip, ap->a_vpp);
2200                 hammer_rel_inode(nip, 0);
2201                 if (error == 0) {
2202                         cache_setunresolved(ap->a_nch);
2203                         cache_setvp(ap->a_nch, *ap->a_vpp);
2204                         hammer_knote(ap->a_dvp, NOTE_WRITE);
2205                 }
2206         }
2207         hammer_done_transaction(&trans);
2208         return (error);
2209 }
2210
2211 /*
2212  * hammer_vop_nwhiteout { nch, dvp, cred, flags }
2213  */
2214 static
2215 int
2216 hammer_vop_nwhiteout(struct vop_nwhiteout_args *ap)
2217 {
2218         struct hammer_transaction trans;
2219         struct hammer_inode *dip;
2220         int error;
2221
2222         dip = VTOI(ap->a_dvp);
2223
2224         if (hammer_nohistory(dip) == 0 &&
2225             (error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0) {
2226                 return (error);
2227         }
2228
2229         hammer_start_transaction(&trans, dip->hmp);
2230         ++hammer_stats_file_iopsw;
2231         error = hammer_dounlink(&trans, ap->a_nch, ap->a_dvp,
2232                                 ap->a_cred, ap->a_flags, -1);
2233         hammer_done_transaction(&trans);
2234
2235         return (error);
2236 }
2237
2238 /*
2239  * hammer_vop_ioctl { vp, command, data, fflag, cred }
2240  */
2241 static
2242 int
2243 hammer_vop_ioctl(struct vop_ioctl_args *ap)
2244 {
2245         struct hammer_inode *ip = ap->a_vp->v_data;
2246
2247         ++hammer_stats_file_iopsr;
2248         return(hammer_ioctl(ip, ap->a_command, ap->a_data,
2249                             ap->a_fflag, ap->a_cred));
2250 }
2251
2252 static
2253 int
2254 hammer_vop_mountctl(struct vop_mountctl_args *ap)
2255 {
2256         static const struct mountctl_opt extraopt[] = {
2257                 { HMNT_NOHISTORY,       "nohistory" },
2258                 { HMNT_MASTERID,        "master" },
2259                 { 0, NULL}
2260
2261         };
2262         struct hammer_mount *hmp;
2263         struct mount *mp;
2264         int usedbytes;
2265         int error;
2266
2267         error = 0;
2268         usedbytes = 0;
2269         mp = ap->a_head.a_ops->head.vv_mount;
2270         KKASSERT(mp->mnt_data != NULL);
2271         hmp = (struct hammer_mount *)mp->mnt_data;
2272
2273         switch(ap->a_op) {
2274
2275         case MOUNTCTL_SET_EXPORT:
2276                 if (ap->a_ctllen != sizeof(struct export_args))
2277                         error = EINVAL;
2278                 else
2279                         error = hammer_vfs_export(mp, ap->a_op,
2280                                       (const struct export_args *)ap->a_ctl);
2281                 break;
2282         case MOUNTCTL_MOUNTFLAGS:
2283         {
2284                 /*
2285                  * Call standard mountctl VOP function
2286                  * so we get user mount flags.
2287                  */
2288                 error = vop_stdmountctl(ap);
2289                 if (error)
2290                         break;
2291
2292                 usedbytes = *ap->a_res;
2293
2294                 if (usedbytes > 0 && usedbytes < ap->a_buflen) {
2295                         usedbytes += vfs_flagstostr(hmp->hflags, extraopt, ap->a_buf,
2296                                                     ap->a_buflen - usedbytes,
2297                                                     &error);
2298                 }
2299
2300                 *ap->a_res += usedbytes;
2301                 break;
2302         }
2303         default:
2304                 error = vop_stdmountctl(ap);
2305                 break;
2306         }
2307         return(error);
2308 }
2309
2310 /*
2311  * hammer_vop_strategy { vp, bio }
2312  *
2313  * Strategy call, used for regular file read & write only.  Note that the
2314  * bp may represent a cluster.
2315  *
2316  * To simplify operation and allow better optimizations in the future,
2317  * this code does not make any assumptions with regards to buffer alignment
2318  * or size.
2319  */
2320 static
2321 int
2322 hammer_vop_strategy(struct vop_strategy_args *ap)
2323 {
2324         struct buf *bp;
2325         int error;
2326
2327         bp = ap->a_bio->bio_buf;
2328
2329         switch(bp->b_cmd) {
2330         case BUF_CMD_READ:
2331                 error = hammer_vop_strategy_read(ap);
2332                 break;
2333         case BUF_CMD_WRITE:
2334                 error = hammer_vop_strategy_write(ap);
2335                 break;
2336         default:
2337                 bp->b_error = error = EINVAL;
2338                 bp->b_flags |= B_ERROR;
2339                 biodone(ap->a_bio);
2340                 break;
2341         }
2342         return (error);
2343 }
2344
2345 /*
2346  * Read from a regular file.  Iterate the related records and fill in the
2347  * BIO/BUF.  Gaps are zero-filled.
2348  *
2349  * The support code in hammer_object.c should be used to deal with mixed
2350  * in-memory and on-disk records.
2351  *
2352  * NOTE: Can be called from the cluster code with an oversized buf.
2353  *
2354  * XXX atime update
2355  */
2356 static
2357 int
2358 hammer_vop_strategy_read(struct vop_strategy_args *ap)
2359 {
2360         struct hammer_transaction trans;
2361         struct hammer_inode *ip;
2362         struct hammer_inode *dip;
2363         struct hammer_cursor cursor;
2364         hammer_base_elm_t base;
2365         hammer_off_t disk_offset;
2366         struct bio *bio;
2367         struct bio *nbio;
2368         struct buf *bp;
2369         int64_t rec_offset;
2370         int64_t ran_end;
2371         int64_t tmp64;
2372         int error;
2373         int boff;
2374         int roff;
2375         int n;
2376
2377         bio = ap->a_bio;
2378         bp = bio->bio_buf;
2379         ip = ap->a_vp->v_data;
2380
2381         /*
2382          * The zone-2 disk offset may have been set by the cluster code via
2383          * a BMAP operation, or else should be NOOFFSET.
2384          *
2385          * Checking the high bits for a match against zone-2 should suffice.
2386          */
2387         nbio = push_bio(bio);
2388         if ((nbio->bio_offset & HAMMER_OFF_ZONE_MASK) ==
2389             HAMMER_ZONE_LARGE_DATA) {
2390                 error = hammer_io_direct_read(ip->hmp, nbio, NULL);
2391                 return (error);
2392         }
2393
2394         /*
2395          * Well, that sucked.  Do it the hard way.  If all the stars are
2396          * aligned we may still be able to issue a direct-read.
2397          */
2398         hammer_simple_transaction(&trans, ip->hmp);
2399         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
2400
2401         /*
2402          * Key range (begin and end inclusive) to scan.  Note that the key's
2403          * stored in the actual records represent BASE+LEN, not BASE.  The
2404          * first record containing bio_offset will have a key > bio_offset.
2405          */
2406         cursor.key_beg.localization = ip->obj_localization +
2407                                       HAMMER_LOCALIZE_MISC;
2408         cursor.key_beg.obj_id = ip->obj_id;
2409         cursor.key_beg.create_tid = 0;
2410         cursor.key_beg.delete_tid = 0;
2411         cursor.key_beg.obj_type = 0;
2412         cursor.key_beg.key = bio->bio_offset + 1;
2413         cursor.asof = ip->obj_asof;
2414         cursor.flags |= HAMMER_CURSOR_ASOF;
2415
2416         cursor.key_end = cursor.key_beg;
2417         KKASSERT(ip->ino_data.obj_type == HAMMER_OBJTYPE_REGFILE);
2418 #if 0
2419         if (ip->ino_data.obj_type == HAMMER_OBJTYPE_DBFILE) {
2420                 cursor.key_beg.rec_type = HAMMER_RECTYPE_DB;
2421                 cursor.key_end.rec_type = HAMMER_RECTYPE_DB;
2422                 cursor.key_end.key = 0x7FFFFFFFFFFFFFFFLL;
2423         } else
2424 #endif
2425         {
2426                 ran_end = bio->bio_offset + bp->b_bufsize;
2427                 cursor.key_beg.rec_type = HAMMER_RECTYPE_DATA;
2428                 cursor.key_end.rec_type = HAMMER_RECTYPE_DATA;
2429                 tmp64 = ran_end + MAXPHYS + 1;  /* work-around GCC-4 bug */
2430                 if (tmp64 < ran_end)
2431                         cursor.key_end.key = 0x7FFFFFFFFFFFFFFFLL;
2432                 else
2433                         cursor.key_end.key = ran_end + MAXPHYS + 1;
2434         }
2435         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE;
2436
2437         error = hammer_ip_first(&cursor);
2438         boff = 0;
2439
2440         while (error == 0) {
2441                 /*
2442                  * Get the base file offset of the record.  The key for
2443                  * data records is (base + bytes) rather then (base).
2444                  */
2445                 base = &cursor.leaf->base;
2446                 rec_offset = base->key - cursor.leaf->data_len;
2447
2448                 /*
2449                  * Calculate the gap, if any, and zero-fill it.
2450                  *
2451                  * n is the offset of the start of the record verses our
2452                  * current seek offset in the bio.
2453                  */
2454                 n = (int)(rec_offset - (bio->bio_offset + boff));
2455                 if (n > 0) {
2456                         if (n > bp->b_bufsize - boff)
2457                                 n = bp->b_bufsize - boff;
2458                         bzero((char *)bp->b_data + boff, n);
2459                         boff += n;
2460                         n = 0;
2461                 }
2462
2463                 /*
2464                  * Calculate the data offset in the record and the number
2465                  * of bytes we can copy.
2466                  *
2467                  * There are two degenerate cases.  First, boff may already
2468                  * be at bp->b_bufsize.  Secondly, the data offset within
2469                  * the record may exceed the record's size.
2470                  */
2471                 roff = -n;
2472                 rec_offset += roff;
2473                 n = cursor.leaf->data_len - roff;
2474                 if (n <= 0) {
2475                         kprintf("strategy_read: bad n=%d roff=%d\n", n, roff);
2476                         n = 0;
2477                 } else if (n > bp->b_bufsize - boff) {
2478                         n = bp->b_bufsize - boff;
2479                 }
2480
2481                 /*
2482                  * Deal with cached truncations.  This cool bit of code
2483                  * allows truncate()/ftruncate() to avoid having to sync
2484                  * the file.
2485                  *
2486                  * If the frontend is truncated then all backend records are
2487                  * subject to the frontend's truncation.
2488                  *
2489                  * If the backend is truncated then backend records on-disk
2490                  * (but not in-memory) are subject to the backend's
2491                  * truncation.  In-memory records owned by the backend
2492                  * represent data written after the truncation point on the
2493                  * backend and must not be truncated.
2494                  *
2495                  * Truncate operations deal with frontend buffer cache
2496                  * buffers and frontend-owned in-memory records synchronously.
2497                  */
2498                 if (ip->flags & HAMMER_INODE_TRUNCATED) {
2499                         if (hammer_cursor_ondisk(&cursor) ||
2500                             cursor.iprec->flush_state == HAMMER_FST_FLUSH) {
2501                                 if (ip->trunc_off <= rec_offset)
2502                                         n = 0;
2503                                 else if (ip->trunc_off < rec_offset + n)
2504                                         n = (int)(ip->trunc_off - rec_offset);
2505                         }
2506                 }
2507                 if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
2508                         if (hammer_cursor_ondisk(&cursor)) {
2509                                 if (ip->sync_trunc_off <= rec_offset)
2510                                         n = 0;
2511                                 else if (ip->sync_trunc_off < rec_offset + n)
2512                                         n = (int)(ip->sync_trunc_off - rec_offset);
2513                         }
2514                 }
2515
2516                 /*
2517                  * Try to issue a direct read into our bio if possible,
2518                  * otherwise resolve the element data into a hammer_buffer
2519                  * and copy.
2520                  *
2521                  * The buffer on-disk should be zerod past any real
2522                  * truncation point, but may not be for any synthesized
2523                  * truncation point from above.
2524                  */
2525                 disk_offset = cursor.leaf->data_offset + roff;
2526                 if (boff == 0 && n == bp->b_bufsize &&
2527                     hammer_cursor_ondisk(&cursor) &&
2528                     (disk_offset & HAMMER_BUFMASK) == 0) {
2529                         KKASSERT((disk_offset & HAMMER_OFF_ZONE_MASK) ==
2530                                  HAMMER_ZONE_LARGE_DATA);
2531                         nbio->bio_offset = disk_offset;
2532                         error = hammer_io_direct_read(trans.hmp, nbio,
2533                                                       cursor.leaf);
2534                         goto done;
2535                 } else if (n) {
2536                         error = hammer_ip_resolve_data(&cursor);
2537                         if (error == 0) {
2538                                 bcopy((char *)cursor.data + roff,
2539                                       (char *)bp->b_data + boff, n);
2540                         }
2541                 }
2542                 if (error)
2543                         break;
2544
2545                 /*
2546                  * Iterate until we have filled the request.
2547                  */
2548                 boff += n;
2549                 if (boff == bp->b_bufsize)
2550                         break;
2551                 error = hammer_ip_next(&cursor);
2552         }
2553
2554         /*
2555          * There may have been a gap after the last record
2556          */
2557         if (error == ENOENT)
2558                 error = 0;
2559         if (error == 0 && boff != bp->b_bufsize) {
2560                 KKASSERT(boff < bp->b_bufsize);
2561                 bzero((char *)bp->b_data + boff, bp->b_bufsize - boff);
2562                 /* boff = bp->b_bufsize; */
2563         }
2564         bp->b_resid = 0;
2565         bp->b_error = error;
2566         if (error)
2567                 bp->b_flags |= B_ERROR;
2568         biodone(ap->a_bio);
2569
2570 done:
2571         /*
2572          * Cache the b-tree node for the last data read in cache[1].
2573          *
2574          * If we hit the file EOF then also cache the node in the
2575          * governing director's cache[3], it will be used to initialize
2576          * the inode's cache[1] for any inodes looked up via the directory.
2577          *
2578          * This doesn't reduce disk accesses since the B-Tree chain is
2579          * likely cached, but it does reduce cpu overhead when looking
2580          * up file offsets for cpdup/tar/cpio style iterations.
2581          */
2582         if (cursor.node)
2583                 hammer_cache_node(&ip->cache[1], cursor.node);
2584         if (ran_end >= ip->ino_data.size) {
2585                 dip = hammer_find_inode(&trans, ip->ino_data.parent_obj_id,
2586                                         ip->obj_asof, ip->obj_localization);
2587                 if (dip) {
2588                         hammer_cache_node(&dip->cache[3], cursor.node);
2589                         hammer_rel_inode(dip, 0);
2590                 }
2591         }
2592         hammer_done_cursor(&cursor);
2593         hammer_done_transaction(&trans);
2594         return(error);
2595 }
2596
2597 /*
2598  * BMAP operation - used to support cluster_read() only.
2599  *
2600  * (struct vnode *vp, off_t loffset, off_t *doffsetp, int *runp, int *runb)
2601  *
2602  * This routine may return EOPNOTSUPP if the opration is not supported for
2603  * the specified offset.  The contents of the pointer arguments do not
2604  * need to be initialized in that case.
2605  *
2606  * If a disk address is available and properly aligned return 0 with
2607  * *doffsetp set to the zone-2 address, and *runp / *runb set appropriately
2608  * to the run-length relative to that offset.  Callers may assume that
2609  * *doffsetp is valid if 0 is returned, even if *runp is not sufficiently
2610  * large, so return EOPNOTSUPP if it is not sufficiently large.
2611  */
2612 static
2613 int
2614 hammer_vop_bmap(struct vop_bmap_args *ap)
2615 {
2616         struct hammer_transaction trans;
2617         struct hammer_inode *ip;
2618         struct hammer_cursor cursor;
2619         hammer_base_elm_t base;
2620         int64_t rec_offset;
2621         int64_t ran_end;
2622         int64_t tmp64;
2623         int64_t base_offset;
2624         int64_t base_disk_offset;
2625         int64_t last_offset;
2626         hammer_off_t last_disk_offset;
2627         hammer_off_t disk_offset;
2628         int     rec_len;
2629         int     error;
2630         int     blksize;
2631
2632         ++hammer_stats_file_iopsr;
2633         ip = ap->a_vp->v_data;
2634
2635         /*
2636          * We can only BMAP regular files.  We can't BMAP database files,
2637          * directories, etc.
2638          */
2639         if (ip->ino_data.obj_type != HAMMER_OBJTYPE_REGFILE)
2640                 return(EOPNOTSUPP);
2641
2642         /*
2643          * bmap is typically called with runp/runb both NULL when used
2644          * for writing.  We do not support BMAP for writing atm.
2645          */
2646         if (ap->a_cmd != BUF_CMD_READ)
2647                 return(EOPNOTSUPP);
2648
2649         /*
2650          * Scan the B-Tree to acquire blockmap addresses, then translate
2651          * to raw addresses.
2652          */
2653         hammer_simple_transaction(&trans, ip->hmp);
2654 #if 0
2655         kprintf("bmap_beg %016llx ip->cache %p\n",
2656                 (long long)ap->a_loffset, ip->cache[1]);
2657 #endif
2658         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
2659
2660         /*
2661          * Key range (begin and end inclusive) to scan.  Note that the key's
2662          * stored in the actual records represent BASE+LEN, not BASE.  The
2663          * first record containing bio_offset will have a key > bio_offset.
2664          */
2665         cursor.key_beg.localization = ip->obj_localization +
2666                                       HAMMER_LOCALIZE_MISC;
2667         cursor.key_beg.obj_id = ip->obj_id;
2668         cursor.key_beg.create_tid = 0;
2669         cursor.key_beg.delete_tid = 0;
2670         cursor.key_beg.obj_type = 0;
2671         if (ap->a_runb)
2672                 cursor.key_beg.key = ap->a_loffset - MAXPHYS + 1;
2673         else
2674                 cursor.key_beg.key = ap->a_loffset + 1;
2675         if (cursor.key_beg.key < 0)
2676                 cursor.key_beg.key = 0;
2677         cursor.asof = ip->obj_asof;
2678         cursor.flags |= HAMMER_CURSOR_ASOF;
2679
2680         cursor.key_end = cursor.key_beg;
2681         KKASSERT(ip->ino_data.obj_type == HAMMER_OBJTYPE_REGFILE);
2682
2683         ran_end = ap->a_loffset + MAXPHYS;
2684         cursor.key_beg.rec_type = HAMMER_RECTYPE_DATA;
2685         cursor.key_end.rec_type = HAMMER_RECTYPE_DATA;
2686         tmp64 = ran_end + MAXPHYS + 1;  /* work-around GCC-4 bug */
2687         if (tmp64 < ran_end)
2688                 cursor.key_end.key = 0x7FFFFFFFFFFFFFFFLL;
2689         else
2690                 cursor.key_end.key = ran_end + MAXPHYS + 1;
2691
2692         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE;
2693
2694         error = hammer_ip_first(&cursor);
2695         base_offset = last_offset = 0;
2696         base_disk_offset = last_disk_offset = 0;
2697
2698         while (error == 0) {
2699                 /*
2700                  * Get the base file offset of the record.  The key for
2701                  * data records is (base + bytes) rather then (base).
2702                  *
2703                  * NOTE: rec_offset + rec_len may exceed the end-of-file.
2704                  * The extra bytes should be zero on-disk and the BMAP op
2705                  * should still be ok.
2706                  */
2707                 base = &cursor.leaf->base;
2708                 rec_offset = base->key - cursor.leaf->data_len;
2709                 rec_len    = cursor.leaf->data_len;
2710
2711                 /*
2712                  * Incorporate any cached truncation.
2713                  *
2714                  * NOTE: Modifications to rec_len based on synthesized
2715                  * truncation points remove the guarantee that any extended
2716                  * data on disk is zero (since the truncations may not have
2717                  * taken place on-media yet).
2718                  */
2719                 if (ip->flags & HAMMER_INODE_TRUNCATED) {
2720                         if (hammer_cursor_ondisk(&cursor) ||
2721                             cursor.iprec->flush_state == HAMMER_FST_FLUSH) {
2722                                 if (ip->trunc_off <= rec_offset)
2723                                         rec_len = 0;
2724                                 else if (ip->trunc_off < rec_offset + rec_len)
2725                                         rec_len = (int)(ip->trunc_off - rec_offset);
2726                         }
2727                 }
2728                 if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
2729                         if (hammer_cursor_ondisk(&cursor)) {
2730                                 if (ip->sync_trunc_off <= rec_offset)
2731                                         rec_len = 0;
2732                                 else if (ip->sync_trunc_off < rec_offset + rec_len)
2733                                         rec_len = (int)(ip->sync_trunc_off - rec_offset);
2734                         }
2735                 }
2736
2737                 /*
2738                  * Accumulate information.  If we have hit a discontiguous
2739                  * block reset base_offset unless we are already beyond the
2740                  * requested offset.  If we are, that's it, we stop.
2741                  */
2742                 if (error)
2743                         break;
2744                 if (hammer_cursor_ondisk(&cursor)) {
2745                         disk_offset = cursor.leaf->data_offset;
2746                         if (rec_offset != last_offset ||
2747                             disk_offset != last_disk_offset) {
2748                                 if (rec_offset > ap->a_loffset)
2749                                         break;
2750                                 base_offset = rec_offset;
2751                                 base_disk_offset = disk_offset;
2752                         }
2753                         last_offset = rec_offset + rec_len;
2754                         last_disk_offset = disk_offset + rec_len;
2755                 }
2756                 error = hammer_ip_next(&cursor);
2757         }
2758
2759 #if 0
2760         kprintf("BMAP %016llx:  %016llx - %016llx\n",
2761                 (long long)ap->a_loffset,
2762                 (long long)base_offset,
2763                 (long long)last_offset);
2764         kprintf("BMAP %16s:  %016llx - %016llx\n", "",
2765                 (long long)base_disk_offset,
2766                 (long long)last_disk_offset);
2767 #endif
2768
2769         if (cursor.node) {
2770                 hammer_cache_node(&ip->cache[1], cursor.node);
2771 #if 0
2772                 kprintf("bmap_end2 %016llx ip->cache %p\n",
2773                         (long long)ap->a_loffset, ip->cache[1]);
2774 #endif
2775         }
2776         hammer_done_cursor(&cursor);
2777         hammer_done_transaction(&trans);
2778
2779         /*
2780          * If we couldn't find any records or the records we did find were
2781          * all behind the requested offset, return failure.  A forward
2782          * truncation can leave a hole w/ no on-disk records.
2783          */
2784         if (last_offset == 0 || last_offset < ap->a_loffset)
2785                 return (EOPNOTSUPP);
2786
2787         /*
2788          * Figure out the block size at the requested offset and adjust
2789          * our limits so the cluster_read() does not create inappropriately
2790          * sized buffer cache buffers.
2791          */
2792         blksize = hammer_blocksize(ap->a_loffset);
2793         if (hammer_blocksize(base_offset) != blksize) {
2794                 base_offset = hammer_blockdemarc(base_offset, ap->a_loffset);
2795         }
2796         if (last_offset != ap->a_loffset &&
2797             hammer_blocksize(last_offset - 1) != blksize) {
2798                 last_offset = hammer_blockdemarc(ap->a_loffset,
2799                                                  last_offset - 1);
2800         }
2801
2802         /*
2803          * Returning EOPNOTSUPP simply prevents the direct-IO optimization
2804          * from occuring.
2805          */
2806         disk_offset = base_disk_offset + (ap->a_loffset - base_offset);
2807
2808         if ((disk_offset & HAMMER_OFF_ZONE_MASK) != HAMMER_ZONE_LARGE_DATA) {
2809                 /*
2810                  * Only large-data zones can be direct-IOd
2811                  */
2812                 error = EOPNOTSUPP;
2813         } else if ((disk_offset & HAMMER_BUFMASK) ||
2814                    (last_offset - ap->a_loffset) < blksize) {
2815                 /*
2816                  * doffsetp is not aligned or the forward run size does
2817                  * not cover a whole buffer, disallow the direct I/O.
2818                  */
2819                 error = EOPNOTSUPP;
2820         } else {
2821                 /*
2822                  * We're good.
2823                  */
2824                 *ap->a_doffsetp = disk_offset;
2825                 if (ap->a_runb) {
2826                         *ap->a_runb = ap->a_loffset - base_offset;
2827                         KKASSERT(*ap->a_runb >= 0);
2828                 }
2829                 if (ap->a_runp) {
2830                         *ap->a_runp = last_offset - ap->a_loffset;
2831                         KKASSERT(*ap->a_runp >= 0);
2832                 }
2833                 error = 0;
2834         }
2835         return(error);
2836 }
2837
2838 /*
2839  * Write to a regular file.   Because this is a strategy call the OS is
2840  * trying to actually get data onto the media.
2841  */
2842 static
2843 int
2844 hammer_vop_strategy_write(struct vop_strategy_args *ap)
2845 {
2846         hammer_record_t record;
2847         hammer_mount_t hmp;
2848         hammer_inode_t ip;
2849         struct bio *bio;
2850         struct buf *bp;
2851         int blksize;
2852         int bytes;
2853         int error;
2854
2855         bio = ap->a_bio;
2856         bp = bio->bio_buf;
2857         ip = ap->a_vp->v_data;
2858         hmp = ip->hmp;
2859
2860         blksize = hammer_blocksize(bio->bio_offset);
2861         KKASSERT(bp->b_bufsize == blksize);
2862
2863         if (ip->flags & HAMMER_INODE_RO) {
2864                 bp->b_error = EROFS;
2865                 bp->b_flags |= B_ERROR;
2866                 biodone(ap->a_bio);
2867                 return(EROFS);
2868         }
2869
2870         /*
2871          * Interlock with inode destruction (no in-kernel or directory
2872          * topology visibility).  If we queue new IO while trying to
2873          * destroy the inode we can deadlock the vtrunc call in
2874          * hammer_inode_unloadable_check().
2875          *
2876          * Besides, there's no point flushing a bp associated with an
2877          * inode that is being destroyed on-media and has no kernel
2878          * references.
2879          */
2880         if ((ip->flags | ip->sync_flags) &
2881             (HAMMER_INODE_DELETING|HAMMER_INODE_DELETED)) {
2882                 bp->b_resid = 0;
2883                 biodone(ap->a_bio);
2884                 return(0);
2885         }
2886
2887         /*
2888          * Reserve space and issue a direct-write from the front-end.
2889          * NOTE: The direct_io code will hammer_bread/bcopy smaller
2890          * allocations.
2891          *
2892          * An in-memory record will be installed to reference the storage
2893          * until the flusher can get to it.
2894          *
2895          * Since we own the high level bio the front-end will not try to
2896          * do a direct-read until the write completes.
2897          *
2898          * NOTE: The only time we do not reserve a full-sized buffers
2899          * worth of data is if the file is small.  We do not try to
2900          * allocate a fragment (from the small-data zone) at the end of
2901          * an otherwise large file as this can lead to wildly separated
2902          * data.
2903          */
2904         KKASSERT((bio->bio_offset & HAMMER_BUFMASK) == 0);
2905         KKASSERT(bio->bio_offset < ip->ino_data.size);
2906         if (bio->bio_offset || ip->ino_data.size > HAMMER_BUFSIZE / 2)
2907                 bytes = bp->b_bufsize;
2908         else
2909                 bytes = ((int)ip->ino_data.size + 15) & ~15;
2910
2911         record = hammer_ip_add_bulk(ip, bio->bio_offset, bp->b_data,
2912                                     bytes, &error);
2913         if (record) {
2914                 hammer_io_direct_write(hmp, record, bio);
2915                 if (ip->rsv_recs > 1 && hmp->rsv_recs > hammer_limit_recs)
2916                         hammer_flush_inode(ip, 0);
2917         } else {
2918                 bp->b_bio2.bio_offset = NOOFFSET;
2919                 bp->b_error = error;
2920                 bp->b_flags |= B_ERROR;
2921                 biodone(ap->a_bio);
2922         }
2923         return(error);
2924 }
2925
2926 /*
2927  * dounlink - disconnect a directory entry
2928  *
2929  * XXX whiteout support not really in yet
2930  */
2931 static int
2932 hammer_dounlink(hammer_transaction_t trans, struct nchandle *nch,
2933                 struct vnode *dvp, struct ucred *cred,
2934                 int flags, int isdir)
2935 {
2936         struct namecache *ncp;
2937         hammer_inode_t dip;
2938         hammer_inode_t ip;
2939         struct hammer_cursor cursor;
2940         int64_t namekey;
2941         u_int32_t max_iterations;
2942         int nlen, error;
2943
2944         /*
2945          * Calculate the namekey and setup the key range for the scan.  This
2946          * works kinda like a chained hash table where the lower 32 bits
2947          * of the namekey synthesize the chain.
2948          *
2949          * The key range is inclusive of both key_beg and key_end.
2950          */
2951         dip = VTOI(dvp);
2952         ncp = nch->ncp;
2953
2954         if (dip->flags & HAMMER_INODE_RO)
2955                 return (EROFS);
2956
2957         namekey = hammer_directory_namekey(dip, ncp->nc_name, ncp->nc_nlen,
2958                                            &max_iterations);
2959 retry:
2960         hammer_init_cursor(trans, &cursor, &dip->cache[1], dip);
2961         cursor.key_beg.localization = dip->obj_localization +
2962                                       hammer_dir_localization(dip);
2963         cursor.key_beg.obj_id = dip->obj_id;
2964         cursor.key_beg.key = namekey;
2965         cursor.key_beg.create_tid = 0;
2966         cursor.key_beg.delete_tid = 0;
2967         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
2968         cursor.key_beg.obj_type = 0;
2969
2970         cursor.key_end = cursor.key_beg;
2971         cursor.key_end.key += max_iterations;
2972         cursor.asof = dip->obj_asof;
2973         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
2974
2975         /*
2976          * Scan all matching records (the chain), locate the one matching
2977          * the requested path component.  info->last_error contains the
2978          * error code on search termination and could be 0, ENOENT, or
2979          * something else.
2980          *
2981          * The hammer_ip_*() functions merge in-memory records with on-disk
2982          * records for the purposes of the search.
2983          */
2984         error = hammer_ip_first(&cursor);
2985
2986         while (error == 0) {
2987                 error = hammer_ip_resolve_data(&cursor);
2988                 if (error)
2989                         break;
2990                 nlen = cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF;
2991                 KKASSERT(nlen > 0);
2992                 if (ncp->nc_nlen == nlen &&
2993                     bcmp(ncp->nc_name, cursor.data->entry.name, nlen) == 0) {
2994                         break;
2995                 }
2996                 error = hammer_ip_next(&cursor);
2997         }
2998
2999         /*
3000          * If all is ok we have to get the inode so we can adjust nlinks.
3001          * To avoid a deadlock with the flusher we must release the inode
3002          * lock on the directory when acquiring the inode for the entry.
3003          *
3004          * If the target is a directory, it must be empty.
3005          */
3006         if (error == 0) {
3007                 hammer_unlock(&cursor.ip->lock);
3008                 ip = hammer_get_inode(trans, dip, cursor.data->entry.obj_id,
3009                                       dip->hmp->asof,
3010                                       cursor.data->entry.localization,
3011                                       0, &error);
3012                 hammer_lock_sh(&cursor.ip->lock);
3013                 if (error == ENOENT) {
3014                         kprintf("HAMMER: WARNING: Removing "
3015                                 "dirent w/missing inode \"%s\"\n"
3016                                 "\tobj_id = %016llx\n",
3017                                 ncp->nc_name,
3018                                 (long long)cursor.data->entry.obj_id);
3019                         error = 0;
3020                 }
3021
3022                 /*
3023                  * If isdir >= 0 we validate that the entry is or is not a
3024                  * directory.  If isdir < 0 we don't care.
3025                  */
3026                 if (error == 0 && isdir >= 0 && ip) {
3027                         if (isdir &&
3028                             ip->ino_data.obj_type != HAMMER_OBJTYPE_DIRECTORY) {
3029                                 error = ENOTDIR;
3030                         } else if (isdir == 0 &&
3031                             ip->ino_data.obj_type == HAMMER_OBJTYPE_DIRECTORY) {
3032                                 error = EISDIR;
3033                         }
3034                 }
3035
3036                 /*
3037                  * If we are trying to remove a directory the directory must
3038                  * be empty.
3039                  *
3040                  * The check directory code can loop and deadlock/retry.  Our
3041                  * own cursor's node locks must be released to avoid a 3-way
3042                  * deadlock with the flusher if the check directory code
3043                  * blocks.
3044                  *
3045                  * If any changes whatsoever have been made to the cursor
3046                  * set EDEADLK and retry.
3047                  */
3048                 if (error == 0 && ip && ip->ino_data.obj_type ==
3049                                         HAMMER_OBJTYPE_DIRECTORY) {
3050                         hammer_unlock_cursor(&cursor);
3051                         error = hammer_ip_check_directory_empty(trans, ip);
3052                         hammer_lock_cursor(&cursor);
3053                         if (cursor.flags & HAMMER_CURSOR_RETEST) {
3054                                 kprintf("HAMMER: Warning: avoided deadlock "
3055                                         "on rmdir '%s'\n",
3056                                         ncp->nc_name);
3057                                 error = EDEADLK;
3058                         }
3059                 }
3060
3061                 /*
3062                  * Delete the directory entry.
3063                  *
3064                  * WARNING: hammer_ip_del_directory() may have to terminate
3065                  * the cursor to avoid a deadlock.  It is ok to call
3066                  * hammer_done_cursor() twice.
3067                  */
3068                 if (error == 0) {
3069                         error = hammer_ip_del_directory(trans, &cursor,
3070                                                         dip, ip);
3071                 }
3072                 hammer_done_cursor(&cursor);
3073                 if (error == 0) {
3074                         cache_setunresolved(nch);
3075                         cache_setvp(nch, NULL);
3076                         /* XXX locking */
3077                         if (ip && ip->vp) {
3078                                 hammer_knote(ip->vp, NOTE_DELETE);
3079                                 cache_inval_vp(ip->vp, CINV_DESTROY);
3080                         }
3081                 }
3082                 if (ip)
3083                         hammer_rel_inode(ip, 0);
3084         } else {
3085                 hammer_done_cursor(&cursor);
3086         }
3087         if (error == EDEADLK)
3088                 goto retry;
3089
3090         return (error);
3091 }
3092
3093 /************************************************************************
3094  *                          FIFO AND SPECFS OPS                         *
3095  ************************************************************************
3096  *
3097  */
3098
3099 static int
3100 hammer_vop_fifoclose (struct vop_close_args *ap)
3101 {
3102         /* XXX update itimes */
3103         return (VOCALL(&fifo_vnode_vops, &ap->a_head));
3104 }
3105
3106 static int
3107 hammer_vop_fiforead (struct vop_read_args *ap)
3108 {
3109         int error;
3110
3111         error = VOCALL(&fifo_vnode_vops, &ap->a_head);
3112         /* XXX update access time */
3113         return (error);
3114 }
3115
3116 static int
3117 hammer_vop_fifowrite (struct vop_write_args *ap)
3118 {
3119         int error;
3120
3121         error = VOCALL(&fifo_vnode_vops, &ap->a_head);
3122         /* XXX update access time */
3123         return (error);
3124 }
3125
3126 static
3127 int
3128 hammer_vop_fifokqfilter(struct vop_kqfilter_args *ap)
3129 {
3130         int error;
3131
3132         error = VOCALL(&fifo_vnode_vops, &ap->a_head);
3133         if (error)
3134                 error = hammer_vop_kqfilter(ap);
3135         return(error);
3136 }
3137
3138 /************************************************************************
3139  *                          KQFILTER OPS                                *
3140  ************************************************************************
3141  *
3142  */
3143 static void filt_hammerdetach(struct knote *kn);
3144 static int filt_hammerread(struct knote *kn, long hint);
3145 static int filt_hammerwrite(struct knote *kn, long hint);
3146 static int filt_hammervnode(struct knote *kn, long hint);
3147
3148 static struct filterops hammerread_filtops =
3149         { 1, NULL, filt_hammerdetach, filt_hammerread };
3150 static struct filterops hammerwrite_filtops =
3151         { 1, NULL, filt_hammerdetach, filt_hammerwrite };
3152 static struct filterops hammervnode_filtops =
3153         { 1, NULL, filt_hammerdetach, filt_hammervnode };
3154
3155 static
3156 int
3157 hammer_vop_kqfilter(struct vop_kqfilter_args *ap)
3158 {
3159         struct vnode *vp = ap->a_vp;
3160         struct knote *kn = ap->a_kn;
3161         lwkt_tokref vlock;
3162
3163         switch (kn->kn_filter) {
3164         case EVFILT_READ:
3165                 kn->kn_fop = &hammerread_filtops;
3166                 break;
3167         case EVFILT_WRITE:
3168                 kn->kn_fop = &hammerwrite_filtops;
3169                 break;
3170         case EVFILT_VNODE:
3171                 kn->kn_fop = &hammervnode_filtops;
3172                 break;
3173         default:
3174                 return (1);
3175         }
3176
3177         kn->kn_hook = (caddr_t)vp;
3178
3179         lwkt_gettoken(&vlock, &vp->v_token);
3180         SLIST_INSERT_HEAD(&vp->v_pollinfo.vpi_selinfo.si_note, kn, kn_selnext);
3181         lwkt_reltoken(&vlock);
3182
3183         return(0);
3184 }
3185
3186 static void
3187 filt_hammerdetach(struct knote *kn)
3188 {
3189         struct vnode *vp = (void *)kn->kn_hook;
3190         lwkt_tokref vlock;
3191
3192         lwkt_gettoken(&vlock, &vp->v_token);
3193         SLIST_REMOVE(&vp->v_pollinfo.vpi_selinfo.si_note,
3194                      kn, knote, kn_selnext);
3195         lwkt_reltoken(&vlock);
3196 }
3197
3198 static int
3199 filt_hammerread(struct knote *kn, long hint)
3200 {
3201         struct vnode *vp = (void *)kn->kn_hook;
3202         hammer_inode_t ip = VTOI(vp);
3203
3204         if (hint == NOTE_REVOKE) {
3205                 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
3206                 return(1);
3207         }
3208         kn->kn_data = ip->ino_data.size - kn->kn_fp->f_offset;
3209         return (kn->kn_data != 0);
3210 }
3211
3212 static int
3213 filt_hammerwrite(struct knote *kn, long hint)
3214 {
3215         if (hint == NOTE_REVOKE)
3216                 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
3217         kn->kn_data = 0;
3218         return (1);
3219 }
3220
3221 static int
3222 filt_hammervnode(struct knote *kn, long hint)
3223 {
3224         if (kn->kn_sfflags & hint)
3225                 kn->kn_fflags |= hint;
3226         if (hint == NOTE_REVOKE) {
3227                 kn->kn_flags |= EV_EOF;
3228                 return (1);
3229         }
3230         return (kn->kn_fflags != 0);
3231 }
3232