sys/vfs/hammer/hammer_vnops.c

   1 /*
   2  * Copyright (c) 2007-2008 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/vfs/hammer/hammer_vnops.c,v 1.102 2008/10/16 17:24:16 dillon Exp $
  35  */
  36
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/kernel.h>
  40 #include <sys/fcntl.h>
  41 #include <sys/namecache.h>
  42 #include <sys/vnode.h>
  43 #include <sys/lockf.h>
  44 #include <sys/event.h>
  45 #include <sys/stat.h>
  46 #include <sys/dirent.h>
  47 #include <sys/file.h>
  48 #include <vm/vm_extern.h>
  49 #include <vfs/fifofs/fifo.h>
  50 #include "hammer.h"
  51
  52 /*
  53  * USERFS VNOPS
  54  */
  55 /*static int hammer_vop_vnoperate(struct vop_generic_args *);*/
  56 static int hammer_vop_fsync(struct vop_fsync_args *);
  57 static int hammer_vop_read(struct vop_read_args *);
  58 static int hammer_vop_write(struct vop_write_args *);
  59 static int hammer_vop_access(struct vop_access_args *);
  60 static int hammer_vop_advlock(struct vop_advlock_args *);
  61 static int hammer_vop_close(struct vop_close_args *);
  62 static int hammer_vop_ncreate(struct vop_ncreate_args *);
  63 static int hammer_vop_getattr(struct vop_getattr_args *);
  64 static int hammer_vop_nresolve(struct vop_nresolve_args *);
  65 static int hammer_vop_nlookupdotdot(struct vop_nlookupdotdot_args *);
  66 static int hammer_vop_nlink(struct vop_nlink_args *);
  67 static int hammer_vop_nmkdir(struct vop_nmkdir_args *);
  68 static int hammer_vop_nmknod(struct vop_nmknod_args *);
  69 static int hammer_vop_open(struct vop_open_args *);
  70 static int hammer_vop_print(struct vop_print_args *);
  71 static int hammer_vop_readdir(struct vop_readdir_args *);
  72 static int hammer_vop_readlink(struct vop_readlink_args *);
  73 static int hammer_vop_nremove(struct vop_nremove_args *);
  74 static int hammer_vop_nrename(struct vop_nrename_args *);
  75 static int hammer_vop_nrmdir(struct vop_nrmdir_args *);
  76 static int hammer_vop_markatime(struct vop_markatime_args *);
  77 static int hammer_vop_setattr(struct vop_setattr_args *);
  78 static int hammer_vop_strategy(struct vop_strategy_args *);
  79 static int hammer_vop_bmap(struct vop_bmap_args *ap);
  80 static int hammer_vop_nsymlink(struct vop_nsymlink_args *);
  81 static int hammer_vop_nwhiteout(struct vop_nwhiteout_args *);
  82 static int hammer_vop_ioctl(struct vop_ioctl_args *);
  83 static int hammer_vop_mountctl(struct vop_mountctl_args *);
  84 static int hammer_vop_kqfilter (struct vop_kqfilter_args *);
  85
  86 static int hammer_vop_fifoclose (struct vop_close_args *);
  87 static int hammer_vop_fiforead (struct vop_read_args *);
  88 static int hammer_vop_fifowrite (struct vop_write_args *);
  89 static int hammer_vop_fifokqfilter (struct vop_kqfilter_args *);
  90
  91 static int hammer_vop_specclose (struct vop_close_args *);
  92 static int hammer_vop_specread (struct vop_read_args *);
  93 static int hammer_vop_specwrite (struct vop_write_args *);
  94
  95 struct vop_ops hammer_vnode_vops = {
  96         .vop_default =          vop_defaultop,
  97         .vop_fsync =            hammer_vop_fsync,
  98         .vop_getpages =         vop_stdgetpages,
  99         .vop_putpages =         vop_stdputpages,
 100         .vop_read =             hammer_vop_read,
 101         .vop_write =            hammer_vop_write,
 102         .vop_access =           hammer_vop_access,
 103         .vop_advlock =          hammer_vop_advlock,
 104         .vop_close =            hammer_vop_close,
 105         .vop_ncreate =          hammer_vop_ncreate,
 106         .vop_getattr =          hammer_vop_getattr,
 107         .vop_inactive =         hammer_vop_inactive,
 108         .vop_reclaim =          hammer_vop_reclaim,
 109         .vop_nresolve =         hammer_vop_nresolve,
 110         .vop_nlookupdotdot =    hammer_vop_nlookupdotdot,
 111         .vop_nlink =            hammer_vop_nlink,
 112         .vop_nmkdir =           hammer_vop_nmkdir,
 113         .vop_nmknod =           hammer_vop_nmknod,
 114         .vop_open =             hammer_vop_open,
 115         .vop_pathconf =         vop_stdpathconf,
 116         .vop_print =            hammer_vop_print,
 117         .vop_readdir =          hammer_vop_readdir,
 118         .vop_readlink =         hammer_vop_readlink,
 119         .vop_nremove =          hammer_vop_nremove,
 120         .vop_nrename =          hammer_vop_nrename,
 121         .vop_nrmdir =           hammer_vop_nrmdir,
 122         .vop_markatime =        hammer_vop_markatime,
 123         .vop_setattr =          hammer_vop_setattr,
 124         .vop_bmap =             hammer_vop_bmap,
 125         .vop_strategy =         hammer_vop_strategy,
 126         .vop_nsymlink =         hammer_vop_nsymlink,
 127         .vop_nwhiteout =        hammer_vop_nwhiteout,
 128         .vop_ioctl =            hammer_vop_ioctl,
 129         .vop_mountctl =         hammer_vop_mountctl,
 130         .vop_kqfilter =         hammer_vop_kqfilter
 131 };
 132
 133 struct vop_ops hammer_spec_vops = {
 134         .vop_default =          spec_vnoperate,
 135         .vop_fsync =            hammer_vop_fsync,
 136         .vop_read =             hammer_vop_specread,
 137         .vop_write =            hammer_vop_specwrite,
 138         .vop_access =           hammer_vop_access,
 139         .vop_close =            hammer_vop_specclose,
 140         .vop_markatime =        hammer_vop_markatime,
 141         .vop_getattr =          hammer_vop_getattr,
 142         .vop_inactive =         hammer_vop_inactive,
 143         .vop_reclaim =          hammer_vop_reclaim,
 144         .vop_setattr =          hammer_vop_setattr
 145 };
 146
 147 struct vop_ops hammer_fifo_vops = {
 148         .vop_default =          fifo_vnoperate,
 149         .vop_fsync =            hammer_vop_fsync,
 150         .vop_read =             hammer_vop_fiforead,
 151         .vop_write =            hammer_vop_fifowrite,
 152         .vop_access =           hammer_vop_access,
 153         .vop_close =            hammer_vop_fifoclose,
 154         .vop_markatime =        hammer_vop_markatime,
 155         .vop_getattr =          hammer_vop_getattr,
 156         .vop_inactive =         hammer_vop_inactive,
 157         .vop_reclaim =          hammer_vop_reclaim,
 158         .vop_setattr =          hammer_vop_setattr,
 159         .vop_kqfilter =         hammer_vop_fifokqfilter
 160 };
 161
 162 static __inline
 163 void
 164 hammer_knote(struct vnode *vp, int flags)
 165 {
 166         if (flags)
 167                 KNOTE(&vp->v_pollinfo.vpi_selinfo.si_note, flags);
 168 }
 169
 170 #ifdef DEBUG_TRUNCATE
 171 struct hammer_inode *HammerTruncIp;
 172 #endif
 173
 174 static int hammer_dounlink(hammer_transaction_t trans, struct nchandle *nch,
 175                            struct vnode *dvp, struct ucred *cred,
 176                            int flags, int isdir);
 177 static int hammer_vop_strategy_read(struct vop_strategy_args *ap);
 178 static int hammer_vop_strategy_write(struct vop_strategy_args *ap);
 179
 180 #if 0
 181 static
 182 int
 183 hammer_vop_vnoperate(struct vop_generic_args *)
 184 {
 185         return (VOCALL(&hammer_vnode_vops, ap));
 186 }
 187 #endif
 188
 189 /*
 190  * hammer_vop_fsync { vp, waitfor }
 191  *
 192  * fsync() an inode to disk and wait for it to be completely committed
 193  * such that the information would not be undone if a crash occured after
 194  * return.
 195  */
 196 static
 197 int
 198 hammer_vop_fsync(struct vop_fsync_args *ap)
 199 {
 200         hammer_inode_t ip = VTOI(ap->a_vp);
 201
 202         ++hammer_count_fsyncs;
 203         vfsync(ap->a_vp, ap->a_waitfor, 1, NULL, NULL);
 204         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 205         if (ap->a_waitfor == MNT_WAIT) {
 206                 vn_unlock(ap->a_vp);
 207                 hammer_wait_inode(ip);
 208                 vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
 209         }
 210         return (ip->error);
 211 }
 212
 213 /*
 214  * hammer_vop_read { vp, uio, ioflag, cred }
 215  */
 216 static
 217 int
 218 hammer_vop_read(struct vop_read_args *ap)
 219 {
 220         struct hammer_transaction trans;
 221         hammer_inode_t ip;
 222         off_t offset;
 223         struct buf *bp;
 224         struct uio *uio;
 225         int error;
 226         int n;
 227         int seqcount;
 228         int ioseqcount;
 229         int blksize;
 230
 231         if (ap->a_vp->v_type != VREG)
 232                 return (EINVAL);
 233         ip = VTOI(ap->a_vp);
 234         error = 0;
 235         uio = ap->a_uio;
 236
 237         /*
 238          * Allow the UIO's size to override the sequential heuristic.
 239          */
 240         blksize = hammer_blocksize(uio->uio_offset);
 241         seqcount = (uio->uio_resid + (blksize - 1)) / blksize;
 242         ioseqcount = ap->a_ioflag >> 16;
 243         if (seqcount < ioseqcount)
 244                 seqcount = ioseqcount;
 245
 246         hammer_start_transaction(&trans, ip->hmp);
 247
 248         /*
 249          * Access the data typically in HAMMER_BUFSIZE blocks via the
 250          * buffer cache, but HAMMER may use a variable block size based
 251          * on the offset.
 252          */
 253         while (uio->uio_resid > 0 && uio->uio_offset < ip->ino_data.size) {
 254                 int64_t base_offset;
 255                 int64_t file_limit;
 256
 257                 blksize = hammer_blocksize(uio->uio_offset);
 258                 offset = (int)uio->uio_offset & (blksize - 1);
 259                 base_offset = uio->uio_offset - offset;
 260
 261                 if (hammer_cluster_enable) {
 262                         /*
 263                          * Use file_limit to prevent cluster_read() from
 264                          * creating buffers of the wrong block size past
 265                          * the demarc.
 266                          */
 267                         file_limit = ip->ino_data.size;
 268                         if (base_offset < HAMMER_XDEMARC &&
 269                             file_limit > HAMMER_XDEMARC) {
 270                                 file_limit = HAMMER_XDEMARC;
 271                         }
 272                         error = cluster_read(ap->a_vp,
 273                                              file_limit, base_offset,
 274                                              blksize, MAXPHYS,
 275                                              seqcount, &bp);
 276                 } else {
 277                         error = bread(ap->a_vp, base_offset, blksize, &bp);
 278                 }
 279                 if (error) {
 280                         kprintf("error %d\n", error);
 281                         brelse(bp);
 282                         break;
 283                 }
 284
 285                 /* bp->b_flags |= B_CLUSTEROK; temporarily disabled */
 286                 n = blksize - offset;
 287                 if (n > uio->uio_resid)
 288                         n = uio->uio_resid;
 289                 if (n > ip->ino_data.size - uio->uio_offset)
 290                         n = (int)(ip->ino_data.size - uio->uio_offset);
 291                 error = uiomove((char *)bp->b_data + offset, n, uio);
 292
 293                 /* data has a lower priority then meta-data */
 294                 bp->b_flags |= B_AGE;
 295                 bqrelse(bp);
 296                 if (error)
 297                         break;
 298                 hammer_stats_file_read += n;
 299         }
 300         if ((ip->flags & HAMMER_INODE_RO) == 0 &&
 301             (ip->hmp->mp->mnt_flag & MNT_NOATIME) == 0) {
 302                 ip->ino_data.atime = trans.time;
 303                 hammer_modify_inode(ip, HAMMER_INODE_ATIME);
 304         }
 305         hammer_done_transaction(&trans);
 306         return (error);
 307 }
 308
 309 /*
 310  * hammer_vop_write { vp, uio, ioflag, cred }
 311  */
 312 static
 313 int
 314 hammer_vop_write(struct vop_write_args *ap)
 315 {
 316         struct hammer_transaction trans;
 317         struct hammer_inode *ip;
 318         hammer_mount_t hmp;
 319         struct uio *uio;
 320         int offset;
 321         off_t base_offset;
 322         struct buf *bp;
 323         int kflags;
 324         int error;
 325         int n;
 326         int flags;
 327         int delta;
 328         int seqcount;
 329
 330         if (ap->a_vp->v_type != VREG)
 331                 return (EINVAL);
 332         ip = VTOI(ap->a_vp);
 333         hmp = ip->hmp;
 334         error = 0;
 335         kflags = 0;
 336         seqcount = ap->a_ioflag >> 16;
 337
 338         if (ip->flags & HAMMER_INODE_RO)
 339                 return (EROFS);
 340
 341         /*
 342          * Create a transaction to cover the operations we perform.
 343          */
 344         hammer_start_transaction(&trans, hmp);
 345         uio = ap->a_uio;
 346
 347         /*
 348          * Check append mode
 349          */
 350         if (ap->a_ioflag & IO_APPEND)
 351                 uio->uio_offset = ip->ino_data.size;
 352
 353         /*
 354          * Check for illegal write offsets.  Valid range is 0...2^63-1.
 355          *
 356          * NOTE: the base_off assignment is required to work around what
 357          * I consider to be a GCC-4 optimization bug.
 358          */
 359         if (uio->uio_offset < 0) {
 360                 hammer_done_transaction(&trans);
 361                 return (EFBIG);
 362         }
 363         base_offset = uio->uio_offset + uio->uio_resid; /* work around gcc-4 */
 364         if (uio->uio_resid > 0 && base_offset <= 0) {
 365                 hammer_done_transaction(&trans);
 366                 return (EFBIG);
 367         }
 368
 369         /*
 370          * Access the data typically in HAMMER_BUFSIZE blocks via the
 371          * buffer cache, but HAMMER may use a variable block size based
 372          * on the offset.
 373          */
 374         while (uio->uio_resid > 0) {
 375                 int fixsize = 0;
 376                 int blksize;
 377                 int blkmask;
 378
 379                 if ((error = hammer_checkspace(hmp, HAMMER_CHKSPC_WRITE)) != 0)
 380                         break;
 381
 382                 blksize = hammer_blocksize(uio->uio_offset);
 383
 384                 /*
 385                  * Do not allow HAMMER to blow out the buffer cache.  Very
 386                  * large UIOs can lockout other processes due to bwillwrite()
 387                  * mechanics.
 388                  *
 389                  * The hammer inode is not locked during these operations.
 390                  * The vnode is locked which can interfere with the pageout
 391                  * daemon for non-UIO_NOCOPY writes but should not interfere
 392                  * with the buffer cache.  Even so, we cannot afford to
 393                  * allow the pageout daemon to build up too many dirty buffer
 394                  * cache buffers.
 395                  *
 396                  * Only call this if we aren't being recursively called from
 397                  * a virtual disk device (vn), else we may deadlock.
 398                  */
 399                 if ((ap->a_ioflag & IO_RECURSE) == 0)
 400                         bwillwrite(blksize);
 401
 402                 /*
 403                  * Do not allow HAMMER to blow out system memory by
 404                  * accumulating too many records.   Records are so well
 405                  * decoupled from the buffer cache that it is possible
 406                  * for userland to push data out to the media via
 407                  * direct-write, but build up the records queued to the
 408                  * backend faster then the backend can flush them out.
 409                  * HAMMER has hit its write limit but the frontend has
 410                  * no pushback to slow it down.
 411                  */
 412                 if (hmp->rsv_recs > hammer_limit_recs / 2) {
 413                         /*
 414                          * Get the inode on the flush list
 415                          */
 416                         if (ip->rsv_recs >= 64)
 417                                 hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
 418                         else if (ip->rsv_recs >= 16)
 419                                 hammer_flush_inode(ip, 0);
 420
 421                         /*
 422                          * Keep the flusher going if the system keeps
 423                          * queueing records.
 424                          */
 425                         delta = hmp->count_newrecords -
 426                                 hmp->last_newrecords;
 427                         if (delta < 0 || delta > hammer_limit_recs / 2) {
 428                                 hmp->last_newrecords = hmp->count_newrecords;
 429                                 hammer_sync_hmp(hmp, MNT_NOWAIT);
 430                         }
 431
 432                         /*
 433                          * If we have gotten behind start slowing
 434                          * down the writers.
 435                          */
 436                         delta = (hmp->rsv_recs - hammer_limit_recs) *
 437                                 hz / hammer_limit_recs;
 438                         if (delta > 0)
 439                                 tsleep(&trans, 0, "hmrslo", delta);
 440                 }
 441
 442                 /*
 443                  * Calculate the blocksize at the current offset and figure
 444                  * out how much we can actually write.
 445                  */
 446                 blkmask = blksize - 1;
 447                 offset = (int)uio->uio_offset & blkmask;
 448                 base_offset = uio->uio_offset & ~(int64_t)blkmask;
 449                 n = blksize - offset;
 450                 if (n > uio->uio_resid)
 451                         n = uio->uio_resid;
 452                 if (uio->uio_offset + n > ip->ino_data.size) {
 453                         vnode_pager_setsize(ap->a_vp, uio->uio_offset + n);
 454                         fixsize = 1;
 455                         kflags |= NOTE_EXTEND;
 456                 }
 457
 458                 if (uio->uio_segflg == UIO_NOCOPY) {
 459                         /*
 460                          * Issuing a write with the same data backing the
 461                          * buffer.  Instantiate the buffer to collect the
 462                          * backing vm pages, then read-in any missing bits.
 463                          *
 464                          * This case is used by vop_stdputpages().
 465                          */
 466                         bp = getblk(ap->a_vp, base_offset,
 467                                     blksize, GETBLK_BHEAVY, 0);
 468                         if ((bp->b_flags & B_CACHE) == 0) {
 469                                 bqrelse(bp);
 470                                 error = bread(ap->a_vp, base_offset,
 471                                               blksize, &bp);
 472                         }
 473                 } else if (offset == 0 && uio->uio_resid >= blksize) {
 474                         /*
 475                          * Even though we are entirely overwriting the buffer
 476                          * we may still have to zero it out to avoid a
 477                          * mmap/write visibility issue.
 478                          */
 479                         bp = getblk(ap->a_vp, base_offset, blksize, GETBLK_BHEAVY, 0);
 480                         if ((bp->b_flags & B_CACHE) == 0)
 481                                 vfs_bio_clrbuf(bp);
 482                 } else if (base_offset >= ip->ino_data.size) {
 483                         /*
 484                          * If the base offset of the buffer is beyond the
 485                          * file EOF, we don't have to issue a read.
 486                          */
 487                         bp = getblk(ap->a_vp, base_offset,
 488                                     blksize, GETBLK_BHEAVY, 0);
 489                         vfs_bio_clrbuf(bp);
 490                 } else {
 491                         /*
 492                          * Partial overwrite, read in any missing bits then
 493                          * replace the portion being written.
 494                          */
 495                         error = bread(ap->a_vp, base_offset, blksize, &bp);
 496                         if (error == 0)
 497                                 bheavy(bp);
 498                 }
 499                 if (error == 0) {
 500                         error = uiomove((char *)bp->b_data + offset,
 501                                         n, uio);
 502                 }
 503
 504                 /*
 505                  * If we screwed up we have to undo any VM size changes we
 506                  * made.
 507                  */
 508                 if (error) {
 509                         brelse(bp);
 510                         if (fixsize) {
 511                                 vtruncbuf(ap->a_vp, ip->ino_data.size,
 512                                           hammer_blocksize(ip->ino_data.size));
 513                         }
 514                         break;
 515                 }
 516                 kflags |= NOTE_WRITE;
 517                 hammer_stats_file_write += n;
 518                 /* bp->b_flags |= B_CLUSTEROK; temporarily disabled */
 519                 if (ip->ino_data.size < uio->uio_offset) {
 520                         ip->ino_data.size = uio->uio_offset;
 521                         flags = HAMMER_INODE_DDIRTY;
 522                         vnode_pager_setsize(ap->a_vp, ip->ino_data.size);
 523                 } else {
 524                         flags = 0;
 525                 }
 526                 ip->ino_data.mtime = trans.time;
 527                 flags |= HAMMER_INODE_MTIME | HAMMER_INODE_BUFS;
 528                 hammer_modify_inode(ip, flags);
 529
 530                 /*
 531                  * Once we dirty the buffer any cached zone-X offset
 532                  * becomes invalid.  HAMMER NOTE: no-history mode cannot
 533                  * allow overwriting over the same data sector unless
 534                  * we provide UNDOs for the old data, which we don't.
 535                  */
 536                 bp->b_bio2.bio_offset = NOOFFSET;
 537
 538                 /*
 539                  * Final buffer disposition.
 540                  */
 541                 bp->b_flags |= B_AGE;
 542                 if (ap->a_ioflag & IO_SYNC) {
 543                         bwrite(bp);
 544                 } else if (ap->a_ioflag & IO_DIRECT) {
 545                         bawrite(bp);
 546                 } else {
 547                         bdwrite(bp);
 548                 }
 549         }
 550         hammer_done_transaction(&trans);
 551         hammer_knote(ap->a_vp, kflags);
 552         return (error);
 553 }
 554
 555 /*
 556  * hammer_vop_access { vp, mode, cred }
 557  */
 558 static
 559 int
 560 hammer_vop_access(struct vop_access_args *ap)
 561 {
 562         struct hammer_inode *ip = VTOI(ap->a_vp);
 563         uid_t uid;
 564         gid_t gid;
 565         int error;
 566
 567         ++hammer_stats_file_iopsr;
 568         uid = hammer_to_unix_xid(&ip->ino_data.uid);
 569         gid = hammer_to_unix_xid(&ip->ino_data.gid);
 570
 571         error = vop_helper_access(ap, uid, gid, ip->ino_data.mode,
 572                                   ip->ino_data.uflags);
 573         return (error);
 574 }
 575
 576 /*
 577  * hammer_vop_advlock { vp, id, op, fl, flags }
 578  */
 579 static
 580 int
 581 hammer_vop_advlock(struct vop_advlock_args *ap)
 582 {
 583         hammer_inode_t ip = VTOI(ap->a_vp);
 584
 585         return (lf_advlock(ap, &ip->advlock, ip->ino_data.size));
 586 }
 587
 588 /*
 589  * hammer_vop_close { vp, fflag }
 590  */
 591 static
 592 int
 593 hammer_vop_close(struct vop_close_args *ap)
 594 {
 595         /*hammer_inode_t ip = VTOI(ap->a_vp);*/
 596         return (vop_stdclose(ap));
 597 }
 598
 599 /*
 600  * hammer_vop_ncreate { nch, dvp, vpp, cred, vap }
 601  *
 602  * The operating system has already ensured that the directory entry
 603  * does not exist and done all appropriate namespace locking.
 604  */
 605 static
 606 int
 607 hammer_vop_ncreate(struct vop_ncreate_args *ap)
 608 {
 609         struct hammer_transaction trans;
 610         struct hammer_inode *dip;
 611         struct hammer_inode *nip;
 612         struct nchandle *nch;
 613         int error;
 614
 615         nch = ap->a_nch;
 616         dip = VTOI(ap->a_dvp);
 617
 618         if (dip->flags & HAMMER_INODE_RO)
 619                 return (EROFS);
 620         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
 621                 return (error);
 622
 623         /*
 624          * Create a transaction to cover the operations we perform.
 625          */
 626         hammer_start_transaction(&trans, dip->hmp);
 627         ++hammer_stats_file_iopsw;
 628
 629         /*
 630          * Create a new filesystem object of the requested type.  The
 631          * returned inode will be referenced and shared-locked to prevent
 632          * it from being moved to the flusher.
 633          */
 634
 635         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
 636                                     dip, NULL, &nip);
 637         if (error) {
 638                 hkprintf("hammer_create_inode error %d\n", error);
 639                 hammer_done_transaction(&trans);
 640                 *ap->a_vpp = NULL;
 641                 return (error);
 642         }
 643
 644         /*
 645          * Add the new filesystem object to the directory.  This will also
 646          * bump the inode's link count.
 647          */
 648         error = hammer_ip_add_directory(&trans, dip,
 649                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
 650                                         nip);
 651         if (error)
 652                 hkprintf("hammer_ip_add_directory error %d\n", error);
 653
 654         /*
 655          * Finish up.
 656          */
 657         if (error) {
 658                 hammer_rel_inode(nip, 0);
 659                 hammer_done_transaction(&trans);
 660                 *ap->a_vpp = NULL;
 661         } else {
 662                 error = hammer_get_vnode(nip, ap->a_vpp);
 663                 hammer_done_transaction(&trans);
 664                 hammer_rel_inode(nip, 0);
 665                 if (error == 0) {
 666                         cache_setunresolved(ap->a_nch);
 667                         cache_setvp(ap->a_nch, *ap->a_vpp);
 668                 }
 669                 hammer_knote(ap->a_dvp, NOTE_WRITE);
 670         }
 671         return (error);
 672 }
 673
 674 /*
 675  * hammer_vop_getattr { vp, vap }
 676  *
 677  * Retrieve an inode's attribute information.  When accessing inodes
 678  * historically we fake the atime field to ensure consistent results.
 679  * The atime field is stored in the B-Tree element and allowed to be
 680  * updated without cycling the element.
 681  */
 682 static
 683 int
 684 hammer_vop_getattr(struct vop_getattr_args *ap)
 685 {
 686         struct hammer_inode *ip = VTOI(ap->a_vp);
 687         struct vattr *vap = ap->a_vap;
 688
 689         /*
 690          * We want the fsid to be different when accessing a filesystem
 691          * with different as-of's so programs like diff don't think
 692          * the files are the same.
 693          *
 694          * We also want the fsid to be the same when comparing snapshots,
 695          * or when comparing mirrors (which might be backed by different
 696          * physical devices).  HAMMER fsids are based on the PFS's
 697          * shared_uuid field.
 698          *
 699          * XXX there is a chance of collision here.  The va_fsid reported
 700          * by stat is different from the more involved fsid used in the
 701          * mount structure.
 702          */
 703         ++hammer_stats_file_iopsr;
 704         vap->va_fsid = ip->pfsm->fsid_udev ^ (u_int32_t)ip->obj_asof ^
 705                        (u_int32_t)(ip->obj_asof >> 32);
 706
 707         vap->va_fileid = ip->ino_leaf.base.obj_id;
 708         vap->va_mode = ip->ino_data.mode;
 709         vap->va_nlink = ip->ino_data.nlinks;
 710         vap->va_uid = hammer_to_unix_xid(&ip->ino_data.uid);
 711         vap->va_gid = hammer_to_unix_xid(&ip->ino_data.gid);
 712         vap->va_rmajor = 0;
 713         vap->va_rminor = 0;
 714         vap->va_size = ip->ino_data.size;
 715
 716         /*
 717          * Special case for @@PFS softlinks.  The actual size of the
 718          * expanded softlink is "@@0x%016llx:%05d" == 26 bytes.
 719          * or for MAX_TID is    "@@-1:%05d" == 10 bytes.
 720          */
 721         if (ip->ino_data.obj_type == HAMMER_OBJTYPE_SOFTLINK &&
 722             ip->ino_data.size == 10 &&
 723             ip->obj_asof == HAMMER_MAX_TID &&
 724             ip->obj_localization == 0 &&
 725             strncmp(ip->ino_data.ext.symlink, "@@PFS", 5) == 0) {
 726                     if (ip->pfsm->pfsd.mirror_flags & HAMMER_PFSD_SLAVE)
 727                             vap->va_size = 26;
 728                     else
 729                             vap->va_size = 10;
 730         }
 731
 732         /*
 733          * We must provide a consistent atime and mtime for snapshots
 734          * so people can do a 'tar cf - ... | md5' on them and get
 735          * consistent results.
 736          */
 737         if (ip->flags & HAMMER_INODE_RO) {
 738                 hammer_time_to_timespec(ip->ino_data.ctime, &vap->va_atime);
 739                 hammer_time_to_timespec(ip->ino_data.ctime, &vap->va_mtime);
 740         } else {
 741                 hammer_time_to_timespec(ip->ino_data.atime, &vap->va_atime);
 742                 hammer_time_to_timespec(ip->ino_data.mtime, &vap->va_mtime);
 743         }
 744         hammer_time_to_timespec(ip->ino_data.ctime, &vap->va_ctime);
 745         vap->va_flags = ip->ino_data.uflags;
 746         vap->va_gen = 1;        /* hammer inums are unique for all time */
 747         vap->va_blocksize = HAMMER_BUFSIZE;
 748         if (ip->ino_data.size >= HAMMER_XDEMARC) {
 749                 vap->va_bytes = (ip->ino_data.size + HAMMER_XBUFMASK64) &
 750                                 ~HAMMER_XBUFMASK64;
 751         } else if (ip->ino_data.size > HAMMER_BUFSIZE / 2) {
 752                 vap->va_bytes = (ip->ino_data.size + HAMMER_BUFMASK64) &
 753                                 ~HAMMER_BUFMASK64;
 754         } else {
 755                 vap->va_bytes = (ip->ino_data.size + 15) & ~15;
 756         }
 757
 758         vap->va_type = hammer_get_vnode_type(ip->ino_data.obj_type);
 759         vap->va_filerev = 0;    /* XXX */
 760         /* mtime uniquely identifies any adjustments made to the file XXX */
 761         vap->va_fsmid = ip->ino_data.mtime;
 762         vap->va_uid_uuid = ip->ino_data.uid;
 763         vap->va_gid_uuid = ip->ino_data.gid;
 764         vap->va_fsid_uuid = ip->hmp->fsid;
 765         vap->va_vaflags = VA_UID_UUID_VALID | VA_GID_UUID_VALID |
 766                           VA_FSID_UUID_VALID;
 767
 768         switch (ip->ino_data.obj_type) {
 769         case HAMMER_OBJTYPE_CDEV:
 770         case HAMMER_OBJTYPE_BDEV:
 771                 vap->va_rmajor = ip->ino_data.rmajor;
 772                 vap->va_rminor = ip->ino_data.rminor;
 773                 break;
 774         default:
 775                 break;
 776         }
 777         return(0);
 778 }
 779
 780 /*
 781  * hammer_vop_nresolve { nch, dvp, cred }
 782  *
 783  * Locate the requested directory entry.
 784  */
 785 static
 786 int
 787 hammer_vop_nresolve(struct vop_nresolve_args *ap)
 788 {
 789         struct hammer_transaction trans;
 790         struct namecache *ncp;
 791         hammer_inode_t dip;
 792         hammer_inode_t ip;
 793         hammer_tid_t asof;
 794         struct hammer_cursor cursor;
 795         struct vnode *vp;
 796         int64_t namekey;
 797         int error;
 798         int i;
 799         int nlen;
 800         int flags;
 801         int ispfs;
 802         int64_t obj_id;
 803         u_int32_t localization;
 804         u_int32_t max_iterations;
 805
 806         /*
 807          * Misc initialization, plus handle as-of name extensions.  Look for
 808          * the '@@' extension.  Note that as-of files and directories cannot
 809          * be modified.
 810          */
 811         dip = VTOI(ap->a_dvp);
 812         ncp = ap->a_nch->ncp;
 813         asof = dip->obj_asof;
 814         localization = dip->obj_localization;   /* for code consistency */
 815         nlen = ncp->nc_nlen;
 816         flags = dip->flags & HAMMER_INODE_RO;
 817         ispfs = 0;
 818
 819         hammer_simple_transaction(&trans, dip->hmp);
 820         ++hammer_stats_file_iopsr;
 821
 822         for (i = 0; i < nlen; ++i) {
 823                 if (ncp->nc_name[i] == '@' && ncp->nc_name[i+1] == '@') {
 824                         error = hammer_str_to_tid(ncp->nc_name + i + 2,
 825                                                   &ispfs, &asof, &localization);
 826                         if (error != 0) {
 827                                 i = nlen;
 828                                 break;
 829                         }
 830                         if (asof != HAMMER_MAX_TID)
 831                                 flags |= HAMMER_INODE_RO;
 832                         break;
 833                 }
 834         }
 835         nlen = i;
 836
 837         /*
 838          * If this is a PFS softlink we dive into the PFS
 839          */
 840         if (ispfs && nlen == 0) {
 841                 ip = hammer_get_inode(&trans, dip, HAMMER_OBJID_ROOT,
 842                                       asof, localization,
 843                                       flags, &error);
 844                 if (error == 0) {
 845                         error = hammer_get_vnode(ip, &vp);
 846                         hammer_rel_inode(ip, 0);
 847                 } else {
 848                         vp = NULL;
 849                 }
 850                 if (error == 0) {
 851                         vn_unlock(vp);
 852                         cache_setvp(ap->a_nch, vp);
 853                         vrele(vp);
 854                 }
 855                 goto done;
 856         }
 857
 858         /*
 859          * If there is no path component the time extension is relative to dip.
 860          * e.g. "fubar/@@<snapshot>"
 861          *
 862          * "." is handled by the kernel, but ".@@<snapshot>" is not.
 863          * e.g. "fubar/.@@<snapshot>"
 864          *
 865          * ".." is handled by the kernel.  We do not currently handle
 866          * "..@<snapshot>".
 867          */
 868         if (nlen == 0 || (nlen == 1 && ncp->nc_name[0] == '.')) {
 869                 ip = hammer_get_inode(&trans, dip, dip->obj_id,
 870                                       asof, dip->obj_localization,
 871                                       flags, &error);
 872                 if (error == 0) {
 873                         error = hammer_get_vnode(ip, &vp);
 874                         hammer_rel_inode(ip, 0);
 875                 } else {
 876                         vp = NULL;
 877                 }
 878                 if (error == 0) {
 879                         vn_unlock(vp);
 880                         cache_setvp(ap->a_nch, vp);
 881                         vrele(vp);
 882                 }
 883                 goto done;
 884         }
 885
 886         /*
 887          * Calculate the namekey and setup the key range for the scan.  This
 888          * works kinda like a chained hash table where the lower 32 bits
 889          * of the namekey synthesize the chain.
 890          *
 891          * The key range is inclusive of both key_beg and key_end.
 892          */
 893         namekey = hammer_directory_namekey(dip, ncp->nc_name, nlen,
 894                                            &max_iterations);
 895
 896         error = hammer_init_cursor(&trans, &cursor, &dip->cache[1], dip);
 897         cursor.key_beg.localization = dip->obj_localization +
 898                                       HAMMER_LOCALIZE_MISC;
 899         cursor.key_beg.obj_id = dip->obj_id;
 900         cursor.key_beg.key = namekey;
 901         cursor.key_beg.create_tid = 0;
 902         cursor.key_beg.delete_tid = 0;
 903         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
 904         cursor.key_beg.obj_type = 0;
 905
 906         cursor.key_end = cursor.key_beg;
 907         cursor.key_end.key += max_iterations;
 908         cursor.asof = asof;
 909         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
 910
 911         /*
 912          * Scan all matching records (the chain), locate the one matching
 913          * the requested path component.
 914          *
 915          * The hammer_ip_*() functions merge in-memory records with on-disk
 916          * records for the purposes of the search.
 917          */
 918         obj_id = 0;
 919         localization = HAMMER_DEF_LOCALIZATION;
 920
 921         if (error == 0) {
 922                 error = hammer_ip_first(&cursor);
 923                 while (error == 0) {
 924                         error = hammer_ip_resolve_data(&cursor);
 925                         if (error)
 926                                 break;
 927                         if (nlen == cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF &&
 928                             bcmp(ncp->nc_name, cursor.data->entry.name, nlen) == 0) {
 929                                 obj_id = cursor.data->entry.obj_id;
 930                                 localization = cursor.data->entry.localization;
 931                                 break;
 932                         }
 933                         error = hammer_ip_next(&cursor);
 934                 }
 935         }
 936         hammer_done_cursor(&cursor);
 937         if (error == 0) {
 938                 ip = hammer_get_inode(&trans, dip, obj_id,
 939                                       asof, localization,
 940                                       flags, &error);
 941                 if (error == 0) {
 942                         error = hammer_get_vnode(ip, &vp);
 943                         hammer_rel_inode(ip, 0);
 944                 } else {
 945                         vp = NULL;
 946                 }
 947                 if (error == 0) {
 948                         vn_unlock(vp);
 949                         cache_setvp(ap->a_nch, vp);
 950                         vrele(vp);
 951                 }
 952         } else if (error == ENOENT) {
 953                 cache_setvp(ap->a_nch, NULL);
 954         }
 955 done:
 956         hammer_done_transaction(&trans);
 957         return (error);
 958 }
 959
 960 /*
 961  * hammer_vop_nlookupdotdot { dvp, vpp, cred }
 962  *
 963  * Locate the parent directory of a directory vnode.
 964  *
 965  * dvp is referenced but not locked.  *vpp must be returned referenced and
 966  * locked.  A parent_obj_id of 0 does not necessarily indicate that we are
 967  * at the root, instead it could indicate that the directory we were in was
 968  * removed.
 969  *
 970  * NOTE: as-of sequences are not linked into the directory structure.  If
 971  * we are at the root with a different asof then the mount point, reload
 972  * the same directory with the mount point's asof.   I'm not sure what this
 973  * will do to NFS.  We encode ASOF stamps in NFS file handles so it might not
 974  * get confused, but it hasn't been tested.
 975  */
 976 static
 977 int
 978 hammer_vop_nlookupdotdot(struct vop_nlookupdotdot_args *ap)
 979 {
 980         struct hammer_transaction trans;
 981         struct hammer_inode *dip;
 982         struct hammer_inode *ip;
 983         int64_t parent_obj_id;
 984         u_int32_t parent_obj_localization;
 985         hammer_tid_t asof;
 986         int error;
 987
 988         dip = VTOI(ap->a_dvp);
 989         asof = dip->obj_asof;
 990
 991         /*
 992          * Whos are parent?  This could be the root of a pseudo-filesystem
 993          * whos parent is in another localization domain.
 994          */
 995         parent_obj_id = dip->ino_data.parent_obj_id;
 996         if (dip->obj_id == HAMMER_OBJID_ROOT)
 997                 parent_obj_localization = dip->ino_data.ext.obj.parent_obj_localization;
 998         else
 999                 parent_obj_localization = dip->obj_localization;
1000
1001         if (parent_obj_id == 0) {
1002                 if (dip->obj_id == HAMMER_OBJID_ROOT &&
1003                    asof != dip->hmp->asof) {
1004                         parent_obj_id = dip->obj_id;
1005                         asof = dip->hmp->asof;
1006                         *ap->a_fakename = kmalloc(19, M_TEMP, M_WAITOK);
1007                         ksnprintf(*ap->a_fakename, 19, "0x%016llx",
1008                                    dip->obj_asof);
1009                 } else {
1010                         *ap->a_vpp = NULL;
1011                         return ENOENT;
1012                 }
1013         }
1014
1015         hammer_simple_transaction(&trans, dip->hmp);
1016         ++hammer_stats_file_iopsr;
1017
1018         ip = hammer_get_inode(&trans, dip, parent_obj_id,
1019                               asof, parent_obj_localization,
1020                               dip->flags, &error);
1021         if (ip) {
1022                 error = hammer_get_vnode(ip, ap->a_vpp);
1023                 hammer_rel_inode(ip, 0);
1024         } else {
1025                 *ap->a_vpp = NULL;
1026         }
1027         hammer_done_transaction(&trans);
1028         return (error);
1029 }
1030
1031 /*
1032  * hammer_vop_nlink { nch, dvp, vp, cred }
1033  */
1034 static
1035 int
1036 hammer_vop_nlink(struct vop_nlink_args *ap)
1037 {
1038         struct hammer_transaction trans;
1039         struct hammer_inode *dip;
1040         struct hammer_inode *ip;
1041         struct nchandle *nch;
1042         int error;
1043
1044         if (ap->a_dvp->v_mount != ap->a_vp->v_mount)
1045                 return(EXDEV);
1046
1047         nch = ap->a_nch;
1048         dip = VTOI(ap->a_dvp);
1049         ip = VTOI(ap->a_vp);
1050
1051         if (dip->obj_localization != ip->obj_localization)
1052                 return(EXDEV);
1053
1054         if (dip->flags & HAMMER_INODE_RO)
1055                 return (EROFS);
1056         if (ip->flags & HAMMER_INODE_RO)
1057                 return (EROFS);
1058         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1059                 return (error);
1060
1061         /*
1062          * Create a transaction to cover the operations we perform.
1063          */
1064         hammer_start_transaction(&trans, dip->hmp);
1065         ++hammer_stats_file_iopsw;
1066
1067         /*
1068          * Add the filesystem object to the directory.  Note that neither
1069          * dip nor ip are referenced or locked, but their vnodes are
1070          * referenced.  This function will bump the inode's link count.
1071          */
1072         error = hammer_ip_add_directory(&trans, dip,
1073                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1074                                         ip);
1075
1076         /*
1077          * Finish up.
1078          */
1079         if (error == 0) {
1080                 cache_setunresolved(nch);
1081                 cache_setvp(nch, ap->a_vp);
1082         }
1083         hammer_done_transaction(&trans);
1084         hammer_knote(ap->a_vp, NOTE_LINK);
1085         hammer_knote(ap->a_dvp, NOTE_WRITE);
1086         return (error);
1087 }
1088
1089 /*
1090  * hammer_vop_nmkdir { nch, dvp, vpp, cred, vap }
1091  *
1092  * The operating system has already ensured that the directory entry
1093  * does not exist and done all appropriate namespace locking.
1094  */
1095 static
1096 int
1097 hammer_vop_nmkdir(struct vop_nmkdir_args *ap)
1098 {
1099         struct hammer_transaction trans;
1100         struct hammer_inode *dip;
1101         struct hammer_inode *nip;
1102         struct nchandle *nch;
1103         int error;
1104
1105         nch = ap->a_nch;
1106         dip = VTOI(ap->a_dvp);
1107
1108         if (dip->flags & HAMMER_INODE_RO)
1109                 return (EROFS);
1110         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1111                 return (error);
1112
1113         /*
1114          * Create a transaction to cover the operations we perform.
1115          */
1116         hammer_start_transaction(&trans, dip->hmp);
1117         ++hammer_stats_file_iopsw;
1118
1119         /*
1120          * Create a new filesystem object of the requested type.  The
1121          * returned inode will be referenced but not locked.
1122          */
1123         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
1124                                     dip, NULL, &nip);
1125         if (error) {
1126                 hkprintf("hammer_mkdir error %d\n", error);
1127                 hammer_done_transaction(&trans);
1128                 *ap->a_vpp = NULL;
1129                 return (error);
1130         }
1131         /*
1132          * Add the new filesystem object to the directory.  This will also
1133          * bump the inode's link count.
1134          */
1135         error = hammer_ip_add_directory(&trans, dip,
1136                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1137                                         nip);
1138         if (error)
1139                 hkprintf("hammer_mkdir (add) error %d\n", error);
1140
1141         /*
1142          * Finish up.
1143          */
1144         if (error) {
1145                 hammer_rel_inode(nip, 0);
1146                 *ap->a_vpp = NULL;
1147         } else {
1148                 error = hammer_get_vnode(nip, ap->a_vpp);
1149                 hammer_rel_inode(nip, 0);
1150                 if (error == 0) {
1151                         cache_setunresolved(ap->a_nch);
1152                         cache_setvp(ap->a_nch, *ap->a_vpp);
1153                 }
1154         }
1155         hammer_done_transaction(&trans);
1156         if (error == 0)
1157                 hammer_knote(ap->a_dvp, NOTE_WRITE | NOTE_LINK);
1158         return (error);
1159 }
1160
1161 /*
1162  * hammer_vop_nmknod { nch, dvp, vpp, cred, vap }
1163  *
1164  * The operating system has already ensured that the directory entry
1165  * does not exist and done all appropriate namespace locking.
1166  */
1167 static
1168 int
1169 hammer_vop_nmknod(struct vop_nmknod_args *ap)
1170 {
1171         struct hammer_transaction trans;
1172         struct hammer_inode *dip;
1173         struct hammer_inode *nip;
1174         struct nchandle *nch;
1175         int error;
1176
1177         nch = ap->a_nch;
1178         dip = VTOI(ap->a_dvp);
1179
1180         if (dip->flags & HAMMER_INODE_RO)
1181                 return (EROFS);
1182         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1183                 return (error);
1184
1185         /*
1186          * Create a transaction to cover the operations we perform.
1187          */
1188         hammer_start_transaction(&trans, dip->hmp);
1189         ++hammer_stats_file_iopsw;
1190
1191         /*
1192          * Create a new filesystem object of the requested type.  The
1193          * returned inode will be referenced but not locked.
1194          *
1195          * If mknod specifies a directory a pseudo-fs is created.
1196          */
1197         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
1198                                     dip, NULL, &nip);
1199         if (error) {
1200                 hammer_done_transaction(&trans);
1201                 *ap->a_vpp = NULL;
1202                 return (error);
1203         }
1204
1205         /*
1206          * Add the new filesystem object to the directory.  This will also
1207          * bump the inode's link count.
1208          */
1209         error = hammer_ip_add_directory(&trans, dip,
1210                                         nch->ncp->nc_name, nch->ncp->nc_nlen,
1211                                         nip);
1212
1213         /*
1214          * Finish up.
1215          */
1216         if (error) {
1217                 hammer_rel_inode(nip, 0);
1218                 *ap->a_vpp = NULL;
1219         } else {
1220                 error = hammer_get_vnode(nip, ap->a_vpp);
1221                 hammer_rel_inode(nip, 0);
1222                 if (error == 0) {
1223                         cache_setunresolved(ap->a_nch);
1224                         cache_setvp(ap->a_nch, *ap->a_vpp);
1225                 }
1226         }
1227         hammer_done_transaction(&trans);
1228         if (error == 0)
1229                 hammer_knote(ap->a_dvp, NOTE_WRITE);
1230         return (error);
1231 }
1232
1233 /*
1234  * hammer_vop_open { vp, mode, cred, fp }
1235  */
1236 static
1237 int
1238 hammer_vop_open(struct vop_open_args *ap)
1239 {
1240         hammer_inode_t ip;
1241
1242         ++hammer_stats_file_iopsr;
1243         ip = VTOI(ap->a_vp);
1244
1245         if ((ap->a_mode & FWRITE) && (ip->flags & HAMMER_INODE_RO))
1246                 return (EROFS);
1247         return(vop_stdopen(ap));
1248 }
1249
1250 /*
1251  * hammer_vop_print { vp }
1252  */
1253 static
1254 int
1255 hammer_vop_print(struct vop_print_args *ap)
1256 {
1257         return EOPNOTSUPP;
1258 }
1259
1260 /*
1261  * hammer_vop_readdir { vp, uio, cred, *eofflag, *ncookies, off_t **cookies }
1262  */
1263 static
1264 int
1265 hammer_vop_readdir(struct vop_readdir_args *ap)
1266 {
1267         struct hammer_transaction trans;
1268         struct hammer_cursor cursor;
1269         struct hammer_inode *ip;
1270         struct uio *uio;
1271         hammer_base_elm_t base;
1272         int error;
1273         int cookie_index;
1274         int ncookies;
1275         off_t *cookies;
1276         off_t saveoff;
1277         int r;
1278         int dtype;
1279
1280         ++hammer_stats_file_iopsr;
1281         ip = VTOI(ap->a_vp);
1282         uio = ap->a_uio;
1283         saveoff = uio->uio_offset;
1284
1285         if (ap->a_ncookies) {
1286                 ncookies = uio->uio_resid / 16 + 1;
1287                 if (ncookies > 1024)
1288                         ncookies = 1024;
1289                 cookies = kmalloc(ncookies * sizeof(off_t), M_TEMP, M_WAITOK);
1290                 cookie_index = 0;
1291         } else {
1292                 ncookies = -1;
1293                 cookies = NULL;
1294                 cookie_index = 0;
1295         }
1296
1297         hammer_simple_transaction(&trans, ip->hmp);
1298
1299         /*
1300          * Handle artificial entries
1301          */
1302         error = 0;
1303         if (saveoff == 0) {
1304                 r = vop_write_dirent(&error, uio, ip->obj_id, DT_DIR, 1, ".");
1305                 if (r)
1306                         goto done;
1307                 if (cookies)
1308                         cookies[cookie_index] = saveoff;
1309                 ++saveoff;
1310                 ++cookie_index;
1311                 if (cookie_index == ncookies)
1312                         goto done;
1313         }
1314         if (saveoff == 1) {
1315                 if (ip->ino_data.parent_obj_id) {
1316                         r = vop_write_dirent(&error, uio,
1317                                              ip->ino_data.parent_obj_id,
1318                                              DT_DIR, 2, "..");
1319                 } else {
1320                         r = vop_write_dirent(&error, uio,
1321                                              ip->obj_id, DT_DIR, 2, "..");
1322                 }
1323                 if (r)
1324                         goto done;
1325                 if (cookies)
1326                         cookies[cookie_index] = saveoff;
1327                 ++saveoff;
1328                 ++cookie_index;
1329                 if (cookie_index == ncookies)
1330                         goto done;
1331         }
1332
1333         /*
1334          * Key range (begin and end inclusive) to scan.  Directory keys
1335          * directly translate to a 64 bit 'seek' position.
1336          */
1337         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
1338         cursor.key_beg.localization = ip->obj_localization +
1339                                       HAMMER_LOCALIZE_MISC;
1340         cursor.key_beg.obj_id = ip->obj_id;
1341         cursor.key_beg.create_tid = 0;
1342         cursor.key_beg.delete_tid = 0;
1343         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
1344         cursor.key_beg.obj_type = 0;
1345         cursor.key_beg.key = saveoff;
1346
1347         cursor.key_end = cursor.key_beg;
1348         cursor.key_end.key = HAMMER_MAX_KEY;
1349         cursor.asof = ip->obj_asof;
1350         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
1351
1352         error = hammer_ip_first(&cursor);
1353
1354         while (error == 0) {
1355                 error = hammer_ip_resolve_data(&cursor);
1356                 if (error)
1357                         break;
1358                 base = &cursor.leaf->base;
1359                 saveoff = base->key;
1360                 KKASSERT(cursor.leaf->data_len > HAMMER_ENTRY_NAME_OFF);
1361
1362                 if (base->obj_id != ip->obj_id)
1363                         panic("readdir: bad record at %p", cursor.node);
1364
1365                 /*
1366                  * Convert pseudo-filesystems into softlinks
1367                  */
1368                 dtype = hammer_get_dtype(cursor.leaf->base.obj_type);
1369                 r = vop_write_dirent(
1370                              &error, uio, cursor.data->entry.obj_id,
1371                              dtype,
1372                              cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF ,
1373                              (void *)cursor.data->entry.name);
1374                 if (r)
1375                         break;
1376                 ++saveoff;
1377                 if (cookies)
1378                         cookies[cookie_index] = base->key;
1379                 ++cookie_index;
1380                 if (cookie_index == ncookies)
1381                         break;
1382                 error = hammer_ip_next(&cursor);
1383         }
1384         hammer_done_cursor(&cursor);
1385
1386 done:
1387         hammer_done_transaction(&trans);
1388
1389         if (ap->a_eofflag)
1390                 *ap->a_eofflag = (error == ENOENT);
1391         uio->uio_offset = saveoff;
1392         if (error && cookie_index == 0) {
1393                 if (error == ENOENT)
1394                         error = 0;
1395                 if (cookies) {
1396                         kfree(cookies, M_TEMP);
1397                         *ap->a_ncookies = 0;
1398                         *ap->a_cookies = NULL;
1399                 }
1400         } else {
1401                 if (error == ENOENT)
1402                         error = 0;
1403                 if (cookies) {
1404                         *ap->a_ncookies = cookie_index;
1405                         *ap->a_cookies = cookies;
1406                 }
1407         }
1408         return(error);
1409 }
1410
1411 /*
1412  * hammer_vop_readlink { vp, uio, cred }
1413  */
1414 static
1415 int
1416 hammer_vop_readlink(struct vop_readlink_args *ap)
1417 {
1418         struct hammer_transaction trans;
1419         struct hammer_cursor cursor;
1420         struct hammer_inode *ip;
1421         char buf[32];
1422         u_int32_t localization;
1423         hammer_pseudofs_inmem_t pfsm;
1424         int error;
1425
1426         ip = VTOI(ap->a_vp);
1427
1428         /*
1429          * Shortcut if the symlink data was stuffed into ino_data.
1430          *
1431          * Also expand special "@@PFS%05d" softlinks (expansion only
1432          * occurs for non-historical (current) accesses made from the
1433          * primary filesystem).
1434          */
1435         if (ip->ino_data.size <= HAMMER_INODE_BASESYMLEN) {
1436                 char *ptr;
1437                 int bytes;
1438
1439                 ptr = ip->ino_data.ext.symlink;
1440                 bytes = (int)ip->ino_data.size;
1441                 if (bytes == 10 &&
1442                     ip->obj_asof == HAMMER_MAX_TID &&
1443                     ip->obj_localization == 0 &&
1444                     strncmp(ptr, "@@PFS", 5) == 0) {
1445                         hammer_simple_transaction(&trans, ip->hmp);
1446                         bcopy(ptr + 5, buf, 5);
1447                         buf[5] = 0;
1448                         localization = strtoul(buf, NULL, 10) << 16;
1449                         pfsm = hammer_load_pseudofs(&trans, localization,
1450                                                     &error);
1451                         if (error == 0) {
1452                                 if (pfsm->pfsd.mirror_flags &
1453                                     HAMMER_PFSD_SLAVE) {
1454                                         /* vap->va_size == 26 */
1455                                         ksnprintf(buf, sizeof(buf),
1456                                                   "@@0x%016llx:%05d",
1457                                                   pfsm->pfsd.sync_end_tid,
1458                                                   localization >> 16);
1459                                 } else {
1460                                         /* vap->va_size == 10 */
1461                                         ksnprintf(buf, sizeof(buf),
1462                                                   "@@-1:%05d",
1463                                                   localization >> 16);
1464 #if 0
1465                                         ksnprintf(buf, sizeof(buf),
1466                                                   "@@0x%016llx:%05d",
1467                                                   HAMMER_MAX_TID,
1468                                                   localization >> 16);
1469 #endif
1470                                 }
1471                                 ptr = buf;
1472                                 bytes = strlen(buf);
1473                         }
1474                         if (pfsm)
1475                                 hammer_rel_pseudofs(trans.hmp, pfsm);
1476                         hammer_done_transaction(&trans);
1477                 }
1478                 error = uiomove(ptr, bytes, ap->a_uio);
1479                 return(error);
1480         }
1481
1482         /*
1483          * Long version
1484          */
1485         hammer_simple_transaction(&trans, ip->hmp);
1486         ++hammer_stats_file_iopsr;
1487         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
1488
1489         /*
1490          * Key range (begin and end inclusive) to scan.  Directory keys
1491          * directly translate to a 64 bit 'seek' position.
1492          */
1493         cursor.key_beg.localization = ip->obj_localization +
1494                                       HAMMER_LOCALIZE_MISC;
1495         cursor.key_beg.obj_id = ip->obj_id;
1496         cursor.key_beg.create_tid = 0;
1497         cursor.key_beg.delete_tid = 0;
1498         cursor.key_beg.rec_type = HAMMER_RECTYPE_FIX;
1499         cursor.key_beg.obj_type = 0;
1500         cursor.key_beg.key = HAMMER_FIXKEY_SYMLINK;
1501         cursor.asof = ip->obj_asof;
1502         cursor.flags |= HAMMER_CURSOR_ASOF;
1503
1504         error = hammer_ip_lookup(&cursor);
1505         if (error == 0) {
1506                 error = hammer_ip_resolve_data(&cursor);
1507                 if (error == 0) {
1508                         KKASSERT(cursor.leaf->data_len >=
1509                                  HAMMER_SYMLINK_NAME_OFF);
1510                         error = uiomove(cursor.data->symlink.name,
1511                                         cursor.leaf->data_len -
1512                                                 HAMMER_SYMLINK_NAME_OFF,
1513                                         ap->a_uio);
1514                 }
1515         }
1516         hammer_done_cursor(&cursor);
1517         hammer_done_transaction(&trans);
1518         return(error);
1519 }
1520
1521 /*
1522  * hammer_vop_nremove { nch, dvp, cred }
1523  */
1524 static
1525 int
1526 hammer_vop_nremove(struct vop_nremove_args *ap)
1527 {
1528         struct hammer_transaction trans;
1529         struct hammer_inode *dip;
1530         int error;
1531
1532         dip = VTOI(ap->a_dvp);
1533
1534         if (hammer_nohistory(dip) == 0 &&
1535             (error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_REMOVE)) != 0) {
1536                 return (error);
1537         }
1538
1539         hammer_start_transaction(&trans, dip->hmp);
1540         ++hammer_stats_file_iopsw;
1541         error = hammer_dounlink(&trans, ap->a_nch, ap->a_dvp, ap->a_cred, 0, 0);
1542         hammer_done_transaction(&trans);
1543         if (error == 0)
1544                 hammer_knote(ap->a_dvp, NOTE_WRITE);
1545         return (error);
1546 }
1547
1548 /*
1549  * hammer_vop_nrename { fnch, tnch, fdvp, tdvp, cred }
1550  */
1551 static
1552 int
1553 hammer_vop_nrename(struct vop_nrename_args *ap)
1554 {
1555         struct hammer_transaction trans;
1556         struct namecache *fncp;
1557         struct namecache *tncp;
1558         struct hammer_inode *fdip;
1559         struct hammer_inode *tdip;
1560         struct hammer_inode *ip;
1561         struct hammer_cursor cursor;
1562         int64_t namekey;
1563         u_int32_t max_iterations;
1564         int nlen, error;
1565
1566         if (ap->a_fdvp->v_mount != ap->a_tdvp->v_mount)
1567                 return(EXDEV);
1568         if (ap->a_fdvp->v_mount != ap->a_fnch->ncp->nc_vp->v_mount)
1569                 return(EXDEV);
1570
1571         fdip = VTOI(ap->a_fdvp);
1572         tdip = VTOI(ap->a_tdvp);
1573         fncp = ap->a_fnch->ncp;
1574         tncp = ap->a_tnch->ncp;
1575         ip = VTOI(fncp->nc_vp);
1576         KKASSERT(ip != NULL);
1577
1578         if (fdip->obj_localization != tdip->obj_localization)
1579                 return(EXDEV);
1580         if (fdip->obj_localization != ip->obj_localization)
1581                 return(EXDEV);
1582
1583         if (fdip->flags & HAMMER_INODE_RO)
1584                 return (EROFS);
1585         if (tdip->flags & HAMMER_INODE_RO)
1586                 return (EROFS);
1587         if (ip->flags & HAMMER_INODE_RO)
1588                 return (EROFS);
1589         if ((error = hammer_checkspace(fdip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1590                 return (error);
1591
1592         hammer_start_transaction(&trans, fdip->hmp);
1593         ++hammer_stats_file_iopsw;
1594
1595         /*
1596          * Remove tncp from the target directory and then link ip as
1597          * tncp. XXX pass trans to dounlink
1598          *
1599          * Force the inode sync-time to match the transaction so it is
1600          * in-sync with the creation of the target directory entry.
1601          */
1602         error = hammer_dounlink(&trans, ap->a_tnch, ap->a_tdvp,
1603                                 ap->a_cred, 0, -1);
1604         if (error == 0 || error == ENOENT) {
1605                 error = hammer_ip_add_directory(&trans, tdip,
1606                                                 tncp->nc_name, tncp->nc_nlen,
1607                                                 ip);
1608                 if (error == 0) {
1609                         ip->ino_data.parent_obj_id = tdip->obj_id;
1610                         hammer_modify_inode(ip, HAMMER_INODE_DDIRTY);
1611                 }
1612         }
1613         if (error)
1614                 goto failed; /* XXX */
1615
1616         /*
1617          * Locate the record in the originating directory and remove it.
1618          *
1619          * Calculate the namekey and setup the key range for the scan.  This
1620          * works kinda like a chained hash table where the lower 32 bits
1621          * of the namekey synthesize the chain.
1622          *
1623          * The key range is inclusive of both key_beg and key_end.
1624          */
1625         namekey = hammer_directory_namekey(fdip, fncp->nc_name, fncp->nc_nlen,
1626                                            &max_iterations);
1627 retry:
1628         hammer_init_cursor(&trans, &cursor, &fdip->cache[1], fdip);
1629         cursor.key_beg.localization = fdip->obj_localization +
1630                                       HAMMER_LOCALIZE_MISC;
1631         cursor.key_beg.obj_id = fdip->obj_id;
1632         cursor.key_beg.key = namekey;
1633         cursor.key_beg.create_tid = 0;
1634         cursor.key_beg.delete_tid = 0;
1635         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
1636         cursor.key_beg.obj_type = 0;
1637
1638         cursor.key_end = cursor.key_beg;
1639         cursor.key_end.key += max_iterations;
1640         cursor.asof = fdip->obj_asof;
1641         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
1642
1643         /*
1644          * Scan all matching records (the chain), locate the one matching
1645          * the requested path component.
1646          *
1647          * The hammer_ip_*() functions merge in-memory records with on-disk
1648          * records for the purposes of the search.
1649          */
1650         error = hammer_ip_first(&cursor);
1651         while (error == 0) {
1652                 if (hammer_ip_resolve_data(&cursor) != 0)
1653                         break;
1654                 nlen = cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF;
1655                 KKASSERT(nlen > 0);
1656                 if (fncp->nc_nlen == nlen &&
1657                     bcmp(fncp->nc_name, cursor.data->entry.name, nlen) == 0) {
1658                         break;
1659                 }
1660                 error = hammer_ip_next(&cursor);
1661         }
1662
1663         /*
1664          * If all is ok we have to get the inode so we can adjust nlinks.
1665          *
1666          * WARNING: hammer_ip_del_directory() may have to terminate the
1667          * cursor to avoid a recursion.  It's ok to call hammer_done_cursor()
1668          * twice.
1669          */
1670         if (error == 0)
1671                 error = hammer_ip_del_directory(&trans, &cursor, fdip, ip);
1672
1673         /*
1674          * XXX A deadlock here will break rename's atomicy for the purposes
1675          * of crash recovery.
1676          */
1677         if (error == EDEADLK) {
1678                 hammer_done_cursor(&cursor);
1679                 goto retry;
1680         }
1681
1682         /*
1683          * Cleanup and tell the kernel that the rename succeeded.
1684          */
1685         hammer_done_cursor(&cursor);
1686         if (error == 0) {
1687                 cache_rename(ap->a_fnch, ap->a_tnch);
1688                 hammer_knote(ap->a_fdvp, NOTE_WRITE);
1689                 hammer_knote(ap->a_tdvp, NOTE_WRITE);
1690                 if (ip->vp)
1691                         hammer_knote(ip->vp, NOTE_RENAME);
1692         }
1693
1694 failed:
1695         hammer_done_transaction(&trans);
1696         return (error);
1697 }
1698
1699 /*
1700  * hammer_vop_nrmdir { nch, dvp, cred }
1701  */
1702 static
1703 int
1704 hammer_vop_nrmdir(struct vop_nrmdir_args *ap)
1705 {
1706         struct hammer_transaction trans;
1707         struct hammer_inode *dip;
1708         int error;
1709
1710         dip = VTOI(ap->a_dvp);
1711
1712         if (hammer_nohistory(dip) == 0 &&
1713             (error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_REMOVE)) != 0) {
1714                 return (error);
1715         }
1716
1717         hammer_start_transaction(&trans, dip->hmp);
1718         ++hammer_stats_file_iopsw;
1719         error = hammer_dounlink(&trans, ap->a_nch, ap->a_dvp, ap->a_cred, 0, 1);
1720         hammer_done_transaction(&trans);
1721         if (error == 0)
1722                 hammer_knote(ap->a_dvp, NOTE_WRITE | NOTE_LINK);
1723         return (error);
1724 }
1725
1726 /*
1727  * hammer_vop_markatime { vp, cred }
1728  */
1729 static
1730 int
1731 hammer_vop_markatime(struct vop_markatime_args *ap)
1732 {
1733         struct hammer_transaction trans;
1734         struct hammer_inode *ip;
1735
1736         ip = VTOI(ap->a_vp);
1737         if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
1738                 return (EROFS);
1739         if (ip->flags & HAMMER_INODE_RO)
1740                 return (EROFS);
1741         if (ip->hmp->mp->mnt_flag & MNT_NOATIME)
1742                 return (0);
1743         hammer_start_transaction(&trans, ip->hmp);
1744         ++hammer_stats_file_iopsw;
1745
1746         ip->ino_data.atime = trans.time;
1747         hammer_modify_inode(ip, HAMMER_INODE_ATIME);
1748         hammer_done_transaction(&trans);
1749         hammer_knote(ap->a_vp, NOTE_ATTRIB);
1750         return (0);
1751 }
1752
1753 /*
1754  * hammer_vop_setattr { vp, vap, cred }
1755  */
1756 static
1757 int
1758 hammer_vop_setattr(struct vop_setattr_args *ap)
1759 {
1760         struct hammer_transaction trans;
1761         struct vattr *vap;
1762         struct hammer_inode *ip;
1763         int modflags;
1764         int error;
1765         int truncating;
1766         int blksize;
1767         int kflags;
1768         int64_t aligned_size;
1769         u_int32_t flags;
1770
1771         vap = ap->a_vap;
1772         ip = ap->a_vp->v_data;
1773         modflags = 0;
1774         kflags = 0;
1775
1776         if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
1777                 return(EROFS);
1778         if (ip->flags & HAMMER_INODE_RO)
1779                 return (EROFS);
1780         if (hammer_nohistory(ip) == 0 &&
1781             (error = hammer_checkspace(ip->hmp, HAMMER_CHKSPC_REMOVE)) != 0) {
1782                 return (error);
1783         }
1784
1785         hammer_start_transaction(&trans, ip->hmp);
1786         ++hammer_stats_file_iopsw;
1787         error = 0;
1788
1789         if (vap->va_flags != VNOVAL) {
1790                 flags = ip->ino_data.uflags;
1791                 error = vop_helper_setattr_flags(&flags, vap->va_flags,
1792                                          hammer_to_unix_xid(&ip->ino_data.uid),
1793                                          ap->a_cred);
1794                 if (error == 0) {
1795                         if (ip->ino_data.uflags != flags) {
1796                                 ip->ino_data.uflags = flags;
1797                                 modflags |= HAMMER_INODE_DDIRTY;
1798                                 kflags |= NOTE_ATTRIB;
1799                         }
1800                         if (ip->ino_data.uflags & (IMMUTABLE | APPEND)) {
1801                                 error = 0;
1802                                 goto done;
1803                         }
1804                 }
1805                 goto done;
1806         }
1807         if (ip->ino_data.uflags & (IMMUTABLE | APPEND)) {
1808                 error = EPERM;
1809                 goto done;
1810         }
1811         if (vap->va_uid != (uid_t)VNOVAL || vap->va_gid != (gid_t)VNOVAL) {
1812                 mode_t cur_mode = ip->ino_data.mode;
1813                 uid_t cur_uid = hammer_to_unix_xid(&ip->ino_data.uid);
1814                 gid_t cur_gid = hammer_to_unix_xid(&ip->ino_data.gid);
1815                 uuid_t uuid_uid;
1816                 uuid_t uuid_gid;
1817
1818                 error = vop_helper_chown(ap->a_vp, vap->va_uid, vap->va_gid,
1819                                          ap->a_cred,
1820                                          &cur_uid, &cur_gid, &cur_mode);
1821                 if (error == 0) {
1822                         hammer_guid_to_uuid(&uuid_uid, cur_uid);
1823                         hammer_guid_to_uuid(&uuid_gid, cur_gid);
1824                         if (bcmp(&uuid_uid, &ip->ino_data.uid,
1825                                  sizeof(uuid_uid)) ||
1826                             bcmp(&uuid_gid, &ip->ino_data.gid,
1827                                  sizeof(uuid_gid)) ||
1828                             ip->ino_data.mode != cur_mode
1829                         ) {
1830                                 ip->ino_data.uid = uuid_uid;
1831                                 ip->ino_data.gid = uuid_gid;
1832                                 ip->ino_data.mode = cur_mode;
1833                         }
1834                         modflags |= HAMMER_INODE_DDIRTY;
1835                         kflags |= NOTE_ATTRIB;
1836                 }
1837         }
1838         while (vap->va_size != VNOVAL && ip->ino_data.size != vap->va_size) {
1839                 switch(ap->a_vp->v_type) {
1840                 case VREG:
1841                         if (vap->va_size == ip->ino_data.size)
1842                                 break;
1843                         /*
1844                          * XXX break atomicy, we can deadlock the backend
1845                          * if we do not release the lock.  Probably not a
1846                          * big deal here.
1847                          */
1848                         blksize = hammer_blocksize(vap->va_size);
1849                         if (vap->va_size < ip->ino_data.size) {
1850                                 vtruncbuf(ap->a_vp, vap->va_size, blksize);
1851                                 truncating = 1;
1852                                 kflags |= NOTE_WRITE;
1853                         } else {
1854                                 vnode_pager_setsize(ap->a_vp, vap->va_size);
1855                                 truncating = 0;
1856                                 kflags |= NOTE_WRITE | NOTE_EXTEND;
1857                         }
1858                         ip->ino_data.size = vap->va_size;
1859                         modflags |= HAMMER_INODE_DDIRTY;
1860
1861                         /*
1862                          * on-media truncation is cached in the inode until
1863                          * the inode is synchronized.
1864                          */
1865                         if (truncating) {
1866                                 hammer_ip_frontend_trunc(ip, vap->va_size);
1867 #ifdef DEBUG_TRUNCATE
1868                                 if (HammerTruncIp == NULL)
1869                                         HammerTruncIp = ip;
1870 #endif
1871                                 if ((ip->flags & HAMMER_INODE_TRUNCATED) == 0) {
1872                                         ip->flags |= HAMMER_INODE_TRUNCATED;
1873                                         ip->trunc_off = vap->va_size;
1874 #ifdef DEBUG_TRUNCATE
1875                                         if (ip == HammerTruncIp)
1876                                         kprintf("truncate1 %016llx\n", ip->trunc_off);
1877 #endif
1878                                 } else if (ip->trunc_off > vap->va_size) {
1879                                         ip->trunc_off = vap->va_size;
1880 #ifdef DEBUG_TRUNCATE
1881                                         if (ip == HammerTruncIp)
1882                                         kprintf("truncate2 %016llx\n", ip->trunc_off);
1883 #endif
1884                                 } else {
1885 #ifdef DEBUG_TRUNCATE
1886                                         if (ip == HammerTruncIp)
1887                                         kprintf("truncate3 %016llx (ignored)\n", vap->va_size);
1888 #endif
1889                                 }
1890                         }
1891
1892                         /*
1893                          * If truncating we have to clean out a portion of
1894                          * the last block on-disk.  We do this in the
1895                          * front-end buffer cache.
1896                          */
1897                         aligned_size = (vap->va_size + (blksize - 1)) &
1898                                        ~(int64_t)(blksize - 1);
1899                         if (truncating && vap->va_size < aligned_size) {
1900                                 struct buf *bp;
1901                                 int offset;
1902
1903                                 aligned_size -= blksize;
1904
1905                                 offset = (int)vap->va_size & (blksize - 1);
1906                                 error = bread(ap->a_vp, aligned_size,
1907                                               blksize, &bp);
1908                                 hammer_ip_frontend_trunc(ip, aligned_size);
1909                                 if (error == 0) {
1910                                         bzero(bp->b_data + offset,
1911                                               blksize - offset);
1912                                         /* must de-cache direct-io offset */
1913                                         bp->b_bio2.bio_offset = NOOFFSET;
1914                                         bdwrite(bp);
1915                                 } else {
1916                                         kprintf("ERROR %d\n", error);
1917                                         brelse(bp);
1918                                 }
1919                         }
1920                         break;
1921                 case VDATABASE:
1922                         if ((ip->flags & HAMMER_INODE_TRUNCATED) == 0) {
1923                                 ip->flags |= HAMMER_INODE_TRUNCATED;
1924                                 ip->trunc_off = vap->va_size;
1925                         } else if (ip->trunc_off > vap->va_size) {
1926                                 ip->trunc_off = vap->va_size;
1927                         }
1928                         hammer_ip_frontend_trunc(ip, vap->va_size);
1929                         ip->ino_data.size = vap->va_size;
1930                         modflags |= HAMMER_INODE_DDIRTY;
1931                         kflags |= NOTE_ATTRIB;
1932                         break;
1933                 default:
1934                         error = EINVAL;
1935                         goto done;
1936                 }
1937                 break;
1938         }
1939         if (vap->va_atime.tv_sec != VNOVAL) {
1940                 ip->ino_data.atime =
1941                         hammer_timespec_to_time(&vap->va_atime);
1942                 modflags |= HAMMER_INODE_ATIME;
1943                 kflags |= NOTE_ATTRIB;
1944         }
1945         if (vap->va_mtime.tv_sec != VNOVAL) {
1946                 ip->ino_data.mtime =
1947                         hammer_timespec_to_time(&vap->va_mtime);
1948                 modflags |= HAMMER_INODE_MTIME;
1949                 kflags |= NOTE_ATTRIB;
1950         }
1951         if (vap->va_mode != (mode_t)VNOVAL) {
1952                 mode_t   cur_mode = ip->ino_data.mode;
1953                 uid_t cur_uid = hammer_to_unix_xid(&ip->ino_data.uid);
1954                 gid_t cur_gid = hammer_to_unix_xid(&ip->ino_data.gid);
1955
1956                 error = vop_helper_chmod(ap->a_vp, vap->va_mode, ap->a_cred,
1957                                          cur_uid, cur_gid, &cur_mode);
1958                 if (error == 0 && ip->ino_data.mode != cur_mode) {
1959                         ip->ino_data.mode = cur_mode;
1960                         modflags |= HAMMER_INODE_DDIRTY;
1961                         kflags |= NOTE_ATTRIB;
1962                 }
1963         }
1964 done:
1965         if (error == 0)
1966                 hammer_modify_inode(ip, modflags);
1967         hammer_done_transaction(&trans);
1968         hammer_knote(ap->a_vp, kflags);
1969         return (error);
1970 }
1971
1972 /*
1973  * hammer_vop_nsymlink { nch, dvp, vpp, cred, vap, target }
1974  */
1975 static
1976 int
1977 hammer_vop_nsymlink(struct vop_nsymlink_args *ap)
1978 {
1979         struct hammer_transaction trans;
1980         struct hammer_inode *dip;
1981         struct hammer_inode *nip;
1982         struct nchandle *nch;
1983         hammer_record_t record;
1984         int error;
1985         int bytes;
1986
1987         ap->a_vap->va_type = VLNK;
1988
1989         nch = ap->a_nch;
1990         dip = VTOI(ap->a_dvp);
1991
1992         if (dip->flags & HAMMER_INODE_RO)
1993                 return (EROFS);
1994         if ((error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0)
1995                 return (error);
1996
1997         /*
1998          * Create a transaction to cover the operations we perform.
1999          */
2000         hammer_start_transaction(&trans, dip->hmp);
2001         ++hammer_stats_file_iopsw;
2002
2003         /*
2004          * Create a new filesystem object of the requested type.  The
2005          * returned inode will be referenced but not locked.
2006          */
2007
2008         error = hammer_create_inode(&trans, ap->a_vap, ap->a_cred,
2009                                     dip, NULL, &nip);
2010         if (error) {
2011                 hammer_done_transaction(&trans);
2012                 *ap->a_vpp = NULL;
2013                 return (error);
2014         }
2015
2016         /*
2017          * Add a record representing the symlink.  symlink stores the link
2018          * as pure data, not a string, and is no \0 terminated.
2019          */
2020         if (error == 0) {
2021                 bytes = strlen(ap->a_target);
2022
2023                 if (bytes <= HAMMER_INODE_BASESYMLEN) {
2024                         bcopy(ap->a_target, nip->ino_data.ext.symlink, bytes);
2025                 } else {
2026                         record = hammer_alloc_mem_record(nip, bytes);
2027                         record->type = HAMMER_MEM_RECORD_GENERAL;
2028
2029                         record->leaf.base.localization = nip->obj_localization +
2030                                                          HAMMER_LOCALIZE_MISC;
2031                         record->leaf.base.key = HAMMER_FIXKEY_SYMLINK;
2032                         record->leaf.base.rec_type = HAMMER_RECTYPE_FIX;
2033                         record->leaf.data_len = bytes;
2034                         KKASSERT(HAMMER_SYMLINK_NAME_OFF == 0);
2035                         bcopy(ap->a_target, record->data->symlink.name, bytes);
2036                         error = hammer_ip_add_record(&trans, record);
2037                 }
2038
2039                 /*
2040                  * Set the file size to the length of the link.
2041                  */
2042                 if (error == 0) {
2043                         nip->ino_data.size = bytes;
2044                         hammer_modify_inode(nip, HAMMER_INODE_DDIRTY);
2045                 }
2046         }
2047         if (error == 0)
2048                 error = hammer_ip_add_directory(&trans, dip, nch->ncp->nc_name,
2049                                                 nch->ncp->nc_nlen, nip);
2050
2051         /*
2052          * Finish up.
2053          */
2054         if (error) {
2055                 hammer_rel_inode(nip, 0);
2056                 *ap->a_vpp = NULL;
2057         } else {
2058                 error = hammer_get_vnode(nip, ap->a_vpp);
2059                 hammer_rel_inode(nip, 0);
2060                 if (error == 0) {
2061                         cache_setunresolved(ap->a_nch);
2062                         cache_setvp(ap->a_nch, *ap->a_vpp);
2063                         hammer_knote(ap->a_dvp, NOTE_WRITE);
2064                 }
2065         }
2066         hammer_done_transaction(&trans);
2067         return (error);
2068 }
2069
2070 /*
2071  * hammer_vop_nwhiteout { nch, dvp, cred, flags }
2072  */
2073 static
2074 int
2075 hammer_vop_nwhiteout(struct vop_nwhiteout_args *ap)
2076 {
2077         struct hammer_transaction trans;
2078         struct hammer_inode *dip;
2079         int error;
2080
2081         dip = VTOI(ap->a_dvp);
2082
2083         if (hammer_nohistory(dip) == 0 &&
2084             (error = hammer_checkspace(dip->hmp, HAMMER_CHKSPC_CREATE)) != 0) {
2085                 return (error);
2086         }
2087
2088         hammer_start_transaction(&trans, dip->hmp);
2089         ++hammer_stats_file_iopsw;
2090         error = hammer_dounlink(&trans, ap->a_nch, ap->a_dvp,
2091                                 ap->a_cred, ap->a_flags, -1);
2092         hammer_done_transaction(&trans);
2093
2094         return (error);
2095 }
2096
2097 /*
2098  * hammer_vop_ioctl { vp, command, data, fflag, cred }
2099  */
2100 static
2101 int
2102 hammer_vop_ioctl(struct vop_ioctl_args *ap)
2103 {
2104         struct hammer_inode *ip = ap->a_vp->v_data;
2105
2106         ++hammer_stats_file_iopsr;
2107         return(hammer_ioctl(ip, ap->a_command, ap->a_data,
2108                             ap->a_fflag, ap->a_cred));
2109 }
2110
2111 static
2112 int
2113 hammer_vop_mountctl(struct vop_mountctl_args *ap)
2114 {
2115         struct mount *mp;
2116         int error;
2117
2118         mp = ap->a_head.a_ops->head.vv_mount;
2119
2120         switch(ap->a_op) {
2121         case MOUNTCTL_SET_EXPORT:
2122                 if (ap->a_ctllen != sizeof(struct export_args))
2123                         error = EINVAL;
2124                 else
2125                         error = hammer_vfs_export(mp, ap->a_op,
2126                                       (const struct export_args *)ap->a_ctl);
2127                 break;
2128         default:
2129                 error = journal_mountctl(ap);
2130                 break;
2131         }
2132         return(error);
2133 }
2134
2135 /*
2136  * hammer_vop_strategy { vp, bio }
2137  *
2138  * Strategy call, used for regular file read & write only.  Note that the
2139  * bp may represent a cluster.
2140  *
2141  * To simplify operation and allow better optimizations in the future,
2142  * this code does not make any assumptions with regards to buffer alignment
2143  * or size.
2144  */
2145 static
2146 int
2147 hammer_vop_strategy(struct vop_strategy_args *ap)
2148 {
2149         struct buf *bp;
2150         int error;
2151
2152         bp = ap->a_bio->bio_buf;
2153
2154         switch(bp->b_cmd) {
2155         case BUF_CMD_READ:
2156                 error = hammer_vop_strategy_read(ap);
2157                 break;
2158         case BUF_CMD_WRITE:
2159                 error = hammer_vop_strategy_write(ap);
2160                 break;
2161         default:
2162                 bp->b_error = error = EINVAL;
2163                 bp->b_flags |= B_ERROR;
2164                 biodone(ap->a_bio);
2165                 break;
2166         }
2167         return (error);
2168 }
2169
2170 /*
2171  * Read from a regular file.  Iterate the related records and fill in the
2172  * BIO/BUF.  Gaps are zero-filled.
2173  *
2174  * The support code in hammer_object.c should be used to deal with mixed
2175  * in-memory and on-disk records.
2176  *
2177  * NOTE: Can be called from the cluster code with an oversized buf.
2178  *
2179  * XXX atime update
2180  */
2181 static
2182 int
2183 hammer_vop_strategy_read(struct vop_strategy_args *ap)
2184 {
2185         struct hammer_transaction trans;
2186         struct hammer_inode *ip;
2187         struct hammer_cursor cursor;
2188         hammer_base_elm_t base;
2189         hammer_off_t disk_offset;
2190         struct bio *bio;
2191         struct bio *nbio;
2192         struct buf *bp;
2193         int64_t rec_offset;
2194         int64_t ran_end;
2195         int64_t tmp64;
2196         int error;
2197         int boff;
2198         int roff;
2199         int n;
2200
2201         bio = ap->a_bio;
2202         bp = bio->bio_buf;
2203         ip = ap->a_vp->v_data;
2204
2205         /*
2206          * The zone-2 disk offset may have been set by the cluster code via
2207          * a BMAP operation, or else should be NOOFFSET.
2208          *
2209          * Checking the high bits for a match against zone-2 should suffice.
2210          */
2211         nbio = push_bio(bio);
2212         if ((nbio->bio_offset & HAMMER_OFF_ZONE_MASK) ==
2213             HAMMER_ZONE_LARGE_DATA) {
2214                 error = hammer_io_direct_read(ip->hmp, nbio, NULL);
2215                 return (error);
2216         }
2217
2218         /*
2219          * Well, that sucked.  Do it the hard way.  If all the stars are
2220          * aligned we may still be able to issue a direct-read.
2221          */
2222         hammer_simple_transaction(&trans, ip->hmp);
2223         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
2224
2225         /*
2226          * Key range (begin and end inclusive) to scan.  Note that the key's
2227          * stored in the actual records represent BASE+LEN, not BASE.  The
2228          * first record containing bio_offset will have a key > bio_offset.
2229          */
2230         cursor.key_beg.localization = ip->obj_localization +
2231                                       HAMMER_LOCALIZE_MISC;
2232         cursor.key_beg.obj_id = ip->obj_id;
2233         cursor.key_beg.create_tid = 0;
2234         cursor.key_beg.delete_tid = 0;
2235         cursor.key_beg.obj_type = 0;
2236         cursor.key_beg.key = bio->bio_offset + 1;
2237         cursor.asof = ip->obj_asof;
2238         cursor.flags |= HAMMER_CURSOR_ASOF;
2239
2240         cursor.key_end = cursor.key_beg;
2241         KKASSERT(ip->ino_data.obj_type == HAMMER_OBJTYPE_REGFILE);
2242 #if 0
2243         if (ip->ino_data.obj_type == HAMMER_OBJTYPE_DBFILE) {
2244                 cursor.key_beg.rec_type = HAMMER_RECTYPE_DB;
2245                 cursor.key_end.rec_type = HAMMER_RECTYPE_DB;
2246                 cursor.key_end.key = 0x7FFFFFFFFFFFFFFFLL;
2247         } else
2248 #endif
2249         {
2250                 ran_end = bio->bio_offset + bp->b_bufsize;
2251                 cursor.key_beg.rec_type = HAMMER_RECTYPE_DATA;
2252                 cursor.key_end.rec_type = HAMMER_RECTYPE_DATA;
2253                 tmp64 = ran_end + MAXPHYS + 1;  /* work-around GCC-4 bug */
2254                 if (tmp64 < ran_end)
2255                         cursor.key_end.key = 0x7FFFFFFFFFFFFFFFLL;
2256                 else
2257                         cursor.key_end.key = ran_end + MAXPHYS + 1;
2258         }
2259         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE;
2260
2261         error = hammer_ip_first(&cursor);
2262         boff = 0;
2263
2264         while (error == 0) {
2265                 /*
2266                  * Get the base file offset of the record.  The key for
2267                  * data records is (base + bytes) rather then (base).
2268                  */
2269                 base = &cursor.leaf->base;
2270                 rec_offset = base->key - cursor.leaf->data_len;
2271
2272                 /*
2273                  * Calculate the gap, if any, and zero-fill it.
2274                  *
2275                  * n is the offset of the start of the record verses our
2276                  * current seek offset in the bio.
2277                  */
2278                 n = (int)(rec_offset - (bio->bio_offset + boff));
2279                 if (n > 0) {
2280                         if (n > bp->b_bufsize - boff)
2281                                 n = bp->b_bufsize - boff;
2282                         bzero((char *)bp->b_data + boff, n);
2283                         boff += n;
2284                         n = 0;
2285                 }
2286
2287                 /*
2288                  * Calculate the data offset in the record and the number
2289                  * of bytes we can copy.
2290                  *
2291                  * There are two degenerate cases.  First, boff may already
2292                  * be at bp->b_bufsize.  Secondly, the data offset within
2293                  * the record may exceed the record's size.
2294                  */
2295                 roff = -n;
2296                 rec_offset += roff;
2297                 n = cursor.leaf->data_len - roff;
2298                 if (n <= 0) {
2299                         kprintf("strategy_read: bad n=%d roff=%d\n", n, roff);
2300                         n = 0;
2301                 } else if (n > bp->b_bufsize - boff) {
2302                         n = bp->b_bufsize - boff;
2303                 }
2304
2305                 /*
2306                  * Deal with cached truncations.  This cool bit of code
2307                  * allows truncate()/ftruncate() to avoid having to sync
2308                  * the file.
2309                  *
2310                  * If the frontend is truncated then all backend records are
2311                  * subject to the frontend's truncation.
2312                  *
2313                  * If the backend is truncated then backend records on-disk
2314                  * (but not in-memory) are subject to the backend's
2315                  * truncation.  In-memory records owned by the backend
2316                  * represent data written after the truncation point on the
2317                  * backend and must not be truncated.
2318                  *
2319                  * Truncate operations deal with frontend buffer cache
2320                  * buffers and frontend-owned in-memory records synchronously.
2321                  */
2322                 if (ip->flags & HAMMER_INODE_TRUNCATED) {
2323                         if (hammer_cursor_ondisk(&cursor) ||
2324                             cursor.iprec->flush_state == HAMMER_FST_FLUSH) {
2325                                 if (ip->trunc_off <= rec_offset)
2326                                         n = 0;
2327                                 else if (ip->trunc_off < rec_offset + n)
2328                                         n = (int)(ip->trunc_off - rec_offset);
2329                         }
2330                 }
2331                 if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
2332                         if (hammer_cursor_ondisk(&cursor)) {
2333                                 if (ip->sync_trunc_off <= rec_offset)
2334                                         n = 0;
2335                                 else if (ip->sync_trunc_off < rec_offset + n)
2336                                         n = (int)(ip->sync_trunc_off - rec_offset);
2337                         }
2338                 }
2339
2340                 /*
2341                  * Try to issue a direct read into our bio if possible,
2342                  * otherwise resolve the element data into a hammer_buffer
2343                  * and copy.
2344                  *
2345                  * The buffer on-disk should be zerod past any real
2346                  * truncation point, but may not be for any synthesized
2347                  * truncation point from above.
2348                  */
2349                 disk_offset = cursor.leaf->data_offset + roff;
2350                 if (boff == 0 && n == bp->b_bufsize &&
2351                     hammer_cursor_ondisk(&cursor) &&
2352                     (disk_offset & HAMMER_BUFMASK) == 0) {
2353                         KKASSERT((disk_offset & HAMMER_OFF_ZONE_MASK) ==
2354                                  HAMMER_ZONE_LARGE_DATA);
2355                         nbio->bio_offset = disk_offset;
2356                         error = hammer_io_direct_read(trans.hmp, nbio,
2357                                                       cursor.leaf);
2358                         goto done;
2359                 } else if (n) {
2360                         error = hammer_ip_resolve_data(&cursor);
2361                         if (error == 0) {
2362                                 bcopy((char *)cursor.data + roff,
2363                                       (char *)bp->b_data + boff, n);
2364                         }
2365                 }
2366                 if (error)
2367                         break;
2368
2369                 /*
2370                  * Iterate until we have filled the request.
2371                  */
2372                 boff += n;
2373                 if (boff == bp->b_bufsize)
2374                         break;
2375                 error = hammer_ip_next(&cursor);
2376         }
2377
2378         /*
2379          * There may have been a gap after the last record
2380          */
2381         if (error == ENOENT)
2382                 error = 0;
2383         if (error == 0 && boff != bp->b_bufsize) {
2384                 KKASSERT(boff < bp->b_bufsize);
2385                 bzero((char *)bp->b_data + boff, bp->b_bufsize - boff);
2386                 /* boff = bp->b_bufsize; */
2387         }
2388         bp->b_resid = 0;
2389         bp->b_error = error;
2390         if (error)
2391                 bp->b_flags |= B_ERROR;
2392         biodone(ap->a_bio);
2393
2394 done:
2395         if (cursor.node)
2396                 hammer_cache_node(&ip->cache[1], cursor.node);
2397         hammer_done_cursor(&cursor);
2398         hammer_done_transaction(&trans);
2399         return(error);
2400 }
2401
2402 /*
2403  * BMAP operation - used to support cluster_read() only.
2404  *
2405  * (struct vnode *vp, off_t loffset, off_t *doffsetp, int *runp, int *runb)
2406  *
2407  * This routine may return EOPNOTSUPP if the opration is not supported for
2408  * the specified offset.  The contents of the pointer arguments do not
2409  * need to be initialized in that case.
2410  *
2411  * If a disk address is available and properly aligned return 0 with
2412  * *doffsetp set to the zone-2 address, and *runp / *runb set appropriately
2413  * to the run-length relative to that offset.  Callers may assume that
2414  * *doffsetp is valid if 0 is returned, even if *runp is not sufficiently
2415  * large, so return EOPNOTSUPP if it is not sufficiently large.
2416  */
2417 static
2418 int
2419 hammer_vop_bmap(struct vop_bmap_args *ap)
2420 {
2421         struct hammer_transaction trans;
2422         struct hammer_inode *ip;
2423         struct hammer_cursor cursor;
2424         hammer_base_elm_t base;
2425         int64_t rec_offset;
2426         int64_t ran_end;
2427         int64_t tmp64;
2428         int64_t base_offset;
2429         int64_t base_disk_offset;
2430         int64_t last_offset;
2431         hammer_off_t last_disk_offset;
2432         hammer_off_t disk_offset;
2433         int     rec_len;
2434         int     error;
2435         int     blksize;
2436
2437         ++hammer_stats_file_iopsr;
2438         ip = ap->a_vp->v_data;
2439
2440         /*
2441          * We can only BMAP regular files.  We can't BMAP database files,
2442          * directories, etc.
2443          */
2444         if (ip->ino_data.obj_type != HAMMER_OBJTYPE_REGFILE)
2445                 return(EOPNOTSUPP);
2446
2447         /*
2448          * bmap is typically called with runp/runb both NULL when used
2449          * for writing.  We do not support BMAP for writing atm.
2450          */
2451         if (ap->a_cmd != BUF_CMD_READ)
2452                 return(EOPNOTSUPP);
2453
2454         /*
2455          * Scan the B-Tree to acquire blockmap addresses, then translate
2456          * to raw addresses.
2457          */
2458         hammer_simple_transaction(&trans, ip->hmp);
2459 #if 0
2460         kprintf("bmap_beg %016llx ip->cache %p\n", ap->a_loffset, ip->cache[1]);
2461 #endif
2462         hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
2463
2464         /*
2465          * Key range (begin and end inclusive) to scan.  Note that the key's
2466          * stored in the actual records represent BASE+LEN, not BASE.  The
2467          * first record containing bio_offset will have a key > bio_offset.
2468          */
2469         cursor.key_beg.localization = ip->obj_localization +
2470                                       HAMMER_LOCALIZE_MISC;
2471         cursor.key_beg.obj_id = ip->obj_id;
2472         cursor.key_beg.create_tid = 0;
2473         cursor.key_beg.delete_tid = 0;
2474         cursor.key_beg.obj_type = 0;
2475         if (ap->a_runb)
2476                 cursor.key_beg.key = ap->a_loffset - MAXPHYS + 1;
2477         else
2478                 cursor.key_beg.key = ap->a_loffset + 1;
2479         if (cursor.key_beg.key < 0)
2480                 cursor.key_beg.key = 0;
2481         cursor.asof = ip->obj_asof;
2482         cursor.flags |= HAMMER_CURSOR_ASOF;
2483
2484         cursor.key_end = cursor.key_beg;
2485         KKASSERT(ip->ino_data.obj_type == HAMMER_OBJTYPE_REGFILE);
2486
2487         ran_end = ap->a_loffset + MAXPHYS;
2488         cursor.key_beg.rec_type = HAMMER_RECTYPE_DATA;
2489         cursor.key_end.rec_type = HAMMER_RECTYPE_DATA;
2490         tmp64 = ran_end + MAXPHYS + 1;  /* work-around GCC-4 bug */
2491         if (tmp64 < ran_end)
2492                 cursor.key_end.key = 0x7FFFFFFFFFFFFFFFLL;
2493         else
2494                 cursor.key_end.key = ran_end + MAXPHYS + 1;
2495
2496         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE;
2497
2498         error = hammer_ip_first(&cursor);
2499         base_offset = last_offset = 0;
2500         base_disk_offset = last_disk_offset = 0;
2501
2502         while (error == 0) {
2503                 /*
2504                  * Get the base file offset of the record.  The key for
2505                  * data records is (base + bytes) rather then (base).
2506                  *
2507                  * NOTE: rec_offset + rec_len may exceed the end-of-file.
2508                  * The extra bytes should be zero on-disk and the BMAP op
2509                  * should still be ok.
2510                  */
2511                 base = &cursor.leaf->base;
2512                 rec_offset = base->key - cursor.leaf->data_len;
2513                 rec_len    = cursor.leaf->data_len;
2514
2515                 /*
2516                  * Incorporate any cached truncation.
2517                  *
2518                  * NOTE: Modifications to rec_len based on synthesized
2519                  * truncation points remove the guarantee that any extended
2520                  * data on disk is zero (since the truncations may not have
2521                  * taken place on-media yet).
2522                  */
2523                 if (ip->flags & HAMMER_INODE_TRUNCATED) {
2524                         if (hammer_cursor_ondisk(&cursor) ||
2525                             cursor.iprec->flush_state == HAMMER_FST_FLUSH) {
2526                                 if (ip->trunc_off <= rec_offset)
2527                                         rec_len = 0;
2528                                 else if (ip->trunc_off < rec_offset + rec_len)
2529                                         rec_len = (int)(ip->trunc_off - rec_offset);
2530                         }
2531                 }
2532                 if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
2533                         if (hammer_cursor_ondisk(&cursor)) {
2534                                 if (ip->sync_trunc_off <= rec_offset)
2535                                         rec_len = 0;
2536                                 else if (ip->sync_trunc_off < rec_offset + rec_len)
2537                                         rec_len = (int)(ip->sync_trunc_off - rec_offset);
2538                         }
2539                 }
2540
2541                 /*
2542                  * Accumulate information.  If we have hit a discontiguous
2543                  * block reset base_offset unless we are already beyond the
2544                  * requested offset.  If we are, that's it, we stop.
2545                  */
2546                 if (error)
2547                         break;
2548                 if (hammer_cursor_ondisk(&cursor)) {
2549                         disk_offset = cursor.leaf->data_offset;
2550                         if (rec_offset != last_offset ||
2551                             disk_offset != last_disk_offset) {
2552                                 if (rec_offset > ap->a_loffset)
2553                                         break;
2554                                 base_offset = rec_offset;
2555                                 base_disk_offset = disk_offset;
2556                         }
2557                         last_offset = rec_offset + rec_len;
2558                         last_disk_offset = disk_offset + rec_len;
2559                 }
2560                 error = hammer_ip_next(&cursor);
2561         }
2562
2563 #if 0
2564         kprintf("BMAP %016llx:  %016llx - %016llx\n",
2565                 ap->a_loffset, base_offset, last_offset);
2566         kprintf("BMAP %16s:  %016llx - %016llx\n",
2567                 "", base_disk_offset, last_disk_offset);
2568 #endif
2569
2570         if (cursor.node) {
2571                 hammer_cache_node(&ip->cache[1], cursor.node);
2572 #if 0
2573                 kprintf("bmap_end2 %016llx ip->cache %p\n", ap->a_loffset, ip->cache[1]);
2574 #endif
2575         }
2576         hammer_done_cursor(&cursor);
2577         hammer_done_transaction(&trans);
2578
2579         /*
2580          * If we couldn't find any records or the records we did find were
2581          * all behind the requested offset, return failure.  A forward
2582          * truncation can leave a hole w/ no on-disk records.
2583          */
2584         if (last_offset == 0 || last_offset < ap->a_loffset)
2585                 return (EOPNOTSUPP);
2586
2587         /*
2588          * Figure out the block size at the requested offset and adjust
2589          * our limits so the cluster_read() does not create inappropriately
2590          * sized buffer cache buffers.
2591          */
2592         blksize = hammer_blocksize(ap->a_loffset);
2593         if (hammer_blocksize(base_offset) != blksize) {
2594                 base_offset = hammer_blockdemarc(base_offset, ap->a_loffset);
2595         }
2596         if (last_offset != ap->a_loffset &&
2597             hammer_blocksize(last_offset - 1) != blksize) {
2598                 last_offset = hammer_blockdemarc(ap->a_loffset,
2599                                                  last_offset - 1);
2600         }
2601
2602         /*
2603          * Returning EOPNOTSUPP simply prevents the direct-IO optimization
2604          * from occuring.
2605          */
2606         disk_offset = base_disk_offset + (ap->a_loffset - base_offset);
2607
2608         if ((disk_offset & HAMMER_OFF_ZONE_MASK) != HAMMER_ZONE_LARGE_DATA) {
2609                 /*
2610                  * Only large-data zones can be direct-IOd
2611                  */
2612                 error = EOPNOTSUPP;
2613         } else if ((disk_offset & HAMMER_BUFMASK) ||
2614                    (last_offset - ap->a_loffset) < blksize) {
2615                 /*
2616                  * doffsetp is not aligned or the forward run size does
2617                  * not cover a whole buffer, disallow the direct I/O.
2618                  */
2619                 error = EOPNOTSUPP;
2620         } else {
2621                 /*
2622                  * We're good.
2623                  */
2624                 *ap->a_doffsetp = disk_offset;
2625                 if (ap->a_runb) {
2626                         *ap->a_runb = ap->a_loffset - base_offset;
2627                         KKASSERT(*ap->a_runb >= 0);
2628                 }
2629                 if (ap->a_runp) {
2630                         *ap->a_runp = last_offset - ap->a_loffset;
2631                         KKASSERT(*ap->a_runp >= 0);
2632                 }
2633                 error = 0;
2634         }
2635         return(error);
2636 }
2637
2638 /*
2639  * Write to a regular file.   Because this is a strategy call the OS is
2640  * trying to actually get data onto the media.
2641  */
2642 static
2643 int
2644 hammer_vop_strategy_write(struct vop_strategy_args *ap)
2645 {
2646         hammer_record_t record;
2647         hammer_mount_t hmp;
2648         hammer_inode_t ip;
2649         struct bio *bio;
2650         struct buf *bp;
2651         int blksize;
2652         int bytes;
2653         int error;
2654
2655         bio = ap->a_bio;
2656         bp = bio->bio_buf;
2657         ip = ap->a_vp->v_data;
2658         hmp = ip->hmp;
2659
2660         blksize = hammer_blocksize(bio->bio_offset);
2661         KKASSERT(bp->b_bufsize == blksize);
2662
2663         if (ip->flags & HAMMER_INODE_RO) {
2664                 bp->b_error = EROFS;
2665                 bp->b_flags |= B_ERROR;
2666                 biodone(ap->a_bio);
2667                 return(EROFS);
2668         }
2669
2670         /*
2671          * Interlock with inode destruction (no in-kernel or directory
2672          * topology visibility).  If we queue new IO while trying to
2673          * destroy the inode we can deadlock the vtrunc call in
2674          * hammer_inode_unloadable_check().
2675          *
2676          * Besides, there's no point flushing a bp associated with an
2677          * inode that is being destroyed on-media and has no kernel
2678          * references.
2679          */
2680         if ((ip->flags | ip->sync_flags) &
2681             (HAMMER_INODE_DELETING|HAMMER_INODE_DELETED)) {
2682                 bp->b_resid = 0;
2683                 biodone(ap->a_bio);
2684                 return(0);
2685         }
2686
2687         /*
2688          * Reserve space and issue a direct-write from the front-end.
2689          * NOTE: The direct_io code will hammer_bread/bcopy smaller
2690          * allocations.
2691          *
2692          * An in-memory record will be installed to reference the storage
2693          * until the flusher can get to it.
2694          *
2695          * Since we own the high level bio the front-end will not try to
2696          * do a direct-read until the write completes.
2697          *
2698          * NOTE: The only time we do not reserve a full-sized buffers
2699          * worth of data is if the file is small.  We do not try to
2700          * allocate a fragment (from the small-data zone) at the end of
2701          * an otherwise large file as this can lead to wildly separated
2702          * data.
2703          */
2704         KKASSERT((bio->bio_offset & HAMMER_BUFMASK) == 0);
2705         KKASSERT(bio->bio_offset < ip->ino_data.size);
2706         if (bio->bio_offset || ip->ino_data.size > HAMMER_BUFSIZE / 2)
2707                 bytes = bp->b_bufsize;
2708         else
2709                 bytes = ((int)ip->ino_data.size + 15) & ~15;
2710
2711         record = hammer_ip_add_bulk(ip, bio->bio_offset, bp->b_data,
2712                                     bytes, &error);
2713         if (record) {
2714                 hammer_io_direct_write(hmp, record, bio);
2715                 if (ip->rsv_recs > 1 && hmp->rsv_recs > hammer_limit_recs)
2716                         hammer_flush_inode(ip, 0);
2717         } else {
2718                 bp->b_bio2.bio_offset = NOOFFSET;
2719                 bp->b_error = error;
2720                 bp->b_flags |= B_ERROR;
2721                 biodone(ap->a_bio);
2722         }
2723         return(error);
2724 }
2725
2726 /*
2727  * dounlink - disconnect a directory entry
2728  *
2729  * XXX whiteout support not really in yet
2730  */
2731 static int
2732 hammer_dounlink(hammer_transaction_t trans, struct nchandle *nch,
2733                 struct vnode *dvp, struct ucred *cred,
2734                 int flags, int isdir)
2735 {
2736         struct namecache *ncp;
2737         hammer_inode_t dip;
2738         hammer_inode_t ip;
2739         struct hammer_cursor cursor;
2740         int64_t namekey;
2741         u_int32_t max_iterations;
2742         int nlen, error;
2743
2744         /*
2745          * Calculate the namekey and setup the key range for the scan.  This
2746          * works kinda like a chained hash table where the lower 32 bits
2747          * of the namekey synthesize the chain.
2748          *
2749          * The key range is inclusive of both key_beg and key_end.
2750          */
2751         dip = VTOI(dvp);
2752         ncp = nch->ncp;
2753
2754         if (dip->flags & HAMMER_INODE_RO)
2755                 return (EROFS);
2756
2757         namekey = hammer_directory_namekey(dip, ncp->nc_name, ncp->nc_nlen,
2758                                            &max_iterations);
2759 retry:
2760         hammer_init_cursor(trans, &cursor, &dip->cache[1], dip);
2761         cursor.key_beg.localization = dip->obj_localization +
2762                                       HAMMER_LOCALIZE_MISC;
2763         cursor.key_beg.obj_id = dip->obj_id;
2764         cursor.key_beg.key = namekey;
2765         cursor.key_beg.create_tid = 0;
2766         cursor.key_beg.delete_tid = 0;
2767         cursor.key_beg.rec_type = HAMMER_RECTYPE_DIRENTRY;
2768         cursor.key_beg.obj_type = 0;
2769
2770         cursor.key_end = cursor.key_beg;
2771         cursor.key_end.key += max_iterations;
2772         cursor.asof = dip->obj_asof;
2773         cursor.flags |= HAMMER_CURSOR_END_INCLUSIVE | HAMMER_CURSOR_ASOF;
2774
2775         /*
2776          * Scan all matching records (the chain), locate the one matching
2777          * the requested path component.  info->last_error contains the
2778          * error code on search termination and could be 0, ENOENT, or
2779          * something else.
2780          *
2781          * The hammer_ip_*() functions merge in-memory records with on-disk
2782          * records for the purposes of the search.
2783          */
2784         error = hammer_ip_first(&cursor);
2785
2786         while (error == 0) {
2787                 error = hammer_ip_resolve_data(&cursor);
2788                 if (error)
2789                         break;
2790                 nlen = cursor.leaf->data_len - HAMMER_ENTRY_NAME_OFF;
2791                 KKASSERT(nlen > 0);
2792                 if (ncp->nc_nlen == nlen &&
2793                     bcmp(ncp->nc_name, cursor.data->entry.name, nlen) == 0) {
2794                         break;
2795                 }
2796                 error = hammer_ip_next(&cursor);
2797         }
2798
2799         /*
2800          * If all is ok we have to get the inode so we can adjust nlinks.
2801          * To avoid a deadlock with the flusher we must release the inode
2802          * lock on the directory when acquiring the inode for the entry.
2803          *
2804          * If the target is a directory, it must be empty.
2805          */
2806         if (error == 0) {
2807                 hammer_unlock(&cursor.ip->lock);
2808                 ip = hammer_get_inode(trans, dip, cursor.data->entry.obj_id,
2809                                       dip->hmp->asof,
2810                                       cursor.data->entry.localization,
2811                                       0, &error);
2812                 hammer_lock_sh(&cursor.ip->lock);
2813                 if (error == ENOENT) {
2814                         kprintf("obj_id %016llx\n", cursor.data->entry.obj_id);
2815                         Debugger("ENOENT unlinking object that should exist");
2816                 }
2817
2818                 /*
2819                  * If isdir >= 0 we validate that the entry is or is not a
2820                  * directory.  If isdir < 0 we don't care.
2821                  */
2822                 if (error == 0 && isdir >= 0) {
2823                         if (isdir &&
2824                             ip->ino_data.obj_type != HAMMER_OBJTYPE_DIRECTORY) {
2825                                 error = ENOTDIR;
2826                         } else if (isdir == 0 &&
2827                             ip->ino_data.obj_type == HAMMER_OBJTYPE_DIRECTORY) {
2828                                 error = EISDIR;
2829                         }
2830                 }
2831
2832                 /*
2833                  * If we are trying to remove a directory the directory must
2834                  * be empty.
2835                  *
2836                  * The check directory code can loop and deadlock/retry.  Our
2837                  * own cursor's node locks must be released to avoid a 3-way
2838                  * deadlock with the flusher if the check directory code
2839                  * blocks.
2840                  *
2841                  * If any changes whatsoever have been made to the cursor
2842                  * set EDEADLK and retry.
2843                  */
2844                 if (error == 0 && ip->ino_data.obj_type ==
2845                                   HAMMER_OBJTYPE_DIRECTORY) {
2846                         hammer_unlock_cursor(&cursor);
2847                         error = hammer_ip_check_directory_empty(trans, ip);
2848                         hammer_lock_cursor(&cursor);
2849                         if (cursor.flags & HAMMER_CURSOR_RETEST) {
2850                                 kprintf("HAMMER: Warning: avoided deadlock "
2851                                         "on rmdir '%s'\n",
2852                                         ncp->nc_name);
2853                                 error = EDEADLK;
2854                         }
2855                 }
2856
2857                 /*
2858                  * Delete the directory entry.
2859                  *
2860                  * WARNING: hammer_ip_del_directory() may have to terminate
2861                  * the cursor to avoid a deadlock.  It is ok to call
2862                  * hammer_done_cursor() twice.
2863                  */
2864                 if (error == 0) {
2865                         error = hammer_ip_del_directory(trans, &cursor,
2866                                                         dip, ip);
2867                 }
2868                 hammer_done_cursor(&cursor);
2869                 if (error == 0) {
2870                         cache_setunresolved(nch);
2871                         cache_setvp(nch, NULL);
2872                         /* XXX locking */
2873                         if (ip->vp) {
2874                                 hammer_knote(ip->vp, NOTE_DELETE);
2875                                 cache_inval_vp(ip->vp, CINV_DESTROY);
2876                         }
2877                 }
2878                 if (ip)
2879                         hammer_rel_inode(ip, 0);
2880         } else {
2881                 hammer_done_cursor(&cursor);
2882         }
2883         if (error == EDEADLK)
2884                 goto retry;
2885
2886         return (error);
2887 }
2888
2889 /************************************************************************
2890  *                          FIFO AND SPECFS OPS                         *
2891  ************************************************************************
2892  *
2893  */
2894
2895 static int
2896 hammer_vop_fifoclose (struct vop_close_args *ap)
2897 {
2898         /* XXX update itimes */
2899         return (VOCALL(&fifo_vnode_vops, &ap->a_head));
2900 }
2901
2902 static int
2903 hammer_vop_fiforead (struct vop_read_args *ap)
2904 {
2905         int error;
2906
2907         error = VOCALL(&fifo_vnode_vops, &ap->a_head);
2908         /* XXX update access time */
2909         return (error);
2910 }
2911
2912 static int
2913 hammer_vop_fifowrite (struct vop_write_args *ap)
2914 {
2915         int error;
2916
2917         error = VOCALL(&fifo_vnode_vops, &ap->a_head);
2918         /* XXX update access time */
2919         return (error);
2920 }
2921
2922 static
2923 int
2924 hammer_vop_fifokqfilter(struct vop_kqfilter_args *ap)
2925 {
2926         int error;
2927
2928         error = VOCALL(&fifo_vnode_vops, &ap->a_head);
2929         if (error)
2930                 error = hammer_vop_kqfilter(ap);
2931         return(error);
2932 }
2933
2934 static int
2935 hammer_vop_specclose (struct vop_close_args *ap)
2936 {
2937         /* XXX update itimes */
2938         return (VOCALL(&spec_vnode_vops, &ap->a_head));
2939 }
2940
2941 static int
2942 hammer_vop_specread (struct vop_read_args *ap)
2943 {
2944         /* XXX update access time */
2945         return (VOCALL(&spec_vnode_vops, &ap->a_head));
2946 }
2947
2948 static int
2949 hammer_vop_specwrite (struct vop_write_args *ap)
2950 {
2951         /* XXX update last change time */
2952         return (VOCALL(&spec_vnode_vops, &ap->a_head));
2953 }
2954
2955 /************************************************************************
2956  *                          KQFILTER OPS                                *
2957  ************************************************************************
2958  *
2959  */
2960 static void filt_hammerdetach(struct knote *kn);
2961 static int filt_hammerread(struct knote *kn, long hint);
2962 static int filt_hammerwrite(struct knote *kn, long hint);
2963 static int filt_hammervnode(struct knote *kn, long hint);
2964
2965 static struct filterops hammerread_filtops =
2966         { 1, NULL, filt_hammerdetach, filt_hammerread };
2967 static struct filterops hammerwrite_filtops =
2968         { 1, NULL, filt_hammerdetach, filt_hammerwrite };
2969 static struct filterops hammervnode_filtops =
2970         { 1, NULL, filt_hammerdetach, filt_hammervnode };
2971
2972 static
2973 int
2974 hammer_vop_kqfilter(struct vop_kqfilter_args *ap)
2975 {
2976         struct vnode *vp = ap->a_vp;
2977         struct knote *kn = ap->a_kn;
2978         lwkt_tokref ilock;
2979
2980         switch (kn->kn_filter) {
2981         case EVFILT_READ:
2982                 kn->kn_fop = &hammerread_filtops;
2983                 break;
2984         case EVFILT_WRITE:
2985                 kn->kn_fop = &hammerwrite_filtops;
2986                 break;
2987         case EVFILT_VNODE:
2988                 kn->kn_fop = &hammervnode_filtops;
2989                 break;
2990         default:
2991                 return (1);
2992         }
2993
2994         kn->kn_hook = (caddr_t)vp;
2995
2996         lwkt_gettoken(&ilock, &vp->v_pollinfo.vpi_token);
2997         SLIST_INSERT_HEAD(&vp->v_pollinfo.vpi_selinfo.si_note, kn, kn_selnext);
2998         lwkt_reltoken(&ilock);
2999
3000         return(0);
3001 }
3002
3003 static void
3004 filt_hammerdetach(struct knote *kn)
3005 {
3006         struct vnode *vp = (void *)kn->kn_hook;
3007         lwkt_tokref ilock;
3008
3009         lwkt_gettoken(&ilock, &vp->v_pollinfo.vpi_token);
3010         SLIST_REMOVE(&vp->v_pollinfo.vpi_selinfo.si_note,
3011                      kn, knote, kn_selnext);
3012         lwkt_reltoken(&ilock);
3013 }
3014
3015 static int
3016 filt_hammerread(struct knote *kn, long hint)
3017 {
3018         struct vnode *vp = (void *)kn->kn_hook;
3019         hammer_inode_t ip = VTOI(vp);
3020
3021         if (hint == NOTE_REVOKE) {
3022                 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
3023                 return(1);
3024         }
3025         kn->kn_data = ip->ino_data.size - kn->kn_fp->f_offset;
3026         return (kn->kn_data != 0);
3027 }
3028
3029 static int
3030 filt_hammerwrite(struct knote *kn, long hint)
3031 {
3032         if (hint == NOTE_REVOKE)
3033                 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
3034         kn->kn_data = 0;
3035         return (1);
3036 }
3037
3038 static int
3039 filt_hammervnode(struct knote *kn, long hint)
3040 {
3041         if (kn->kn_sfflags & hint)
3042                 kn->kn_fflags |= hint;
3043         if (hint == NOTE_REVOKE) {
3044                 kn->kn_flags |= EV_EOF;
3045                 return (1);
3046         }
3047         return (kn->kn_fflags != 0);
3048 }
3049