sys/vfs/hammer/hammer_inode.c

   1 /*
   2  * Copyright (c) 2007-2008 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/vfs/hammer/hammer_inode.c,v 1.81 2008/06/21 20:21:58 dillon Exp $
  35  */
  36
  37 #include "hammer.h"
  38 #include <vm/vm_extern.h>
  39 #include <sys/buf.h>
  40 #include <sys/buf2.h>
  41
  42 static int      hammer_unload_inode(struct hammer_inode *ip);
  43 static void     hammer_flush_inode_core(hammer_inode_t ip, int flags);
  44 static int      hammer_setup_child_callback(hammer_record_t rec, void *data);
  45 static int      hammer_setup_parent_inodes(hammer_inode_t ip);
  46 static int      hammer_setup_parent_inodes_helper(hammer_record_t record);
  47 static void     hammer_inode_wakereclaims(hammer_inode_t ip);
  48
  49 #ifdef DEBUG_TRUNCATE
  50 extern struct hammer_inode *HammerTruncIp;
  51 #endif
  52
  53 /*
  54  * Red-Black tree support for inode structures.
  55  *
  56  * Insertions
  57  */
  58 int
  59 hammer_ino_rb_compare(hammer_inode_t ip1, hammer_inode_t ip2)
  60 {
  61         if (ip1->obj_localization < ip2->obj_localization)
  62                 return(-1);
  63         if (ip1->obj_localization > ip2->obj_localization)
  64                 return(1);
  65         if (ip1->obj_id < ip2->obj_id)
  66                 return(-1);
  67         if (ip1->obj_id > ip2->obj_id)
  68                 return(1);
  69         if (ip1->obj_asof < ip2->obj_asof)
  70                 return(-1);
  71         if (ip1->obj_asof > ip2->obj_asof)
  72                 return(1);
  73         return(0);
  74 }
  75
  76 /*
  77  * LOOKUP_INFO
  78  */
  79 static int
  80 hammer_inode_info_cmp(hammer_inode_info_t info, hammer_inode_t ip)
  81 {
  82         if (info->obj_localization < ip->obj_localization)
  83                 return(-1);
  84         if (info->obj_localization > ip->obj_localization)
  85                 return(1);
  86         if (info->obj_id < ip->obj_id)
  87                 return(-1);
  88         if (info->obj_id > ip->obj_id)
  89                 return(1);
  90         if (info->obj_asof < ip->obj_asof)
  91                 return(-1);
  92         if (info->obj_asof > ip->obj_asof)
  93                 return(1);
  94         return(0);
  95 }
  96
  97 /*
  98  * Used by hammer_scan_inode_snapshots() to locate all of an object's
  99  * snapshots.  Note that the asof field is not tested, which we can get
 100  * away with because it is the lowest-priority field.
 101  */
 102 static int
 103 hammer_inode_info_cmp_all_history(hammer_inode_t ip, void *data)
 104 {
 105         hammer_inode_info_t info = data;
 106
 107         if (ip->obj_localization > info->obj_localization)
 108                 return(1);
 109         if (ip->obj_localization < info->obj_localization)
 110                 return(-1);
 111         if (ip->obj_id > info->obj_id)
 112                 return(1);
 113         if (ip->obj_id < info->obj_id)
 114                 return(-1);
 115         return(0);
 116 }
 117
 118 RB_GENERATE(hammer_ino_rb_tree, hammer_inode, rb_node, hammer_ino_rb_compare);
 119 RB_GENERATE_XLOOKUP(hammer_ino_rb_tree, INFO, hammer_inode, rb_node,
 120                 hammer_inode_info_cmp, hammer_inode_info_t);
 121
 122 /*
 123  * The kernel is not actively referencing this vnode but is still holding
 124  * it cached.
 125  *
 126  * This is called from the frontend.
 127  */
 128 int
 129 hammer_vop_inactive(struct vop_inactive_args *ap)
 130 {
 131         struct hammer_inode *ip = VTOI(ap->a_vp);
 132
 133         /*
 134          * Degenerate case
 135          */
 136         if (ip == NULL) {
 137                 vrecycle(ap->a_vp);
 138                 return(0);
 139         }
 140
 141         /*
 142          * If the inode no longer has visibility in the filesystem try to
 143          * recycle it immediately, even if the inode is dirty.  Recycling
 144          * it quickly allows the system to reclaim buffer cache and VM
 145          * resources which can matter a lot in a heavily loaded system.
 146          *
 147          * This can deadlock in vfsync() if we aren't careful.
 148          *
 149          * Do not queue the inode to the flusher if we still have visibility,
 150          * otherwise namespace calls such as chmod will unnecessarily generate
 151          * multiple inode updates.
 152          */
 153         hammer_inode_unloadable_check(ip, 0);
 154         if (ip->ino_data.nlinks == 0) {
 155                 if (ip->flags & HAMMER_INODE_MODMASK)
 156                         hammer_flush_inode(ip, 0);
 157                 vrecycle(ap->a_vp);
 158         }
 159         return(0);
 160 }
 161
 162 /*
 163  * Release the vnode association.  This is typically (but not always)
 164  * the last reference on the inode.
 165  *
 166  * Once the association is lost we are on our own with regards to
 167  * flushing the inode.
 168  */
 169 int
 170 hammer_vop_reclaim(struct vop_reclaim_args *ap)
 171 {
 172         struct hammer_inode *ip;
 173         hammer_mount_t hmp;
 174         struct vnode *vp;
 175
 176         vp = ap->a_vp;
 177
 178         if ((ip = vp->v_data) != NULL) {
 179                 hmp = ip->hmp;
 180                 vp->v_data = NULL;
 181                 ip->vp = NULL;
 182
 183                 if ((ip->flags & HAMMER_INODE_RECLAIM) == 0) {
 184                         ++hammer_count_reclaiming;
 185                         ++hmp->inode_reclaims;
 186                         ip->flags |= HAMMER_INODE_RECLAIM;
 187                         if (hmp->inode_reclaims > HAMMER_RECLAIM_FLUSH &&
 188                             (hmp->inode_reclaims & 255) == 0) {
 189                                 hammer_flusher_async(hmp);
 190                         }
 191                 }
 192                 hammer_rel_inode(ip, 1);
 193         }
 194         return(0);
 195 }
 196
 197 /*
 198  * Return a locked vnode for the specified inode.  The inode must be
 199  * referenced but NOT LOCKED on entry and will remain referenced on
 200  * return.
 201  *
 202  * Called from the frontend.
 203  */
 204 int
 205 hammer_get_vnode(struct hammer_inode *ip, struct vnode **vpp)
 206 {
 207         hammer_mount_t hmp;
 208         struct vnode *vp;
 209         int error = 0;
 210
 211         hmp = ip->hmp;
 212
 213         for (;;) {
 214                 if ((vp = ip->vp) == NULL) {
 215                         error = getnewvnode(VT_HAMMER, hmp->mp, vpp, 0, 0);
 216                         if (error)
 217                                 break;
 218                         hammer_lock_ex(&ip->lock);
 219                         if (ip->vp != NULL) {
 220                                 hammer_unlock(&ip->lock);
 221                                 vp->v_type = VBAD;
 222                                 vx_put(vp);
 223                                 continue;
 224                         }
 225                         hammer_ref(&ip->lock);
 226                         vp = *vpp;
 227                         ip->vp = vp;
 228                         vp->v_type =
 229                                 hammer_get_vnode_type(ip->ino_data.obj_type);
 230
 231                         hammer_inode_wakereclaims(ip);
 232
 233                         switch(ip->ino_data.obj_type) {
 234                         case HAMMER_OBJTYPE_CDEV:
 235                         case HAMMER_OBJTYPE_BDEV:
 236                                 vp->v_ops = &hmp->mp->mnt_vn_spec_ops;
 237                                 addaliasu(vp, ip->ino_data.rmajor,
 238                                           ip->ino_data.rminor);
 239                                 break;
 240                         case HAMMER_OBJTYPE_FIFO:
 241                                 vp->v_ops = &hmp->mp->mnt_vn_fifo_ops;
 242                                 break;
 243                         default:
 244                                 break;
 245                         }
 246
 247                         /*
 248                          * Only mark as the root vnode if the ip is not
 249                          * historical, otherwise the VFS cache will get
 250                          * confused.  The other half of the special handling
 251                          * is in hammer_vop_nlookupdotdot().
 252                          *
 253                          * Pseudo-filesystem roots also do not count.
 254                          */
 255                         if (ip->obj_id == HAMMER_OBJID_ROOT &&
 256                             ip->obj_asof == hmp->asof &&
 257                             ip->obj_localization == 0) {
 258                                 vp->v_flag |= VROOT;
 259                         }
 260
 261                         vp->v_data = (void *)ip;
 262                         /* vnode locked by getnewvnode() */
 263                         /* make related vnode dirty if inode dirty? */
 264                         hammer_unlock(&ip->lock);
 265                         if (vp->v_type == VREG)
 266                                 vinitvmio(vp, ip->ino_data.size);
 267                         break;
 268                 }
 269
 270                 /*
 271                  * loop if the vget fails (aka races), or if the vp
 272                  * no longer matches ip->vp.
 273                  */
 274                 if (vget(vp, LK_EXCLUSIVE) == 0) {
 275                         if (vp == ip->vp)
 276                                 break;
 277                         vput(vp);
 278                 }
 279         }
 280         *vpp = vp;
 281         return(error);
 282 }
 283
 284 /*
 285  * Locate all copies of the inode for obj_id compatible with the specified
 286  * asof, reference, and issue the related call-back.  This routine is used
 287  * for direct-io invalidation and does not create any new inodes.
 288  */
 289 void
 290 hammer_scan_inode_snapshots(hammer_mount_t hmp, hammer_inode_info_t iinfo,
 291                             int (*callback)(hammer_inode_t ip, void *data),
 292                             void *data)
 293 {
 294         hammer_ino_rb_tree_RB_SCAN(&hmp->rb_inos_root,
 295                                    hammer_inode_info_cmp_all_history,
 296                                    callback, iinfo);
 297 }
 298
 299 /*
 300  * Acquire a HAMMER inode.  The returned inode is not locked.  These functions
 301  * do not attach or detach the related vnode (use hammer_get_vnode() for
 302  * that).
 303  *
 304  * The flags argument is only applied for newly created inodes, and only
 305  * certain flags are inherited.
 306  *
 307  * Called from the frontend.
 308  */
 309 struct hammer_inode *
 310 hammer_get_inode(hammer_transaction_t trans, hammer_inode_t dip,
 311                  u_int64_t obj_id, hammer_tid_t asof, u_int32_t localization,
 312                  int flags, int *errorp)
 313 {
 314         hammer_mount_t hmp = trans->hmp;
 315         struct hammer_inode_info iinfo;
 316         struct hammer_cursor cursor;
 317         struct hammer_inode *ip;
 318
 319         /*
 320          * Determine if we already have an inode cached.  If we do then
 321          * we are golden.
 322          */
 323         iinfo.obj_id = obj_id;
 324         iinfo.obj_asof = asof;
 325         iinfo.obj_localization = localization;
 326 loop:
 327         ip = hammer_ino_rb_tree_RB_LOOKUP_INFO(&hmp->rb_inos_root, &iinfo);
 328         if (ip) {
 329                 hammer_ref(&ip->lock);
 330                 *errorp = 0;
 331                 return(ip);
 332         }
 333
 334         /*
 335          * Allocate a new inode structure and deal with races later.
 336          */
 337         ip = kmalloc(sizeof(*ip), M_HAMMER, M_WAITOK|M_ZERO);
 338         ++hammer_count_inodes;
 339         ++hmp->count_inodes;
 340         ip->obj_id = obj_id;
 341         ip->obj_asof = iinfo.obj_asof;
 342         ip->obj_localization = localization;
 343         ip->hmp = hmp;
 344         ip->flags = flags & HAMMER_INODE_RO;
 345         ip->cache[0].ip = ip;
 346         ip->cache[1].ip = ip;
 347         if (hmp->ronly)
 348                 ip->flags |= HAMMER_INODE_RO;
 349         ip->sync_trunc_off = ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
 350         RB_INIT(&ip->rec_tree);
 351         TAILQ_INIT(&ip->target_list);
 352
 353         /*
 354          * Locate the on-disk inode.
 355          */
 356 retry:
 357         hammer_init_cursor(trans, &cursor, (dip ? &dip->cache[0] : NULL), NULL);
 358         cursor.key_beg.localization = HAMMER_LOCALIZE_INODE;
 359         cursor.key_beg.obj_id = ip->obj_id;
 360         cursor.key_beg.key = 0;
 361         cursor.key_beg.create_tid = 0;
 362         cursor.key_beg.delete_tid = 0;
 363         cursor.key_beg.rec_type = HAMMER_RECTYPE_INODE;
 364         cursor.key_beg.obj_type = 0;
 365         cursor.asof = iinfo.obj_asof;
 366         cursor.flags = HAMMER_CURSOR_GET_LEAF | HAMMER_CURSOR_GET_DATA |
 367                        HAMMER_CURSOR_ASOF;
 368
 369         *errorp = hammer_btree_lookup(&cursor);
 370         if (*errorp == EDEADLK) {
 371                 hammer_done_cursor(&cursor);
 372                 goto retry;
 373         }
 374
 375         /*
 376          * On success the B-Tree lookup will hold the appropriate
 377          * buffer cache buffers and provide a pointer to the requested
 378          * information.  Copy the information to the in-memory inode
 379          * and cache the B-Tree node to improve future operations.
 380          */
 381         if (*errorp == 0) {
 382                 ip->ino_leaf = cursor.node->ondisk->elms[cursor.index].leaf;
 383                 ip->ino_data = cursor.data->inode;
 384
 385                 /*
 386                  * cache[0] tries to cache the location of the object inode.
 387                  * The assumption is that it is near the directory inode.
 388                  *
 389                  * cache[1] tries to cache the location of the object data.
 390                  * The assumption is that it is near the directory data.
 391                  */
 392                 hammer_cache_node(&ip->cache[0], cursor.node);
 393                 if (dip && dip->cache[1].node)
 394                         hammer_cache_node(&ip->cache[1], dip->cache[1].node);
 395
 396                 /*
 397                  * The file should not contain any data past the file size
 398                  * stored in the inode.  Setting sync_trunc_off to the
 399                  * file size instead of max reduces B-Tree lookup overheads
 400                  * on append by allowing the flusher to avoid checking for
 401                  * record overwrites.
 402                  */
 403                 ip->sync_trunc_off = ip->ino_data.size;
 404         }
 405
 406         /*
 407          * The inode is placed on the red-black tree and will be synced to
 408          * the media when flushed or by the filesystem sync.  If this races
 409          * another instantiation/lookup the insertion will fail.
 410          */
 411         if (*errorp == 0) {
 412                 hammer_ref(&ip->lock);
 413                 if (RB_INSERT(hammer_ino_rb_tree, &hmp->rb_inos_root, ip)) {
 414                         hammer_uncache_node(&ip->cache[0]);
 415                         hammer_uncache_node(&ip->cache[1]);
 416                         KKASSERT(ip->lock.refs == 1);
 417                         --hammer_count_inodes;
 418                         --hmp->count_inodes;
 419                         kfree(ip, M_HAMMER);
 420                         hammer_done_cursor(&cursor);
 421                         goto loop;
 422                 }
 423                 ip->flags |= HAMMER_INODE_ONDISK;
 424         } else {
 425                 /*
 426                  * Do not panic on read-only accesses which fail, particularly
 427                  * historical accesses where the snapshot might not have
 428                  * complete connectivity.
 429                  */
 430                 if ((flags & HAMMER_INODE_RO) == 0) {
 431                         kprintf("hammer_get_inode: failed ip %p obj_id %016llx cursor %p error %d\n",
 432                                 ip, ip->obj_id, &cursor, *errorp);
 433                         Debugger("x");
 434                 }
 435                 if (ip->flags & HAMMER_INODE_RSV_INODES) {
 436                         ip->flags &= ~HAMMER_INODE_RSV_INODES; /* sanity */
 437                         --hmp->rsv_inodes;
 438                 }
 439                 hmp->rsv_databufs -= ip->rsv_databufs;
 440                 ip->rsv_databufs = 0;                          /* sanity */
 441
 442                 --hammer_count_inodes;
 443                 --hmp->count_inodes;
 444                 kfree(ip, M_HAMMER);
 445                 ip = NULL;
 446         }
 447         hammer_done_cursor(&cursor);
 448         return (ip);
 449 }
 450
 451 /*
 452  * Create a new filesystem object, returning the inode in *ipp.  The
 453  * returned inode will be referenced.
 454  *
 455  * The inode is created in-memory.
 456  */
 457 int
 458 hammer_create_inode(hammer_transaction_t trans, struct vattr *vap,
 459                     struct ucred *cred, hammer_inode_t dip,
 460                     struct hammer_inode **ipp)
 461 {
 462         hammer_mount_t hmp;
 463         hammer_inode_t ip;
 464         uid_t xuid;
 465
 466         hmp = trans->hmp;
 467         ip = kmalloc(sizeof(*ip), M_HAMMER, M_WAITOK|M_ZERO);
 468         ++hammer_count_inodes;
 469         ++hmp->count_inodes;
 470         ip->obj_id = hammer_alloc_objid(trans, dip);
 471         KKASSERT(ip->obj_id != 0);
 472         ip->obj_asof = hmp->asof;
 473         ip->obj_localization = dip->obj_localization;
 474         ip->hmp = hmp;
 475         ip->flush_state = HAMMER_FST_IDLE;
 476         ip->flags = HAMMER_INODE_DDIRTY |
 477                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME;
 478         ip->cache[0].ip = ip;
 479         ip->cache[1].ip = ip;
 480
 481         ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
 482         RB_INIT(&ip->rec_tree);
 483         TAILQ_INIT(&ip->target_list);
 484
 485         ip->ino_data.atime = trans->time;
 486         ip->ino_data.mtime = trans->time;
 487         ip->ino_data.size = 0;
 488         ip->ino_data.nlinks = 0;
 489
 490         /*
 491          * A nohistory designator on the parent directory is inherited by
 492          * the child.
 493          */
 494         ip->ino_data.uflags = dip->ino_data.uflags &
 495                               (SF_NOHISTORY|UF_NOHISTORY|UF_NODUMP);
 496
 497         ip->ino_leaf.base.btype = HAMMER_BTREE_TYPE_RECORD;
 498         ip->ino_leaf.base.localization = HAMMER_LOCALIZE_INODE;
 499         ip->ino_leaf.base.obj_id = ip->obj_id;
 500         ip->ino_leaf.base.key = 0;
 501         ip->ino_leaf.base.create_tid = 0;
 502         ip->ino_leaf.base.delete_tid = 0;
 503         ip->ino_leaf.base.rec_type = HAMMER_RECTYPE_INODE;
 504         ip->ino_leaf.base.obj_type = hammer_get_obj_type(vap->va_type);
 505
 506         ip->ino_data.obj_type = ip->ino_leaf.base.obj_type;
 507         ip->ino_data.version = HAMMER_INODE_DATA_VERSION;
 508         ip->ino_data.mode = vap->va_mode;
 509         ip->ino_data.ctime = trans->time;
 510         ip->ino_data.parent_obj_id = (dip) ? dip->ino_leaf.base.obj_id : 0;
 511
 512         switch(ip->ino_leaf.base.obj_type) {
 513         case HAMMER_OBJTYPE_CDEV:
 514         case HAMMER_OBJTYPE_BDEV:
 515                 ip->ino_data.rmajor = vap->va_rmajor;
 516                 ip->ino_data.rminor = vap->va_rminor;
 517                 break;
 518         default:
 519                 break;
 520         }
 521
 522         /*
 523          * Calculate default uid/gid and overwrite with information from
 524          * the vap.
 525          */
 526         xuid = hammer_to_unix_xid(&dip->ino_data.uid);
 527         xuid = vop_helper_create_uid(hmp->mp, dip->ino_data.mode, xuid, cred,
 528                                      &vap->va_mode);
 529         ip->ino_data.mode = vap->va_mode;
 530
 531         if (vap->va_vaflags & VA_UID_UUID_VALID)
 532                 ip->ino_data.uid = vap->va_uid_uuid;
 533         else if (vap->va_uid != (uid_t)VNOVAL)
 534                 hammer_guid_to_uuid(&ip->ino_data.uid, vap->va_uid);
 535         else
 536                 hammer_guid_to_uuid(&ip->ino_data.uid, xuid);
 537
 538         if (vap->va_vaflags & VA_GID_UUID_VALID)
 539                 ip->ino_data.gid = vap->va_gid_uuid;
 540         else if (vap->va_gid != (gid_t)VNOVAL)
 541                 hammer_guid_to_uuid(&ip->ino_data.gid, vap->va_gid);
 542         else
 543                 ip->ino_data.gid = dip->ino_data.gid;
 544
 545         hammer_ref(&ip->lock);
 546         if (RB_INSERT(hammer_ino_rb_tree, &hmp->rb_inos_root, ip)) {
 547                 hammer_unref(&ip->lock);
 548                 panic("hammer_create_inode: duplicate obj_id %llx", ip->obj_id);
 549         }
 550         *ipp = ip;
 551         return(0);
 552 }
 553
 554 /*
 555  * Called by hammer_sync_inode().
 556  */
 557 static int
 558 hammer_update_inode(hammer_cursor_t cursor, hammer_inode_t ip)
 559 {
 560         hammer_transaction_t trans = cursor->trans;
 561         hammer_record_t record;
 562         int error;
 563
 564 retry:
 565         error = 0;
 566
 567         /*
 568          * If the inode has a presence on-disk then locate it and mark
 569          * it deleted, setting DELONDISK.
 570          *
 571          * The record may or may not be physically deleted, depending on
 572          * the retention policy.
 573          */
 574         if ((ip->flags & (HAMMER_INODE_ONDISK|HAMMER_INODE_DELONDISK)) ==
 575             HAMMER_INODE_ONDISK) {
 576                 hammer_normalize_cursor(cursor);
 577                 cursor->key_beg.localization = HAMMER_LOCALIZE_INODE;
 578                 cursor->key_beg.obj_id = ip->obj_id;
 579                 cursor->key_beg.key = 0;
 580                 cursor->key_beg.create_tid = 0;
 581                 cursor->key_beg.delete_tid = 0;
 582                 cursor->key_beg.rec_type = HAMMER_RECTYPE_INODE;
 583                 cursor->key_beg.obj_type = 0;
 584                 cursor->asof = ip->obj_asof;
 585                 cursor->flags &= ~HAMMER_CURSOR_INITMASK;
 586                 cursor->flags |= HAMMER_CURSOR_GET_LEAF | HAMMER_CURSOR_ASOF;
 587                 cursor->flags |= HAMMER_CURSOR_BACKEND;
 588
 589                 error = hammer_btree_lookup(cursor);
 590                 if (hammer_debug_inode)
 591                         kprintf("IPDEL %p %08x %d", ip, ip->flags, error);
 592                 if (error) {
 593                         kprintf("error %d\n", error);
 594                         Debugger("hammer_update_inode");
 595                 }
 596
 597                 if (error == 0) {
 598                         error = hammer_ip_delete_record(cursor, ip, trans->tid);
 599                         if (hammer_debug_inode)
 600                                 kprintf(" error %d\n", error);
 601                         if (error && error != EDEADLK) {
 602                                 kprintf("error %d\n", error);
 603                                 Debugger("hammer_update_inode2");
 604                         }
 605                         if (error == 0) {
 606                                 ip->flags |= HAMMER_INODE_DELONDISK;
 607                         }
 608                         if (cursor->node)
 609                                 hammer_cache_node(&ip->cache[0], cursor->node);
 610                 }
 611                 if (error == EDEADLK) {
 612                         hammer_done_cursor(cursor);
 613                         error = hammer_init_cursor(trans, cursor,
 614                                                    &ip->cache[0], ip);
 615                         if (hammer_debug_inode)
 616                                 kprintf("IPDED %p %d\n", ip, error);
 617                         if (error == 0)
 618                                 goto retry;
 619                 }
 620         }
 621
 622         /*
 623          * Ok, write out the initial record or a new record (after deleting
 624          * the old one), unless the DELETED flag is set.  This routine will
 625          * clear DELONDISK if it writes out a record.
 626          *
 627          * Update our inode statistics if this is the first application of
 628          * the inode on-disk.
 629          */
 630         if (error == 0 && (ip->flags & HAMMER_INODE_DELETED) == 0) {
 631                 /*
 632                  * Generate a record and write it to the media
 633                  */
 634                 record = hammer_alloc_mem_record(ip, 0);
 635                 record->type = HAMMER_MEM_RECORD_INODE;
 636                 record->flush_state = HAMMER_FST_FLUSH;
 637                 record->leaf = ip->sync_ino_leaf;
 638                 record->leaf.base.create_tid = trans->tid;
 639                 record->leaf.data_len = sizeof(ip->sync_ino_data);
 640                 record->data = (void *)&ip->sync_ino_data;
 641                 record->flags |= HAMMER_RECF_INTERLOCK_BE;
 642                 for (;;) {
 643                         error = hammer_ip_sync_record_cursor(cursor, record);
 644                         if (hammer_debug_inode)
 645                                 kprintf("GENREC %p rec %08x %d\n",
 646                                         ip, record->flags, error);
 647                         if (error != EDEADLK)
 648                                 break;
 649                         hammer_done_cursor(cursor);
 650                         error = hammer_init_cursor(trans, cursor,
 651                                                    &ip->cache[0], ip);
 652                         if (hammer_debug_inode)
 653                                 kprintf("GENREC reinit %d\n", error);
 654                         if (error)
 655                                 break;
 656                 }
 657                 if (error) {
 658                         kprintf("error %d\n", error);
 659                         Debugger("hammer_update_inode3");
 660                 }
 661
 662                 /*
 663                  * The record isn't managed by the inode's record tree,
 664                  * destroy it whether we succeed or fail.
 665                  */
 666                 record->flags &= ~HAMMER_RECF_INTERLOCK_BE;
 667                 record->flags |= HAMMER_RECF_DELETED_FE;
 668                 record->flush_state = HAMMER_FST_IDLE;
 669                 hammer_rel_mem_record(record);
 670
 671                 /*
 672                  * Finish up.
 673                  */
 674                 if (error == 0) {
 675                         if (hammer_debug_inode)
 676                                 kprintf("CLEANDELOND %p %08x\n", ip, ip->flags);
 677                         ip->sync_flags &= ~(HAMMER_INODE_DDIRTY |
 678                                             HAMMER_INODE_ATIME |
 679                                             HAMMER_INODE_MTIME);
 680                         ip->flags &= ~HAMMER_INODE_DELONDISK;
 681
 682                         /*
 683                          * Root volume count of inodes
 684                          */
 685                         if ((ip->flags & HAMMER_INODE_ONDISK) == 0) {
 686                                 hammer_modify_volume_field(trans,
 687                                                            trans->rootvol,
 688                                                            vol0_stat_inodes);
 689                                 ++ip->hmp->rootvol->ondisk->vol0_stat_inodes;
 690                                 hammer_modify_volume_done(trans->rootvol);
 691                                 ip->flags |= HAMMER_INODE_ONDISK;
 692                                 if (hammer_debug_inode)
 693                                         kprintf("NOWONDISK %p\n", ip);
 694                         }
 695                 }
 696         }
 697
 698         /*
 699          * If the inode has been destroyed, clean out any left-over flags
 700          * that may have been set by the frontend.
 701          */
 702         if (error == 0 && (ip->flags & HAMMER_INODE_DELETED)) {
 703                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY |
 704                                     HAMMER_INODE_ATIME |
 705                                     HAMMER_INODE_MTIME);
 706         }
 707         return(error);
 708 }
 709
 710 /*
 711  * Update only the itimes fields.
 712  *
 713  * ATIME can be updated without generating any UNDO.  MTIME is updated
 714  * with UNDO so it is guaranteed to be synchronized properly in case of
 715  * a crash.
 716  *
 717  * Neither field is included in the B-Tree leaf element's CRC, which is how
 718  * we can get away with updating ATIME the way we do.
 719  */
 720 static int
 721 hammer_update_itimes(hammer_cursor_t cursor, hammer_inode_t ip)
 722 {
 723         hammer_transaction_t trans = cursor->trans;
 724         int error;
 725
 726 retry:
 727         if ((ip->flags & (HAMMER_INODE_ONDISK|HAMMER_INODE_DELONDISK)) !=
 728             HAMMER_INODE_ONDISK) {
 729                 return(0);
 730         }
 731
 732         hammer_normalize_cursor(cursor);
 733         cursor->key_beg.localization = HAMMER_LOCALIZE_INODE;
 734         cursor->key_beg.obj_id = ip->obj_id;
 735         cursor->key_beg.key = 0;
 736         cursor->key_beg.create_tid = 0;
 737         cursor->key_beg.delete_tid = 0;
 738         cursor->key_beg.rec_type = HAMMER_RECTYPE_INODE;
 739         cursor->key_beg.obj_type = 0;
 740         cursor->asof = ip->obj_asof;
 741         cursor->flags &= ~HAMMER_CURSOR_INITMASK;
 742         cursor->flags |= HAMMER_CURSOR_ASOF;
 743         cursor->flags |= HAMMER_CURSOR_GET_LEAF;
 744         cursor->flags |= HAMMER_CURSOR_GET_DATA;
 745         cursor->flags |= HAMMER_CURSOR_BACKEND;
 746
 747         error = hammer_btree_lookup(cursor);
 748         if (error) {
 749                 kprintf("error %d\n", error);
 750                 Debugger("hammer_update_itimes1");
 751         }
 752         if (error == 0) {
 753                 hammer_cache_node(&ip->cache[0], cursor->node);
 754                 if (ip->sync_flags & HAMMER_INODE_MTIME) {
 755                         /*
 756                          * Updating MTIME requires an UNDO.  Just cover
 757                          * both atime and mtime.
 758                          */
 759                         hammer_modify_buffer(trans, cursor->data_buffer,
 760                                      HAMMER_ITIMES_BASE(&cursor->data->inode),
 761                                      HAMMER_ITIMES_BYTES);
 762                         cursor->data->inode.atime = ip->sync_ino_data.atime;
 763                         cursor->data->inode.mtime = ip->sync_ino_data.mtime;
 764                         hammer_modify_buffer_done(cursor->data_buffer);
 765                 } else if (ip->sync_flags & HAMMER_INODE_ATIME) {
 766                         /*
 767                          * Updating atime only can be done in-place with
 768                          * no UNDO.
 769                          */
 770                         hammer_modify_buffer(trans, cursor->data_buffer,
 771                                              NULL, 0);
 772                         cursor->data->inode.atime = ip->sync_ino_data.atime;
 773                         hammer_modify_buffer_done(cursor->data_buffer);
 774                 }
 775                 ip->sync_flags &= ~(HAMMER_INODE_ATIME | HAMMER_INODE_MTIME);
 776         }
 777         if (error == EDEADLK) {
 778                 hammer_done_cursor(cursor);
 779                 error = hammer_init_cursor(trans, cursor,
 780                                            &ip->cache[0], ip);
 781                 if (error == 0)
 782                         goto retry;
 783         }
 784         return(error);
 785 }
 786
 787 /*
 788  * Release a reference on an inode, flush as requested.
 789  *
 790  * On the last reference we queue the inode to the flusher for its final
 791  * disposition.
 792  */
 793 void
 794 hammer_rel_inode(struct hammer_inode *ip, int flush)
 795 {
 796         hammer_mount_t hmp = ip->hmp;
 797
 798         /*
 799          * Handle disposition when dropping the last ref.
 800          */
 801         for (;;) {
 802                 if (ip->lock.refs == 1) {
 803                         /*
 804                          * Determine whether on-disk action is needed for
 805                          * the inode's final disposition.
 806                          */
 807                         KKASSERT(ip->vp == NULL);
 808                         hammer_inode_unloadable_check(ip, 0);
 809                         if (ip->flags & HAMMER_INODE_MODMASK) {
 810                                 if (hmp->rsv_inodes > desiredvnodes) {
 811                                         hammer_flush_inode(ip,
 812                                                            HAMMER_FLUSH_SIGNAL);
 813                                 } else {
 814                                         hammer_flush_inode(ip, 0);
 815                                 }
 816                         } else if (ip->lock.refs == 1) {
 817                                 hammer_unload_inode(ip);
 818                                 break;
 819                         }
 820                 } else {
 821                         if (flush)
 822                                 hammer_flush_inode(ip, 0);
 823
 824                         /*
 825                          * The inode still has multiple refs, try to drop
 826                          * one ref.
 827                          */
 828                         KKASSERT(ip->lock.refs >= 1);
 829                         if (ip->lock.refs > 1) {
 830                                 hammer_unref(&ip->lock);
 831                                 break;
 832                         }
 833                 }
 834         }
 835 }
 836
 837 /*
 838  * Unload and destroy the specified inode.  Must be called with one remaining
 839  * reference.  The reference is disposed of.
 840  *
 841  * This can only be called in the context of the flusher.
 842  */
 843 static int
 844 hammer_unload_inode(struct hammer_inode *ip)
 845 {
 846         hammer_mount_t hmp = ip->hmp;
 847
 848         KASSERT(ip->lock.refs == 1,
 849                 ("hammer_unload_inode: %d refs\n", ip->lock.refs));
 850         KKASSERT(ip->vp == NULL);
 851         KKASSERT(ip->flush_state == HAMMER_FST_IDLE);
 852         KKASSERT(ip->cursor_ip_refs == 0);
 853         KKASSERT(ip->lock.lockcount == 0);
 854         KKASSERT((ip->flags & HAMMER_INODE_MODMASK) == 0);
 855
 856         KKASSERT(RB_EMPTY(&ip->rec_tree));
 857         KKASSERT(TAILQ_EMPTY(&ip->target_list));
 858
 859         RB_REMOVE(hammer_ino_rb_tree, &hmp->rb_inos_root, ip);
 860
 861         hammer_uncache_node(&ip->cache[0]);
 862         hammer_uncache_node(&ip->cache[1]);
 863         if (ip->objid_cache)
 864                 hammer_clear_objid(ip);
 865         --hammer_count_inodes;
 866         --hmp->count_inodes;
 867
 868         hammer_inode_wakereclaims(ip);
 869         kfree(ip, M_HAMMER);
 870
 871         return(0);
 872 }
 873
 874 /*
 875  * Called on mount -u when switching from RW to RO or vise-versa.  Adjust
 876  * the read-only flag for cached inodes.
 877  *
 878  * This routine is called from a RB_SCAN().
 879  */
 880 int
 881 hammer_reload_inode(hammer_inode_t ip, void *arg __unused)
 882 {
 883         hammer_mount_t hmp = ip->hmp;
 884
 885         if (hmp->ronly || hmp->asof != HAMMER_MAX_TID)
 886                 ip->flags |= HAMMER_INODE_RO;
 887         else
 888                 ip->flags &= ~HAMMER_INODE_RO;
 889         return(0);
 890 }
 891
 892 /*
 893  * A transaction has modified an inode, requiring updates as specified by
 894  * the passed flags.
 895  *
 896  * HAMMER_INODE_DDIRTY: Inode data has been updated
 897  * HAMMER_INODE_XDIRTY: Dirty in-memory records
 898  * HAMMER_INODE_BUFS:   Dirty buffer cache buffers
 899  * HAMMER_INODE_DELETED: Inode record/data must be deleted
 900  * HAMMER_INODE_ATIME/MTIME: mtime/atime has been updated
 901  */
 902 void
 903 hammer_modify_inode(hammer_inode_t ip, int flags)
 904 {
 905         KKASSERT ((ip->flags & HAMMER_INODE_RO) == 0 ||
 906                   (flags & (HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
 907                             HAMMER_INODE_BUFS | HAMMER_INODE_DELETED |
 908                             HAMMER_INODE_ATIME | HAMMER_INODE_MTIME)) == 0);
 909         if ((ip->flags & HAMMER_INODE_RSV_INODES) == 0) {
 910                 ip->flags |= HAMMER_INODE_RSV_INODES;
 911                 ++ip->hmp->rsv_inodes;
 912         }
 913
 914         ip->flags |= flags;
 915 }
 916
 917 /*
 918  * Request that an inode be flushed.  This whole mess cannot block and may
 919  * recurse (if not synchronous).  Once requested HAMMER will attempt to
 920  * actively flush the inode until the flush can be done.
 921  *
 922  * The inode may already be flushing, or may be in a setup state.  We can
 923  * place the inode in a flushing state if it is currently idle and flag it
 924  * to reflush if it is currently flushing.
 925  *
 926  * If the HAMMER_FLUSH_SYNCHRONOUS flag is specified we will attempt to
 927  * flush the indoe synchronously using the caller's context.
 928  */
 929 void
 930 hammer_flush_inode(hammer_inode_t ip, int flags)
 931 {
 932         int good;
 933
 934         /*
 935          * Trivial 'nothing to flush' case.  If the inode is ina SETUP
 936          * state we have to put it back into an IDLE state so we can
 937          * drop the extra ref.
 938          */
 939         if ((ip->flags & HAMMER_INODE_MODMASK) == 0) {
 940                 if (ip->flush_state == HAMMER_FST_SETUP) {
 941                         ip->flush_state = HAMMER_FST_IDLE;
 942                         hammer_rel_inode(ip, 0);
 943                 }
 944                 return;
 945         }
 946
 947         /*
 948          * Our flush action will depend on the current state.
 949          */
 950         switch(ip->flush_state) {
 951         case HAMMER_FST_IDLE:
 952                 /*
 953                  * We have no dependancies and can flush immediately.  Some
 954                  * our children may not be flushable so we have to re-test
 955                  * with that additional knowledge.
 956                  */
 957                 hammer_flush_inode_core(ip, flags);
 958                 break;
 959         case HAMMER_FST_SETUP:
 960                 /*
 961                  * Recurse upwards through dependancies via target_list
 962                  * and start their flusher actions going if possible.
 963                  *
 964                  * 'good' is our connectivity.  -1 means we have none and
 965                  * can't flush, 0 means there weren't any dependancies, and
 966                  * 1 means we have good connectivity.
 967                  */
 968                 good = hammer_setup_parent_inodes(ip);
 969
 970                 /*
 971                  * We can continue if good >= 0.  Determine how many records
 972                  * under our inode can be flushed (and mark them).
 973                  */
 974                 if (good >= 0) {
 975                         hammer_flush_inode_core(ip, flags);
 976                 } else {
 977                         ip->flags |= HAMMER_INODE_REFLUSH;
 978                         if (flags & HAMMER_FLUSH_SIGNAL) {
 979                                 ip->flags |= HAMMER_INODE_RESIGNAL;
 980                                 hammer_flusher_async(ip->hmp);
 981                         }
 982                 }
 983                 break;
 984         default:
 985                 /*
 986                  * We are already flushing, flag the inode to reflush
 987                  * if needed after it completes its current flush.
 988                  */
 989                 if ((ip->flags & HAMMER_INODE_REFLUSH) == 0)
 990                         ip->flags |= HAMMER_INODE_REFLUSH;
 991                 if (flags & HAMMER_FLUSH_SIGNAL) {
 992                         ip->flags |= HAMMER_INODE_RESIGNAL;
 993                         hammer_flusher_async(ip->hmp);
 994                 }
 995                 break;
 996         }
 997 }
 998
 999 /*
1000  * Scan ip->target_list, which is a list of records owned by PARENTS to our
1001  * ip which reference our ip.
1002  *
1003  * XXX This is a huge mess of recursive code, but not one bit of it blocks
1004  *     so for now do not ref/deref the structures.  Note that if we use the
1005  *     ref/rel code later, the rel CAN block.
1006  */
1007 static int
1008 hammer_setup_parent_inodes(hammer_inode_t ip)
1009 {
1010         hammer_record_t depend;
1011 #if 0
1012         hammer_record_t next;
1013         hammer_inode_t  pip;
1014 #endif
1015         int good;
1016         int r;
1017
1018         good = 0;
1019         TAILQ_FOREACH(depend, &ip->target_list, target_entry) {
1020                 r = hammer_setup_parent_inodes_helper(depend);
1021                 KKASSERT(depend->target_ip == ip);
1022                 if (r < 0 && good == 0)
1023                         good = -1;
1024                 if (r > 0)
1025                         good = 1;
1026         }
1027         return(good);
1028
1029 #if 0
1030 retry:
1031         good = 0;
1032         next = TAILQ_FIRST(&ip->target_list);
1033         if (next) {
1034                 hammer_ref(&next->lock);
1035                 hammer_ref(&next->ip->lock);
1036         }
1037         while ((depend = next) != NULL) {
1038                 if (depend->target_ip == NULL) {
1039                         pip = depend->ip;
1040                         hammer_rel_mem_record(depend);
1041                         hammer_rel_inode(pip, 0);
1042                         goto retry;
1043                 }
1044                 KKASSERT(depend->target_ip == ip);
1045                 next = TAILQ_NEXT(depend, target_entry);
1046                 if (next) {
1047                         hammer_ref(&next->lock);
1048                         hammer_ref(&next->ip->lock);
1049                 }
1050                 r = hammer_setup_parent_inodes_helper(depend);
1051                 if (r < 0 && good == 0)
1052                         good = -1;
1053                 if (r > 0)
1054                         good = 1;
1055                 pip = depend->ip;
1056                 hammer_rel_mem_record(depend);
1057                 hammer_rel_inode(pip, 0);
1058         }
1059         return(good);
1060 #endif
1061 }
1062
1063 /*
1064  * This helper function takes a record representing the dependancy between
1065  * the parent inode and child inode.
1066  *
1067  * record->ip           = parent inode
1068  * record->target_ip    = child inode
1069  *
1070  * We are asked to recurse upwards and convert the record from SETUP
1071  * to FLUSH if possible.
1072  *
1073  * Return 1 if the record gives us connectivity
1074  *
1075  * Return 0 if the record is not relevant
1076  *
1077  * Return -1 if we can't resolve the dependancy and there is no connectivity.
1078  */
1079 static int
1080 hammer_setup_parent_inodes_helper(hammer_record_t record)
1081 {
1082         hammer_mount_t hmp;
1083         hammer_inode_t pip;
1084         int good;
1085
1086         KKASSERT(record->flush_state != HAMMER_FST_IDLE);
1087         pip = record->ip;
1088         hmp = pip->hmp;
1089
1090         /*
1091          * If the record is already flushing, is it in our flush group?
1092          *
1093          * If it is in our flush group but it is a general record or a
1094          * delete-on-disk, it does not improve our connectivity (return 0),
1095          * and if the target inode is not trying to destroy itself we can't
1096          * allow the operation yet anyway (the second return -1).
1097          */
1098         if (record->flush_state == HAMMER_FST_FLUSH) {
1099                 if (record->flush_group != hmp->flusher.next) {
1100                         pip->flags |= HAMMER_INODE_REFLUSH;
1101                         return(-1);
1102                 }
1103                 if (record->type == HAMMER_MEM_RECORD_ADD)
1104                         return(1);
1105                 /* GENERAL or DEL */
1106                 return(0);
1107         }
1108
1109         /*
1110          * It must be a setup record.  Try to resolve the setup dependancies
1111          * by recursing upwards so we can place ip on the flush list.
1112          */
1113         KKASSERT(record->flush_state == HAMMER_FST_SETUP);
1114
1115         good = hammer_setup_parent_inodes(pip);
1116
1117         /*
1118          * We can't flush ip because it has no connectivity (XXX also check
1119          * nlinks for pre-existing connectivity!).  Flag it so any resolution
1120          * recurses back down.
1121          */
1122         if (good < 0) {
1123                 pip->flags |= HAMMER_INODE_REFLUSH;
1124                 return(good);
1125         }
1126
1127         /*
1128          * We are go, place the parent inode in a flushing state so we can
1129          * place its record in a flushing state.  Note that the parent
1130          * may already be flushing.  The record must be in the same flush
1131          * group as the parent.
1132          */
1133         if (pip->flush_state != HAMMER_FST_FLUSH)
1134                 hammer_flush_inode_core(pip, HAMMER_FLUSH_RECURSION);
1135         KKASSERT(pip->flush_state == HAMMER_FST_FLUSH);
1136         KKASSERT(record->flush_state == HAMMER_FST_SETUP);
1137
1138 #if 0
1139         if (record->type == HAMMER_MEM_RECORD_DEL &&
1140             (record->target_ip->flags & (HAMMER_INODE_DELETED|HAMMER_INODE_DELONDISK)) == 0) {
1141                 /*
1142                  * Regardless of flushing state we cannot sync this path if the
1143                  * record represents a delete-on-disk but the target inode
1144                  * is not ready to sync its own deletion.
1145                  *
1146                  * XXX need to count effective nlinks to determine whether
1147                  * the flush is ok, otherwise removing a hardlink will
1148                  * just leave the DEL record to rot.
1149                  */
1150                 record->target_ip->flags |= HAMMER_INODE_REFLUSH;
1151                 return(-1);
1152         } else
1153 #endif
1154         if (pip->flush_group == pip->hmp->flusher.next) {
1155                 /*
1156                  * This is the record we wanted to synchronize.  If the
1157                  * record went into a flush state while we blocked it
1158                  * had better be in the correct flush group.
1159                  */
1160                 if (record->flush_state != HAMMER_FST_FLUSH) {
1161                         record->flush_state = HAMMER_FST_FLUSH;
1162                         record->flush_group = pip->flush_group;
1163                         hammer_ref(&record->lock);
1164                 } else {
1165                         KKASSERT(record->flush_group == pip->flush_group);
1166                 }
1167                 if (record->type == HAMMER_MEM_RECORD_ADD)
1168                         return(1);
1169
1170                 /*
1171                  * A general or delete-on-disk record does not contribute
1172                  * to our visibility.  We can still flush it, however.
1173                  */
1174                 return(0);
1175         } else {
1176                 /*
1177                  * We couldn't resolve the dependancies, request that the
1178                  * inode be flushed when the dependancies can be resolved.
1179                  */
1180                 pip->flags |= HAMMER_INODE_REFLUSH;
1181                 return(-1);
1182         }
1183 }
1184
1185 /*
1186  * This is the core routine placing an inode into the FST_FLUSH state.
1187  */
1188 static void
1189 hammer_flush_inode_core(hammer_inode_t ip, int flags)
1190 {
1191         int go_count;
1192
1193         /*
1194          * Set flush state and prevent the flusher from cycling into
1195          * the next flush group.  Do not place the ip on the list yet.
1196          * Inodes not in the idle state get an extra reference.
1197          */
1198         KKASSERT(ip->flush_state != HAMMER_FST_FLUSH);
1199         if (ip->flush_state == HAMMER_FST_IDLE)
1200                 hammer_ref(&ip->lock);
1201         ip->flush_state = HAMMER_FST_FLUSH;
1202         ip->flush_group = ip->hmp->flusher.next;
1203         ++ip->hmp->flusher.group_lock;
1204         ++ip->hmp->count_iqueued;
1205         ++hammer_count_iqueued;
1206
1207         /*
1208          * We need to be able to vfsync/truncate from the backend.
1209          */
1210         KKASSERT((ip->flags & HAMMER_INODE_VHELD) == 0);
1211         if (ip->vp && (ip->vp->v_flag & VINACTIVE) == 0) {
1212                 ip->flags |= HAMMER_INODE_VHELD;
1213                 vref(ip->vp);
1214         }
1215
1216         /*
1217          * Figure out how many in-memory records we can actually flush
1218          * (not including inode meta-data, buffers, etc).
1219          */
1220         if (flags & HAMMER_FLUSH_RECURSION) {
1221                 go_count = 1;
1222         } else {
1223                 go_count = RB_SCAN(hammer_rec_rb_tree, &ip->rec_tree, NULL,
1224                                    hammer_setup_child_callback, NULL);
1225         }
1226
1227         /*
1228          * This is a more involved test that includes go_count.  If we
1229          * can't flush, flag the inode and return.  If go_count is 0 we
1230          * were are unable to flush any records in our rec_tree and
1231          * must ignore the XDIRTY flag.
1232          */
1233         if (go_count == 0) {
1234                 if ((ip->flags & HAMMER_INODE_MODMASK_NOXDIRTY) == 0) {
1235                         ip->flags |= HAMMER_INODE_REFLUSH;
1236
1237                         --ip->hmp->count_iqueued;
1238                         --hammer_count_iqueued;
1239
1240                         ip->flush_state = HAMMER_FST_SETUP;
1241                         if (ip->flags & HAMMER_INODE_VHELD) {
1242                                 ip->flags &= ~HAMMER_INODE_VHELD;
1243                                 vrele(ip->vp);
1244                         }
1245                         if (flags & HAMMER_FLUSH_SIGNAL) {
1246                                 ip->flags |= HAMMER_INODE_RESIGNAL;
1247                                 hammer_flusher_async(ip->hmp);
1248                         }
1249                         if (--ip->hmp->flusher.group_lock == 0)
1250                                 wakeup(&ip->hmp->flusher.group_lock);
1251                         return;
1252                 }
1253         }
1254
1255         /*
1256          * Snapshot the state of the inode for the backend flusher.
1257          *
1258          * The truncation must be retained in the frontend until after
1259          * we've actually performed the record deletion.
1260          *
1261          * We continue to retain sync_trunc_off even when all truncations
1262          * have been resolved as an optimization to determine if we can
1263          * skip the B-Tree lookup for overwrite deletions.
1264          *
1265          * NOTE: The DELETING flag is a mod flag, but it is also sticky,
1266          * and stays in ip->flags.  Once set, it stays set until the
1267          * inode is destroyed.
1268          */
1269         ip->sync_flags = (ip->flags & HAMMER_INODE_MODMASK);
1270         if (ip->sync_flags & HAMMER_INODE_TRUNCATED)
1271                 ip->sync_trunc_off = ip->trunc_off;
1272         ip->sync_ino_leaf = ip->ino_leaf;
1273         ip->sync_ino_data = ip->ino_data;
1274         ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
1275         ip->flags &= ~HAMMER_INODE_MODMASK;
1276 #ifdef DEBUG_TRUNCATE
1277         if ((ip->sync_flags & HAMMER_INODE_TRUNCATED) && ip == HammerTruncIp)
1278                 kprintf("truncateS %016llx\n", ip->sync_trunc_off);
1279 #endif
1280
1281         /*
1282          * The flusher list inherits our inode and reference.
1283          */
1284         TAILQ_INSERT_TAIL(&ip->hmp->flush_list, ip, flush_entry);
1285         if (--ip->hmp->flusher.group_lock == 0)
1286                 wakeup(&ip->hmp->flusher.group_lock);
1287
1288         if (flags & HAMMER_FLUSH_SIGNAL) {
1289                 hammer_flusher_async(ip->hmp);
1290         }
1291 }
1292
1293 /*
1294  * Callback for scan of ip->rec_tree.  Try to include each record in our
1295  * flush.  ip->flush_group has been set but the inode has not yet been
1296  * moved into a flushing state.
1297  *
1298  * If we get stuck on a record we have to set HAMMER_INODE_REFLUSH on
1299  * both inodes.
1300  *
1301  * We return 1 for any record placed or found in FST_FLUSH, which prevents
1302  * the caller from shortcutting the flush.
1303  */
1304 static int
1305 hammer_setup_child_callback(hammer_record_t rec, void *data)
1306 {
1307         hammer_inode_t target_ip;
1308         hammer_inode_t ip;
1309         int r;
1310
1311         /*
1312          * Deleted records are ignored.  Note that the flush detects deleted
1313          * front-end records at multiple points to deal with races.  This is
1314          * just the first line of defense.  The only time DELETED_FE cannot
1315          * be set is when HAMMER_RECF_INTERLOCK_BE is set.
1316          *
1317          * Don't get confused between record deletion and, say, directory
1318          * entry deletion.  The deletion of a directory entry that is on
1319          * the media has nothing to do with the record deletion flags.
1320          */
1321         if (rec->flags & (HAMMER_RECF_DELETED_FE|HAMMER_RECF_DELETED_BE))
1322                 return(0);
1323
1324         /*
1325          * If the record is in an idle state it has no dependancies and
1326          * can be flushed.
1327          */
1328         ip = rec->ip;
1329         r = 0;
1330
1331         switch(rec->flush_state) {
1332         case HAMMER_FST_IDLE:
1333                 /*
1334                  * Record has no setup dependancy, we can flush it.
1335                  */
1336                 KKASSERT(rec->target_ip == NULL);
1337                 rec->flush_state = HAMMER_FST_FLUSH;
1338                 rec->flush_group = ip->flush_group;
1339                 hammer_ref(&rec->lock);
1340                 r = 1;
1341                 break;
1342         case HAMMER_FST_SETUP:
1343                 /*
1344                  * Record has a setup dependancy.  Try to include the
1345                  * target ip in the flush.
1346                  *
1347                  * We have to be careful here, if we do not do the right
1348                  * thing we can lose track of dirty inodes and the system
1349                  * will lockup trying to allocate buffers.
1350                  */
1351                 target_ip = rec->target_ip;
1352                 KKASSERT(target_ip != NULL);
1353                 KKASSERT(target_ip->flush_state != HAMMER_FST_IDLE);
1354                 if (target_ip->flush_state == HAMMER_FST_FLUSH) {
1355                         /*
1356                          * If the target IP is already flushing in our group
1357                          * we are golden, otherwise make sure the target
1358                          * reflushes.
1359                          */
1360                         if (target_ip->flush_group == ip->flush_group) {
1361                                 rec->flush_state = HAMMER_FST_FLUSH;
1362                                 rec->flush_group = ip->flush_group;
1363                                 hammer_ref(&rec->lock);
1364                                 r = 1;
1365                         } else {
1366                                 target_ip->flags |= HAMMER_INODE_REFLUSH;
1367                         }
1368                 } else if (rec->type == HAMMER_MEM_RECORD_ADD) {
1369                         /*
1370                          * If the target IP is not flushing we can force
1371                          * it to flush, even if it is unable to write out
1372                          * any of its own records we have at least one in
1373                          * hand that we CAN deal with.
1374                          */
1375                         rec->flush_state = HAMMER_FST_FLUSH;
1376                         rec->flush_group = ip->flush_group;
1377                         hammer_ref(&rec->lock);
1378                         hammer_flush_inode_core(target_ip,
1379                                                 HAMMER_FLUSH_RECURSION);
1380                         r = 1;
1381                 } else {
1382                         /*
1383                          * General or delete-on-disk record.
1384                          *
1385                          * XXX this needs help.  If a delete-on-disk we could
1386                          * disconnect the target.  If the target has its own
1387                          * dependancies they really need to be flushed.
1388                          *
1389                          * XXX
1390                          */
1391                         rec->flush_state = HAMMER_FST_FLUSH;
1392                         rec->flush_group = ip->flush_group;
1393                         hammer_ref(&rec->lock);
1394                         hammer_flush_inode_core(target_ip,
1395                                                 HAMMER_FLUSH_RECURSION);
1396                         r = 1;
1397                 }
1398                 break;
1399         case HAMMER_FST_FLUSH:
1400                 /*
1401                  * Record already associated with a flush group.  It had
1402                  * better be ours.
1403                  */
1404                 KKASSERT(rec->flush_group == ip->flush_group);
1405                 r = 1;
1406                 break;
1407         }
1408         return(r);
1409 }
1410
1411 /*
1412  * Wait for a previously queued flush to complete.  Not only do we need to
1413  * wait for the inode to sync out, we also may have to run the flusher again
1414  * to get it past the UNDO position pertaining to the flush so a crash does
1415  * not 'undo' our flush.
1416  */
1417 void
1418 hammer_wait_inode(hammer_inode_t ip)
1419 {
1420         hammer_mount_t hmp = ip->hmp;
1421         int sync_group;
1422         int waitcount;
1423
1424         sync_group = ip->flush_group;
1425         waitcount = (ip->flags & HAMMER_INODE_REFLUSH) ? 2 : 1;
1426
1427         if (ip->flush_state == HAMMER_FST_SETUP) {
1428                 kprintf("X");
1429                 hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
1430         }
1431         /* XXX can we make this != FST_IDLE ? check SETUP depends */
1432         while (ip->flush_state == HAMMER_FST_FLUSH &&
1433                (ip->flush_group - sync_group) < waitcount) {
1434                 ip->flags |= HAMMER_INODE_FLUSHW;
1435                 tsleep(&ip->flags, 0, "hmrwin", 0);
1436         }
1437         while (hmp->flusher.done - sync_group < waitcount) {
1438                 kprintf("Y");
1439                 hammer_flusher_sync(hmp);
1440         }
1441 }
1442
1443 /*
1444  * Called by the backend code when a flush has been completed.
1445  * The inode has already been removed from the flush list.
1446  *
1447  * A pipelined flush can occur, in which case we must re-enter the
1448  * inode on the list and re-copy its fields.
1449  */
1450 void
1451 hammer_flush_inode_done(hammer_inode_t ip)
1452 {
1453         hammer_mount_t hmp;
1454         int dorel;
1455
1456         KKASSERT(ip->flush_state == HAMMER_FST_FLUSH);
1457
1458         hmp = ip->hmp;
1459
1460         /*
1461          * Merge left-over flags back into the frontend and fix the state.
1462          */
1463         ip->flags |= ip->sync_flags;
1464
1465         /*
1466          * The backend may have adjusted nlinks, so if the adjusted nlinks
1467          * does not match the fronttend set the frontend's RDIRTY flag again.
1468          */
1469         if (ip->ino_data.nlinks != ip->sync_ino_data.nlinks)
1470                 ip->flags |= HAMMER_INODE_DDIRTY;
1471
1472         /*
1473          * Fix up the dirty buffer status.  IO completions will also
1474          * try to clean up rsv_databufs.
1475          */
1476         if (ip->vp && RB_ROOT(&ip->vp->v_rbdirty_tree)) {
1477                 ip->flags |= HAMMER_INODE_BUFS;
1478         } else {
1479                 hmp->rsv_databufs -= ip->rsv_databufs;
1480                 ip->rsv_databufs = 0;
1481         }
1482
1483         /*
1484          * Re-set the XDIRTY flag if some of the inode's in-memory records
1485          * could not be flushed.
1486          */
1487         KKASSERT((RB_EMPTY(&ip->rec_tree) &&
1488                   (ip->flags & HAMMER_INODE_XDIRTY) == 0) ||
1489                  (!RB_EMPTY(&ip->rec_tree) &&
1490                   (ip->flags & HAMMER_INODE_XDIRTY) != 0));
1491
1492         /*
1493          * Do not lose track of inodes which no longer have vnode
1494          * assocations, otherwise they may never get flushed again.
1495          */
1496         if ((ip->flags & HAMMER_INODE_MODMASK) && ip->vp == NULL)
1497                 ip->flags |= HAMMER_INODE_REFLUSH;
1498
1499         /*
1500          * Adjust flush_state.  The target state (idle or setup) shouldn't
1501          * be terribly important since we will reflush if we really need
1502          * to do anything. XXX
1503          */
1504         if (TAILQ_EMPTY(&ip->target_list) && RB_EMPTY(&ip->rec_tree)) {
1505                 ip->flush_state = HAMMER_FST_IDLE;
1506                 dorel = 1;
1507         } else {
1508                 ip->flush_state = HAMMER_FST_SETUP;
1509                 dorel = 0;
1510         }
1511
1512         --hmp->count_iqueued;
1513         --hammer_count_iqueued;
1514
1515         /*
1516          * Clean up the vnode ref
1517          */
1518         if (ip->flags & HAMMER_INODE_VHELD) {
1519                 ip->flags &= ~HAMMER_INODE_VHELD;
1520                 vrele(ip->vp);
1521         }
1522
1523         /*
1524          * If the frontend made more changes and requested another flush,
1525          * then try to get it running.
1526          */
1527         if (ip->flags & HAMMER_INODE_REFLUSH) {
1528                 ip->flags &= ~HAMMER_INODE_REFLUSH;
1529                 if (ip->flags & HAMMER_INODE_RESIGNAL) {
1530                         ip->flags &= ~HAMMER_INODE_RESIGNAL;
1531                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
1532                 } else {
1533                         hammer_flush_inode(ip, 0);
1534                 }
1535         }
1536
1537         /*
1538          * If the inode is now clean drop the space reservation.
1539          */
1540         if ((ip->flags & HAMMER_INODE_MODMASK) == 0 &&
1541             (ip->flags & HAMMER_INODE_RSV_INODES)) {
1542                 ip->flags &= ~HAMMER_INODE_RSV_INODES;
1543                 --hmp->rsv_inodes;
1544         }
1545
1546         /*
1547          * Finally, if the frontend is waiting for a flush to complete,
1548          * wake it up.
1549          */
1550         if (ip->flush_state != HAMMER_FST_FLUSH) {
1551                 if (ip->flags & HAMMER_INODE_FLUSHW) {
1552                         ip->flags &= ~HAMMER_INODE_FLUSHW;
1553                         wakeup(&ip->flags);
1554                 }
1555         }
1556         if (dorel)
1557                 hammer_rel_inode(ip, 0);
1558 }
1559
1560 /*
1561  * Called from hammer_sync_inode() to synchronize in-memory records
1562  * to the media.
1563  */
1564 static int
1565 hammer_sync_record_callback(hammer_record_t record, void *data)
1566 {
1567         hammer_cursor_t cursor = data;
1568         hammer_transaction_t trans = cursor->trans;
1569         int error;
1570
1571         /*
1572          * Skip records that do not belong to the current flush.
1573          */
1574         ++hammer_stats_record_iterations;
1575         if (record->flush_state != HAMMER_FST_FLUSH)
1576                 return(0);
1577
1578 #if 1
1579         if (record->flush_group != record->ip->flush_group) {
1580                 kprintf("sync_record %p ip %p bad flush group %d %d\n", record, record->ip, record->flush_group ,record->ip->flush_group);
1581                 Debugger("blah2");
1582                 return(0);
1583         }
1584 #endif
1585         KKASSERT(record->flush_group == record->ip->flush_group);
1586
1587         /*
1588          * Interlock the record using the BE flag.  Once BE is set the
1589          * frontend cannot change the state of FE.
1590          *
1591          * NOTE: If FE is set prior to us setting BE we still sync the
1592          * record out, but the flush completion code converts it to
1593          * a delete-on-disk record instead of destroying it.
1594          */
1595         KKASSERT((record->flags & HAMMER_RECF_INTERLOCK_BE) == 0);
1596         record->flags |= HAMMER_RECF_INTERLOCK_BE;
1597
1598         /*
1599          * The backend may have already disposed of the record.
1600          */
1601         if (record->flags & HAMMER_RECF_DELETED_BE) {
1602                 error = 0;
1603                 goto done;
1604         }
1605
1606         /*
1607          * If the whole inode is being deleting all on-disk records will
1608          * be deleted very soon, we can't sync any new records to disk
1609          * because they will be deleted in the same transaction they were
1610          * created in (delete_tid == create_tid), which will assert.
1611          *
1612          * XXX There may be a case with RECORD_ADD with DELETED_FE set
1613          * that we currently panic on.
1614          */
1615         if (record->ip->sync_flags & HAMMER_INODE_DELETING) {
1616                 switch(record->type) {
1617                 case HAMMER_MEM_RECORD_DATA:
1618                         /*
1619                          * We don't have to do anything, if the record was
1620                          * committed the space will have been accounted for
1621                          * in the blockmap.
1622                          */
1623                         /* fall through */
1624                 case HAMMER_MEM_RECORD_GENERAL:
1625                         record->flags |= HAMMER_RECF_DELETED_FE;
1626                         record->flags |= HAMMER_RECF_DELETED_BE;
1627                         error = 0;
1628                         goto done;
1629                 case HAMMER_MEM_RECORD_ADD:
1630                         panic("hammer_sync_record_callback: illegal add "
1631                               "during inode deletion record %p", record);
1632                         break; /* NOT REACHED */
1633                 case HAMMER_MEM_RECORD_INODE:
1634                         panic("hammer_sync_record_callback: attempt to "
1635                               "sync inode record %p?", record);
1636                         break; /* NOT REACHED */
1637                 case HAMMER_MEM_RECORD_DEL:
1638                         /*
1639                          * Follow through and issue the on-disk deletion
1640                          */
1641                         break;
1642                 }
1643         }
1644
1645         /*
1646          * If DELETED_FE is set special handling is needed for directory
1647          * entries.  Dependant pieces related to the directory entry may
1648          * have already been synced to disk.  If this occurs we have to
1649          * sync the directory entry and then change the in-memory record
1650          * from an ADD to a DELETE to cover the fact that it's been
1651          * deleted by the frontend.
1652          *
1653          * A directory delete covering record (MEM_RECORD_DEL) can never
1654          * be deleted by the frontend.
1655          *
1656          * Any other record type (aka DATA) can be deleted by the frontend.
1657          * XXX At the moment the flusher must skip it because there may
1658          * be another data record in the flush group for the same block,
1659          * meaning that some frontend data changes can leak into the backend's
1660          * synchronization point.
1661          */
1662         if (record->flags & HAMMER_RECF_DELETED_FE) {
1663                 if (record->type == HAMMER_MEM_RECORD_ADD) {
1664                         record->flags |= HAMMER_RECF_CONVERT_DELETE;
1665                 } else {
1666                         KKASSERT(record->type != HAMMER_MEM_RECORD_DEL);
1667                         record->flags |= HAMMER_RECF_DELETED_BE;
1668                         error = 0;
1669                         goto done;
1670                 }
1671         }
1672
1673         /*
1674          * Assign the create_tid for new records.  Deletions already
1675          * have the record's entire key properly set up.
1676          */
1677         if (record->type != HAMMER_MEM_RECORD_DEL)
1678                 record->leaf.base.create_tid = trans->tid;
1679         for (;;) {
1680                 error = hammer_ip_sync_record_cursor(cursor, record);
1681                 if (error != EDEADLK)
1682                         break;
1683                 hammer_done_cursor(cursor);
1684                 error = hammer_init_cursor(trans, cursor, &record->ip->cache[0],
1685                                            record->ip);
1686                 if (error)
1687                         break;
1688         }
1689         record->flags &= ~HAMMER_RECF_CONVERT_DELETE;
1690
1691         if (error) {
1692                 error = -error;
1693                 if (error != -ENOSPC) {
1694                         kprintf("hammer_sync_record_callback: sync failed rec "
1695                                 "%p, error %d\n", record, error);
1696                         Debugger("sync failed rec");
1697                 }
1698         }
1699 done:
1700         hammer_flush_record_done(record, error);
1701         return(error);
1702 }
1703
1704 /*
1705  * XXX error handling
1706  */
1707 int
1708 hammer_sync_inode(hammer_inode_t ip)
1709 {
1710         struct hammer_transaction trans;
1711         struct hammer_cursor cursor;
1712         hammer_node_t tmp_node;
1713         hammer_record_t depend;
1714         hammer_record_t next;
1715         int error, tmp_error;
1716         u_int64_t nlinks;
1717
1718         if ((ip->sync_flags & HAMMER_INODE_MODMASK) == 0)
1719                 return(0);
1720
1721         hammer_start_transaction_fls(&trans, ip->hmp);
1722         error = hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
1723         if (error)
1724                 goto done;
1725
1726         /*
1727          * Any directory records referencing this inode which are not in
1728          * our current flush group must adjust our nlink count for the
1729          * purposes of synchronization to disk.
1730          *
1731          * Records which are in our flush group can be unlinked from our
1732          * inode now, potentially allowing the inode to be physically
1733          * deleted.
1734          *
1735          * This cannot block.
1736          */
1737         nlinks = ip->ino_data.nlinks;
1738         next = TAILQ_FIRST(&ip->target_list);
1739         while ((depend = next) != NULL) {
1740                 next = TAILQ_NEXT(depend, target_entry);
1741                 if (depend->flush_state == HAMMER_FST_FLUSH &&
1742                     depend->flush_group == ip->hmp->flusher.act) {
1743                         /*
1744                          * If this is an ADD that was deleted by the frontend
1745                          * the frontend nlinks count will have already been
1746                          * decremented, but the backend is going to sync its
1747                          * directory entry and must account for it.  The
1748                          * record will be converted to a delete-on-disk when
1749                          * it gets synced.
1750                          *
1751                          * If the ADD was not deleted by the frontend we
1752                          * can remove the dependancy from our target_list.
1753                          */
1754                         if (depend->flags & HAMMER_RECF_DELETED_FE) {
1755                                 ++nlinks;
1756                         } else {
1757                                 TAILQ_REMOVE(&ip->target_list, depend,
1758                                              target_entry);
1759                                 depend->target_ip = NULL;
1760                         }
1761                 } else if ((depend->flags & HAMMER_RECF_DELETED_FE) == 0) {
1762                         /*
1763                          * Not part of our flush group
1764                          */
1765                         KKASSERT((depend->flags & HAMMER_RECF_DELETED_BE) == 0);
1766                         switch(depend->type) {
1767                         case HAMMER_MEM_RECORD_ADD:
1768                                 --nlinks;
1769                                 break;
1770                         case HAMMER_MEM_RECORD_DEL:
1771                                 ++nlinks;
1772                                 break;
1773                         default:
1774                                 break;
1775                         }
1776                 }
1777         }
1778
1779         /*
1780          * Set dirty if we had to modify the link count.
1781          */
1782         if (ip->sync_ino_data.nlinks != nlinks) {
1783                 KKASSERT((int64_t)nlinks >= 0);
1784                 ip->sync_ino_data.nlinks = nlinks;
1785                 ip->sync_flags |= HAMMER_INODE_DDIRTY;
1786         }
1787
1788         /*
1789          * If there is a trunction queued destroy any data past the (aligned)
1790          * truncation point.  Userland will have dealt with the buffer
1791          * containing the truncation point for us.
1792          *
1793          * We don't flush pending frontend data buffers until after we've
1794          * dealt with the truncation.
1795          */
1796         if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
1797                 /*
1798                  * Interlock trunc_off.  The VOP front-end may continue to
1799                  * make adjustments to it while we are blocked.
1800                  */
1801                 off_t trunc_off;
1802                 off_t aligned_trunc_off;
1803                 int blkmask;
1804
1805                 trunc_off = ip->sync_trunc_off;
1806                 blkmask = hammer_blocksize(trunc_off) - 1;
1807                 aligned_trunc_off = (trunc_off + blkmask) & ~(int64_t)blkmask;
1808
1809                 /*
1810                  * Delete any whole blocks on-media.  The front-end has
1811                  * already cleaned out any partial block and made it
1812                  * pending.  The front-end may have updated trunc_off
1813                  * while we were blocked so we only use sync_trunc_off.
1814                  */
1815                 error = hammer_ip_delete_range(&cursor, ip,
1816                                                 aligned_trunc_off,
1817                                                 0x7FFFFFFFFFFFFFFFLL, 1);
1818                 if (error)
1819                         Debugger("hammer_ip_delete_range errored");
1820
1821                 /*
1822                  * Clear the truncation flag on the backend after we have
1823                  * complete the deletions.  Backend data is now good again
1824                  * (including new records we are about to sync, below).
1825                  *
1826                  * Leave sync_trunc_off intact.  As we write additional
1827                  * records the backend will update sync_trunc_off.  This
1828                  * tells the backend whether it can skip the overwrite
1829                  * test.  This should work properly even when the backend
1830                  * writes full blocks where the truncation point straddles
1831                  * the block because the comparison is against the base
1832                  * offset of the record.
1833                  */
1834                 ip->sync_flags &= ~HAMMER_INODE_TRUNCATED;
1835                 /* ip->sync_trunc_off = 0x7FFFFFFFFFFFFFFFLL; */
1836         } else {
1837                 error = 0;
1838         }
1839
1840         /*
1841          * Now sync related records.  These will typically be directory
1842          * entries or delete-on-disk records.
1843          *
1844          * Not all records will be flushed, but clear XDIRTY anyway.  We
1845          * will set it again in the frontend hammer_flush_inode_done()
1846          * if records remain.
1847          */
1848         if (error == 0) {
1849                 tmp_error = RB_SCAN(hammer_rec_rb_tree, &ip->rec_tree, NULL,
1850                                     hammer_sync_record_callback, &cursor);
1851                 if (tmp_error < 0)
1852                         tmp_error = -error;
1853                 if (tmp_error)
1854                         error = tmp_error;
1855         }
1856         hammer_cache_node(&ip->cache[1], cursor.node);
1857
1858         /*
1859          * Re-seek for inode update, assuming our cache hasn't been ripped
1860          * out from under us.
1861          */
1862         if (error == 0) {
1863                 tmp_node = hammer_ref_node_safe(ip->hmp, &ip->cache[0], &error);
1864                 if (tmp_node) {
1865                         if ((tmp_node->flags & HAMMER_NODE_DELETED) == 0)
1866                                 hammer_cursor_seek(&cursor, tmp_node, 0);
1867                         hammer_rel_node(tmp_node);
1868                 }
1869                 error = 0;
1870         }
1871
1872         /*
1873          * If we are deleting the inode the frontend had better not have
1874          * any active references on elements making up the inode.
1875          */
1876         if (error == 0 && ip->sync_ino_data.nlinks == 0 &&
1877                 RB_EMPTY(&ip->rec_tree)  &&
1878             (ip->sync_flags & HAMMER_INODE_DELETING) &&
1879             (ip->flags & HAMMER_INODE_DELETED) == 0) {
1880                 int count1 = 0;
1881
1882                 ip->flags |= HAMMER_INODE_DELETED;
1883                 error = hammer_ip_delete_range_all(&cursor, ip, &count1);
1884                 if (error == 0) {
1885                         ip->sync_flags &= ~HAMMER_INODE_DELETING;
1886                         ip->sync_flags &= ~HAMMER_INODE_TRUNCATED;
1887                         KKASSERT(RB_EMPTY(&ip->rec_tree));
1888
1889                         /*
1890                          * Set delete_tid in both the frontend and backend
1891                          * copy of the inode record.  The DELETED flag handles
1892                          * this, do not set RDIRTY.
1893                          */
1894                         ip->ino_leaf.base.delete_tid = trans.tid;
1895                         ip->sync_ino_leaf.base.delete_tid = trans.tid;
1896
1897                         /*
1898                          * Adjust the inode count in the volume header
1899                          */
1900                         if (ip->flags & HAMMER_INODE_ONDISK) {
1901                                 hammer_modify_volume_field(&trans,
1902                                                            trans.rootvol,
1903                                                            vol0_stat_inodes);
1904                                 --ip->hmp->rootvol->ondisk->vol0_stat_inodes;
1905                                 hammer_modify_volume_done(trans.rootvol);
1906                         }
1907                 } else {
1908                         ip->flags &= ~HAMMER_INODE_DELETED;
1909                         Debugger("hammer_ip_delete_range_all errored");
1910                 }
1911         }
1912
1913         ip->sync_flags &= ~HAMMER_INODE_BUFS;
1914
1915         if (error)
1916                 Debugger("RB_SCAN errored");
1917
1918         /*
1919          * Now update the inode's on-disk inode-data and/or on-disk record.
1920          * DELETED and ONDISK are managed only in ip->flags.
1921          */
1922         switch(ip->flags & (HAMMER_INODE_DELETED | HAMMER_INODE_ONDISK)) {
1923         case HAMMER_INODE_DELETED|HAMMER_INODE_ONDISK:
1924                 /*
1925                  * If deleted and on-disk, don't set any additional flags.
1926                  * the delete flag takes care of things.
1927                  *
1928                  * Clear flags which may have been set by the frontend.
1929                  */
1930                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
1931                                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME |
1932                                     HAMMER_INODE_DELETING);
1933                 break;
1934         case HAMMER_INODE_DELETED:
1935                 /*
1936                  * Take care of the case where a deleted inode was never
1937                  * flushed to the disk in the first place.
1938                  *
1939                  * Clear flags which may have been set by the frontend.
1940                  */
1941                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
1942                                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME |
1943                                     HAMMER_INODE_DELETING);
1944                 while (RB_ROOT(&ip->rec_tree)) {
1945                         hammer_record_t record = RB_ROOT(&ip->rec_tree);
1946                         hammer_ref(&record->lock);
1947                         KKASSERT(record->lock.refs == 1);
1948                         record->flags |= HAMMER_RECF_DELETED_FE;
1949                         record->flags |= HAMMER_RECF_DELETED_BE;
1950                         hammer_rel_mem_record(record);
1951                 }
1952                 break;
1953         case HAMMER_INODE_ONDISK:
1954                 /*
1955                  * If already on-disk, do not set any additional flags.
1956                  */
1957                 break;
1958         default:
1959                 /*
1960                  * If not on-disk and not deleted, set DDIRTY to force
1961                  * an initial record to be written.
1962                  *
1963                  * Also set the create_tid in both the frontend and backend
1964                  * copy of the inode record.
1965                  */
1966                 ip->ino_leaf.base.create_tid = trans.tid;
1967                 ip->sync_ino_leaf.base.create_tid = trans.tid;
1968                 ip->sync_flags |= HAMMER_INODE_DDIRTY;
1969                 break;
1970         }
1971
1972         /*
1973          * If RDIRTY or DDIRTY is set, write out a new record.  If the inode
1974          * is already on-disk the old record is marked as deleted.
1975          *
1976          * If DELETED is set hammer_update_inode() will delete the existing
1977          * record without writing out a new one.
1978          *
1979          * If *ONLY* the ITIMES flag is set we can update the record in-place.
1980          */
1981         if (ip->flags & HAMMER_INODE_DELETED) {
1982                 error = hammer_update_inode(&cursor, ip);
1983         } else
1984         if ((ip->sync_flags & HAMMER_INODE_DDIRTY) == 0 &&
1985             (ip->sync_flags & (HAMMER_INODE_ATIME | HAMMER_INODE_MTIME))) {
1986                 error = hammer_update_itimes(&cursor, ip);
1987         } else
1988         if (ip->sync_flags & (HAMMER_INODE_DDIRTY | HAMMER_INODE_ATIME | HAMMER_INODE_MTIME)) {
1989                 error = hammer_update_inode(&cursor, ip);
1990         }
1991         if (error)
1992                 Debugger("hammer_update_itimes/inode errored");
1993 done:
1994         /*
1995          * Save the TID we used to sync the inode with to make sure we
1996          * do not improperly reuse it.
1997          */
1998         hammer_done_cursor(&cursor);
1999         hammer_done_transaction(&trans);
2000         return(error);
2001 }
2002
2003 /*
2004  * This routine is called when the OS is no longer actively referencing
2005  * the inode (but might still be keeping it cached), or when releasing
2006  * the last reference to an inode.
2007  *
2008  * At this point if the inode's nlinks count is zero we want to destroy
2009  * it, which may mean destroying it on-media too.
2010  */
2011 void
2012 hammer_inode_unloadable_check(hammer_inode_t ip, int getvp)
2013 {
2014         struct vnode *vp;
2015
2016         /*
2017          * Set the DELETING flag when the link count drops to 0 and the
2018          * OS no longer has any opens on the inode.
2019          *
2020          * The backend will clear DELETING (a mod flag) and set DELETED
2021          * (a state flag) when it is actually able to perform the
2022          * operation.
2023          */
2024         if (ip->ino_data.nlinks == 0 &&
2025             (ip->flags & (HAMMER_INODE_DELETING|HAMMER_INODE_DELETED)) == 0) {
2026                 ip->flags |= HAMMER_INODE_DELETING;
2027                 ip->flags |= HAMMER_INODE_TRUNCATED;
2028                 ip->trunc_off = 0;
2029                 vp = NULL;
2030                 if (getvp) {
2031                         if (hammer_get_vnode(ip, &vp) != 0)
2032                                 return;
2033                 }
2034
2035                 /*
2036                  * Final cleanup
2037                  */
2038                 if (ip->vp) {
2039                         vtruncbuf(ip->vp, 0, HAMMER_BUFSIZE);
2040                         vnode_pager_setsize(ip->vp, 0);
2041                 }
2042                 if (getvp) {
2043                         vput(vp);
2044                 }
2045         }
2046 }
2047
2048 /*
2049  * Re-test an inode when a dependancy had gone away to see if we
2050  * can chain flush it.
2051  */
2052 void
2053 hammer_test_inode(hammer_inode_t ip)
2054 {
2055         if (ip->flags & HAMMER_INODE_REFLUSH) {
2056                 ip->flags &= ~HAMMER_INODE_REFLUSH;
2057                 hammer_ref(&ip->lock);
2058                 if (ip->flags & HAMMER_INODE_RESIGNAL) {
2059                         ip->flags &= ~HAMMER_INODE_RESIGNAL;
2060                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
2061                 } else {
2062                         hammer_flush_inode(ip, 0);
2063                 }
2064                 hammer_rel_inode(ip, 0);
2065         }
2066 }
2067
2068 /*
2069  * Clear the RECLAIM flag on an inode.  This occurs when the inode is
2070  * reassociated with a vp or just before it gets freed.
2071  *
2072  * Wakeup one thread blocked waiting on reclaims to complete.  Note that
2073  * the inode the thread is waiting on behalf of is a different inode then
2074  * the inode we are called with.  This is to create a pipeline.
2075  */
2076 static void
2077 hammer_inode_wakereclaims(hammer_inode_t ip)
2078 {
2079         struct hammer_reclaim *reclaim;
2080         hammer_mount_t hmp = ip->hmp;
2081
2082         if ((ip->flags & HAMMER_INODE_RECLAIM) == 0)
2083                 return;
2084
2085         --hammer_count_reclaiming;
2086         --hmp->inode_reclaims;
2087         ip->flags &= ~HAMMER_INODE_RECLAIM;
2088
2089         if ((reclaim = TAILQ_FIRST(&hmp->reclaim_list)) != NULL) {
2090                 TAILQ_REMOVE(&hmp->reclaim_list, reclaim, entry);
2091                 reclaim->okydoky = 1;
2092                 wakeup(reclaim);
2093         }
2094 }
2095
2096 /*
2097  * Setup our reclaim pipeline.  We only let so many detached (and dirty)
2098  * inodes build up before we start blocking.
2099  *
2100  * When we block we don't care *which* inode has finished reclaiming,
2101  * as lone as one does.  This is somewhat heuristical... we also put a
2102  * cap on how long we are willing to wait.
2103  */
2104 void
2105 hammer_inode_waitreclaims(hammer_mount_t hmp)
2106 {
2107         struct hammer_reclaim reclaim;
2108         int delay;
2109
2110         if (hmp->inode_reclaims > HAMMER_RECLAIM_WAIT) {
2111                 reclaim.okydoky = 0;
2112                 TAILQ_INSERT_TAIL(&hmp->reclaim_list,
2113                                   &reclaim, entry);
2114         } else {
2115                 reclaim.okydoky = 1;
2116         }
2117
2118         if (reclaim.okydoky == 0) {
2119                 delay = (hmp->inode_reclaims - HAMMER_RECLAIM_WAIT) * hz /
2120                         HAMMER_RECLAIM_WAIT;
2121                 if (delay >= 0)
2122                         tsleep(&reclaim, 0, "hmrrcm", delay + 1);
2123                 if (reclaim.okydoky == 0)
2124                         TAILQ_REMOVE(&hmp->reclaim_list, &reclaim, entry);
2125         }
2126 }
2127