sys/vfs/hammer/hammer_inode.c

   1 /*
   2  * Copyright (c) 2007-2008 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/vfs/hammer/hammer_inode.c,v 1.79 2008/06/20 21:24:53 dillon Exp $
  35  */
  36
  37 #include "hammer.h"
  38 #include <vm/vm_extern.h>
  39 #include <sys/buf.h>
  40 #include <sys/buf2.h>
  41
  42 static int      hammer_unload_inode(struct hammer_inode *ip);
  43 static void     hammer_flush_inode_core(hammer_inode_t ip, int flags);
  44 static int      hammer_setup_child_callback(hammer_record_t rec, void *data);
  45 static int      hammer_setup_parent_inodes(hammer_inode_t ip);
  46 static int      hammer_setup_parent_inodes_helper(hammer_record_t record);
  47 static void     hammer_inode_wakereclaims(hammer_inode_t ip);
  48
  49 #ifdef DEBUG_TRUNCATE
  50 extern struct hammer_inode *HammerTruncIp;
  51 #endif
  52
  53 /*
  54  * The kernel is not actively referencing this vnode but is still holding
  55  * it cached.
  56  *
  57  * This is called from the frontend.
  58  */
  59 int
  60 hammer_vop_inactive(struct vop_inactive_args *ap)
  61 {
  62         struct hammer_inode *ip = VTOI(ap->a_vp);
  63
  64         /*
  65          * Degenerate case
  66          */
  67         if (ip == NULL) {
  68                 vrecycle(ap->a_vp);
  69                 return(0);
  70         }
  71
  72         /*
  73          * If the inode no longer has visibility in the filesystem try to
  74          * recycle it immediately, even if the inode is dirty.  Recycling
  75          * it quickly allows the system to reclaim buffer cache and VM
  76          * resources which can matter a lot in a heavily loaded system.
  77          *
  78          * This can deadlock in vfsync() if we aren't careful.
  79          *
  80          * Do not queue the inode to the flusher if we still have visibility,
  81          * otherwise namespace calls such as chmod will unnecessarily generate
  82          * multiple inode updates.
  83          */
  84         hammer_inode_unloadable_check(ip, 0);
  85         if (ip->ino_data.nlinks == 0) {
  86                 if (ip->flags & HAMMER_INODE_MODMASK)
  87                         hammer_flush_inode(ip, 0);
  88                 vrecycle(ap->a_vp);
  89         }
  90         return(0);
  91 }
  92
  93 /*
  94  * Release the vnode association.  This is typically (but not always)
  95  * the last reference on the inode.
  96  *
  97  * Once the association is lost we are on our own with regards to
  98  * flushing the inode.
  99  */
 100 int
 101 hammer_vop_reclaim(struct vop_reclaim_args *ap)
 102 {
 103         struct hammer_inode *ip;
 104         hammer_mount_t hmp;
 105         struct vnode *vp;
 106
 107         vp = ap->a_vp;
 108
 109         if ((ip = vp->v_data) != NULL) {
 110                 hmp = ip->hmp;
 111                 vp->v_data = NULL;
 112                 ip->vp = NULL;
 113
 114                 if ((ip->flags & HAMMER_INODE_RECLAIM) == 0) {
 115                         ++hammer_count_reclaiming;
 116                         ++hmp->inode_reclaims;
 117                         ip->flags |= HAMMER_INODE_RECLAIM;
 118                         if (hmp->inode_reclaims > HAMMER_RECLAIM_FLUSH &&
 119                             (hmp->inode_reclaims & 255) == 0) {
 120                                 hammer_flusher_async(hmp);
 121                         }
 122                 }
 123                 hammer_rel_inode(ip, 1);
 124         }
 125         return(0);
 126 }
 127
 128 /*
 129  * Return a locked vnode for the specified inode.  The inode must be
 130  * referenced but NOT LOCKED on entry and will remain referenced on
 131  * return.
 132  *
 133  * Called from the frontend.
 134  */
 135 int
 136 hammer_get_vnode(struct hammer_inode *ip, struct vnode **vpp)
 137 {
 138         hammer_mount_t hmp;
 139         struct vnode *vp;
 140         int error = 0;
 141
 142         hmp = ip->hmp;
 143
 144         for (;;) {
 145                 if ((vp = ip->vp) == NULL) {
 146                         error = getnewvnode(VT_HAMMER, hmp->mp, vpp, 0, 0);
 147                         if (error)
 148                                 break;
 149                         hammer_lock_ex(&ip->lock);
 150                         if (ip->vp != NULL) {
 151                                 hammer_unlock(&ip->lock);
 152                                 vp->v_type = VBAD;
 153                                 vx_put(vp);
 154                                 continue;
 155                         }
 156                         hammer_ref(&ip->lock);
 157                         vp = *vpp;
 158                         ip->vp = vp;
 159                         vp->v_type =
 160                                 hammer_get_vnode_type(ip->ino_data.obj_type);
 161
 162                         hammer_inode_wakereclaims(ip);
 163
 164                         switch(ip->ino_data.obj_type) {
 165                         case HAMMER_OBJTYPE_CDEV:
 166                         case HAMMER_OBJTYPE_BDEV:
 167                                 vp->v_ops = &hmp->mp->mnt_vn_spec_ops;
 168                                 addaliasu(vp, ip->ino_data.rmajor,
 169                                           ip->ino_data.rminor);
 170                                 break;
 171                         case HAMMER_OBJTYPE_FIFO:
 172                                 vp->v_ops = &hmp->mp->mnt_vn_fifo_ops;
 173                                 break;
 174                         default:
 175                                 break;
 176                         }
 177
 178                         /*
 179                          * Only mark as the root vnode if the ip is not
 180                          * historical, otherwise the VFS cache will get
 181                          * confused.  The other half of the special handling
 182                          * is in hammer_vop_nlookupdotdot().
 183                          *
 184                          * Pseudo-filesystem roots also do not count.
 185                          */
 186                         if (ip->obj_id == HAMMER_OBJID_ROOT &&
 187                             ip->obj_asof == hmp->asof &&
 188                             ip->obj_localization == 0) {
 189                                 vp->v_flag |= VROOT;
 190                         }
 191
 192                         vp->v_data = (void *)ip;
 193                         /* vnode locked by getnewvnode() */
 194                         /* make related vnode dirty if inode dirty? */
 195                         hammer_unlock(&ip->lock);
 196                         if (vp->v_type == VREG)
 197                                 vinitvmio(vp, ip->ino_data.size);
 198                         break;
 199                 }
 200
 201                 /*
 202                  * loop if the vget fails (aka races), or if the vp
 203                  * no longer matches ip->vp.
 204                  */
 205                 if (vget(vp, LK_EXCLUSIVE) == 0) {
 206                         if (vp == ip->vp)
 207                                 break;
 208                         vput(vp);
 209                 }
 210         }
 211         *vpp = vp;
 212         return(error);
 213 }
 214
 215 /*
 216  * Acquire a HAMMER inode.  The returned inode is not locked.  These functions
 217  * do not attach or detach the related vnode (use hammer_get_vnode() for
 218  * that).
 219  *
 220  * The flags argument is only applied for newly created inodes, and only
 221  * certain flags are inherited.
 222  *
 223  * Called from the frontend.
 224  */
 225 struct hammer_inode *
 226 hammer_get_inode(hammer_transaction_t trans, hammer_inode_t dip,
 227                  u_int64_t obj_id, hammer_tid_t asof, u_int32_t localization,
 228                  int flags, int *errorp)
 229 {
 230         hammer_mount_t hmp = trans->hmp;
 231         struct hammer_inode_info iinfo;
 232         struct hammer_cursor cursor;
 233         struct hammer_inode *ip;
 234
 235         /*
 236          * Determine if we already have an inode cached.  If we do then
 237          * we are golden.
 238          */
 239         iinfo.obj_id = obj_id;
 240         iinfo.obj_asof = asof;
 241         iinfo.obj_localization = localization;
 242 loop:
 243         ip = hammer_ino_rb_tree_RB_LOOKUP_INFO(&hmp->rb_inos_root, &iinfo);
 244         if (ip) {
 245                 hammer_ref(&ip->lock);
 246                 *errorp = 0;
 247                 return(ip);
 248         }
 249
 250         /*
 251          * Allocate a new inode structure and deal with races later.
 252          */
 253         ip = kmalloc(sizeof(*ip), M_HAMMER, M_WAITOK|M_ZERO);
 254         ++hammer_count_inodes;
 255         ++hmp->count_inodes;
 256         ip->obj_id = obj_id;
 257         ip->obj_asof = iinfo.obj_asof;
 258         ip->obj_localization = localization;
 259         ip->hmp = hmp;
 260         ip->flags = flags & HAMMER_INODE_RO;
 261         ip->cache[0].ip = ip;
 262         ip->cache[1].ip = ip;
 263         if (hmp->ronly)
 264                 ip->flags |= HAMMER_INODE_RO;
 265         ip->sync_trunc_off = ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
 266         RB_INIT(&ip->rec_tree);
 267         TAILQ_INIT(&ip->target_list);
 268
 269         /*
 270          * Locate the on-disk inode.
 271          */
 272 retry:
 273         hammer_init_cursor(trans, &cursor, (dip ? &dip->cache[0] : NULL), NULL);
 274         cursor.key_beg.localization = HAMMER_LOCALIZE_INODE;
 275         cursor.key_beg.obj_id = ip->obj_id;
 276         cursor.key_beg.key = 0;
 277         cursor.key_beg.create_tid = 0;
 278         cursor.key_beg.delete_tid = 0;
 279         cursor.key_beg.rec_type = HAMMER_RECTYPE_INODE;
 280         cursor.key_beg.obj_type = 0;
 281         cursor.asof = iinfo.obj_asof;
 282         cursor.flags = HAMMER_CURSOR_GET_LEAF | HAMMER_CURSOR_GET_DATA |
 283                        HAMMER_CURSOR_ASOF;
 284
 285         *errorp = hammer_btree_lookup(&cursor);
 286         if (*errorp == EDEADLK) {
 287                 hammer_done_cursor(&cursor);
 288                 goto retry;
 289         }
 290
 291         /*
 292          * On success the B-Tree lookup will hold the appropriate
 293          * buffer cache buffers and provide a pointer to the requested
 294          * information.  Copy the information to the in-memory inode
 295          * and cache the B-Tree node to improve future operations.
 296          */
 297         if (*errorp == 0) {
 298                 ip->ino_leaf = cursor.node->ondisk->elms[cursor.index].leaf;
 299                 ip->ino_data = cursor.data->inode;
 300
 301                 /*
 302                  * cache[0] tries to cache the location of the object inode.
 303                  * The assumption is that it is near the directory inode.
 304                  *
 305                  * cache[1] tries to cache the location of the object data.
 306                  * The assumption is that it is near the directory data.
 307                  */
 308                 hammer_cache_node(&ip->cache[0], cursor.node);
 309                 if (dip && dip->cache[1].node)
 310                         hammer_cache_node(&ip->cache[1], dip->cache[1].node);
 311
 312                 /*
 313                  * The file should not contain any data past the file size
 314                  * stored in the inode.  Setting sync_trunc_off to the
 315                  * file size instead of max reduces B-Tree lookup overheads
 316                  * on append by allowing the flusher to avoid checking for
 317                  * record overwrites.
 318                  */
 319                 ip->sync_trunc_off = ip->ino_data.size;
 320         }
 321
 322         /*
 323          * The inode is placed on the red-black tree and will be synced to
 324          * the media when flushed or by the filesystem sync.  If this races
 325          * another instantiation/lookup the insertion will fail.
 326          */
 327         if (*errorp == 0) {
 328                 hammer_ref(&ip->lock);
 329                 if (RB_INSERT(hammer_ino_rb_tree, &hmp->rb_inos_root, ip)) {
 330                         hammer_uncache_node(&ip->cache[0]);
 331                         hammer_uncache_node(&ip->cache[1]);
 332                         KKASSERT(ip->lock.refs == 1);
 333                         --hammer_count_inodes;
 334                         --hmp->count_inodes;
 335                         kfree(ip, M_HAMMER);
 336                         hammer_done_cursor(&cursor);
 337                         goto loop;
 338                 }
 339                 ip->flags |= HAMMER_INODE_ONDISK;
 340         } else {
 341                 /*
 342                  * Do not panic on read-only accesses which fail, particularly
 343                  * historical accesses where the snapshot might not have
 344                  * complete connectivity.
 345                  */
 346                 if ((flags & HAMMER_INODE_RO) == 0) {
 347                         kprintf("hammer_get_inode: failed ip %p obj_id %016llx cursor %p error %d\n",
 348                                 ip, ip->obj_id, &cursor, *errorp);
 349                         Debugger("x");
 350                 }
 351                 if (ip->flags & HAMMER_INODE_RSV_INODES) {
 352                         ip->flags &= ~HAMMER_INODE_RSV_INODES; /* sanity */
 353                         --hmp->rsv_inodes;
 354                 }
 355                 hmp->rsv_databufs -= ip->rsv_databufs;
 356                 ip->rsv_databufs = 0;                          /* sanity */
 357
 358                 --hammer_count_inodes;
 359                 --hmp->count_inodes;
 360                 kfree(ip, M_HAMMER);
 361                 ip = NULL;
 362         }
 363         hammer_done_cursor(&cursor);
 364         return (ip);
 365 }
 366
 367 /*
 368  * Create a new filesystem object, returning the inode in *ipp.  The
 369  * returned inode will be referenced.
 370  *
 371  * The inode is created in-memory.
 372  */
 373 int
 374 hammer_create_inode(hammer_transaction_t trans, struct vattr *vap,
 375                     struct ucred *cred, hammer_inode_t dip,
 376                     struct hammer_inode **ipp)
 377 {
 378         hammer_mount_t hmp;
 379         hammer_inode_t ip;
 380         uid_t xuid;
 381
 382         hmp = trans->hmp;
 383         ip = kmalloc(sizeof(*ip), M_HAMMER, M_WAITOK|M_ZERO);
 384         ++hammer_count_inodes;
 385         ++hmp->count_inodes;
 386         ip->obj_id = hammer_alloc_objid(trans, dip);
 387         KKASSERT(ip->obj_id != 0);
 388         ip->obj_asof = hmp->asof;
 389         ip->obj_localization = dip->obj_localization;
 390         ip->hmp = hmp;
 391         ip->flush_state = HAMMER_FST_IDLE;
 392         ip->flags = HAMMER_INODE_DDIRTY |
 393                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME;
 394         ip->cache[0].ip = ip;
 395         ip->cache[1].ip = ip;
 396
 397         ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
 398         RB_INIT(&ip->rec_tree);
 399         TAILQ_INIT(&ip->target_list);
 400
 401         ip->ino_data.atime = trans->time;
 402         ip->ino_data.mtime = trans->time;
 403         ip->ino_data.size = 0;
 404         ip->ino_data.nlinks = 0;
 405
 406         /*
 407          * A nohistory designator on the parent directory is inherited by
 408          * the child.
 409          */
 410         ip->ino_data.uflags = dip->ino_data.uflags &
 411                               (SF_NOHISTORY|UF_NOHISTORY|UF_NODUMP);
 412
 413         ip->ino_leaf.base.btype = HAMMER_BTREE_TYPE_RECORD;
 414         ip->ino_leaf.base.localization = HAMMER_LOCALIZE_INODE;
 415         ip->ino_leaf.base.obj_id = ip->obj_id;
 416         ip->ino_leaf.base.key = 0;
 417         ip->ino_leaf.base.create_tid = 0;
 418         ip->ino_leaf.base.delete_tid = 0;
 419         ip->ino_leaf.base.rec_type = HAMMER_RECTYPE_INODE;
 420         ip->ino_leaf.base.obj_type = hammer_get_obj_type(vap->va_type);
 421
 422         ip->ino_data.obj_type = ip->ino_leaf.base.obj_type;
 423         ip->ino_data.version = HAMMER_INODE_DATA_VERSION;
 424         ip->ino_data.mode = vap->va_mode;
 425         ip->ino_data.ctime = trans->time;
 426         ip->ino_data.parent_obj_id = (dip) ? dip->ino_leaf.base.obj_id : 0;
 427
 428         switch(ip->ino_leaf.base.obj_type) {
 429         case HAMMER_OBJTYPE_CDEV:
 430         case HAMMER_OBJTYPE_BDEV:
 431                 ip->ino_data.rmajor = vap->va_rmajor;
 432                 ip->ino_data.rminor = vap->va_rminor;
 433                 break;
 434         default:
 435                 break;
 436         }
 437
 438         /*
 439          * Calculate default uid/gid and overwrite with information from
 440          * the vap.
 441          */
 442         xuid = hammer_to_unix_xid(&dip->ino_data.uid);
 443         xuid = vop_helper_create_uid(hmp->mp, dip->ino_data.mode, xuid, cred,
 444                                      &vap->va_mode);
 445         ip->ino_data.mode = vap->va_mode;
 446
 447         if (vap->va_vaflags & VA_UID_UUID_VALID)
 448                 ip->ino_data.uid = vap->va_uid_uuid;
 449         else if (vap->va_uid != (uid_t)VNOVAL)
 450                 hammer_guid_to_uuid(&ip->ino_data.uid, vap->va_uid);
 451         else
 452                 hammer_guid_to_uuid(&ip->ino_data.uid, xuid);
 453
 454         if (vap->va_vaflags & VA_GID_UUID_VALID)
 455                 ip->ino_data.gid = vap->va_gid_uuid;
 456         else if (vap->va_gid != (gid_t)VNOVAL)
 457                 hammer_guid_to_uuid(&ip->ino_data.gid, vap->va_gid);
 458         else
 459                 ip->ino_data.gid = dip->ino_data.gid;
 460
 461         hammer_ref(&ip->lock);
 462         if (RB_INSERT(hammer_ino_rb_tree, &hmp->rb_inos_root, ip)) {
 463                 hammer_unref(&ip->lock);
 464                 panic("hammer_create_inode: duplicate obj_id %llx", ip->obj_id);
 465         }
 466         *ipp = ip;
 467         return(0);
 468 }
 469
 470 /*
 471  * Called by hammer_sync_inode().
 472  */
 473 static int
 474 hammer_update_inode(hammer_cursor_t cursor, hammer_inode_t ip)
 475 {
 476         hammer_transaction_t trans = cursor->trans;
 477         hammer_record_t record;
 478         int error;
 479
 480 retry:
 481         error = 0;
 482
 483         /*
 484          * If the inode has a presence on-disk then locate it and mark
 485          * it deleted, setting DELONDISK.
 486          *
 487          * The record may or may not be physically deleted, depending on
 488          * the retention policy.
 489          */
 490         if ((ip->flags & (HAMMER_INODE_ONDISK|HAMMER_INODE_DELONDISK)) ==
 491             HAMMER_INODE_ONDISK) {
 492                 hammer_normalize_cursor(cursor);
 493                 cursor->key_beg.localization = HAMMER_LOCALIZE_INODE;
 494                 cursor->key_beg.obj_id = ip->obj_id;
 495                 cursor->key_beg.key = 0;
 496                 cursor->key_beg.create_tid = 0;
 497                 cursor->key_beg.delete_tid = 0;
 498                 cursor->key_beg.rec_type = HAMMER_RECTYPE_INODE;
 499                 cursor->key_beg.obj_type = 0;
 500                 cursor->asof = ip->obj_asof;
 501                 cursor->flags &= ~HAMMER_CURSOR_INITMASK;
 502                 cursor->flags |= HAMMER_CURSOR_GET_LEAF | HAMMER_CURSOR_ASOF;
 503                 cursor->flags |= HAMMER_CURSOR_BACKEND;
 504
 505                 error = hammer_btree_lookup(cursor);
 506                 if (hammer_debug_inode)
 507                         kprintf("IPDEL %p %08x %d", ip, ip->flags, error);
 508                 if (error) {
 509                         kprintf("error %d\n", error);
 510                         Debugger("hammer_update_inode");
 511                 }
 512
 513                 if (error == 0) {
 514                         error = hammer_ip_delete_record(cursor, ip, trans->tid);
 515                         if (hammer_debug_inode)
 516                                 kprintf(" error %d\n", error);
 517                         if (error && error != EDEADLK) {
 518                                 kprintf("error %d\n", error);
 519                                 Debugger("hammer_update_inode2");
 520                         }
 521                         if (error == 0) {
 522                                 ip->flags |= HAMMER_INODE_DELONDISK;
 523                         }
 524                         if (cursor->node)
 525                                 hammer_cache_node(&ip->cache[0], cursor->node);
 526                 }
 527                 if (error == EDEADLK) {
 528                         hammer_done_cursor(cursor);
 529                         error = hammer_init_cursor(trans, cursor,
 530                                                    &ip->cache[0], ip);
 531                         if (hammer_debug_inode)
 532                                 kprintf("IPDED %p %d\n", ip, error);
 533                         if (error == 0)
 534                                 goto retry;
 535                 }
 536         }
 537
 538         /*
 539          * Ok, write out the initial record or a new record (after deleting
 540          * the old one), unless the DELETED flag is set.  This routine will
 541          * clear DELONDISK if it writes out a record.
 542          *
 543          * Update our inode statistics if this is the first application of
 544          * the inode on-disk.
 545          */
 546         if (error == 0 && (ip->flags & HAMMER_INODE_DELETED) == 0) {
 547                 /*
 548                  * Generate a record and write it to the media
 549                  */
 550                 record = hammer_alloc_mem_record(ip, 0);
 551                 record->type = HAMMER_MEM_RECORD_INODE;
 552                 record->flush_state = HAMMER_FST_FLUSH;
 553                 record->leaf = ip->sync_ino_leaf;
 554                 record->leaf.base.create_tid = trans->tid;
 555                 record->leaf.data_len = sizeof(ip->sync_ino_data);
 556                 record->data = (void *)&ip->sync_ino_data;
 557                 record->flags |= HAMMER_RECF_INTERLOCK_BE;
 558                 for (;;) {
 559                         error = hammer_ip_sync_record_cursor(cursor, record);
 560                         if (hammer_debug_inode)
 561                                 kprintf("GENREC %p rec %08x %d\n",
 562                                         ip, record->flags, error);
 563                         if (error != EDEADLK)
 564                                 break;
 565                         hammer_done_cursor(cursor);
 566                         error = hammer_init_cursor(trans, cursor,
 567                                                    &ip->cache[0], ip);
 568                         if (hammer_debug_inode)
 569                                 kprintf("GENREC reinit %d\n", error);
 570                         if (error)
 571                                 break;
 572                 }
 573                 if (error) {
 574                         kprintf("error %d\n", error);
 575                         Debugger("hammer_update_inode3");
 576                 }
 577
 578                 /*
 579                  * The record isn't managed by the inode's record tree,
 580                  * destroy it whether we succeed or fail.
 581                  */
 582                 record->flags &= ~HAMMER_RECF_INTERLOCK_BE;
 583                 record->flags |= HAMMER_RECF_DELETED_FE;
 584                 record->flush_state = HAMMER_FST_IDLE;
 585                 hammer_rel_mem_record(record);
 586
 587                 /*
 588                  * Finish up.
 589                  */
 590                 if (error == 0) {
 591                         if (hammer_debug_inode)
 592                                 kprintf("CLEANDELOND %p %08x\n", ip, ip->flags);
 593                         ip->sync_flags &= ~(HAMMER_INODE_DDIRTY |
 594                                             HAMMER_INODE_ATIME |
 595                                             HAMMER_INODE_MTIME);
 596                         ip->flags &= ~HAMMER_INODE_DELONDISK;
 597
 598                         /*
 599                          * Root volume count of inodes
 600                          */
 601                         if ((ip->flags & HAMMER_INODE_ONDISK) == 0) {
 602                                 hammer_modify_volume_field(trans,
 603                                                            trans->rootvol,
 604                                                            vol0_stat_inodes);
 605                                 ++ip->hmp->rootvol->ondisk->vol0_stat_inodes;
 606                                 hammer_modify_volume_done(trans->rootvol);
 607                                 ip->flags |= HAMMER_INODE_ONDISK;
 608                                 if (hammer_debug_inode)
 609                                         kprintf("NOWONDISK %p\n", ip);
 610                         }
 611                 }
 612         }
 613
 614         /*
 615          * If the inode has been destroyed, clean out any left-over flags
 616          * that may have been set by the frontend.
 617          */
 618         if (error == 0 && (ip->flags & HAMMER_INODE_DELETED)) {
 619                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY |
 620                                     HAMMER_INODE_ATIME |
 621                                     HAMMER_INODE_MTIME);
 622         }
 623         return(error);
 624 }
 625
 626 /*
 627  * Update only the itimes fields.
 628  *
 629  * ATIME can be updated without generating any UNDO.  MTIME is updated
 630  * with UNDO so it is guaranteed to be synchronized properly in case of
 631  * a crash.
 632  *
 633  * Neither field is included in the B-Tree leaf element's CRC, which is how
 634  * we can get away with updating ATIME the way we do.
 635  */
 636 static int
 637 hammer_update_itimes(hammer_cursor_t cursor, hammer_inode_t ip)
 638 {
 639         hammer_transaction_t trans = cursor->trans;
 640         int error;
 641
 642 retry:
 643         if ((ip->flags & (HAMMER_INODE_ONDISK|HAMMER_INODE_DELONDISK)) !=
 644             HAMMER_INODE_ONDISK) {
 645                 return(0);
 646         }
 647
 648         hammer_normalize_cursor(cursor);
 649         cursor->key_beg.localization = HAMMER_LOCALIZE_INODE;
 650         cursor->key_beg.obj_id = ip->obj_id;
 651         cursor->key_beg.key = 0;
 652         cursor->key_beg.create_tid = 0;
 653         cursor->key_beg.delete_tid = 0;
 654         cursor->key_beg.rec_type = HAMMER_RECTYPE_INODE;
 655         cursor->key_beg.obj_type = 0;
 656         cursor->asof = ip->obj_asof;
 657         cursor->flags &= ~HAMMER_CURSOR_INITMASK;
 658         cursor->flags |= HAMMER_CURSOR_ASOF;
 659         cursor->flags |= HAMMER_CURSOR_GET_LEAF;
 660         cursor->flags |= HAMMER_CURSOR_GET_DATA;
 661         cursor->flags |= HAMMER_CURSOR_BACKEND;
 662
 663         error = hammer_btree_lookup(cursor);
 664         if (error) {
 665                 kprintf("error %d\n", error);
 666                 Debugger("hammer_update_itimes1");
 667         }
 668         if (error == 0) {
 669                 hammer_cache_node(&ip->cache[0], cursor->node);
 670                 if (ip->sync_flags & HAMMER_INODE_MTIME) {
 671                         /*
 672                          * Updating MTIME requires an UNDO.  Just cover
 673                          * both atime and mtime.
 674                          */
 675                         hammer_modify_buffer(trans, cursor->data_buffer,
 676                                      HAMMER_ITIMES_BASE(&cursor->data->inode),
 677                                      HAMMER_ITIMES_BYTES);
 678                         cursor->data->inode.atime = ip->sync_ino_data.atime;
 679                         cursor->data->inode.mtime = ip->sync_ino_data.mtime;
 680                         hammer_modify_buffer_done(cursor->data_buffer);
 681                 } else if (ip->sync_flags & HAMMER_INODE_ATIME) {
 682                         /*
 683                          * Updating atime only can be done in-place with
 684                          * no UNDO.
 685                          */
 686                         hammer_modify_buffer(trans, cursor->data_buffer,
 687                                              NULL, 0);
 688                         cursor->data->inode.atime = ip->sync_ino_data.atime;
 689                         hammer_modify_buffer_done(cursor->data_buffer);
 690                 }
 691                 ip->sync_flags &= ~(HAMMER_INODE_ATIME | HAMMER_INODE_MTIME);
 692         }
 693         if (error == EDEADLK) {
 694                 hammer_done_cursor(cursor);
 695                 error = hammer_init_cursor(trans, cursor,
 696                                            &ip->cache[0], ip);
 697                 if (error == 0)
 698                         goto retry;
 699         }
 700         return(error);
 701 }
 702
 703 /*
 704  * Release a reference on an inode, flush as requested.
 705  *
 706  * On the last reference we queue the inode to the flusher for its final
 707  * disposition.
 708  */
 709 void
 710 hammer_rel_inode(struct hammer_inode *ip, int flush)
 711 {
 712         hammer_mount_t hmp = ip->hmp;
 713
 714         /*
 715          * Handle disposition when dropping the last ref.
 716          */
 717         for (;;) {
 718                 if (ip->lock.refs == 1) {
 719                         /*
 720                          * Determine whether on-disk action is needed for
 721                          * the inode's final disposition.
 722                          */
 723                         KKASSERT(ip->vp == NULL);
 724                         hammer_inode_unloadable_check(ip, 0);
 725                         if (ip->flags & HAMMER_INODE_MODMASK) {
 726                                 if (hmp->rsv_inodes > desiredvnodes) {
 727                                         hammer_flush_inode(ip,
 728                                                            HAMMER_FLUSH_SIGNAL);
 729                                 } else {
 730                                         hammer_flush_inode(ip, 0);
 731                                 }
 732                         } else if (ip->lock.refs == 1) {
 733                                 hammer_unload_inode(ip);
 734                                 break;
 735                         }
 736                 } else {
 737                         if (flush)
 738                                 hammer_flush_inode(ip, 0);
 739
 740                         /*
 741                          * The inode still has multiple refs, try to drop
 742                          * one ref.
 743                          */
 744                         KKASSERT(ip->lock.refs >= 1);
 745                         if (ip->lock.refs > 1) {
 746                                 hammer_unref(&ip->lock);
 747                                 break;
 748                         }
 749                 }
 750         }
 751 }
 752
 753 /*
 754  * Unload and destroy the specified inode.  Must be called with one remaining
 755  * reference.  The reference is disposed of.
 756  *
 757  * This can only be called in the context of the flusher.
 758  */
 759 static int
 760 hammer_unload_inode(struct hammer_inode *ip)
 761 {
 762         hammer_mount_t hmp = ip->hmp;
 763
 764         KASSERT(ip->lock.refs == 1,
 765                 ("hammer_unload_inode: %d refs\n", ip->lock.refs));
 766         KKASSERT(ip->vp == NULL);
 767         KKASSERT(ip->flush_state == HAMMER_FST_IDLE);
 768         KKASSERT(ip->cursor_ip_refs == 0);
 769         KKASSERT(ip->lock.lockcount == 0);
 770         KKASSERT((ip->flags & HAMMER_INODE_MODMASK) == 0);
 771
 772         KKASSERT(RB_EMPTY(&ip->rec_tree));
 773         KKASSERT(TAILQ_EMPTY(&ip->target_list));
 774
 775         RB_REMOVE(hammer_ino_rb_tree, &hmp->rb_inos_root, ip);
 776
 777         hammer_uncache_node(&ip->cache[0]);
 778         hammer_uncache_node(&ip->cache[1]);
 779         if (ip->objid_cache)
 780                 hammer_clear_objid(ip);
 781         --hammer_count_inodes;
 782         --hmp->count_inodes;
 783
 784         hammer_inode_wakereclaims(ip);
 785         kfree(ip, M_HAMMER);
 786
 787         return(0);
 788 }
 789
 790 /*
 791  * Called on mount -u when switching from RW to RO or vise-versa.  Adjust
 792  * the read-only flag for cached inodes.
 793  *
 794  * This routine is called from a RB_SCAN().
 795  */
 796 int
 797 hammer_reload_inode(hammer_inode_t ip, void *arg __unused)
 798 {
 799         hammer_mount_t hmp = ip->hmp;
 800
 801         if (hmp->ronly || hmp->asof != HAMMER_MAX_TID)
 802                 ip->flags |= HAMMER_INODE_RO;
 803         else
 804                 ip->flags &= ~HAMMER_INODE_RO;
 805         return(0);
 806 }
 807
 808 /*
 809  * A transaction has modified an inode, requiring updates as specified by
 810  * the passed flags.
 811  *
 812  * HAMMER_INODE_DDIRTY: Inode data has been updated
 813  * HAMMER_INODE_XDIRTY: Dirty in-memory records
 814  * HAMMER_INODE_BUFS:   Dirty buffer cache buffers
 815  * HAMMER_INODE_DELETED: Inode record/data must be deleted
 816  * HAMMER_INODE_ATIME/MTIME: mtime/atime has been updated
 817  */
 818 void
 819 hammer_modify_inode(hammer_inode_t ip, int flags)
 820 {
 821         KKASSERT ((ip->flags & HAMMER_INODE_RO) == 0 ||
 822                   (flags & (HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
 823                             HAMMER_INODE_BUFS | HAMMER_INODE_DELETED |
 824                             HAMMER_INODE_ATIME | HAMMER_INODE_MTIME)) == 0);
 825         if ((ip->flags & HAMMER_INODE_RSV_INODES) == 0) {
 826                 ip->flags |= HAMMER_INODE_RSV_INODES;
 827                 ++ip->hmp->rsv_inodes;
 828         }
 829
 830         ip->flags |= flags;
 831 }
 832
 833 /*
 834  * Request that an inode be flushed.  This whole mess cannot block and may
 835  * recurse (if not synchronous).  Once requested HAMMER will attempt to
 836  * actively flush the inode until the flush can be done.
 837  *
 838  * The inode may already be flushing, or may be in a setup state.  We can
 839  * place the inode in a flushing state if it is currently idle and flag it
 840  * to reflush if it is currently flushing.
 841  *
 842  * If the HAMMER_FLUSH_SYNCHRONOUS flag is specified we will attempt to
 843  * flush the indoe synchronously using the caller's context.
 844  */
 845 void
 846 hammer_flush_inode(hammer_inode_t ip, int flags)
 847 {
 848         int good;
 849
 850         /*
 851          * Trivial 'nothing to flush' case.  If the inode is ina SETUP
 852          * state we have to put it back into an IDLE state so we can
 853          * drop the extra ref.
 854          */
 855         if ((ip->flags & HAMMER_INODE_MODMASK) == 0) {
 856                 if (ip->flush_state == HAMMER_FST_SETUP) {
 857                         ip->flush_state = HAMMER_FST_IDLE;
 858                         hammer_rel_inode(ip, 0);
 859                 }
 860                 return;
 861         }
 862
 863         /*
 864          * Our flush action will depend on the current state.
 865          */
 866         switch(ip->flush_state) {
 867         case HAMMER_FST_IDLE:
 868                 /*
 869                  * We have no dependancies and can flush immediately.  Some
 870                  * our children may not be flushable so we have to re-test
 871                  * with that additional knowledge.
 872                  */
 873                 hammer_flush_inode_core(ip, flags);
 874                 break;
 875         case HAMMER_FST_SETUP:
 876                 /*
 877                  * Recurse upwards through dependancies via target_list
 878                  * and start their flusher actions going if possible.
 879                  *
 880                  * 'good' is our connectivity.  -1 means we have none and
 881                  * can't flush, 0 means there weren't any dependancies, and
 882                  * 1 means we have good connectivity.
 883                  */
 884                 good = hammer_setup_parent_inodes(ip);
 885
 886                 /*
 887                  * We can continue if good >= 0.  Determine how many records
 888                  * under our inode can be flushed (and mark them).
 889                  */
 890                 if (good >= 0) {
 891                         hammer_flush_inode_core(ip, flags);
 892                 } else {
 893                         ip->flags |= HAMMER_INODE_REFLUSH;
 894                         if (flags & HAMMER_FLUSH_SIGNAL) {
 895                                 ip->flags |= HAMMER_INODE_RESIGNAL;
 896                                 hammer_flusher_async(ip->hmp);
 897                         }
 898                 }
 899                 break;
 900         default:
 901                 /*
 902                  * We are already flushing, flag the inode to reflush
 903                  * if needed after it completes its current flush.
 904                  */
 905                 if ((ip->flags & HAMMER_INODE_REFLUSH) == 0)
 906                         ip->flags |= HAMMER_INODE_REFLUSH;
 907                 if (flags & HAMMER_FLUSH_SIGNAL) {
 908                         ip->flags |= HAMMER_INODE_RESIGNAL;
 909                         hammer_flusher_async(ip->hmp);
 910                 }
 911                 break;
 912         }
 913 }
 914
 915 /*
 916  * Scan ip->target_list, which is a list of records owned by PARENTS to our
 917  * ip which reference our ip.
 918  *
 919  * XXX This is a huge mess of recursive code, but not one bit of it blocks
 920  *     so for now do not ref/deref the structures.  Note that if we use the
 921  *     ref/rel code later, the rel CAN block.
 922  */
 923 static int
 924 hammer_setup_parent_inodes(hammer_inode_t ip)
 925 {
 926         hammer_record_t depend;
 927 #if 0
 928         hammer_record_t next;
 929         hammer_inode_t  pip;
 930 #endif
 931         int good;
 932         int r;
 933
 934         good = 0;
 935         TAILQ_FOREACH(depend, &ip->target_list, target_entry) {
 936                 r = hammer_setup_parent_inodes_helper(depend);
 937                 KKASSERT(depend->target_ip == ip);
 938                 if (r < 0 && good == 0)
 939                         good = -1;
 940                 if (r > 0)
 941                         good = 1;
 942         }
 943         return(good);
 944
 945 #if 0
 946 retry:
 947         good = 0;
 948         next = TAILQ_FIRST(&ip->target_list);
 949         if (next) {
 950                 hammer_ref(&next->lock);
 951                 hammer_ref(&next->ip->lock);
 952         }
 953         while ((depend = next) != NULL) {
 954                 if (depend->target_ip == NULL) {
 955                         pip = depend->ip;
 956                         hammer_rel_mem_record(depend);
 957                         hammer_rel_inode(pip, 0);
 958                         goto retry;
 959                 }
 960                 KKASSERT(depend->target_ip == ip);
 961                 next = TAILQ_NEXT(depend, target_entry);
 962                 if (next) {
 963                         hammer_ref(&next->lock);
 964                         hammer_ref(&next->ip->lock);
 965                 }
 966                 r = hammer_setup_parent_inodes_helper(depend);
 967                 if (r < 0 && good == 0)
 968                         good = -1;
 969                 if (r > 0)
 970                         good = 1;
 971                 pip = depend->ip;
 972                 hammer_rel_mem_record(depend);
 973                 hammer_rel_inode(pip, 0);
 974         }
 975         return(good);
 976 #endif
 977 }
 978
 979 /*
 980  * This helper function takes a record representing the dependancy between
 981  * the parent inode and child inode.
 982  *
 983  * record->ip           = parent inode
 984  * record->target_ip    = child inode
 985  *
 986  * We are asked to recurse upwards and convert the record from SETUP
 987  * to FLUSH if possible.
 988  *
 989  * Return 1 if the record gives us connectivity
 990  *
 991  * Return 0 if the record is not relevant
 992  *
 993  * Return -1 if we can't resolve the dependancy and there is no connectivity.
 994  */
 995 static int
 996 hammer_setup_parent_inodes_helper(hammer_record_t record)
 997 {
 998         hammer_mount_t hmp;
 999         hammer_inode_t pip;
1000         int good;
1001
1002         KKASSERT(record->flush_state != HAMMER_FST_IDLE);
1003         pip = record->ip;
1004         hmp = pip->hmp;
1005
1006         /*
1007          * If the record is already flushing, is it in our flush group?
1008          *
1009          * If it is in our flush group but it is a general record or a
1010          * delete-on-disk, it does not improve our connectivity (return 0),
1011          * and if the target inode is not trying to destroy itself we can't
1012          * allow the operation yet anyway (the second return -1).
1013          */
1014         if (record->flush_state == HAMMER_FST_FLUSH) {
1015                 if (record->flush_group != hmp->flusher.next) {
1016                         pip->flags |= HAMMER_INODE_REFLUSH;
1017                         return(-1);
1018                 }
1019                 if (record->type == HAMMER_MEM_RECORD_ADD)
1020                         return(1);
1021                 /* GENERAL or DEL */
1022                 return(0);
1023         }
1024
1025         /*
1026          * It must be a setup record.  Try to resolve the setup dependancies
1027          * by recursing upwards so we can place ip on the flush list.
1028          */
1029         KKASSERT(record->flush_state == HAMMER_FST_SETUP);
1030
1031         good = hammer_setup_parent_inodes(pip);
1032
1033         /*
1034          * We can't flush ip because it has no connectivity (XXX also check
1035          * nlinks for pre-existing connectivity!).  Flag it so any resolution
1036          * recurses back down.
1037          */
1038         if (good < 0) {
1039                 pip->flags |= HAMMER_INODE_REFLUSH;
1040                 return(good);
1041         }
1042
1043         /*
1044          * We are go, place the parent inode in a flushing state so we can
1045          * place its record in a flushing state.  Note that the parent
1046          * may already be flushing.  The record must be in the same flush
1047          * group as the parent.
1048          */
1049         if (pip->flush_state != HAMMER_FST_FLUSH)
1050                 hammer_flush_inode_core(pip, HAMMER_FLUSH_RECURSION);
1051         KKASSERT(pip->flush_state == HAMMER_FST_FLUSH);
1052         KKASSERT(record->flush_state == HAMMER_FST_SETUP);
1053
1054 #if 0
1055         if (record->type == HAMMER_MEM_RECORD_DEL &&
1056             (record->target_ip->flags & (HAMMER_INODE_DELETED|HAMMER_INODE_DELONDISK)) == 0) {
1057                 /*
1058                  * Regardless of flushing state we cannot sync this path if the
1059                  * record represents a delete-on-disk but the target inode
1060                  * is not ready to sync its own deletion.
1061                  *
1062                  * XXX need to count effective nlinks to determine whether
1063                  * the flush is ok, otherwise removing a hardlink will
1064                  * just leave the DEL record to rot.
1065                  */
1066                 record->target_ip->flags |= HAMMER_INODE_REFLUSH;
1067                 return(-1);
1068         } else
1069 #endif
1070         if (pip->flush_group == pip->hmp->flusher.next) {
1071                 /*
1072                  * This is the record we wanted to synchronize.  If the
1073                  * record went into a flush state while we blocked it
1074                  * had better be in the correct flush group.
1075                  */
1076                 if (record->flush_state != HAMMER_FST_FLUSH) {
1077                         record->flush_state = HAMMER_FST_FLUSH;
1078                         record->flush_group = pip->flush_group;
1079                         hammer_ref(&record->lock);
1080                 } else {
1081                         KKASSERT(record->flush_group == pip->flush_group);
1082                 }
1083                 if (record->type == HAMMER_MEM_RECORD_ADD)
1084                         return(1);
1085
1086                 /*
1087                  * A general or delete-on-disk record does not contribute
1088                  * to our visibility.  We can still flush it, however.
1089                  */
1090                 return(0);
1091         } else {
1092                 /*
1093                  * We couldn't resolve the dependancies, request that the
1094                  * inode be flushed when the dependancies can be resolved.
1095                  */
1096                 pip->flags |= HAMMER_INODE_REFLUSH;
1097                 return(-1);
1098         }
1099 }
1100
1101 /*
1102  * This is the core routine placing an inode into the FST_FLUSH state.
1103  */
1104 static void
1105 hammer_flush_inode_core(hammer_inode_t ip, int flags)
1106 {
1107         int go_count;
1108
1109         /*
1110          * Set flush state and prevent the flusher from cycling into
1111          * the next flush group.  Do not place the ip on the list yet.
1112          * Inodes not in the idle state get an extra reference.
1113          */
1114         KKASSERT(ip->flush_state != HAMMER_FST_FLUSH);
1115         if (ip->flush_state == HAMMER_FST_IDLE)
1116                 hammer_ref(&ip->lock);
1117         ip->flush_state = HAMMER_FST_FLUSH;
1118         ip->flush_group = ip->hmp->flusher.next;
1119         ++ip->hmp->flusher.group_lock;
1120         ++ip->hmp->count_iqueued;
1121         ++hammer_count_iqueued;
1122
1123         /*
1124          * We need to be able to vfsync/truncate from the backend.
1125          */
1126         KKASSERT((ip->flags & HAMMER_INODE_VHELD) == 0);
1127         if (ip->vp && (ip->vp->v_flag & VINACTIVE) == 0) {
1128                 ip->flags |= HAMMER_INODE_VHELD;
1129                 vref(ip->vp);
1130         }
1131
1132         /*
1133          * Figure out how many in-memory records we can actually flush
1134          * (not including inode meta-data, buffers, etc).
1135          */
1136         if (flags & HAMMER_FLUSH_RECURSION) {
1137                 go_count = 1;
1138         } else {
1139                 go_count = RB_SCAN(hammer_rec_rb_tree, &ip->rec_tree, NULL,
1140                                    hammer_setup_child_callback, NULL);
1141         }
1142
1143         /*
1144          * This is a more involved test that includes go_count.  If we
1145          * can't flush, flag the inode and return.  If go_count is 0 we
1146          * were are unable to flush any records in our rec_tree and
1147          * must ignore the XDIRTY flag.
1148          */
1149         if (go_count == 0) {
1150                 if ((ip->flags & HAMMER_INODE_MODMASK_NOXDIRTY) == 0) {
1151                         ip->flags |= HAMMER_INODE_REFLUSH;
1152
1153                         --ip->hmp->count_iqueued;
1154                         --hammer_count_iqueued;
1155
1156                         ip->flush_state = HAMMER_FST_SETUP;
1157                         if (ip->flags & HAMMER_INODE_VHELD) {
1158                                 ip->flags &= ~HAMMER_INODE_VHELD;
1159                                 vrele(ip->vp);
1160                         }
1161                         if (flags & HAMMER_FLUSH_SIGNAL) {
1162                                 ip->flags |= HAMMER_INODE_RESIGNAL;
1163                                 hammer_flusher_async(ip->hmp);
1164                         }
1165                         if (--ip->hmp->flusher.group_lock == 0)
1166                                 wakeup(&ip->hmp->flusher.group_lock);
1167                         return;
1168                 }
1169         }
1170
1171         /*
1172          * Snapshot the state of the inode for the backend flusher.
1173          *
1174          * The truncation must be retained in the frontend until after
1175          * we've actually performed the record deletion.
1176          *
1177          * We continue to retain sync_trunc_off even when all truncations
1178          * have been resolved as an optimization to determine if we can
1179          * skip the B-Tree lookup for overwrite deletions.
1180          *
1181          * NOTE: The DELETING flag is a mod flag, but it is also sticky,
1182          * and stays in ip->flags.  Once set, it stays set until the
1183          * inode is destroyed.
1184          */
1185         ip->sync_flags = (ip->flags & HAMMER_INODE_MODMASK);
1186         if (ip->sync_flags & HAMMER_INODE_TRUNCATED)
1187                 ip->sync_trunc_off = ip->trunc_off;
1188         ip->sync_ino_leaf = ip->ino_leaf;
1189         ip->sync_ino_data = ip->ino_data;
1190         ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
1191         ip->flags &= ~HAMMER_INODE_MODMASK;
1192 #ifdef DEBUG_TRUNCATE
1193         if ((ip->sync_flags & HAMMER_INODE_TRUNCATED) && ip == HammerTruncIp)
1194                 kprintf("truncateS %016llx\n", ip->sync_trunc_off);
1195 #endif
1196
1197         /*
1198          * The flusher list inherits our inode and reference.
1199          */
1200         TAILQ_INSERT_TAIL(&ip->hmp->flush_list, ip, flush_entry);
1201         if (--ip->hmp->flusher.group_lock == 0)
1202                 wakeup(&ip->hmp->flusher.group_lock);
1203
1204         if (flags & HAMMER_FLUSH_SIGNAL) {
1205                 hammer_flusher_async(ip->hmp);
1206         }
1207 }
1208
1209 /*
1210  * Callback for scan of ip->rec_tree.  Try to include each record in our
1211  * flush.  ip->flush_group has been set but the inode has not yet been
1212  * moved into a flushing state.
1213  *
1214  * If we get stuck on a record we have to set HAMMER_INODE_REFLUSH on
1215  * both inodes.
1216  *
1217  * We return 1 for any record placed or found in FST_FLUSH, which prevents
1218  * the caller from shortcutting the flush.
1219  */
1220 static int
1221 hammer_setup_child_callback(hammer_record_t rec, void *data)
1222 {
1223         hammer_inode_t target_ip;
1224         hammer_inode_t ip;
1225         int r;
1226
1227         /*
1228          * Deleted records are ignored.  Note that the flush detects deleted
1229          * front-end records at multiple points to deal with races.  This is
1230          * just the first line of defense.  The only time DELETED_FE cannot
1231          * be set is when HAMMER_RECF_INTERLOCK_BE is set.
1232          *
1233          * Don't get confused between record deletion and, say, directory
1234          * entry deletion.  The deletion of a directory entry that is on
1235          * the media has nothing to do with the record deletion flags.
1236          */
1237         if (rec->flags & (HAMMER_RECF_DELETED_FE|HAMMER_RECF_DELETED_BE))
1238                 return(0);
1239
1240         /*
1241          * If the record is in an idle state it has no dependancies and
1242          * can be flushed.
1243          */
1244         ip = rec->ip;
1245         r = 0;
1246
1247         switch(rec->flush_state) {
1248         case HAMMER_FST_IDLE:
1249                 /*
1250                  * Record has no setup dependancy, we can flush it.
1251                  */
1252                 KKASSERT(rec->target_ip == NULL);
1253                 rec->flush_state = HAMMER_FST_FLUSH;
1254                 rec->flush_group = ip->flush_group;
1255                 hammer_ref(&rec->lock);
1256                 r = 1;
1257                 break;
1258         case HAMMER_FST_SETUP:
1259                 /*
1260                  * Record has a setup dependancy.  Try to include the
1261                  * target ip in the flush.
1262                  *
1263                  * We have to be careful here, if we do not do the right
1264                  * thing we can lose track of dirty inodes and the system
1265                  * will lockup trying to allocate buffers.
1266                  */
1267                 target_ip = rec->target_ip;
1268                 KKASSERT(target_ip != NULL);
1269                 KKASSERT(target_ip->flush_state != HAMMER_FST_IDLE);
1270                 if (target_ip->flush_state == HAMMER_FST_FLUSH) {
1271                         /*
1272                          * If the target IP is already flushing in our group
1273                          * we are golden, otherwise make sure the target
1274                          * reflushes.
1275                          */
1276                         if (target_ip->flush_group == ip->flush_group) {
1277                                 rec->flush_state = HAMMER_FST_FLUSH;
1278                                 rec->flush_group = ip->flush_group;
1279                                 hammer_ref(&rec->lock);
1280                                 r = 1;
1281                         } else {
1282                                 target_ip->flags |= HAMMER_INODE_REFLUSH;
1283                         }
1284                 } else if (rec->type == HAMMER_MEM_RECORD_ADD) {
1285                         /*
1286                          * If the target IP is not flushing we can force
1287                          * it to flush, even if it is unable to write out
1288                          * any of its own records we have at least one in
1289                          * hand that we CAN deal with.
1290                          */
1291                         rec->flush_state = HAMMER_FST_FLUSH;
1292                         rec->flush_group = ip->flush_group;
1293                         hammer_ref(&rec->lock);
1294                         hammer_flush_inode_core(target_ip,
1295                                                 HAMMER_FLUSH_RECURSION);
1296                         r = 1;
1297                 } else {
1298                         /*
1299                          * General or delete-on-disk record.
1300                          *
1301                          * XXX this needs help.  If a delete-on-disk we could
1302                          * disconnect the target.  If the target has its own
1303                          * dependancies they really need to be flushed.
1304                          *
1305                          * XXX
1306                          */
1307                         rec->flush_state = HAMMER_FST_FLUSH;
1308                         rec->flush_group = ip->flush_group;
1309                         hammer_ref(&rec->lock);
1310                         hammer_flush_inode_core(target_ip,
1311                                                 HAMMER_FLUSH_RECURSION);
1312                         r = 1;
1313                 }
1314                 break;
1315         case HAMMER_FST_FLUSH:
1316                 /*
1317                  * Record already associated with a flush group.  It had
1318                  * better be ours.
1319                  */
1320                 KKASSERT(rec->flush_group == ip->flush_group);
1321                 r = 1;
1322                 break;
1323         }
1324         return(r);
1325 }
1326
1327 /*
1328  * Wait for a previously queued flush to complete.  Not only do we need to
1329  * wait for the inode to sync out, we also may have to run the flusher again
1330  * to get it past the UNDO position pertaining to the flush so a crash does
1331  * not 'undo' our flush.
1332  */
1333 void
1334 hammer_wait_inode(hammer_inode_t ip)
1335 {
1336         hammer_mount_t hmp = ip->hmp;
1337         int sync_group;
1338         int waitcount;
1339
1340         sync_group = ip->flush_group;
1341         waitcount = (ip->flags & HAMMER_INODE_REFLUSH) ? 2 : 1;
1342
1343         while (ip->flush_state != HAMMER_FST_IDLE &&
1344                (ip->flush_group - sync_group) < 2) {
1345                 if (ip->flush_state == HAMMER_FST_SETUP) {
1346                         kprintf("X");
1347                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
1348                 } else {
1349                         ip->flags |= HAMMER_INODE_FLUSHW;
1350                         tsleep(&ip->flags, 0, "hmrwin", 0);
1351                 }
1352         }
1353         while (hmp->flusher.done - sync_group < waitcount) {
1354                 kprintf("Y");
1355                 hammer_flusher_sync(hmp);
1356         }
1357 }
1358
1359 /*
1360  * Called by the backend code when a flush has been completed.
1361  * The inode has already been removed from the flush list.
1362  *
1363  * A pipelined flush can occur, in which case we must re-enter the
1364  * inode on the list and re-copy its fields.
1365  */
1366 void
1367 hammer_flush_inode_done(hammer_inode_t ip)
1368 {
1369         hammer_mount_t hmp;
1370         int dorel;
1371
1372         KKASSERT(ip->flush_state == HAMMER_FST_FLUSH);
1373
1374         hmp = ip->hmp;
1375
1376         /*
1377          * Merge left-over flags back into the frontend and fix the state.
1378          */
1379         ip->flags |= ip->sync_flags;
1380
1381         /*
1382          * The backend may have adjusted nlinks, so if the adjusted nlinks
1383          * does not match the fronttend set the frontend's RDIRTY flag again.
1384          */
1385         if (ip->ino_data.nlinks != ip->sync_ino_data.nlinks)
1386                 ip->flags |= HAMMER_INODE_DDIRTY;
1387
1388         /*
1389          * Fix up the dirty buffer status.  IO completions will also
1390          * try to clean up rsv_databufs.
1391          */
1392         if (ip->vp && RB_ROOT(&ip->vp->v_rbdirty_tree)) {
1393                 ip->flags |= HAMMER_INODE_BUFS;
1394         } else {
1395                 hmp->rsv_databufs -= ip->rsv_databufs;
1396                 ip->rsv_databufs = 0;
1397         }
1398
1399         /*
1400          * Re-set the XDIRTY flag if some of the inode's in-memory records
1401          * could not be flushed.
1402          */
1403         KKASSERT((RB_EMPTY(&ip->rec_tree) &&
1404                   (ip->flags & HAMMER_INODE_XDIRTY) == 0) ||
1405                  (!RB_EMPTY(&ip->rec_tree) &&
1406                   (ip->flags & HAMMER_INODE_XDIRTY) != 0));
1407
1408         /*
1409          * Do not lose track of inodes which no longer have vnode
1410          * assocations, otherwise they may never get flushed again.
1411          */
1412         if ((ip->flags & HAMMER_INODE_MODMASK) && ip->vp == NULL)
1413                 ip->flags |= HAMMER_INODE_REFLUSH;
1414
1415         /*
1416          * Adjust flush_state.  The target state (idle or setup) shouldn't
1417          * be terribly important since we will reflush if we really need
1418          * to do anything. XXX
1419          */
1420         if (TAILQ_EMPTY(&ip->target_list) && RB_EMPTY(&ip->rec_tree)) {
1421                 ip->flush_state = HAMMER_FST_IDLE;
1422                 dorel = 1;
1423         } else {
1424                 ip->flush_state = HAMMER_FST_SETUP;
1425                 dorel = 0;
1426         }
1427
1428         --hmp->count_iqueued;
1429         --hammer_count_iqueued;
1430
1431         /*
1432          * Clean up the vnode ref
1433          */
1434         if (ip->flags & HAMMER_INODE_VHELD) {
1435                 ip->flags &= ~HAMMER_INODE_VHELD;
1436                 vrele(ip->vp);
1437         }
1438
1439         /*
1440          * If the frontend made more changes and requested another flush,
1441          * then try to get it running.
1442          */
1443         if (ip->flags & HAMMER_INODE_REFLUSH) {
1444                 ip->flags &= ~HAMMER_INODE_REFLUSH;
1445                 if (ip->flags & HAMMER_INODE_RESIGNAL) {
1446                         ip->flags &= ~HAMMER_INODE_RESIGNAL;
1447                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
1448                 } else {
1449                         hammer_flush_inode(ip, 0);
1450                 }
1451         }
1452
1453         /*
1454          * If the inode is now clean drop the space reservation.
1455          */
1456         if ((ip->flags & HAMMER_INODE_MODMASK) == 0 &&
1457             (ip->flags & HAMMER_INODE_RSV_INODES)) {
1458                 ip->flags &= ~HAMMER_INODE_RSV_INODES;
1459                 --hmp->rsv_inodes;
1460         }
1461
1462         /*
1463          * Finally, if the frontend is waiting for a flush to complete,
1464          * wake it up.
1465          */
1466         if (ip->flush_state != HAMMER_FST_FLUSH) {
1467                 if (ip->flags & HAMMER_INODE_FLUSHW) {
1468                         ip->flags &= ~HAMMER_INODE_FLUSHW;
1469                         wakeup(&ip->flags);
1470                 }
1471         }
1472         if (dorel)
1473                 hammer_rel_inode(ip, 0);
1474 }
1475
1476 /*
1477  * Called from hammer_sync_inode() to synchronize in-memory records
1478  * to the media.
1479  */
1480 static int
1481 hammer_sync_record_callback(hammer_record_t record, void *data)
1482 {
1483         hammer_cursor_t cursor = data;
1484         hammer_transaction_t trans = cursor->trans;
1485         int error;
1486
1487         /*
1488          * Skip records that do not belong to the current flush.
1489          */
1490         ++hammer_stats_record_iterations;
1491         if (record->flush_state != HAMMER_FST_FLUSH)
1492                 return(0);
1493
1494 #if 1
1495         if (record->flush_group != record->ip->flush_group) {
1496                 kprintf("sync_record %p ip %p bad flush group %d %d\n", record, record->ip, record->flush_group ,record->ip->flush_group);
1497                 Debugger("blah2");
1498                 return(0);
1499         }
1500 #endif
1501         KKASSERT(record->flush_group == record->ip->flush_group);
1502
1503         /*
1504          * Interlock the record using the BE flag.  Once BE is set the
1505          * frontend cannot change the state of FE.
1506          *
1507          * NOTE: If FE is set prior to us setting BE we still sync the
1508          * record out, but the flush completion code converts it to
1509          * a delete-on-disk record instead of destroying it.
1510          */
1511         KKASSERT((record->flags & HAMMER_RECF_INTERLOCK_BE) == 0);
1512         record->flags |= HAMMER_RECF_INTERLOCK_BE;
1513
1514         /*
1515          * The backend may have already disposed of the record.
1516          */
1517         if (record->flags & HAMMER_RECF_DELETED_BE) {
1518                 error = 0;
1519                 goto done;
1520         }
1521
1522         /*
1523          * If the whole inode is being deleting all on-disk records will
1524          * be deleted very soon, we can't sync any new records to disk
1525          * because they will be deleted in the same transaction they were
1526          * created in (delete_tid == create_tid), which will assert.
1527          *
1528          * XXX There may be a case with RECORD_ADD with DELETED_FE set
1529          * that we currently panic on.
1530          */
1531         if (record->ip->sync_flags & HAMMER_INODE_DELETING) {
1532                 switch(record->type) {
1533                 case HAMMER_MEM_RECORD_DATA:
1534                         /*
1535                          * We don't have to do anything, if the record was
1536                          * committed the space will have been accounted for
1537                          * in the blockmap.
1538                          */
1539                         /* fall through */
1540                 case HAMMER_MEM_RECORD_GENERAL:
1541                         record->flags |= HAMMER_RECF_DELETED_FE;
1542                         record->flags |= HAMMER_RECF_DELETED_BE;
1543                         error = 0;
1544                         goto done;
1545                 case HAMMER_MEM_RECORD_ADD:
1546                         panic("hammer_sync_record_callback: illegal add "
1547                               "during inode deletion record %p", record);
1548                         break; /* NOT REACHED */
1549                 case HAMMER_MEM_RECORD_INODE:
1550                         panic("hammer_sync_record_callback: attempt to "
1551                               "sync inode record %p?", record);
1552                         break; /* NOT REACHED */
1553                 case HAMMER_MEM_RECORD_DEL:
1554                         /*
1555                          * Follow through and issue the on-disk deletion
1556                          */
1557                         break;
1558                 }
1559         }
1560
1561         /*
1562          * If DELETED_FE is set special handling is needed for directory
1563          * entries.  Dependant pieces related to the directory entry may
1564          * have already been synced to disk.  If this occurs we have to
1565          * sync the directory entry and then change the in-memory record
1566          * from an ADD to a DELETE to cover the fact that it's been
1567          * deleted by the frontend.
1568          *
1569          * A directory delete covering record (MEM_RECORD_DEL) can never
1570          * be deleted by the frontend.
1571          *
1572          * Any other record type (aka DATA) can be deleted by the frontend.
1573          * XXX At the moment the flusher must skip it because there may
1574          * be another data record in the flush group for the same block,
1575          * meaning that some frontend data changes can leak into the backend's
1576          * synchronization point.
1577          */
1578         if (record->flags & HAMMER_RECF_DELETED_FE) {
1579                 if (record->type == HAMMER_MEM_RECORD_ADD) {
1580                         record->flags |= HAMMER_RECF_CONVERT_DELETE;
1581                 } else {
1582                         KKASSERT(record->type != HAMMER_MEM_RECORD_DEL);
1583                         record->flags |= HAMMER_RECF_DELETED_BE;
1584                         error = 0;
1585                         goto done;
1586                 }
1587         }
1588
1589         /*
1590          * Assign the create_tid for new records.  Deletions already
1591          * have the record's entire key properly set up.
1592          */
1593         if (record->type != HAMMER_MEM_RECORD_DEL)
1594                 record->leaf.base.create_tid = trans->tid;
1595         for (;;) {
1596                 error = hammer_ip_sync_record_cursor(cursor, record);
1597                 if (error != EDEADLK)
1598                         break;
1599                 hammer_done_cursor(cursor);
1600                 error = hammer_init_cursor(trans, cursor, &record->ip->cache[0],
1601                                            record->ip);
1602                 if (error)
1603                         break;
1604         }
1605         record->flags &= ~HAMMER_RECF_CONVERT_DELETE;
1606
1607         if (error) {
1608                 error = -error;
1609                 if (error != -ENOSPC) {
1610                         kprintf("hammer_sync_record_callback: sync failed rec "
1611                                 "%p, error %d\n", record, error);
1612                         Debugger("sync failed rec");
1613                 }
1614         }
1615 done:
1616         hammer_flush_record_done(record, error);
1617         return(error);
1618 }
1619
1620 /*
1621  * XXX error handling
1622  */
1623 int
1624 hammer_sync_inode(hammer_inode_t ip)
1625 {
1626         struct hammer_transaction trans;
1627         struct hammer_cursor cursor;
1628         hammer_node_t tmp_node;
1629         hammer_record_t depend;
1630         hammer_record_t next;
1631         int error, tmp_error;
1632         u_int64_t nlinks;
1633
1634         if ((ip->sync_flags & HAMMER_INODE_MODMASK) == 0)
1635                 return(0);
1636
1637         hammer_start_transaction_fls(&trans, ip->hmp);
1638         error = hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
1639         if (error)
1640                 goto done;
1641
1642         /*
1643          * Any directory records referencing this inode which are not in
1644          * our current flush group must adjust our nlink count for the
1645          * purposes of synchronization to disk.
1646          *
1647          * Records which are in our flush group can be unlinked from our
1648          * inode now, potentially allowing the inode to be physically
1649          * deleted.
1650          *
1651          * This cannot block.
1652          */
1653         nlinks = ip->ino_data.nlinks;
1654         next = TAILQ_FIRST(&ip->target_list);
1655         while ((depend = next) != NULL) {
1656                 next = TAILQ_NEXT(depend, target_entry);
1657                 if (depend->flush_state == HAMMER_FST_FLUSH &&
1658                     depend->flush_group == ip->hmp->flusher.act) {
1659                         /*
1660                          * If this is an ADD that was deleted by the frontend
1661                          * the frontend nlinks count will have already been
1662                          * decremented, but the backend is going to sync its
1663                          * directory entry and must account for it.  The
1664                          * record will be converted to a delete-on-disk when
1665                          * it gets synced.
1666                          *
1667                          * If the ADD was not deleted by the frontend we
1668                          * can remove the dependancy from our target_list.
1669                          */
1670                         if (depend->flags & HAMMER_RECF_DELETED_FE) {
1671                                 ++nlinks;
1672                         } else {
1673                                 TAILQ_REMOVE(&ip->target_list, depend,
1674                                              target_entry);
1675                                 depend->target_ip = NULL;
1676                         }
1677                 } else if ((depend->flags & HAMMER_RECF_DELETED_FE) == 0) {
1678                         /*
1679                          * Not part of our flush group
1680                          */
1681                         KKASSERT((depend->flags & HAMMER_RECF_DELETED_BE) == 0);
1682                         switch(depend->type) {
1683                         case HAMMER_MEM_RECORD_ADD:
1684                                 --nlinks;
1685                                 break;
1686                         case HAMMER_MEM_RECORD_DEL:
1687                                 ++nlinks;
1688                                 break;
1689                         default:
1690                                 break;
1691                         }
1692                 }
1693         }
1694
1695         /*
1696          * Set dirty if we had to modify the link count.
1697          */
1698         if (ip->sync_ino_data.nlinks != nlinks) {
1699                 KKASSERT((int64_t)nlinks >= 0);
1700                 ip->sync_ino_data.nlinks = nlinks;
1701                 ip->sync_flags |= HAMMER_INODE_DDIRTY;
1702         }
1703
1704         /*
1705          * If there is a trunction queued destroy any data past the (aligned)
1706          * truncation point.  Userland will have dealt with the buffer
1707          * containing the truncation point for us.
1708          *
1709          * We don't flush pending frontend data buffers until after we've
1710          * dealt with the truncation.
1711          */
1712         if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
1713                 /*
1714                  * Interlock trunc_off.  The VOP front-end may continue to
1715                  * make adjustments to it while we are blocked.
1716                  */
1717                 off_t trunc_off;
1718                 off_t aligned_trunc_off;
1719                 int blkmask;
1720
1721                 trunc_off = ip->sync_trunc_off;
1722                 blkmask = hammer_blocksize(trunc_off) - 1;
1723                 aligned_trunc_off = (trunc_off + blkmask) & ~(int64_t)blkmask;
1724
1725                 /*
1726                  * Delete any whole blocks on-media.  The front-end has
1727                  * already cleaned out any partial block and made it
1728                  * pending.  The front-end may have updated trunc_off
1729                  * while we were blocked so we only use sync_trunc_off.
1730                  */
1731                 error = hammer_ip_delete_range(&cursor, ip,
1732                                                 aligned_trunc_off,
1733                                                 0x7FFFFFFFFFFFFFFFLL, 1);
1734                 if (error)
1735                         Debugger("hammer_ip_delete_range errored");
1736
1737                 /*
1738                  * Clear the truncation flag on the backend after we have
1739                  * complete the deletions.  Backend data is now good again
1740                  * (including new records we are about to sync, below).
1741                  *
1742                  * Leave sync_trunc_off intact.  As we write additional
1743                  * records the backend will update sync_trunc_off.  This
1744                  * tells the backend whether it can skip the overwrite
1745                  * test.  This should work properly even when the backend
1746                  * writes full blocks where the truncation point straddles
1747                  * the block because the comparison is against the base
1748                  * offset of the record.
1749                  */
1750                 ip->sync_flags &= ~HAMMER_INODE_TRUNCATED;
1751                 /* ip->sync_trunc_off = 0x7FFFFFFFFFFFFFFFLL; */
1752         } else {
1753                 error = 0;
1754         }
1755
1756         /*
1757          * Now sync related records.  These will typically be directory
1758          * entries or delete-on-disk records.
1759          *
1760          * Not all records will be flushed, but clear XDIRTY anyway.  We
1761          * will set it again in the frontend hammer_flush_inode_done()
1762          * if records remain.
1763          */
1764         if (error == 0) {
1765                 tmp_error = RB_SCAN(hammer_rec_rb_tree, &ip->rec_tree, NULL,
1766                                     hammer_sync_record_callback, &cursor);
1767                 if (tmp_error < 0)
1768                         tmp_error = -error;
1769                 if (tmp_error)
1770                         error = tmp_error;
1771         }
1772         hammer_cache_node(&ip->cache[1], cursor.node);
1773
1774         /*
1775          * Re-seek for inode update.
1776          */
1777         if (error == 0) {
1778                 tmp_node = hammer_ref_node_safe(ip->hmp, &ip->cache[0], &error);
1779                 if (tmp_node) {
1780                         hammer_cursor_seek(&cursor, tmp_node, 0);
1781                         hammer_rel_node(tmp_node);
1782                 }
1783                 error = 0;
1784         }
1785
1786         /*
1787          * If we are deleting the inode the frontend had better not have
1788          * any active references on elements making up the inode.
1789          */
1790         if (error == 0 && ip->sync_ino_data.nlinks == 0 &&
1791                 RB_EMPTY(&ip->rec_tree)  &&
1792             (ip->sync_flags & HAMMER_INODE_DELETING) &&
1793             (ip->flags & HAMMER_INODE_DELETED) == 0) {
1794                 int count1 = 0;
1795
1796                 ip->flags |= HAMMER_INODE_DELETED;
1797                 error = hammer_ip_delete_range_all(&cursor, ip, &count1);
1798                 if (error == 0) {
1799                         ip->sync_flags &= ~HAMMER_INODE_DELETING;
1800                         ip->sync_flags &= ~HAMMER_INODE_TRUNCATED;
1801                         KKASSERT(RB_EMPTY(&ip->rec_tree));
1802
1803                         /*
1804                          * Set delete_tid in both the frontend and backend
1805                          * copy of the inode record.  The DELETED flag handles
1806                          * this, do not set RDIRTY.
1807                          */
1808                         ip->ino_leaf.base.delete_tid = trans.tid;
1809                         ip->sync_ino_leaf.base.delete_tid = trans.tid;
1810
1811                         /*
1812                          * Adjust the inode count in the volume header
1813                          */
1814                         if (ip->flags & HAMMER_INODE_ONDISK) {
1815                                 hammer_modify_volume_field(&trans,
1816                                                            trans.rootvol,
1817                                                            vol0_stat_inodes);
1818                                 --ip->hmp->rootvol->ondisk->vol0_stat_inodes;
1819                                 hammer_modify_volume_done(trans.rootvol);
1820                         }
1821                 } else {
1822                         ip->flags &= ~HAMMER_INODE_DELETED;
1823                         Debugger("hammer_ip_delete_range_all errored");
1824                 }
1825         }
1826
1827         ip->sync_flags &= ~HAMMER_INODE_BUFS;
1828
1829         if (error)
1830                 Debugger("RB_SCAN errored");
1831
1832         /*
1833          * Now update the inode's on-disk inode-data and/or on-disk record.
1834          * DELETED and ONDISK are managed only in ip->flags.
1835          */
1836         switch(ip->flags & (HAMMER_INODE_DELETED | HAMMER_INODE_ONDISK)) {
1837         case HAMMER_INODE_DELETED|HAMMER_INODE_ONDISK:
1838                 /*
1839                  * If deleted and on-disk, don't set any additional flags.
1840                  * the delete flag takes care of things.
1841                  *
1842                  * Clear flags which may have been set by the frontend.
1843                  */
1844                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
1845                                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME |
1846                                     HAMMER_INODE_DELETING);
1847                 break;
1848         case HAMMER_INODE_DELETED:
1849                 /*
1850                  * Take care of the case where a deleted inode was never
1851                  * flushed to the disk in the first place.
1852                  *
1853                  * Clear flags which may have been set by the frontend.
1854                  */
1855                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
1856                                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME |
1857                                     HAMMER_INODE_DELETING);
1858                 while (RB_ROOT(&ip->rec_tree)) {
1859                         hammer_record_t record = RB_ROOT(&ip->rec_tree);
1860                         hammer_ref(&record->lock);
1861                         KKASSERT(record->lock.refs == 1);
1862                         record->flags |= HAMMER_RECF_DELETED_FE;
1863                         record->flags |= HAMMER_RECF_DELETED_BE;
1864                         hammer_rel_mem_record(record);
1865                 }
1866                 break;
1867         case HAMMER_INODE_ONDISK:
1868                 /*
1869                  * If already on-disk, do not set any additional flags.
1870                  */
1871                 break;
1872         default:
1873                 /*
1874                  * If not on-disk and not deleted, set DDIRTY to force
1875                  * an initial record to be written.
1876                  *
1877                  * Also set the create_tid in both the frontend and backend
1878                  * copy of the inode record.
1879                  */
1880                 ip->ino_leaf.base.create_tid = trans.tid;
1881                 ip->sync_ino_leaf.base.create_tid = trans.tid;
1882                 ip->sync_flags |= HAMMER_INODE_DDIRTY;
1883                 break;
1884         }
1885
1886         /*
1887          * If RDIRTY or DDIRTY is set, write out a new record.  If the inode
1888          * is already on-disk the old record is marked as deleted.
1889          *
1890          * If DELETED is set hammer_update_inode() will delete the existing
1891          * record without writing out a new one.
1892          *
1893          * If *ONLY* the ITIMES flag is set we can update the record in-place.
1894          */
1895         if (ip->flags & HAMMER_INODE_DELETED) {
1896                 error = hammer_update_inode(&cursor, ip);
1897         } else
1898         if ((ip->sync_flags & HAMMER_INODE_DDIRTY) == 0 &&
1899             (ip->sync_flags & (HAMMER_INODE_ATIME | HAMMER_INODE_MTIME))) {
1900                 error = hammer_update_itimes(&cursor, ip);
1901         } else
1902         if (ip->sync_flags & (HAMMER_INODE_DDIRTY | HAMMER_INODE_ATIME | HAMMER_INODE_MTIME)) {
1903                 error = hammer_update_inode(&cursor, ip);
1904         }
1905         if (error)
1906                 Debugger("hammer_update_itimes/inode errored");
1907 done:
1908         /*
1909          * Save the TID we used to sync the inode with to make sure we
1910          * do not improperly reuse it.
1911          */
1912         hammer_done_cursor(&cursor);
1913         hammer_done_transaction(&trans);
1914         return(error);
1915 }
1916
1917 /*
1918  * This routine is called when the OS is no longer actively referencing
1919  * the inode (but might still be keeping it cached), or when releasing
1920  * the last reference to an inode.
1921  *
1922  * At this point if the inode's nlinks count is zero we want to destroy
1923  * it, which may mean destroying it on-media too.
1924  */
1925 void
1926 hammer_inode_unloadable_check(hammer_inode_t ip, int getvp)
1927 {
1928         struct vnode *vp;
1929
1930         /*
1931          * Set the DELETING flag when the link count drops to 0 and the
1932          * OS no longer has any opens on the inode.
1933          *
1934          * The backend will clear DELETING (a mod flag) and set DELETED
1935          * (a state flag) when it is actually able to perform the
1936          * operation.
1937          */
1938         if (ip->ino_data.nlinks == 0 &&
1939             (ip->flags & (HAMMER_INODE_DELETING|HAMMER_INODE_DELETED)) == 0) {
1940                 ip->flags |= HAMMER_INODE_DELETING;
1941                 ip->flags |= HAMMER_INODE_TRUNCATED;
1942                 ip->trunc_off = 0;
1943                 vp = NULL;
1944                 if (getvp) {
1945                         if (hammer_get_vnode(ip, &vp) != 0)
1946                                 return;
1947                 }
1948
1949                 /*
1950                  * Final cleanup
1951                  */
1952                 if (ip->vp) {
1953                         vtruncbuf(ip->vp, 0, HAMMER_BUFSIZE);
1954                         vnode_pager_setsize(ip->vp, 0);
1955                 }
1956                 if (getvp) {
1957                         vput(vp);
1958                 }
1959         }
1960 }
1961
1962 /*
1963  * Re-test an inode when a dependancy had gone away to see if we
1964  * can chain flush it.
1965  */
1966 void
1967 hammer_test_inode(hammer_inode_t ip)
1968 {
1969         if (ip->flags & HAMMER_INODE_REFLUSH) {
1970                 ip->flags &= ~HAMMER_INODE_REFLUSH;
1971                 hammer_ref(&ip->lock);
1972                 if (ip->flags & HAMMER_INODE_RESIGNAL) {
1973                         ip->flags &= ~HAMMER_INODE_RESIGNAL;
1974                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
1975                 } else {
1976                         hammer_flush_inode(ip, 0);
1977                 }
1978                 hammer_rel_inode(ip, 0);
1979         }
1980 }
1981
1982 /*
1983  * Clear the RECLAIM flag on an inode.  This occurs when the inode is
1984  * reassociated with a vp or just before it gets freed.
1985  *
1986  * Wakeup one thread blocked waiting on reclaims to complete.  Note that
1987  * the inode the thread is waiting on behalf of is a different inode then
1988  * the inode we are called with.  This is to create a pipeline.
1989  */
1990 static void
1991 hammer_inode_wakereclaims(hammer_inode_t ip)
1992 {
1993         struct hammer_reclaim *reclaim;
1994         hammer_mount_t hmp = ip->hmp;
1995
1996         if ((ip->flags & HAMMER_INODE_RECLAIM) == 0)
1997                 return;
1998
1999         --hammer_count_reclaiming;
2000         --hmp->inode_reclaims;
2001         ip->flags &= ~HAMMER_INODE_RECLAIM;
2002
2003         if ((reclaim = TAILQ_FIRST(&hmp->reclaim_list)) != NULL) {
2004                 TAILQ_REMOVE(&hmp->reclaim_list, reclaim, entry);
2005                 reclaim->okydoky = 1;
2006                 wakeup(reclaim);
2007         }
2008 }
2009
2010 /*
2011  * Setup our reclaim pipeline.  We only let so many detached (and dirty)
2012  * inodes build up before we start blocking.
2013  *
2014  * When we block we don't care *which* inode has finished reclaiming,
2015  * as lone as one does.  This is somewhat heuristical... we also put a
2016  * cap on how long we are willing to wait.
2017  */
2018 void
2019 hammer_inode_waitreclaims(hammer_mount_t hmp)
2020 {
2021         struct hammer_reclaim reclaim;
2022         int delay;
2023
2024         if (hmp->inode_reclaims > HAMMER_RECLAIM_WAIT) {
2025                 reclaim.okydoky = 0;
2026                 TAILQ_INSERT_TAIL(&hmp->reclaim_list,
2027                                   &reclaim, entry);
2028         } else {
2029                 reclaim.okydoky = 1;
2030         }
2031
2032         if (reclaim.okydoky == 0) {
2033                 delay = (hmp->inode_reclaims - HAMMER_RECLAIM_WAIT) * hz /
2034                         HAMMER_RECLAIM_WAIT;
2035                 if (delay >= 0)
2036                         tsleep(&reclaim, 0, "hmrrcm", delay + 1);
2037                 if (reclaim.okydoky == 0)
2038                         TAILQ_REMOVE(&hmp->reclaim_list, &reclaim, entry);
2039         }
2040 }
2041