sys/vfs/hammer/hammer_inode.c

   1 /*
   2  * Copyright (c) 2007-2008 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/vfs/hammer/hammer_inode.c,v 1.83 2008/06/23 21:42:48 dillon Exp $
  35  */
  36
  37 #include "hammer.h"
  38 #include <vm/vm_extern.h>
  39 #include <sys/buf.h>
  40 #include <sys/buf2.h>
  41
  42 static int      hammer_unload_inode(struct hammer_inode *ip);
  43 static void     hammer_flush_inode_core(hammer_inode_t ip, int flags);
  44 static int      hammer_setup_child_callback(hammer_record_t rec, void *data);
  45 static int      hammer_setup_parent_inodes(hammer_inode_t ip);
  46 static int      hammer_setup_parent_inodes_helper(hammer_record_t record);
  47 static void     hammer_inode_wakereclaims(hammer_inode_t ip);
  48
  49 #ifdef DEBUG_TRUNCATE
  50 extern struct hammer_inode *HammerTruncIp;
  51 #endif
  52
  53 /*
  54  * Red-Black tree support for inode structures.
  55  *
  56  * Insertions
  57  */
  58 int
  59 hammer_ino_rb_compare(hammer_inode_t ip1, hammer_inode_t ip2)
  60 {
  61         if (ip1->obj_localization < ip2->obj_localization)
  62                 return(-1);
  63         if (ip1->obj_localization > ip2->obj_localization)
  64                 return(1);
  65         if (ip1->obj_id < ip2->obj_id)
  66                 return(-1);
  67         if (ip1->obj_id > ip2->obj_id)
  68                 return(1);
  69         if (ip1->obj_asof < ip2->obj_asof)
  70                 return(-1);
  71         if (ip1->obj_asof > ip2->obj_asof)
  72                 return(1);
  73         return(0);
  74 }
  75
  76 /*
  77  * LOOKUP_INFO
  78  */
  79 static int
  80 hammer_inode_info_cmp(hammer_inode_info_t info, hammer_inode_t ip)
  81 {
  82         if (info->obj_localization < ip->obj_localization)
  83                 return(-1);
  84         if (info->obj_localization > ip->obj_localization)
  85                 return(1);
  86         if (info->obj_id < ip->obj_id)
  87                 return(-1);
  88         if (info->obj_id > ip->obj_id)
  89                 return(1);
  90         if (info->obj_asof < ip->obj_asof)
  91                 return(-1);
  92         if (info->obj_asof > ip->obj_asof)
  93                 return(1);
  94         return(0);
  95 }
  96
  97 /*
  98  * Used by hammer_scan_inode_snapshots() to locate all of an object's
  99  * snapshots.  Note that the asof field is not tested, which we can get
 100  * away with because it is the lowest-priority field.
 101  */
 102 static int
 103 hammer_inode_info_cmp_all_history(hammer_inode_t ip, void *data)
 104 {
 105         hammer_inode_info_t info = data;
 106
 107         if (ip->obj_localization > info->obj_localization)
 108                 return(1);
 109         if (ip->obj_localization < info->obj_localization)
 110                 return(-1);
 111         if (ip->obj_id > info->obj_id)
 112                 return(1);
 113         if (ip->obj_id < info->obj_id)
 114                 return(-1);
 115         return(0);
 116 }
 117
 118 RB_GENERATE(hammer_ino_rb_tree, hammer_inode, rb_node, hammer_ino_rb_compare);
 119 RB_GENERATE_XLOOKUP(hammer_ino_rb_tree, INFO, hammer_inode, rb_node,
 120                 hammer_inode_info_cmp, hammer_inode_info_t);
 121
 122 /*
 123  * The kernel is not actively referencing this vnode but is still holding
 124  * it cached.
 125  *
 126  * This is called from the frontend.
 127  */
 128 int
 129 hammer_vop_inactive(struct vop_inactive_args *ap)
 130 {
 131         struct hammer_inode *ip = VTOI(ap->a_vp);
 132
 133         /*
 134          * Degenerate case
 135          */
 136         if (ip == NULL) {
 137                 vrecycle(ap->a_vp);
 138                 return(0);
 139         }
 140
 141         /*
 142          * If the inode no longer has visibility in the filesystem try to
 143          * recycle it immediately, even if the inode is dirty.  Recycling
 144          * it quickly allows the system to reclaim buffer cache and VM
 145          * resources which can matter a lot in a heavily loaded system.
 146          *
 147          * This can deadlock in vfsync() if we aren't careful.
 148          *
 149          * Do not queue the inode to the flusher if we still have visibility,
 150          * otherwise namespace calls such as chmod will unnecessarily generate
 151          * multiple inode updates.
 152          */
 153         hammer_inode_unloadable_check(ip, 0);
 154         if (ip->ino_data.nlinks == 0) {
 155                 if (ip->flags & HAMMER_INODE_MODMASK)
 156                         hammer_flush_inode(ip, 0);
 157                 vrecycle(ap->a_vp);
 158         }
 159         return(0);
 160 }
 161
 162 /*
 163  * Release the vnode association.  This is typically (but not always)
 164  * the last reference on the inode.
 165  *
 166  * Once the association is lost we are on our own with regards to
 167  * flushing the inode.
 168  */
 169 int
 170 hammer_vop_reclaim(struct vop_reclaim_args *ap)
 171 {
 172         struct hammer_inode *ip;
 173         hammer_mount_t hmp;
 174         struct vnode *vp;
 175
 176         vp = ap->a_vp;
 177
 178         if ((ip = vp->v_data) != NULL) {
 179                 hmp = ip->hmp;
 180                 vp->v_data = NULL;
 181                 ip->vp = NULL;
 182
 183                 if ((ip->flags & HAMMER_INODE_RECLAIM) == 0) {
 184                         ++hammer_count_reclaiming;
 185                         ++hmp->inode_reclaims;
 186                         ip->flags |= HAMMER_INODE_RECLAIM;
 187                         if (hmp->inode_reclaims > HAMMER_RECLAIM_FLUSH &&
 188                             (hmp->inode_reclaims & 255) == 0) {
 189                                 hammer_flusher_async(hmp);
 190                         }
 191                 }
 192                 hammer_rel_inode(ip, 1);
 193         }
 194         return(0);
 195 }
 196
 197 /*
 198  * Return a locked vnode for the specified inode.  The inode must be
 199  * referenced but NOT LOCKED on entry and will remain referenced on
 200  * return.
 201  *
 202  * Called from the frontend.
 203  */
 204 int
 205 hammer_get_vnode(struct hammer_inode *ip, struct vnode **vpp)
 206 {
 207         hammer_mount_t hmp;
 208         struct vnode *vp;
 209         int error = 0;
 210
 211         hmp = ip->hmp;
 212
 213         for (;;) {
 214                 if ((vp = ip->vp) == NULL) {
 215                         error = getnewvnode(VT_HAMMER, hmp->mp, vpp, 0, 0);
 216                         if (error)
 217                                 break;
 218                         hammer_lock_ex(&ip->lock);
 219                         if (ip->vp != NULL) {
 220                                 hammer_unlock(&ip->lock);
 221                                 vp->v_type = VBAD;
 222                                 vx_put(vp);
 223                                 continue;
 224                         }
 225                         hammer_ref(&ip->lock);
 226                         vp = *vpp;
 227                         ip->vp = vp;
 228                         vp->v_type =
 229                                 hammer_get_vnode_type(ip->ino_data.obj_type);
 230
 231                         hammer_inode_wakereclaims(ip);
 232
 233                         switch(ip->ino_data.obj_type) {
 234                         case HAMMER_OBJTYPE_CDEV:
 235                         case HAMMER_OBJTYPE_BDEV:
 236                                 vp->v_ops = &hmp->mp->mnt_vn_spec_ops;
 237                                 addaliasu(vp, ip->ino_data.rmajor,
 238                                           ip->ino_data.rminor);
 239                                 break;
 240                         case HAMMER_OBJTYPE_FIFO:
 241                                 vp->v_ops = &hmp->mp->mnt_vn_fifo_ops;
 242                                 break;
 243                         default:
 244                                 break;
 245                         }
 246
 247                         /*
 248                          * Only mark as the root vnode if the ip is not
 249                          * historical, otherwise the VFS cache will get
 250                          * confused.  The other half of the special handling
 251                          * is in hammer_vop_nlookupdotdot().
 252                          *
 253                          * Pseudo-filesystem roots also do not count.
 254                          */
 255                         if (ip->obj_id == HAMMER_OBJID_ROOT &&
 256                             ip->obj_asof == hmp->asof &&
 257                             ip->obj_localization == 0) {
 258                                 vp->v_flag |= VROOT;
 259                         }
 260
 261                         vp->v_data = (void *)ip;
 262                         /* vnode locked by getnewvnode() */
 263                         /* make related vnode dirty if inode dirty? */
 264                         hammer_unlock(&ip->lock);
 265                         if (vp->v_type == VREG)
 266                                 vinitvmio(vp, ip->ino_data.size);
 267                         break;
 268                 }
 269
 270                 /*
 271                  * loop if the vget fails (aka races), or if the vp
 272                  * no longer matches ip->vp.
 273                  */
 274                 if (vget(vp, LK_EXCLUSIVE) == 0) {
 275                         if (vp == ip->vp)
 276                                 break;
 277                         vput(vp);
 278                 }
 279         }
 280         *vpp = vp;
 281         return(error);
 282 }
 283
 284 /*
 285  * Locate all copies of the inode for obj_id compatible with the specified
 286  * asof, reference, and issue the related call-back.  This routine is used
 287  * for direct-io invalidation and does not create any new inodes.
 288  */
 289 void
 290 hammer_scan_inode_snapshots(hammer_mount_t hmp, hammer_inode_info_t iinfo,
 291                             int (*callback)(hammer_inode_t ip, void *data),
 292                             void *data)
 293 {
 294         hammer_ino_rb_tree_RB_SCAN(&hmp->rb_inos_root,
 295                                    hammer_inode_info_cmp_all_history,
 296                                    callback, iinfo);
 297 }
 298
 299 /*
 300  * Acquire a HAMMER inode.  The returned inode is not locked.  These functions
 301  * do not attach or detach the related vnode (use hammer_get_vnode() for
 302  * that).
 303  *
 304  * The flags argument is only applied for newly created inodes, and only
 305  * certain flags are inherited.
 306  *
 307  * Called from the frontend.
 308  */
 309 struct hammer_inode *
 310 hammer_get_inode(hammer_transaction_t trans, hammer_inode_t dip,
 311                  u_int64_t obj_id, hammer_tid_t asof, u_int32_t localization,
 312                  int flags, int *errorp)
 313 {
 314         hammer_mount_t hmp = trans->hmp;
 315         struct hammer_inode_info iinfo;
 316         struct hammer_cursor cursor;
 317         struct hammer_inode *ip;
 318
 319         /*
 320          * Determine if we already have an inode cached.  If we do then
 321          * we are golden.
 322          */
 323         iinfo.obj_id = obj_id;
 324         iinfo.obj_asof = asof;
 325         iinfo.obj_localization = localization;
 326 loop:
 327         ip = hammer_ino_rb_tree_RB_LOOKUP_INFO(&hmp->rb_inos_root, &iinfo);
 328         if (ip) {
 329                 hammer_ref(&ip->lock);
 330                 *errorp = 0;
 331                 return(ip);
 332         }
 333
 334         /*
 335          * Allocate a new inode structure and deal with races later.
 336          */
 337         ip = kmalloc(sizeof(*ip), M_HAMMER, M_WAITOK|M_ZERO);
 338         ++hammer_count_inodes;
 339         ++hmp->count_inodes;
 340         ip->obj_id = obj_id;
 341         ip->obj_asof = iinfo.obj_asof;
 342         ip->obj_localization = localization;
 343         ip->hmp = hmp;
 344         ip->flags = flags & HAMMER_INODE_RO;
 345         ip->cache[0].ip = ip;
 346         ip->cache[1].ip = ip;
 347         if (hmp->ronly)
 348                 ip->flags |= HAMMER_INODE_RO;
 349         ip->sync_trunc_off = ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
 350         RB_INIT(&ip->rec_tree);
 351         TAILQ_INIT(&ip->target_list);
 352
 353         /*
 354          * Locate the on-disk inode.
 355          */
 356 retry:
 357         hammer_init_cursor(trans, &cursor, (dip ? &dip->cache[0] : NULL), NULL);
 358         cursor.key_beg.localization = localization + HAMMER_LOCALIZE_INODE;
 359         cursor.key_beg.obj_id = ip->obj_id;
 360         cursor.key_beg.key = 0;
 361         cursor.key_beg.create_tid = 0;
 362         cursor.key_beg.delete_tid = 0;
 363         cursor.key_beg.rec_type = HAMMER_RECTYPE_INODE;
 364         cursor.key_beg.obj_type = 0;
 365         cursor.asof = iinfo.obj_asof;
 366         cursor.flags = HAMMER_CURSOR_GET_LEAF | HAMMER_CURSOR_GET_DATA |
 367                        HAMMER_CURSOR_ASOF;
 368
 369         *errorp = hammer_btree_lookup(&cursor);
 370         if (*errorp == EDEADLK) {
 371                 hammer_done_cursor(&cursor);
 372                 goto retry;
 373         }
 374
 375         /*
 376          * On success the B-Tree lookup will hold the appropriate
 377          * buffer cache buffers and provide a pointer to the requested
 378          * information.  Copy the information to the in-memory inode
 379          * and cache the B-Tree node to improve future operations.
 380          */
 381         if (*errorp == 0) {
 382                 ip->ino_leaf = cursor.node->ondisk->elms[cursor.index].leaf;
 383                 ip->ino_data = cursor.data->inode;
 384
 385                 /*
 386                  * cache[0] tries to cache the location of the object inode.
 387                  * The assumption is that it is near the directory inode.
 388                  *
 389                  * cache[1] tries to cache the location of the object data.
 390                  * The assumption is that it is near the directory data.
 391                  */
 392                 hammer_cache_node(&ip->cache[0], cursor.node);
 393                 if (dip && dip->cache[1].node)
 394                         hammer_cache_node(&ip->cache[1], dip->cache[1].node);
 395
 396                 /*
 397                  * The file should not contain any data past the file size
 398                  * stored in the inode.  Setting sync_trunc_off to the
 399                  * file size instead of max reduces B-Tree lookup overheads
 400                  * on append by allowing the flusher to avoid checking for
 401                  * record overwrites.
 402                  */
 403                 ip->sync_trunc_off = ip->ino_data.size;
 404         }
 405
 406         /*
 407          * The inode is placed on the red-black tree and will be synced to
 408          * the media when flushed or by the filesystem sync.  If this races
 409          * another instantiation/lookup the insertion will fail.
 410          */
 411         if (*errorp == 0) {
 412                 hammer_ref(&ip->lock);
 413                 if (RB_INSERT(hammer_ino_rb_tree, &hmp->rb_inos_root, ip)) {
 414                         hammer_uncache_node(&ip->cache[0]);
 415                         hammer_uncache_node(&ip->cache[1]);
 416                         KKASSERT(ip->lock.refs == 1);
 417                         --hammer_count_inodes;
 418                         --hmp->count_inodes;
 419                         kfree(ip, M_HAMMER);
 420                         hammer_done_cursor(&cursor);
 421                         goto loop;
 422                 }
 423                 ip->flags |= HAMMER_INODE_ONDISK;
 424         } else {
 425                 if (ip->flags & HAMMER_INODE_RSV_INODES) {
 426                         ip->flags &= ~HAMMER_INODE_RSV_INODES; /* sanity */
 427                         --hmp->rsv_inodes;
 428                 }
 429                 hmp->rsv_databufs -= ip->rsv_databufs;
 430                 ip->rsv_databufs = 0;                          /* sanity */
 431
 432                 --hammer_count_inodes;
 433                 --hmp->count_inodes;
 434                 kfree(ip, M_HAMMER);
 435                 ip = NULL;
 436         }
 437         hammer_done_cursor(&cursor);
 438         return (ip);
 439 }
 440
 441 /*
 442  * Create a new filesystem object, returning the inode in *ipp.  The
 443  * returned inode will be referenced.
 444  *
 445  * The inode is created in-memory.
 446  */
 447 int
 448 hammer_create_inode(hammer_transaction_t trans, struct vattr *vap,
 449                     struct ucred *cred, hammer_inode_t dip,
 450                     int pseudofs, struct hammer_inode **ipp)
 451 {
 452         hammer_mount_t hmp;
 453         hammer_inode_t ip;
 454         uid_t xuid;
 455         u_int32_t localization;
 456         int error;
 457
 458         hmp = trans->hmp;
 459
 460         /*
 461          * Assign the localization domain.  If if dip is NULL we are creating
 462          * a pseudo-fs and must locate an unused localization domain.
 463          */
 464         if (pseudofs) {
 465                 for (localization = HAMMER_DEF_LOCALIZATION;
 466                      localization < HAMMER_LOCALIZE_PSEUDOFS_MASK;
 467                      localization += HAMMER_LOCALIZE_PSEUDOFS_INC) {
 468                         ip = hammer_get_inode(trans, NULL, HAMMER_OBJID_ROOT,
 469                                               hmp->asof, localization,
 470                                               0, &error);
 471                         if (ip == NULL) {
 472                                 if (error != ENOENT)
 473                                         return(error);
 474                                 break;
 475                         }
 476                         if (ip)
 477                                 hammer_rel_inode(ip, 0);
 478                 }
 479         } else {
 480                 localization = dip->obj_localization;
 481         }
 482
 483         ip = kmalloc(sizeof(*ip), M_HAMMER, M_WAITOK|M_ZERO);
 484         ++hammer_count_inodes;
 485         ++hmp->count_inodes;
 486
 487         /*
 488          * Allocate a new object id.  If creating a new pseudo-fs the
 489          * obj_id is 1.
 490          */
 491         if (pseudofs)
 492                 ip->obj_id = HAMMER_OBJID_ROOT;
 493         else
 494                 ip->obj_id = hammer_alloc_objid(hmp, dip);
 495         ip->obj_localization = localization;
 496
 497         KKASSERT(ip->obj_id != 0);
 498         ip->obj_asof = hmp->asof;
 499         ip->hmp = hmp;
 500         ip->flush_state = HAMMER_FST_IDLE;
 501         ip->flags = HAMMER_INODE_DDIRTY |
 502                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME;
 503         ip->cache[0].ip = ip;
 504         ip->cache[1].ip = ip;
 505
 506         ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
 507         RB_INIT(&ip->rec_tree);
 508         TAILQ_INIT(&ip->target_list);
 509
 510         ip->ino_data.atime = trans->time;
 511         ip->ino_data.mtime = trans->time;
 512         ip->ino_data.size = 0;
 513         ip->ino_data.nlinks = 0;
 514
 515         /*
 516          * A nohistory designator on the parent directory is inherited by
 517          * the child.  We will do this even for pseudo-fs creation... the
 518          * sysad can turn it off.
 519          */
 520         ip->ino_data.uflags = dip->ino_data.uflags &
 521                               (SF_NOHISTORY|UF_NOHISTORY|UF_NODUMP);
 522
 523         ip->ino_leaf.base.btype = HAMMER_BTREE_TYPE_RECORD;
 524         ip->ino_leaf.base.localization = ip->obj_localization +
 525                                          HAMMER_LOCALIZE_INODE;
 526         ip->ino_leaf.base.obj_id = ip->obj_id;
 527         ip->ino_leaf.base.key = 0;
 528         ip->ino_leaf.base.create_tid = 0;
 529         ip->ino_leaf.base.delete_tid = 0;
 530         ip->ino_leaf.base.rec_type = HAMMER_RECTYPE_INODE;
 531         ip->ino_leaf.base.obj_type = hammer_get_obj_type(vap->va_type);
 532
 533         ip->ino_data.obj_type = ip->ino_leaf.base.obj_type;
 534         ip->ino_data.version = HAMMER_INODE_DATA_VERSION;
 535         ip->ino_data.mode = vap->va_mode;
 536         ip->ino_data.ctime = trans->time;
 537
 538         /*
 539          * Setup the ".." pointer.  This only needs to be done for directories
 540          * but we do it for all objects as a recovery aid.
 541          *
 542          * The parent_obj_localization field only applies to pseudo-fs roots.
 543          */
 544         ip->ino_data.parent_obj_id = dip->ino_leaf.base.obj_id;
 545         if (ip->ino_data.obj_type == HAMMER_OBJTYPE_DIRECTORY &&
 546             ip->obj_id == HAMMER_OBJID_ROOT) {
 547                 ip->ino_data.ext.obj.parent_obj_localization =
 548                                                 dip->obj_localization;
 549         }
 550
 551         switch(ip->ino_leaf.base.obj_type) {
 552         case HAMMER_OBJTYPE_CDEV:
 553         case HAMMER_OBJTYPE_BDEV:
 554                 ip->ino_data.rmajor = vap->va_rmajor;
 555                 ip->ino_data.rminor = vap->va_rminor;
 556                 break;
 557         default:
 558                 break;
 559         }
 560
 561         /*
 562          * Calculate default uid/gid and overwrite with information from
 563          * the vap.
 564          */
 565         xuid = hammer_to_unix_xid(&dip->ino_data.uid);
 566         xuid = vop_helper_create_uid(hmp->mp, dip->ino_data.mode, xuid, cred,
 567                                      &vap->va_mode);
 568         ip->ino_data.mode = vap->va_mode;
 569
 570         if (vap->va_vaflags & VA_UID_UUID_VALID)
 571                 ip->ino_data.uid = vap->va_uid_uuid;
 572         else if (vap->va_uid != (uid_t)VNOVAL)
 573                 hammer_guid_to_uuid(&ip->ino_data.uid, vap->va_uid);
 574         else
 575                 hammer_guid_to_uuid(&ip->ino_data.uid, xuid);
 576
 577         if (vap->va_vaflags & VA_GID_UUID_VALID)
 578                 ip->ino_data.gid = vap->va_gid_uuid;
 579         else if (vap->va_gid != (gid_t)VNOVAL)
 580                 hammer_guid_to_uuid(&ip->ino_data.gid, vap->va_gid);
 581         else
 582                 ip->ino_data.gid = dip->ino_data.gid;
 583
 584         hammer_ref(&ip->lock);
 585         if (RB_INSERT(hammer_ino_rb_tree, &hmp->rb_inos_root, ip)) {
 586                 hammer_unref(&ip->lock);
 587                 panic("hammer_create_inode: duplicate obj_id %llx", ip->obj_id);
 588         }
 589         *ipp = ip;
 590         return(0);
 591 }
 592
 593 /*
 594  * Called by hammer_sync_inode().
 595  */
 596 static int
 597 hammer_update_inode(hammer_cursor_t cursor, hammer_inode_t ip)
 598 {
 599         hammer_transaction_t trans = cursor->trans;
 600         hammer_record_t record;
 601         int error;
 602
 603 retry:
 604         error = 0;
 605
 606         /*
 607          * If the inode has a presence on-disk then locate it and mark
 608          * it deleted, setting DELONDISK.
 609          *
 610          * The record may or may not be physically deleted, depending on
 611          * the retention policy.
 612          */
 613         if ((ip->flags & (HAMMER_INODE_ONDISK|HAMMER_INODE_DELONDISK)) ==
 614             HAMMER_INODE_ONDISK) {
 615                 hammer_normalize_cursor(cursor);
 616                 cursor->key_beg.localization = ip->obj_localization +
 617                                                HAMMER_LOCALIZE_INODE;
 618                 cursor->key_beg.obj_id = ip->obj_id;
 619                 cursor->key_beg.key = 0;
 620                 cursor->key_beg.create_tid = 0;
 621                 cursor->key_beg.delete_tid = 0;
 622                 cursor->key_beg.rec_type = HAMMER_RECTYPE_INODE;
 623                 cursor->key_beg.obj_type = 0;
 624                 cursor->asof = ip->obj_asof;
 625                 cursor->flags &= ~HAMMER_CURSOR_INITMASK;
 626                 cursor->flags |= HAMMER_CURSOR_GET_LEAF | HAMMER_CURSOR_ASOF;
 627                 cursor->flags |= HAMMER_CURSOR_BACKEND;
 628
 629                 error = hammer_btree_lookup(cursor);
 630                 if (hammer_debug_inode)
 631                         kprintf("IPDEL %p %08x %d", ip, ip->flags, error);
 632                 if (error) {
 633                         kprintf("error %d\n", error);
 634                         Debugger("hammer_update_inode");
 635                 }
 636
 637                 if (error == 0) {
 638                         error = hammer_ip_delete_record(cursor, ip, trans->tid);
 639                         if (hammer_debug_inode)
 640                                 kprintf(" error %d\n", error);
 641                         if (error && error != EDEADLK) {
 642                                 kprintf("error %d\n", error);
 643                                 Debugger("hammer_update_inode2");
 644                         }
 645                         if (error == 0) {
 646                                 ip->flags |= HAMMER_INODE_DELONDISK;
 647                         }
 648                         if (cursor->node)
 649                                 hammer_cache_node(&ip->cache[0], cursor->node);
 650                 }
 651                 if (error == EDEADLK) {
 652                         hammer_done_cursor(cursor);
 653                         error = hammer_init_cursor(trans, cursor,
 654                                                    &ip->cache[0], ip);
 655                         if (hammer_debug_inode)
 656                                 kprintf("IPDED %p %d\n", ip, error);
 657                         if (error == 0)
 658                                 goto retry;
 659                 }
 660         }
 661
 662         /*
 663          * Ok, write out the initial record or a new record (after deleting
 664          * the old one), unless the DELETED flag is set.  This routine will
 665          * clear DELONDISK if it writes out a record.
 666          *
 667          * Update our inode statistics if this is the first application of
 668          * the inode on-disk.
 669          */
 670         if (error == 0 && (ip->flags & HAMMER_INODE_DELETED) == 0) {
 671                 /*
 672                  * Generate a record and write it to the media
 673                  */
 674                 record = hammer_alloc_mem_record(ip, 0);
 675                 record->type = HAMMER_MEM_RECORD_INODE;
 676                 record->flush_state = HAMMER_FST_FLUSH;
 677                 record->leaf = ip->sync_ino_leaf;
 678                 record->leaf.base.create_tid = trans->tid;
 679                 record->leaf.data_len = sizeof(ip->sync_ino_data);
 680                 record->data = (void *)&ip->sync_ino_data;
 681                 record->flags |= HAMMER_RECF_INTERLOCK_BE;
 682                 for (;;) {
 683                         error = hammer_ip_sync_record_cursor(cursor, record);
 684                         if (hammer_debug_inode)
 685                                 kprintf("GENREC %p rec %08x %d\n",
 686                                         ip, record->flags, error);
 687                         if (error != EDEADLK)
 688                                 break;
 689                         hammer_done_cursor(cursor);
 690                         error = hammer_init_cursor(trans, cursor,
 691                                                    &ip->cache[0], ip);
 692                         if (hammer_debug_inode)
 693                                 kprintf("GENREC reinit %d\n", error);
 694                         if (error)
 695                                 break;
 696                 }
 697                 if (error) {
 698                         kprintf("error %d\n", error);
 699                         Debugger("hammer_update_inode3");
 700                 }
 701
 702                 /*
 703                  * The record isn't managed by the inode's record tree,
 704                  * destroy it whether we succeed or fail.
 705                  */
 706                 record->flags &= ~HAMMER_RECF_INTERLOCK_BE;
 707                 record->flags |= HAMMER_RECF_DELETED_FE;
 708                 record->flush_state = HAMMER_FST_IDLE;
 709                 hammer_rel_mem_record(record);
 710
 711                 /*
 712                  * Finish up.
 713                  */
 714                 if (error == 0) {
 715                         if (hammer_debug_inode)
 716                                 kprintf("CLEANDELOND %p %08x\n", ip, ip->flags);
 717                         ip->sync_flags &= ~(HAMMER_INODE_DDIRTY |
 718                                             HAMMER_INODE_ATIME |
 719                                             HAMMER_INODE_MTIME);
 720                         ip->flags &= ~HAMMER_INODE_DELONDISK;
 721
 722                         /*
 723                          * Root volume count of inodes
 724                          */
 725                         if ((ip->flags & HAMMER_INODE_ONDISK) == 0) {
 726                                 hammer_modify_volume_field(trans,
 727                                                            trans->rootvol,
 728                                                            vol0_stat_inodes);
 729                                 ++ip->hmp->rootvol->ondisk->vol0_stat_inodes;
 730                                 hammer_modify_volume_done(trans->rootvol);
 731                                 ip->flags |= HAMMER_INODE_ONDISK;
 732                                 if (hammer_debug_inode)
 733                                         kprintf("NOWONDISK %p\n", ip);
 734                         }
 735                 }
 736         }
 737
 738         /*
 739          * If the inode has been destroyed, clean out any left-over flags
 740          * that may have been set by the frontend.
 741          */
 742         if (error == 0 && (ip->flags & HAMMER_INODE_DELETED)) {
 743                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY |
 744                                     HAMMER_INODE_ATIME |
 745                                     HAMMER_INODE_MTIME);
 746         }
 747         return(error);
 748 }
 749
 750 /*
 751  * Update only the itimes fields.
 752  *
 753  * ATIME can be updated without generating any UNDO.  MTIME is updated
 754  * with UNDO so it is guaranteed to be synchronized properly in case of
 755  * a crash.
 756  *
 757  * Neither field is included in the B-Tree leaf element's CRC, which is how
 758  * we can get away with updating ATIME the way we do.
 759  */
 760 static int
 761 hammer_update_itimes(hammer_cursor_t cursor, hammer_inode_t ip)
 762 {
 763         hammer_transaction_t trans = cursor->trans;
 764         int error;
 765
 766 retry:
 767         if ((ip->flags & (HAMMER_INODE_ONDISK|HAMMER_INODE_DELONDISK)) !=
 768             HAMMER_INODE_ONDISK) {
 769                 return(0);
 770         }
 771
 772         hammer_normalize_cursor(cursor);
 773         cursor->key_beg.localization = ip->obj_localization +
 774                                        HAMMER_LOCALIZE_INODE;
 775         cursor->key_beg.obj_id = ip->obj_id;
 776         cursor->key_beg.key = 0;
 777         cursor->key_beg.create_tid = 0;
 778         cursor->key_beg.delete_tid = 0;
 779         cursor->key_beg.rec_type = HAMMER_RECTYPE_INODE;
 780         cursor->key_beg.obj_type = 0;
 781         cursor->asof = ip->obj_asof;
 782         cursor->flags &= ~HAMMER_CURSOR_INITMASK;
 783         cursor->flags |= HAMMER_CURSOR_ASOF;
 784         cursor->flags |= HAMMER_CURSOR_GET_LEAF;
 785         cursor->flags |= HAMMER_CURSOR_GET_DATA;
 786         cursor->flags |= HAMMER_CURSOR_BACKEND;
 787
 788         error = hammer_btree_lookup(cursor);
 789         if (error) {
 790                 kprintf("error %d\n", error);
 791                 Debugger("hammer_update_itimes1");
 792         }
 793         if (error == 0) {
 794                 hammer_cache_node(&ip->cache[0], cursor->node);
 795                 if (ip->sync_flags & HAMMER_INODE_MTIME) {
 796                         /*
 797                          * Updating MTIME requires an UNDO.  Just cover
 798                          * both atime and mtime.
 799                          */
 800                         hammer_modify_buffer(trans, cursor->data_buffer,
 801                                      HAMMER_ITIMES_BASE(&cursor->data->inode),
 802                                      HAMMER_ITIMES_BYTES);
 803                         cursor->data->inode.atime = ip->sync_ino_data.atime;
 804                         cursor->data->inode.mtime = ip->sync_ino_data.mtime;
 805                         hammer_modify_buffer_done(cursor->data_buffer);
 806                 } else if (ip->sync_flags & HAMMER_INODE_ATIME) {
 807                         /*
 808                          * Updating atime only can be done in-place with
 809                          * no UNDO.
 810                          */
 811                         hammer_modify_buffer(trans, cursor->data_buffer,
 812                                              NULL, 0);
 813                         cursor->data->inode.atime = ip->sync_ino_data.atime;
 814                         hammer_modify_buffer_done(cursor->data_buffer);
 815                 }
 816                 ip->sync_flags &= ~(HAMMER_INODE_ATIME | HAMMER_INODE_MTIME);
 817         }
 818         if (error == EDEADLK) {
 819                 hammer_done_cursor(cursor);
 820                 error = hammer_init_cursor(trans, cursor,
 821                                            &ip->cache[0], ip);
 822                 if (error == 0)
 823                         goto retry;
 824         }
 825         return(error);
 826 }
 827
 828 /*
 829  * Release a reference on an inode, flush as requested.
 830  *
 831  * On the last reference we queue the inode to the flusher for its final
 832  * disposition.
 833  */
 834 void
 835 hammer_rel_inode(struct hammer_inode *ip, int flush)
 836 {
 837         hammer_mount_t hmp = ip->hmp;
 838
 839         /*
 840          * Handle disposition when dropping the last ref.
 841          */
 842         for (;;) {
 843                 if (ip->lock.refs == 1) {
 844                         /*
 845                          * Determine whether on-disk action is needed for
 846                          * the inode's final disposition.
 847                          */
 848                         KKASSERT(ip->vp == NULL);
 849                         hammer_inode_unloadable_check(ip, 0);
 850                         if (ip->flags & HAMMER_INODE_MODMASK) {
 851                                 if (hmp->rsv_inodes > desiredvnodes) {
 852                                         hammer_flush_inode(ip,
 853                                                            HAMMER_FLUSH_SIGNAL);
 854                                 } else {
 855                                         hammer_flush_inode(ip, 0);
 856                                 }
 857                         } else if (ip->lock.refs == 1) {
 858                                 hammer_unload_inode(ip);
 859                                 break;
 860                         }
 861                 } else {
 862                         if (flush)
 863                                 hammer_flush_inode(ip, 0);
 864
 865                         /*
 866                          * The inode still has multiple refs, try to drop
 867                          * one ref.
 868                          */
 869                         KKASSERT(ip->lock.refs >= 1);
 870                         if (ip->lock.refs > 1) {
 871                                 hammer_unref(&ip->lock);
 872                                 break;
 873                         }
 874                 }
 875         }
 876 }
 877
 878 /*
 879  * Unload and destroy the specified inode.  Must be called with one remaining
 880  * reference.  The reference is disposed of.
 881  *
 882  * This can only be called in the context of the flusher.
 883  */
 884 static int
 885 hammer_unload_inode(struct hammer_inode *ip)
 886 {
 887         hammer_mount_t hmp = ip->hmp;
 888
 889         KASSERT(ip->lock.refs == 1,
 890                 ("hammer_unload_inode: %d refs\n", ip->lock.refs));
 891         KKASSERT(ip->vp == NULL);
 892         KKASSERT(ip->flush_state == HAMMER_FST_IDLE);
 893         KKASSERT(ip->cursor_ip_refs == 0);
 894         KKASSERT(ip->lock.lockcount == 0);
 895         KKASSERT((ip->flags & HAMMER_INODE_MODMASK) == 0);
 896
 897         KKASSERT(RB_EMPTY(&ip->rec_tree));
 898         KKASSERT(TAILQ_EMPTY(&ip->target_list));
 899
 900         RB_REMOVE(hammer_ino_rb_tree, &hmp->rb_inos_root, ip);
 901
 902         hammer_uncache_node(&ip->cache[0]);
 903         hammer_uncache_node(&ip->cache[1]);
 904         if (ip->objid_cache)
 905                 hammer_clear_objid(ip);
 906         --hammer_count_inodes;
 907         --hmp->count_inodes;
 908
 909         hammer_inode_wakereclaims(ip);
 910         kfree(ip, M_HAMMER);
 911
 912         return(0);
 913 }
 914
 915 /*
 916  * Called on mount -u when switching from RW to RO or vise-versa.  Adjust
 917  * the read-only flag for cached inodes.
 918  *
 919  * This routine is called from a RB_SCAN().
 920  */
 921 int
 922 hammer_reload_inode(hammer_inode_t ip, void *arg __unused)
 923 {
 924         hammer_mount_t hmp = ip->hmp;
 925
 926         if (hmp->ronly || hmp->asof != HAMMER_MAX_TID)
 927                 ip->flags |= HAMMER_INODE_RO;
 928         else
 929                 ip->flags &= ~HAMMER_INODE_RO;
 930         return(0);
 931 }
 932
 933 /*
 934  * A transaction has modified an inode, requiring updates as specified by
 935  * the passed flags.
 936  *
 937  * HAMMER_INODE_DDIRTY: Inode data has been updated
 938  * HAMMER_INODE_XDIRTY: Dirty in-memory records
 939  * HAMMER_INODE_BUFS:   Dirty buffer cache buffers
 940  * HAMMER_INODE_DELETED: Inode record/data must be deleted
 941  * HAMMER_INODE_ATIME/MTIME: mtime/atime has been updated
 942  */
 943 void
 944 hammer_modify_inode(hammer_inode_t ip, int flags)
 945 {
 946         KKASSERT ((ip->flags & HAMMER_INODE_RO) == 0 ||
 947                   (flags & (HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
 948                             HAMMER_INODE_BUFS | HAMMER_INODE_DELETED |
 949                             HAMMER_INODE_ATIME | HAMMER_INODE_MTIME)) == 0);
 950         if ((ip->flags & HAMMER_INODE_RSV_INODES) == 0) {
 951                 ip->flags |= HAMMER_INODE_RSV_INODES;
 952                 ++ip->hmp->rsv_inodes;
 953         }
 954
 955         ip->flags |= flags;
 956 }
 957
 958 /*
 959  * Request that an inode be flushed.  This whole mess cannot block and may
 960  * recurse (if not synchronous).  Once requested HAMMER will attempt to
 961  * actively flush the inode until the flush can be done.
 962  *
 963  * The inode may already be flushing, or may be in a setup state.  We can
 964  * place the inode in a flushing state if it is currently idle and flag it
 965  * to reflush if it is currently flushing.
 966  *
 967  * If the HAMMER_FLUSH_SYNCHRONOUS flag is specified we will attempt to
 968  * flush the indoe synchronously using the caller's context.
 969  */
 970 void
 971 hammer_flush_inode(hammer_inode_t ip, int flags)
 972 {
 973         int good;
 974
 975         /*
 976          * Trivial 'nothing to flush' case.  If the inode is ina SETUP
 977          * state we have to put it back into an IDLE state so we can
 978          * drop the extra ref.
 979          */
 980         if ((ip->flags & HAMMER_INODE_MODMASK) == 0) {
 981                 if (ip->flush_state == HAMMER_FST_SETUP) {
 982                         ip->flush_state = HAMMER_FST_IDLE;
 983                         hammer_rel_inode(ip, 0);
 984                 }
 985                 return;
 986         }
 987
 988         /*
 989          * Our flush action will depend on the current state.
 990          */
 991         switch(ip->flush_state) {
 992         case HAMMER_FST_IDLE:
 993                 /*
 994                  * We have no dependancies and can flush immediately.  Some
 995                  * our children may not be flushable so we have to re-test
 996                  * with that additional knowledge.
 997                  */
 998                 hammer_flush_inode_core(ip, flags);
 999                 break;
1000         case HAMMER_FST_SETUP:
1001                 /*
1002                  * Recurse upwards through dependancies via target_list
1003                  * and start their flusher actions going if possible.
1004                  *
1005                  * 'good' is our connectivity.  -1 means we have none and
1006                  * can't flush, 0 means there weren't any dependancies, and
1007                  * 1 means we have good connectivity.
1008                  */
1009                 good = hammer_setup_parent_inodes(ip);
1010
1011                 /*
1012                  * We can continue if good >= 0.  Determine how many records
1013                  * under our inode can be flushed (and mark them).
1014                  */
1015                 if (good >= 0) {
1016                         hammer_flush_inode_core(ip, flags);
1017                 } else {
1018                         ip->flags |= HAMMER_INODE_REFLUSH;
1019                         if (flags & HAMMER_FLUSH_SIGNAL) {
1020                                 ip->flags |= HAMMER_INODE_RESIGNAL;
1021                                 hammer_flusher_async(ip->hmp);
1022                         }
1023                 }
1024                 break;
1025         default:
1026                 /*
1027                  * We are already flushing, flag the inode to reflush
1028                  * if needed after it completes its current flush.
1029                  */
1030                 if ((ip->flags & HAMMER_INODE_REFLUSH) == 0)
1031                         ip->flags |= HAMMER_INODE_REFLUSH;
1032                 if (flags & HAMMER_FLUSH_SIGNAL) {
1033                         ip->flags |= HAMMER_INODE_RESIGNAL;
1034                         hammer_flusher_async(ip->hmp);
1035                 }
1036                 break;
1037         }
1038 }
1039
1040 /*
1041  * Scan ip->target_list, which is a list of records owned by PARENTS to our
1042  * ip which reference our ip.
1043  *
1044  * XXX This is a huge mess of recursive code, but not one bit of it blocks
1045  *     so for now do not ref/deref the structures.  Note that if we use the
1046  *     ref/rel code later, the rel CAN block.
1047  */
1048 static int
1049 hammer_setup_parent_inodes(hammer_inode_t ip)
1050 {
1051         hammer_record_t depend;
1052 #if 0
1053         hammer_record_t next;
1054         hammer_inode_t  pip;
1055 #endif
1056         int good;
1057         int r;
1058
1059         good = 0;
1060         TAILQ_FOREACH(depend, &ip->target_list, target_entry) {
1061                 r = hammer_setup_parent_inodes_helper(depend);
1062                 KKASSERT(depend->target_ip == ip);
1063                 if (r < 0 && good == 0)
1064                         good = -1;
1065                 if (r > 0)
1066                         good = 1;
1067         }
1068         return(good);
1069
1070 #if 0
1071 retry:
1072         good = 0;
1073         next = TAILQ_FIRST(&ip->target_list);
1074         if (next) {
1075                 hammer_ref(&next->lock);
1076                 hammer_ref(&next->ip->lock);
1077         }
1078         while ((depend = next) != NULL) {
1079                 if (depend->target_ip == NULL) {
1080                         pip = depend->ip;
1081                         hammer_rel_mem_record(depend);
1082                         hammer_rel_inode(pip, 0);
1083                         goto retry;
1084                 }
1085                 KKASSERT(depend->target_ip == ip);
1086                 next = TAILQ_NEXT(depend, target_entry);
1087                 if (next) {
1088                         hammer_ref(&next->lock);
1089                         hammer_ref(&next->ip->lock);
1090                 }
1091                 r = hammer_setup_parent_inodes_helper(depend);
1092                 if (r < 0 && good == 0)
1093                         good = -1;
1094                 if (r > 0)
1095                         good = 1;
1096                 pip = depend->ip;
1097                 hammer_rel_mem_record(depend);
1098                 hammer_rel_inode(pip, 0);
1099         }
1100         return(good);
1101 #endif
1102 }
1103
1104 /*
1105  * This helper function takes a record representing the dependancy between
1106  * the parent inode and child inode.
1107  *
1108  * record->ip           = parent inode
1109  * record->target_ip    = child inode
1110  *
1111  * We are asked to recurse upwards and convert the record from SETUP
1112  * to FLUSH if possible.
1113  *
1114  * Return 1 if the record gives us connectivity
1115  *
1116  * Return 0 if the record is not relevant
1117  *
1118  * Return -1 if we can't resolve the dependancy and there is no connectivity.
1119  */
1120 static int
1121 hammer_setup_parent_inodes_helper(hammer_record_t record)
1122 {
1123         hammer_mount_t hmp;
1124         hammer_inode_t pip;
1125         int good;
1126
1127         KKASSERT(record->flush_state != HAMMER_FST_IDLE);
1128         pip = record->ip;
1129         hmp = pip->hmp;
1130
1131         /*
1132          * If the record is already flushing, is it in our flush group?
1133          *
1134          * If it is in our flush group but it is a general record or a
1135          * delete-on-disk, it does not improve our connectivity (return 0),
1136          * and if the target inode is not trying to destroy itself we can't
1137          * allow the operation yet anyway (the second return -1).
1138          */
1139         if (record->flush_state == HAMMER_FST_FLUSH) {
1140                 if (record->flush_group != hmp->flusher.next) {
1141                         pip->flags |= HAMMER_INODE_REFLUSH;
1142                         return(-1);
1143                 }
1144                 if (record->type == HAMMER_MEM_RECORD_ADD)
1145                         return(1);
1146                 /* GENERAL or DEL */
1147                 return(0);
1148         }
1149
1150         /*
1151          * It must be a setup record.  Try to resolve the setup dependancies
1152          * by recursing upwards so we can place ip on the flush list.
1153          */
1154         KKASSERT(record->flush_state == HAMMER_FST_SETUP);
1155
1156         good = hammer_setup_parent_inodes(pip);
1157
1158         /*
1159          * We can't flush ip because it has no connectivity (XXX also check
1160          * nlinks for pre-existing connectivity!).  Flag it so any resolution
1161          * recurses back down.
1162          */
1163         if (good < 0) {
1164                 pip->flags |= HAMMER_INODE_REFLUSH;
1165                 return(good);
1166         }
1167
1168         /*
1169          * We are go, place the parent inode in a flushing state so we can
1170          * place its record in a flushing state.  Note that the parent
1171          * may already be flushing.  The record must be in the same flush
1172          * group as the parent.
1173          */
1174         if (pip->flush_state != HAMMER_FST_FLUSH)
1175                 hammer_flush_inode_core(pip, HAMMER_FLUSH_RECURSION);
1176         KKASSERT(pip->flush_state == HAMMER_FST_FLUSH);
1177         KKASSERT(record->flush_state == HAMMER_FST_SETUP);
1178
1179 #if 0
1180         if (record->type == HAMMER_MEM_RECORD_DEL &&
1181             (record->target_ip->flags & (HAMMER_INODE_DELETED|HAMMER_INODE_DELONDISK)) == 0) {
1182                 /*
1183                  * Regardless of flushing state we cannot sync this path if the
1184                  * record represents a delete-on-disk but the target inode
1185                  * is not ready to sync its own deletion.
1186                  *
1187                  * XXX need to count effective nlinks to determine whether
1188                  * the flush is ok, otherwise removing a hardlink will
1189                  * just leave the DEL record to rot.
1190                  */
1191                 record->target_ip->flags |= HAMMER_INODE_REFLUSH;
1192                 return(-1);
1193         } else
1194 #endif
1195         if (pip->flush_group == pip->hmp->flusher.next) {
1196                 /*
1197                  * This is the record we wanted to synchronize.  If the
1198                  * record went into a flush state while we blocked it
1199                  * had better be in the correct flush group.
1200                  */
1201                 if (record->flush_state != HAMMER_FST_FLUSH) {
1202                         record->flush_state = HAMMER_FST_FLUSH;
1203                         record->flush_group = pip->flush_group;
1204                         hammer_ref(&record->lock);
1205                 } else {
1206                         KKASSERT(record->flush_group == pip->flush_group);
1207                 }
1208                 if (record->type == HAMMER_MEM_RECORD_ADD)
1209                         return(1);
1210
1211                 /*
1212                  * A general or delete-on-disk record does not contribute
1213                  * to our visibility.  We can still flush it, however.
1214                  */
1215                 return(0);
1216         } else {
1217                 /*
1218                  * We couldn't resolve the dependancies, request that the
1219                  * inode be flushed when the dependancies can be resolved.
1220                  */
1221                 pip->flags |= HAMMER_INODE_REFLUSH;
1222                 return(-1);
1223         }
1224 }
1225
1226 /*
1227  * This is the core routine placing an inode into the FST_FLUSH state.
1228  */
1229 static void
1230 hammer_flush_inode_core(hammer_inode_t ip, int flags)
1231 {
1232         int go_count;
1233
1234         /*
1235          * Set flush state and prevent the flusher from cycling into
1236          * the next flush group.  Do not place the ip on the list yet.
1237          * Inodes not in the idle state get an extra reference.
1238          */
1239         KKASSERT(ip->flush_state != HAMMER_FST_FLUSH);
1240         if (ip->flush_state == HAMMER_FST_IDLE)
1241                 hammer_ref(&ip->lock);
1242         ip->flush_state = HAMMER_FST_FLUSH;
1243         ip->flush_group = ip->hmp->flusher.next;
1244         ++ip->hmp->flusher.group_lock;
1245         ++ip->hmp->count_iqueued;
1246         ++hammer_count_iqueued;
1247
1248         /*
1249          * We need to be able to vfsync/truncate from the backend.
1250          */
1251         KKASSERT((ip->flags & HAMMER_INODE_VHELD) == 0);
1252         if (ip->vp && (ip->vp->v_flag & VINACTIVE) == 0) {
1253                 ip->flags |= HAMMER_INODE_VHELD;
1254                 vref(ip->vp);
1255         }
1256
1257         /*
1258          * Figure out how many in-memory records we can actually flush
1259          * (not including inode meta-data, buffers, etc).
1260          */
1261         if (flags & HAMMER_FLUSH_RECURSION) {
1262                 go_count = 1;
1263         } else {
1264                 go_count = RB_SCAN(hammer_rec_rb_tree, &ip->rec_tree, NULL,
1265                                    hammer_setup_child_callback, NULL);
1266         }
1267
1268         /*
1269          * This is a more involved test that includes go_count.  If we
1270          * can't flush, flag the inode and return.  If go_count is 0 we
1271          * were are unable to flush any records in our rec_tree and
1272          * must ignore the XDIRTY flag.
1273          */
1274         if (go_count == 0) {
1275                 if ((ip->flags & HAMMER_INODE_MODMASK_NOXDIRTY) == 0) {
1276                         ip->flags |= HAMMER_INODE_REFLUSH;
1277
1278                         --ip->hmp->count_iqueued;
1279                         --hammer_count_iqueued;
1280
1281                         ip->flush_state = HAMMER_FST_SETUP;
1282                         if (ip->flags & HAMMER_INODE_VHELD) {
1283                                 ip->flags &= ~HAMMER_INODE_VHELD;
1284                                 vrele(ip->vp);
1285                         }
1286                         if (flags & HAMMER_FLUSH_SIGNAL) {
1287                                 ip->flags |= HAMMER_INODE_RESIGNAL;
1288                                 hammer_flusher_async(ip->hmp);
1289                         }
1290                         if (--ip->hmp->flusher.group_lock == 0)
1291                                 wakeup(&ip->hmp->flusher.group_lock);
1292                         return;
1293                 }
1294         }
1295
1296         /*
1297          * Snapshot the state of the inode for the backend flusher.
1298          *
1299          * The truncation must be retained in the frontend until after
1300          * we've actually performed the record deletion.
1301          *
1302          * We continue to retain sync_trunc_off even when all truncations
1303          * have been resolved as an optimization to determine if we can
1304          * skip the B-Tree lookup for overwrite deletions.
1305          *
1306          * NOTE: The DELETING flag is a mod flag, but it is also sticky,
1307          * and stays in ip->flags.  Once set, it stays set until the
1308          * inode is destroyed.
1309          */
1310         ip->sync_flags = (ip->flags & HAMMER_INODE_MODMASK);
1311         if (ip->sync_flags & HAMMER_INODE_TRUNCATED)
1312                 ip->sync_trunc_off = ip->trunc_off;
1313         ip->sync_ino_leaf = ip->ino_leaf;
1314         ip->sync_ino_data = ip->ino_data;
1315         ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
1316         ip->flags &= ~HAMMER_INODE_MODMASK;
1317 #ifdef DEBUG_TRUNCATE
1318         if ((ip->sync_flags & HAMMER_INODE_TRUNCATED) && ip == HammerTruncIp)
1319                 kprintf("truncateS %016llx\n", ip->sync_trunc_off);
1320 #endif
1321
1322         /*
1323          * The flusher list inherits our inode and reference.
1324          */
1325         TAILQ_INSERT_TAIL(&ip->hmp->flush_list, ip, flush_entry);
1326         if (--ip->hmp->flusher.group_lock == 0)
1327                 wakeup(&ip->hmp->flusher.group_lock);
1328
1329         if (flags & HAMMER_FLUSH_SIGNAL) {
1330                 hammer_flusher_async(ip->hmp);
1331         }
1332 }
1333
1334 /*
1335  * Callback for scan of ip->rec_tree.  Try to include each record in our
1336  * flush.  ip->flush_group has been set but the inode has not yet been
1337  * moved into a flushing state.
1338  *
1339  * If we get stuck on a record we have to set HAMMER_INODE_REFLUSH on
1340  * both inodes.
1341  *
1342  * We return 1 for any record placed or found in FST_FLUSH, which prevents
1343  * the caller from shortcutting the flush.
1344  */
1345 static int
1346 hammer_setup_child_callback(hammer_record_t rec, void *data)
1347 {
1348         hammer_inode_t target_ip;
1349         hammer_inode_t ip;
1350         int r;
1351
1352         /*
1353          * Deleted records are ignored.  Note that the flush detects deleted
1354          * front-end records at multiple points to deal with races.  This is
1355          * just the first line of defense.  The only time DELETED_FE cannot
1356          * be set is when HAMMER_RECF_INTERLOCK_BE is set.
1357          *
1358          * Don't get confused between record deletion and, say, directory
1359          * entry deletion.  The deletion of a directory entry that is on
1360          * the media has nothing to do with the record deletion flags.
1361          */
1362         if (rec->flags & (HAMMER_RECF_DELETED_FE|HAMMER_RECF_DELETED_BE))
1363                 return(0);
1364
1365         /*
1366          * If the record is in an idle state it has no dependancies and
1367          * can be flushed.
1368          */
1369         ip = rec->ip;
1370         r = 0;
1371
1372         switch(rec->flush_state) {
1373         case HAMMER_FST_IDLE:
1374                 /*
1375                  * Record has no setup dependancy, we can flush it.
1376                  */
1377                 KKASSERT(rec->target_ip == NULL);
1378                 rec->flush_state = HAMMER_FST_FLUSH;
1379                 rec->flush_group = ip->flush_group;
1380                 hammer_ref(&rec->lock);
1381                 r = 1;
1382                 break;
1383         case HAMMER_FST_SETUP:
1384                 /*
1385                  * Record has a setup dependancy.  Try to include the
1386                  * target ip in the flush.
1387                  *
1388                  * We have to be careful here, if we do not do the right
1389                  * thing we can lose track of dirty inodes and the system
1390                  * will lockup trying to allocate buffers.
1391                  */
1392                 target_ip = rec->target_ip;
1393                 KKASSERT(target_ip != NULL);
1394                 KKASSERT(target_ip->flush_state != HAMMER_FST_IDLE);
1395                 if (target_ip->flush_state == HAMMER_FST_FLUSH) {
1396                         /*
1397                          * If the target IP is already flushing in our group
1398                          * we are golden, otherwise make sure the target
1399                          * reflushes.
1400                          */
1401                         if (target_ip->flush_group == ip->flush_group) {
1402                                 rec->flush_state = HAMMER_FST_FLUSH;
1403                                 rec->flush_group = ip->flush_group;
1404                                 hammer_ref(&rec->lock);
1405                                 r = 1;
1406                         } else {
1407                                 target_ip->flags |= HAMMER_INODE_REFLUSH;
1408                         }
1409                 } else if (rec->type == HAMMER_MEM_RECORD_ADD) {
1410                         /*
1411                          * If the target IP is not flushing we can force
1412                          * it to flush, even if it is unable to write out
1413                          * any of its own records we have at least one in
1414                          * hand that we CAN deal with.
1415                          */
1416                         rec->flush_state = HAMMER_FST_FLUSH;
1417                         rec->flush_group = ip->flush_group;
1418                         hammer_ref(&rec->lock);
1419                         hammer_flush_inode_core(target_ip,
1420                                                 HAMMER_FLUSH_RECURSION);
1421                         r = 1;
1422                 } else {
1423                         /*
1424                          * General or delete-on-disk record.
1425                          *
1426                          * XXX this needs help.  If a delete-on-disk we could
1427                          * disconnect the target.  If the target has its own
1428                          * dependancies they really need to be flushed.
1429                          *
1430                          * XXX
1431                          */
1432                         rec->flush_state = HAMMER_FST_FLUSH;
1433                         rec->flush_group = ip->flush_group;
1434                         hammer_ref(&rec->lock);
1435                         hammer_flush_inode_core(target_ip,
1436                                                 HAMMER_FLUSH_RECURSION);
1437                         r = 1;
1438                 }
1439                 break;
1440         case HAMMER_FST_FLUSH:
1441                 /*
1442                  * Record already associated with a flush group.  It had
1443                  * better be ours.
1444                  */
1445                 KKASSERT(rec->flush_group == ip->flush_group);
1446                 r = 1;
1447                 break;
1448         }
1449         return(r);
1450 }
1451
1452 /*
1453  * Wait for a previously queued flush to complete.  Not only do we need to
1454  * wait for the inode to sync out, we also may have to run the flusher again
1455  * to get it past the UNDO position pertaining to the flush so a crash does
1456  * not 'undo' our flush.
1457  */
1458 void
1459 hammer_wait_inode(hammer_inode_t ip)
1460 {
1461         hammer_mount_t hmp = ip->hmp;
1462         int sync_group;
1463         int waitcount;
1464
1465         sync_group = ip->flush_group;
1466         waitcount = (ip->flags & HAMMER_INODE_REFLUSH) ? 2 : 1;
1467
1468         if (ip->flush_state == HAMMER_FST_SETUP) {
1469                 kprintf("X");
1470                 hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
1471         }
1472         /* XXX can we make this != FST_IDLE ? check SETUP depends */
1473         while (ip->flush_state == HAMMER_FST_FLUSH &&
1474                (ip->flush_group - sync_group) < waitcount) {
1475                 ip->flags |= HAMMER_INODE_FLUSHW;
1476                 tsleep(&ip->flags, 0, "hmrwin", 0);
1477         }
1478         while (hmp->flusher.done - sync_group < waitcount) {
1479                 kprintf("Y");
1480                 hammer_flusher_sync(hmp);
1481         }
1482 }
1483
1484 /*
1485  * Called by the backend code when a flush has been completed.
1486  * The inode has already been removed from the flush list.
1487  *
1488  * A pipelined flush can occur, in which case we must re-enter the
1489  * inode on the list and re-copy its fields.
1490  */
1491 void
1492 hammer_flush_inode_done(hammer_inode_t ip)
1493 {
1494         hammer_mount_t hmp;
1495         int dorel;
1496
1497         KKASSERT(ip->flush_state == HAMMER_FST_FLUSH);
1498
1499         hmp = ip->hmp;
1500
1501         /*
1502          * Merge left-over flags back into the frontend and fix the state.
1503          */
1504         ip->flags |= ip->sync_flags;
1505
1506         /*
1507          * The backend may have adjusted nlinks, so if the adjusted nlinks
1508          * does not match the fronttend set the frontend's RDIRTY flag again.
1509          */
1510         if (ip->ino_data.nlinks != ip->sync_ino_data.nlinks)
1511                 ip->flags |= HAMMER_INODE_DDIRTY;
1512
1513         /*
1514          * Fix up the dirty buffer status.  IO completions will also
1515          * try to clean up rsv_databufs.
1516          */
1517         if (ip->vp && RB_ROOT(&ip->vp->v_rbdirty_tree)) {
1518                 ip->flags |= HAMMER_INODE_BUFS;
1519         } else {
1520                 hmp->rsv_databufs -= ip->rsv_databufs;
1521                 ip->rsv_databufs = 0;
1522         }
1523
1524         /*
1525          * Re-set the XDIRTY flag if some of the inode's in-memory records
1526          * could not be flushed.
1527          */
1528         KKASSERT((RB_EMPTY(&ip->rec_tree) &&
1529                   (ip->flags & HAMMER_INODE_XDIRTY) == 0) ||
1530                  (!RB_EMPTY(&ip->rec_tree) &&
1531                   (ip->flags & HAMMER_INODE_XDIRTY) != 0));
1532
1533         /*
1534          * Do not lose track of inodes which no longer have vnode
1535          * assocations, otherwise they may never get flushed again.
1536          */
1537         if ((ip->flags & HAMMER_INODE_MODMASK) && ip->vp == NULL)
1538                 ip->flags |= HAMMER_INODE_REFLUSH;
1539
1540         /*
1541          * Adjust flush_state.  The target state (idle or setup) shouldn't
1542          * be terribly important since we will reflush if we really need
1543          * to do anything. XXX
1544          */
1545         if (TAILQ_EMPTY(&ip->target_list) && RB_EMPTY(&ip->rec_tree)) {
1546                 ip->flush_state = HAMMER_FST_IDLE;
1547                 dorel = 1;
1548         } else {
1549                 ip->flush_state = HAMMER_FST_SETUP;
1550                 dorel = 0;
1551         }
1552
1553         --hmp->count_iqueued;
1554         --hammer_count_iqueued;
1555
1556         /*
1557          * Clean up the vnode ref
1558          */
1559         if (ip->flags & HAMMER_INODE_VHELD) {
1560                 ip->flags &= ~HAMMER_INODE_VHELD;
1561                 vrele(ip->vp);
1562         }
1563
1564         /*
1565          * If the frontend made more changes and requested another flush,
1566          * then try to get it running.
1567          */
1568         if (ip->flags & HAMMER_INODE_REFLUSH) {
1569                 ip->flags &= ~HAMMER_INODE_REFLUSH;
1570                 if (ip->flags & HAMMER_INODE_RESIGNAL) {
1571                         ip->flags &= ~HAMMER_INODE_RESIGNAL;
1572                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
1573                 } else {
1574                         hammer_flush_inode(ip, 0);
1575                 }
1576         }
1577
1578         /*
1579          * If the inode is now clean drop the space reservation.
1580          */
1581         if ((ip->flags & HAMMER_INODE_MODMASK) == 0 &&
1582             (ip->flags & HAMMER_INODE_RSV_INODES)) {
1583                 ip->flags &= ~HAMMER_INODE_RSV_INODES;
1584                 --hmp->rsv_inodes;
1585         }
1586
1587         /*
1588          * Finally, if the frontend is waiting for a flush to complete,
1589          * wake it up.
1590          */
1591         if (ip->flush_state != HAMMER_FST_FLUSH) {
1592                 if (ip->flags & HAMMER_INODE_FLUSHW) {
1593                         ip->flags &= ~HAMMER_INODE_FLUSHW;
1594                         wakeup(&ip->flags);
1595                 }
1596         }
1597         if (dorel)
1598                 hammer_rel_inode(ip, 0);
1599 }
1600
1601 /*
1602  * Called from hammer_sync_inode() to synchronize in-memory records
1603  * to the media.
1604  */
1605 static int
1606 hammer_sync_record_callback(hammer_record_t record, void *data)
1607 {
1608         hammer_cursor_t cursor = data;
1609         hammer_transaction_t trans = cursor->trans;
1610         int error;
1611
1612         /*
1613          * Skip records that do not belong to the current flush.
1614          */
1615         ++hammer_stats_record_iterations;
1616         if (record->flush_state != HAMMER_FST_FLUSH)
1617                 return(0);
1618
1619 #if 1
1620         if (record->flush_group != record->ip->flush_group) {
1621                 kprintf("sync_record %p ip %p bad flush group %d %d\n", record, record->ip, record->flush_group ,record->ip->flush_group);
1622                 Debugger("blah2");
1623                 return(0);
1624         }
1625 #endif
1626         KKASSERT(record->flush_group == record->ip->flush_group);
1627
1628         /*
1629          * Interlock the record using the BE flag.  Once BE is set the
1630          * frontend cannot change the state of FE.
1631          *
1632          * NOTE: If FE is set prior to us setting BE we still sync the
1633          * record out, but the flush completion code converts it to
1634          * a delete-on-disk record instead of destroying it.
1635          */
1636         KKASSERT((record->flags & HAMMER_RECF_INTERLOCK_BE) == 0);
1637         record->flags |= HAMMER_RECF_INTERLOCK_BE;
1638
1639         /*
1640          * The backend may have already disposed of the record.
1641          */
1642         if (record->flags & HAMMER_RECF_DELETED_BE) {
1643                 error = 0;
1644                 goto done;
1645         }
1646
1647         /*
1648          * If the whole inode is being deleting all on-disk records will
1649          * be deleted very soon, we can't sync any new records to disk
1650          * because they will be deleted in the same transaction they were
1651          * created in (delete_tid == create_tid), which will assert.
1652          *
1653          * XXX There may be a case with RECORD_ADD with DELETED_FE set
1654          * that we currently panic on.
1655          */
1656         if (record->ip->sync_flags & HAMMER_INODE_DELETING) {
1657                 switch(record->type) {
1658                 case HAMMER_MEM_RECORD_DATA:
1659                         /*
1660                          * We don't have to do anything, if the record was
1661                          * committed the space will have been accounted for
1662                          * in the blockmap.
1663                          */
1664                         /* fall through */
1665                 case HAMMER_MEM_RECORD_GENERAL:
1666                         record->flags |= HAMMER_RECF_DELETED_FE;
1667                         record->flags |= HAMMER_RECF_DELETED_BE;
1668                         error = 0;
1669                         goto done;
1670                 case HAMMER_MEM_RECORD_ADD:
1671                         panic("hammer_sync_record_callback: illegal add "
1672                               "during inode deletion record %p", record);
1673                         break; /* NOT REACHED */
1674                 case HAMMER_MEM_RECORD_INODE:
1675                         panic("hammer_sync_record_callback: attempt to "
1676                               "sync inode record %p?", record);
1677                         break; /* NOT REACHED */
1678                 case HAMMER_MEM_RECORD_DEL:
1679                         /*
1680                          * Follow through and issue the on-disk deletion
1681                          */
1682                         break;
1683                 }
1684         }
1685
1686         /*
1687          * If DELETED_FE is set special handling is needed for directory
1688          * entries.  Dependant pieces related to the directory entry may
1689          * have already been synced to disk.  If this occurs we have to
1690          * sync the directory entry and then change the in-memory record
1691          * from an ADD to a DELETE to cover the fact that it's been
1692          * deleted by the frontend.
1693          *
1694          * A directory delete covering record (MEM_RECORD_DEL) can never
1695          * be deleted by the frontend.
1696          *
1697          * Any other record type (aka DATA) can be deleted by the frontend.
1698          * XXX At the moment the flusher must skip it because there may
1699          * be another data record in the flush group for the same block,
1700          * meaning that some frontend data changes can leak into the backend's
1701          * synchronization point.
1702          */
1703         if (record->flags & HAMMER_RECF_DELETED_FE) {
1704                 if (record->type == HAMMER_MEM_RECORD_ADD) {
1705                         record->flags |= HAMMER_RECF_CONVERT_DELETE;
1706                 } else {
1707                         KKASSERT(record->type != HAMMER_MEM_RECORD_DEL);
1708                         record->flags |= HAMMER_RECF_DELETED_BE;
1709                         error = 0;
1710                         goto done;
1711                 }
1712         }
1713
1714         /*
1715          * Assign the create_tid for new records.  Deletions already
1716          * have the record's entire key properly set up.
1717          */
1718         if (record->type != HAMMER_MEM_RECORD_DEL)
1719                 record->leaf.base.create_tid = trans->tid;
1720         for (;;) {
1721                 error = hammer_ip_sync_record_cursor(cursor, record);
1722                 if (error != EDEADLK)
1723                         break;
1724                 hammer_done_cursor(cursor);
1725                 error = hammer_init_cursor(trans, cursor, &record->ip->cache[0],
1726                                            record->ip);
1727                 if (error)
1728                         break;
1729         }
1730         record->flags &= ~HAMMER_RECF_CONVERT_DELETE;
1731
1732         if (error) {
1733                 error = -error;
1734                 if (error != -ENOSPC) {
1735                         kprintf("hammer_sync_record_callback: sync failed rec "
1736                                 "%p, error %d\n", record, error);
1737                         Debugger("sync failed rec");
1738                 }
1739         }
1740 done:
1741         hammer_flush_record_done(record, error);
1742         return(error);
1743 }
1744
1745 /*
1746  * XXX error handling
1747  */
1748 int
1749 hammer_sync_inode(hammer_inode_t ip)
1750 {
1751         struct hammer_transaction trans;
1752         struct hammer_cursor cursor;
1753         hammer_node_t tmp_node;
1754         hammer_record_t depend;
1755         hammer_record_t next;
1756         int error, tmp_error;
1757         u_int64_t nlinks;
1758
1759         if ((ip->sync_flags & HAMMER_INODE_MODMASK) == 0)
1760                 return(0);
1761
1762         hammer_start_transaction_fls(&trans, ip->hmp);
1763         error = hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
1764         if (error)
1765                 goto done;
1766
1767         /*
1768          * Any directory records referencing this inode which are not in
1769          * our current flush group must adjust our nlink count for the
1770          * purposes of synchronization to disk.
1771          *
1772          * Records which are in our flush group can be unlinked from our
1773          * inode now, potentially allowing the inode to be physically
1774          * deleted.
1775          *
1776          * This cannot block.
1777          */
1778         nlinks = ip->ino_data.nlinks;
1779         next = TAILQ_FIRST(&ip->target_list);
1780         while ((depend = next) != NULL) {
1781                 next = TAILQ_NEXT(depend, target_entry);
1782                 if (depend->flush_state == HAMMER_FST_FLUSH &&
1783                     depend->flush_group == ip->hmp->flusher.act) {
1784                         /*
1785                          * If this is an ADD that was deleted by the frontend
1786                          * the frontend nlinks count will have already been
1787                          * decremented, but the backend is going to sync its
1788                          * directory entry and must account for it.  The
1789                          * record will be converted to a delete-on-disk when
1790                          * it gets synced.
1791                          *
1792                          * If the ADD was not deleted by the frontend we
1793                          * can remove the dependancy from our target_list.
1794                          */
1795                         if (depend->flags & HAMMER_RECF_DELETED_FE) {
1796                                 ++nlinks;
1797                         } else {
1798                                 TAILQ_REMOVE(&ip->target_list, depend,
1799                                              target_entry);
1800                                 depend->target_ip = NULL;
1801                         }
1802                 } else if ((depend->flags & HAMMER_RECF_DELETED_FE) == 0) {
1803                         /*
1804                          * Not part of our flush group
1805                          */
1806                         KKASSERT((depend->flags & HAMMER_RECF_DELETED_BE) == 0);
1807                         switch(depend->type) {
1808                         case HAMMER_MEM_RECORD_ADD:
1809                                 --nlinks;
1810                                 break;
1811                         case HAMMER_MEM_RECORD_DEL:
1812                                 ++nlinks;
1813                                 break;
1814                         default:
1815                                 break;
1816                         }
1817                 }
1818         }
1819
1820         /*
1821          * Set dirty if we had to modify the link count.
1822          */
1823         if (ip->sync_ino_data.nlinks != nlinks) {
1824                 KKASSERT((int64_t)nlinks >= 0);
1825                 ip->sync_ino_data.nlinks = nlinks;
1826                 ip->sync_flags |= HAMMER_INODE_DDIRTY;
1827         }
1828
1829         /*
1830          * If there is a trunction queued destroy any data past the (aligned)
1831          * truncation point.  Userland will have dealt with the buffer
1832          * containing the truncation point for us.
1833          *
1834          * We don't flush pending frontend data buffers until after we've
1835          * dealt with the truncation.
1836          */
1837         if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
1838                 /*
1839                  * Interlock trunc_off.  The VOP front-end may continue to
1840                  * make adjustments to it while we are blocked.
1841                  */
1842                 off_t trunc_off;
1843                 off_t aligned_trunc_off;
1844                 int blkmask;
1845
1846                 trunc_off = ip->sync_trunc_off;
1847                 blkmask = hammer_blocksize(trunc_off) - 1;
1848                 aligned_trunc_off = (trunc_off + blkmask) & ~(int64_t)blkmask;
1849
1850                 /*
1851                  * Delete any whole blocks on-media.  The front-end has
1852                  * already cleaned out any partial block and made it
1853                  * pending.  The front-end may have updated trunc_off
1854                  * while we were blocked so we only use sync_trunc_off.
1855                  */
1856                 error = hammer_ip_delete_range(&cursor, ip,
1857                                                 aligned_trunc_off,
1858                                                 0x7FFFFFFFFFFFFFFFLL, 1);
1859                 if (error)
1860                         Debugger("hammer_ip_delete_range errored");
1861
1862                 /*
1863                  * Clear the truncation flag on the backend after we have
1864                  * complete the deletions.  Backend data is now good again
1865                  * (including new records we are about to sync, below).
1866                  *
1867                  * Leave sync_trunc_off intact.  As we write additional
1868                  * records the backend will update sync_trunc_off.  This
1869                  * tells the backend whether it can skip the overwrite
1870                  * test.  This should work properly even when the backend
1871                  * writes full blocks where the truncation point straddles
1872                  * the block because the comparison is against the base
1873                  * offset of the record.
1874                  */
1875                 ip->sync_flags &= ~HAMMER_INODE_TRUNCATED;
1876                 /* ip->sync_trunc_off = 0x7FFFFFFFFFFFFFFFLL; */
1877         } else {
1878                 error = 0;
1879         }
1880
1881         /*
1882          * Now sync related records.  These will typically be directory
1883          * entries or delete-on-disk records.
1884          *
1885          * Not all records will be flushed, but clear XDIRTY anyway.  We
1886          * will set it again in the frontend hammer_flush_inode_done()
1887          * if records remain.
1888          */
1889         if (error == 0) {
1890                 tmp_error = RB_SCAN(hammer_rec_rb_tree, &ip->rec_tree, NULL,
1891                                     hammer_sync_record_callback, &cursor);
1892                 if (tmp_error < 0)
1893                         tmp_error = -error;
1894                 if (tmp_error)
1895                         error = tmp_error;
1896         }
1897         hammer_cache_node(&ip->cache[1], cursor.node);
1898
1899         /*
1900          * Re-seek for inode update, assuming our cache hasn't been ripped
1901          * out from under us.
1902          */
1903         if (error == 0) {
1904                 tmp_node = hammer_ref_node_safe(ip->hmp, &ip->cache[0], &error);
1905                 if (tmp_node) {
1906                         if ((tmp_node->flags & HAMMER_NODE_DELETED) == 0)
1907                                 hammer_cursor_seek(&cursor, tmp_node, 0);
1908                         hammer_rel_node(tmp_node);
1909                 }
1910                 error = 0;
1911         }
1912
1913         /*
1914          * If we are deleting the inode the frontend had better not have
1915          * any active references on elements making up the inode.
1916          */
1917         if (error == 0 && ip->sync_ino_data.nlinks == 0 &&
1918                 RB_EMPTY(&ip->rec_tree)  &&
1919             (ip->sync_flags & HAMMER_INODE_DELETING) &&
1920             (ip->flags & HAMMER_INODE_DELETED) == 0) {
1921                 int count1 = 0;
1922
1923                 ip->flags |= HAMMER_INODE_DELETED;
1924                 error = hammer_ip_delete_range_all(&cursor, ip, &count1);
1925                 if (error == 0) {
1926                         ip->sync_flags &= ~HAMMER_INODE_DELETING;
1927                         ip->sync_flags &= ~HAMMER_INODE_TRUNCATED;
1928                         KKASSERT(RB_EMPTY(&ip->rec_tree));
1929
1930                         /*
1931                          * Set delete_tid in both the frontend and backend
1932                          * copy of the inode record.  The DELETED flag handles
1933                          * this, do not set RDIRTY.
1934                          */
1935                         ip->ino_leaf.base.delete_tid = trans.tid;
1936                         ip->sync_ino_leaf.base.delete_tid = trans.tid;
1937
1938                         /*
1939                          * Adjust the inode count in the volume header
1940                          */
1941                         if (ip->flags & HAMMER_INODE_ONDISK) {
1942                                 hammer_modify_volume_field(&trans,
1943                                                            trans.rootvol,
1944                                                            vol0_stat_inodes);
1945                                 --ip->hmp->rootvol->ondisk->vol0_stat_inodes;
1946                                 hammer_modify_volume_done(trans.rootvol);
1947                         }
1948                 } else {
1949                         ip->flags &= ~HAMMER_INODE_DELETED;
1950                         Debugger("hammer_ip_delete_range_all errored");
1951                 }
1952         }
1953
1954         ip->sync_flags &= ~HAMMER_INODE_BUFS;
1955
1956         if (error)
1957                 Debugger("RB_SCAN errored");
1958
1959         /*
1960          * Now update the inode's on-disk inode-data and/or on-disk record.
1961          * DELETED and ONDISK are managed only in ip->flags.
1962          */
1963         switch(ip->flags & (HAMMER_INODE_DELETED | HAMMER_INODE_ONDISK)) {
1964         case HAMMER_INODE_DELETED|HAMMER_INODE_ONDISK:
1965                 /*
1966                  * If deleted and on-disk, don't set any additional flags.
1967                  * the delete flag takes care of things.
1968                  *
1969                  * Clear flags which may have been set by the frontend.
1970                  */
1971                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
1972                                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME |
1973                                     HAMMER_INODE_DELETING);
1974                 break;
1975         case HAMMER_INODE_DELETED:
1976                 /*
1977                  * Take care of the case where a deleted inode was never
1978                  * flushed to the disk in the first place.
1979                  *
1980                  * Clear flags which may have been set by the frontend.
1981                  */
1982                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
1983                                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME |
1984                                     HAMMER_INODE_DELETING);
1985                 while (RB_ROOT(&ip->rec_tree)) {
1986                         hammer_record_t record = RB_ROOT(&ip->rec_tree);
1987                         hammer_ref(&record->lock);
1988                         KKASSERT(record->lock.refs == 1);
1989                         record->flags |= HAMMER_RECF_DELETED_FE;
1990                         record->flags |= HAMMER_RECF_DELETED_BE;
1991                         hammer_rel_mem_record(record);
1992                 }
1993                 break;
1994         case HAMMER_INODE_ONDISK:
1995                 /*
1996                  * If already on-disk, do not set any additional flags.
1997                  */
1998                 break;
1999         default:
2000                 /*
2001                  * If not on-disk and not deleted, set DDIRTY to force
2002                  * an initial record to be written.
2003                  *
2004                  * Also set the create_tid in both the frontend and backend
2005                  * copy of the inode record.
2006                  */
2007                 ip->ino_leaf.base.create_tid = trans.tid;
2008                 ip->sync_ino_leaf.base.create_tid = trans.tid;
2009                 ip->sync_flags |= HAMMER_INODE_DDIRTY;
2010                 break;
2011         }
2012
2013         /*
2014          * If RDIRTY or DDIRTY is set, write out a new record.  If the inode
2015          * is already on-disk the old record is marked as deleted.
2016          *
2017          * If DELETED is set hammer_update_inode() will delete the existing
2018          * record without writing out a new one.
2019          *
2020          * If *ONLY* the ITIMES flag is set we can update the record in-place.
2021          */
2022         if (ip->flags & HAMMER_INODE_DELETED) {
2023                 error = hammer_update_inode(&cursor, ip);
2024         } else
2025         if ((ip->sync_flags & HAMMER_INODE_DDIRTY) == 0 &&
2026             (ip->sync_flags & (HAMMER_INODE_ATIME | HAMMER_INODE_MTIME))) {
2027                 error = hammer_update_itimes(&cursor, ip);
2028         } else
2029         if (ip->sync_flags & (HAMMER_INODE_DDIRTY | HAMMER_INODE_ATIME | HAMMER_INODE_MTIME)) {
2030                 error = hammer_update_inode(&cursor, ip);
2031         }
2032         if (error)
2033                 Debugger("hammer_update_itimes/inode errored");
2034 done:
2035         /*
2036          * Save the TID we used to sync the inode with to make sure we
2037          * do not improperly reuse it.
2038          */
2039         hammer_done_cursor(&cursor);
2040         hammer_done_transaction(&trans);
2041         return(error);
2042 }
2043
2044 /*
2045  * This routine is called when the OS is no longer actively referencing
2046  * the inode (but might still be keeping it cached), or when releasing
2047  * the last reference to an inode.
2048  *
2049  * At this point if the inode's nlinks count is zero we want to destroy
2050  * it, which may mean destroying it on-media too.
2051  */
2052 void
2053 hammer_inode_unloadable_check(hammer_inode_t ip, int getvp)
2054 {
2055         struct vnode *vp;
2056
2057         /*
2058          * Set the DELETING flag when the link count drops to 0 and the
2059          * OS no longer has any opens on the inode.
2060          *
2061          * The backend will clear DELETING (a mod flag) and set DELETED
2062          * (a state flag) when it is actually able to perform the
2063          * operation.
2064          */
2065         if (ip->ino_data.nlinks == 0 &&
2066             (ip->flags & (HAMMER_INODE_DELETING|HAMMER_INODE_DELETED)) == 0) {
2067                 ip->flags |= HAMMER_INODE_DELETING;
2068                 ip->flags |= HAMMER_INODE_TRUNCATED;
2069                 ip->trunc_off = 0;
2070                 vp = NULL;
2071                 if (getvp) {
2072                         if (hammer_get_vnode(ip, &vp) != 0)
2073                                 return;
2074                 }
2075
2076                 /*
2077                  * Final cleanup
2078                  */
2079                 if (ip->vp) {
2080                         vtruncbuf(ip->vp, 0, HAMMER_BUFSIZE);
2081                         vnode_pager_setsize(ip->vp, 0);
2082                 }
2083                 if (getvp) {
2084                         vput(vp);
2085                 }
2086         }
2087 }
2088
2089 /*
2090  * Re-test an inode when a dependancy had gone away to see if we
2091  * can chain flush it.
2092  */
2093 void
2094 hammer_test_inode(hammer_inode_t ip)
2095 {
2096         if (ip->flags & HAMMER_INODE_REFLUSH) {
2097                 ip->flags &= ~HAMMER_INODE_REFLUSH;
2098                 hammer_ref(&ip->lock);
2099                 if (ip->flags & HAMMER_INODE_RESIGNAL) {
2100                         ip->flags &= ~HAMMER_INODE_RESIGNAL;
2101                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
2102                 } else {
2103                         hammer_flush_inode(ip, 0);
2104                 }
2105                 hammer_rel_inode(ip, 0);
2106         }
2107 }
2108
2109 /*
2110  * Clear the RECLAIM flag on an inode.  This occurs when the inode is
2111  * reassociated with a vp or just before it gets freed.
2112  *
2113  * Wakeup one thread blocked waiting on reclaims to complete.  Note that
2114  * the inode the thread is waiting on behalf of is a different inode then
2115  * the inode we are called with.  This is to create a pipeline.
2116  */
2117 static void
2118 hammer_inode_wakereclaims(hammer_inode_t ip)
2119 {
2120         struct hammer_reclaim *reclaim;
2121         hammer_mount_t hmp = ip->hmp;
2122
2123         if ((ip->flags & HAMMER_INODE_RECLAIM) == 0)
2124                 return;
2125
2126         --hammer_count_reclaiming;
2127         --hmp->inode_reclaims;
2128         ip->flags &= ~HAMMER_INODE_RECLAIM;
2129
2130         if ((reclaim = TAILQ_FIRST(&hmp->reclaim_list)) != NULL) {
2131                 TAILQ_REMOVE(&hmp->reclaim_list, reclaim, entry);
2132                 reclaim->okydoky = 1;
2133                 wakeup(reclaim);
2134         }
2135 }
2136
2137 /*
2138  * Setup our reclaim pipeline.  We only let so many detached (and dirty)
2139  * inodes build up before we start blocking.
2140  *
2141  * When we block we don't care *which* inode has finished reclaiming,
2142  * as lone as one does.  This is somewhat heuristical... we also put a
2143  * cap on how long we are willing to wait.
2144  */
2145 void
2146 hammer_inode_waitreclaims(hammer_mount_t hmp)
2147 {
2148         struct hammer_reclaim reclaim;
2149         int delay;
2150
2151         if (hmp->inode_reclaims > HAMMER_RECLAIM_WAIT) {
2152                 reclaim.okydoky = 0;
2153                 TAILQ_INSERT_TAIL(&hmp->reclaim_list,
2154                                   &reclaim, entry);
2155         } else {
2156                 reclaim.okydoky = 1;
2157         }
2158
2159         if (reclaim.okydoky == 0) {
2160                 delay = (hmp->inode_reclaims - HAMMER_RECLAIM_WAIT) * hz /
2161                         HAMMER_RECLAIM_WAIT;
2162                 if (delay >= 0)
2163                         tsleep(&reclaim, 0, "hmrrcm", delay + 1);
2164                 if (reclaim.okydoky == 0)
2165                         TAILQ_REMOVE(&hmp->reclaim_list, &reclaim, entry);
2166         }
2167 }
2168