081fde7f925297ec430dcb99b8db0eb260b96e55
[dragonfly.git] / sys / vfs / hammer / hammer_inode.c
1 /*
2  * Copyright (c) 2007-2008 The DragonFly Project.  All rights reserved.
3  * 
4  * This code is derived from software contributed to The DragonFly Project
5  * by Matthew Dillon <dillon@backplane.com>
6  * 
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  * 
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  * 
34  * $DragonFly: src/sys/vfs/hammer/hammer_inode.c,v 1.104 2008/07/16 18:30:59 dillon Exp $
35  */
36
37 #include "hammer.h"
38 #include <vm/vm_extern.h>
39 #include <sys/buf.h>
40 #include <sys/buf2.h>
41
42 static int      hammer_unload_inode(struct hammer_inode *ip);
43 static void     hammer_free_inode(hammer_inode_t ip);
44 static void     hammer_flush_inode_core(hammer_inode_t ip,
45                                         hammer_flush_group_t flg, int flags);
46 static int      hammer_setup_child_callback(hammer_record_t rec, void *data);
47 #if 0
48 static int      hammer_syncgrp_child_callback(hammer_record_t rec, void *data);
49 #endif
50 static int      hammer_setup_parent_inodes(hammer_inode_t ip,
51                                         hammer_flush_group_t flg);
52 static int      hammer_setup_parent_inodes_helper(hammer_record_t record,
53                                         hammer_flush_group_t flg);
54 static void     hammer_inode_wakereclaims(hammer_inode_t ip);
55
56 #ifdef DEBUG_TRUNCATE
57 extern struct hammer_inode *HammerTruncIp;
58 #endif
59
60 /*
61  * RB-Tree support for inode structures
62  */
63 int
64 hammer_ino_rb_compare(hammer_inode_t ip1, hammer_inode_t ip2)
65 {
66         if (ip1->obj_localization < ip2->obj_localization)
67                 return(-1);
68         if (ip1->obj_localization > ip2->obj_localization)
69                 return(1);
70         if (ip1->obj_id < ip2->obj_id)
71                 return(-1);
72         if (ip1->obj_id > ip2->obj_id)
73                 return(1);
74         if (ip1->obj_asof < ip2->obj_asof)
75                 return(-1);
76         if (ip1->obj_asof > ip2->obj_asof)
77                 return(1);
78         return(0);
79 }
80
81 /*
82  * RB-Tree support for inode structures / special LOOKUP_INFO
83  */
84 static int
85 hammer_inode_info_cmp(hammer_inode_info_t info, hammer_inode_t ip)
86 {
87         if (info->obj_localization < ip->obj_localization)
88                 return(-1);
89         if (info->obj_localization > ip->obj_localization)
90                 return(1);
91         if (info->obj_id < ip->obj_id)
92                 return(-1);
93         if (info->obj_id > ip->obj_id)
94                 return(1);
95         if (info->obj_asof < ip->obj_asof)
96                 return(-1);
97         if (info->obj_asof > ip->obj_asof)
98                 return(1);
99         return(0);
100 }
101
102 /*
103  * Used by hammer_scan_inode_snapshots() to locate all of an object's
104  * snapshots.  Note that the asof field is not tested, which we can get
105  * away with because it is the lowest-priority field.
106  */
107 static int
108 hammer_inode_info_cmp_all_history(hammer_inode_t ip, void *data)
109 {
110         hammer_inode_info_t info = data;
111
112         if (ip->obj_localization > info->obj_localization)
113                 return(1);
114         if (ip->obj_localization < info->obj_localization)
115                 return(-1);
116         if (ip->obj_id > info->obj_id)
117                 return(1);
118         if (ip->obj_id < info->obj_id)
119                 return(-1);
120         return(0);
121 }
122
123 /*
124  * Used by hammer_unload_pseudofs() to locate all inodes associated with
125  * a particular PFS.
126  */
127 static int
128 hammer_inode_pfs_cmp(hammer_inode_t ip, void *data)
129 {
130         u_int32_t localization = *(u_int32_t *)data;
131         if (ip->obj_localization > localization)
132                 return(1);
133         if (ip->obj_localization < localization)
134                 return(-1);
135         return(0);
136 }
137
138 /*
139  * RB-Tree support for pseudofs structures
140  */
141 static int
142 hammer_pfs_rb_compare(hammer_pseudofs_inmem_t p1, hammer_pseudofs_inmem_t p2)
143 {
144         if (p1->localization < p2->localization)
145                 return(-1);
146         if (p1->localization > p2->localization)
147                 return(1);
148         return(0);
149 }
150
151
152 RB_GENERATE(hammer_ino_rb_tree, hammer_inode, rb_node, hammer_ino_rb_compare);
153 RB_GENERATE_XLOOKUP(hammer_ino_rb_tree, INFO, hammer_inode, rb_node,
154                 hammer_inode_info_cmp, hammer_inode_info_t);
155 RB_GENERATE2(hammer_pfs_rb_tree, hammer_pseudofs_inmem, rb_node,
156              hammer_pfs_rb_compare, u_int32_t, localization);
157
158 /*
159  * The kernel is not actively referencing this vnode but is still holding
160  * it cached.
161  *
162  * This is called from the frontend.
163  */
164 int
165 hammer_vop_inactive(struct vop_inactive_args *ap)
166 {
167         struct hammer_inode *ip = VTOI(ap->a_vp);
168
169         /*
170          * Degenerate case
171          */
172         if (ip == NULL) {
173                 vrecycle(ap->a_vp);
174                 return(0);
175         }
176
177         /*
178          * If the inode no longer has visibility in the filesystem try to
179          * recycle it immediately, even if the inode is dirty.  Recycling
180          * it quickly allows the system to reclaim buffer cache and VM
181          * resources which can matter a lot in a heavily loaded system.
182          *
183          * This can deadlock in vfsync() if we aren't careful.
184          * 
185          * Do not queue the inode to the flusher if we still have visibility,
186          * otherwise namespace calls such as chmod will unnecessarily generate
187          * multiple inode updates.
188          */
189         hammer_inode_unloadable_check(ip, 0);
190         if (ip->ino_data.nlinks == 0) {
191                 if (ip->flags & HAMMER_INODE_MODMASK)
192                         hammer_flush_inode(ip, 0);
193                 vrecycle(ap->a_vp);
194         }
195         return(0);
196 }
197
198 /*
199  * Release the vnode association.  This is typically (but not always)
200  * the last reference on the inode.
201  *
202  * Once the association is lost we are on our own with regards to
203  * flushing the inode.
204  */
205 int
206 hammer_vop_reclaim(struct vop_reclaim_args *ap)
207 {
208         struct hammer_inode *ip;
209         hammer_mount_t hmp;
210         struct vnode *vp;
211
212         vp = ap->a_vp;
213
214         if ((ip = vp->v_data) != NULL) {
215                 hmp = ip->hmp;
216                 vp->v_data = NULL;
217                 ip->vp = NULL;
218
219                 if ((ip->flags & HAMMER_INODE_RECLAIM) == 0) {
220                         ++hammer_count_reclaiming;
221                         ++hmp->inode_reclaims;
222                         ip->flags |= HAMMER_INODE_RECLAIM;
223
224                         /*
225                          * Poke the flusher.  If we don't do this programs
226                          * will start to stall on the reclaiming count.
227                          */
228                         if (hmp->inode_reclaims > HAMMER_RECLAIM_FLUSH &&
229                            (hmp->inode_reclaims & 255) == 0) {
230                                hammer_flusher_async(hmp, NULL);
231                         }
232                 }
233                 hammer_rel_inode(ip, 1);
234         }
235         return(0);
236 }
237
238 /*
239  * Return a locked vnode for the specified inode.  The inode must be
240  * referenced but NOT LOCKED on entry and will remain referenced on
241  * return.
242  *
243  * Called from the frontend.
244  */
245 int
246 hammer_get_vnode(struct hammer_inode *ip, struct vnode **vpp)
247 {
248         hammer_mount_t hmp;
249         struct vnode *vp;
250         int error = 0;
251         u_int8_t obj_type;
252
253         hmp = ip->hmp;
254
255         for (;;) {
256                 if ((vp = ip->vp) == NULL) {
257                         error = getnewvnode(VT_HAMMER, hmp->mp, vpp, 0, 0);
258                         if (error)
259                                 break;
260                         hammer_lock_ex(&ip->lock);
261                         if (ip->vp != NULL) {
262                                 hammer_unlock(&ip->lock);
263                                 vp->v_type = VBAD;
264                                 vx_put(vp);
265                                 continue;
266                         }
267                         hammer_ref(&ip->lock);
268                         vp = *vpp;
269                         ip->vp = vp;
270
271                         obj_type = ip->ino_data.obj_type;
272                         vp->v_type = hammer_get_vnode_type(obj_type);
273
274                         hammer_inode_wakereclaims(ip);
275
276                         switch(ip->ino_data.obj_type) {
277                         case HAMMER_OBJTYPE_CDEV:
278                         case HAMMER_OBJTYPE_BDEV:
279                                 vp->v_ops = &hmp->mp->mnt_vn_spec_ops;
280                                 addaliasu(vp, ip->ino_data.rmajor,
281                                           ip->ino_data.rminor);
282                                 break;
283                         case HAMMER_OBJTYPE_FIFO:
284                                 vp->v_ops = &hmp->mp->mnt_vn_fifo_ops;
285                                 break;
286                         default:
287                                 break;
288                         }
289
290                         /*
291                          * Only mark as the root vnode if the ip is not
292                          * historical, otherwise the VFS cache will get
293                          * confused.  The other half of the special handling
294                          * is in hammer_vop_nlookupdotdot().
295                          *
296                          * Pseudo-filesystem roots also do not count.
297                          */
298                         if (ip->obj_id == HAMMER_OBJID_ROOT &&
299                             ip->obj_asof == hmp->asof &&
300                             ip->obj_localization == 0) {
301                                 vp->v_flag |= VROOT;
302                         }
303
304                         vp->v_data = (void *)ip;
305                         /* vnode locked by getnewvnode() */
306                         /* make related vnode dirty if inode dirty? */
307                         hammer_unlock(&ip->lock);
308                         if (vp->v_type == VREG)
309                                 vinitvmio(vp, ip->ino_data.size);
310                         break;
311                 }
312
313                 /*
314                  * loop if the vget fails (aka races), or if the vp
315                  * no longer matches ip->vp.
316                  */
317                 if (vget(vp, LK_EXCLUSIVE) == 0) {
318                         if (vp == ip->vp)
319                                 break;
320                         vput(vp);
321                 }
322         }
323         *vpp = vp;
324         return(error);
325 }
326
327 /*
328  * Locate all copies of the inode for obj_id compatible with the specified
329  * asof, reference, and issue the related call-back.  This routine is used
330  * for direct-io invalidation and does not create any new inodes.
331  */
332 void
333 hammer_scan_inode_snapshots(hammer_mount_t hmp, hammer_inode_info_t iinfo,
334                             int (*callback)(hammer_inode_t ip, void *data),
335                             void *data)
336 {
337         hammer_ino_rb_tree_RB_SCAN(&hmp->rb_inos_root,
338                                    hammer_inode_info_cmp_all_history,
339                                    callback, iinfo);
340 }
341
342 /*
343  * Acquire a HAMMER inode.  The returned inode is not locked.  These functions
344  * do not attach or detach the related vnode (use hammer_get_vnode() for
345  * that).
346  *
347  * The flags argument is only applied for newly created inodes, and only
348  * certain flags are inherited.
349  *
350  * Called from the frontend.
351  */
352 struct hammer_inode *
353 hammer_get_inode(hammer_transaction_t trans, hammer_inode_t dip,
354                  int64_t obj_id, hammer_tid_t asof, u_int32_t localization,
355                  int flags, int *errorp)
356 {
357         hammer_mount_t hmp = trans->hmp;
358         struct hammer_inode_info iinfo;
359         struct hammer_cursor cursor;
360         struct hammer_inode *ip;
361
362
363         /*
364          * Determine if we already have an inode cached.  If we do then
365          * we are golden.
366          */
367         iinfo.obj_id = obj_id;
368         iinfo.obj_asof = asof;
369         iinfo.obj_localization = localization;
370 loop:
371         ip = hammer_ino_rb_tree_RB_LOOKUP_INFO(&hmp->rb_inos_root, &iinfo);
372         if (ip) {
373                 hammer_ref(&ip->lock);
374                 *errorp = 0;
375                 return(ip);
376         }
377
378         /*
379          * Allocate a new inode structure and deal with races later.
380          */
381         ip = kmalloc(sizeof(*ip), M_HAMMER, M_WAITOK|M_ZERO);
382         ++hammer_count_inodes;
383         ++hmp->count_inodes;
384         ip->obj_id = obj_id;
385         ip->obj_asof = iinfo.obj_asof;
386         ip->obj_localization = localization;
387         ip->hmp = hmp;
388         ip->flags = flags & HAMMER_INODE_RO;
389         ip->cache[0].ip = ip;
390         ip->cache[1].ip = ip;
391         if (hmp->ronly)
392                 ip->flags |= HAMMER_INODE_RO;
393         ip->sync_trunc_off = ip->trunc_off = ip->save_trunc_off =
394                 0x7FFFFFFFFFFFFFFFLL;
395         RB_INIT(&ip->rec_tree);
396         TAILQ_INIT(&ip->target_list);
397         hammer_ref(&ip->lock);
398
399         /*
400          * Locate the on-disk inode.  If this is a PFS root we always
401          * access the current version of the root inode and (if it is not
402          * a master) always access information under it with a snapshot
403          * TID.
404          */
405 retry:
406         hammer_init_cursor(trans, &cursor, (dip ? &dip->cache[0] : NULL), NULL);
407         cursor.key_beg.localization = localization + HAMMER_LOCALIZE_INODE;
408         cursor.key_beg.obj_id = ip->obj_id;
409         cursor.key_beg.key = 0;
410         cursor.key_beg.create_tid = 0;
411         cursor.key_beg.delete_tid = 0;
412         cursor.key_beg.rec_type = HAMMER_RECTYPE_INODE;
413         cursor.key_beg.obj_type = 0;
414
415         cursor.asof = iinfo.obj_asof;
416         cursor.flags = HAMMER_CURSOR_GET_LEAF | HAMMER_CURSOR_GET_DATA |
417                        HAMMER_CURSOR_ASOF;
418
419         *errorp = hammer_btree_lookup(&cursor);
420         if (*errorp == EDEADLK) {
421                 hammer_done_cursor(&cursor);
422                 goto retry;
423         }
424
425         /*
426          * On success the B-Tree lookup will hold the appropriate
427          * buffer cache buffers and provide a pointer to the requested
428          * information.  Copy the information to the in-memory inode
429          * and cache the B-Tree node to improve future operations.
430          */
431         if (*errorp == 0) {
432                 ip->ino_leaf = cursor.node->ondisk->elms[cursor.index].leaf;
433                 ip->ino_data = cursor.data->inode;
434
435                 /*
436                  * cache[0] tries to cache the location of the object inode.
437                  * The assumption is that it is near the directory inode.
438                  *
439                  * cache[1] tries to cache the location of the object data.
440                  * The assumption is that it is near the directory data.
441                  */
442                 hammer_cache_node(&ip->cache[0], cursor.node);
443                 if (dip && dip->cache[1].node)
444                         hammer_cache_node(&ip->cache[1], dip->cache[1].node);
445
446                 /*
447                  * The file should not contain any data past the file size
448                  * stored in the inode.  Setting save_trunc_off to the
449                  * file size instead of max reduces B-Tree lookup overheads
450                  * on append by allowing the flusher to avoid checking for
451                  * record overwrites.
452                  */
453                 ip->save_trunc_off = ip->ino_data.size;
454
455                 /*
456                  * Locate and assign the pseudofs management structure to
457                  * the inode.
458                  */
459                 if (dip && dip->obj_localization == ip->obj_localization) {
460                         ip->pfsm = dip->pfsm;
461                         hammer_ref(&ip->pfsm->lock);
462                 } else {
463                         ip->pfsm = hammer_load_pseudofs(trans,
464                                                         ip->obj_localization,
465                                                         errorp);
466                         *errorp = 0;    /* ignore ENOENT */
467                 }
468         }
469
470         /*
471          * The inode is placed on the red-black tree and will be synced to
472          * the media when flushed or by the filesystem sync.  If this races
473          * another instantiation/lookup the insertion will fail.
474          */
475         if (*errorp == 0) {
476                 if (RB_INSERT(hammer_ino_rb_tree, &hmp->rb_inos_root, ip)) {
477                         hammer_free_inode(ip);
478                         hammer_done_cursor(&cursor);
479                         goto loop;
480                 }
481                 ip->flags |= HAMMER_INODE_ONDISK;
482         } else {
483                 if (ip->flags & HAMMER_INODE_RSV_INODES) {
484                         ip->flags &= ~HAMMER_INODE_RSV_INODES; /* sanity */
485                         --hmp->rsv_inodes;
486                 }
487
488                 hammer_free_inode(ip);
489                 ip = NULL;
490         }
491         hammer_done_cursor(&cursor);
492         return (ip);
493 }
494
495 /*
496  * Create a new filesystem object, returning the inode in *ipp.  The
497  * returned inode will be referenced.  The inode is created in-memory.
498  *
499  * If pfsm is non-NULL the caller wishes to create the root inode for
500  * a master PFS.
501  */
502 int
503 hammer_create_inode(hammer_transaction_t trans, struct vattr *vap,
504                     struct ucred *cred, hammer_inode_t dip,
505                     hammer_pseudofs_inmem_t pfsm, struct hammer_inode **ipp)
506 {
507         hammer_mount_t hmp;
508         hammer_inode_t ip;
509         uid_t xuid;
510         int error;
511
512         hmp = trans->hmp;
513
514         ip = kmalloc(sizeof(*ip), M_HAMMER, M_WAITOK|M_ZERO);
515         ++hammer_count_inodes;
516         ++hmp->count_inodes;
517
518         if (pfsm) {
519                 KKASSERT(pfsm->localization != 0);
520                 ip->obj_id = HAMMER_OBJID_ROOT;
521                 ip->obj_localization = pfsm->localization;
522         } else {
523                 KKASSERT(dip != NULL);
524                 ip->obj_id = hammer_alloc_objid(hmp, dip);
525                 ip->obj_localization = dip->obj_localization;
526         }
527
528         KKASSERT(ip->obj_id != 0);
529         ip->obj_asof = hmp->asof;
530         ip->hmp = hmp;
531         ip->flush_state = HAMMER_FST_IDLE;
532         ip->flags = HAMMER_INODE_DDIRTY |
533                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME;
534         ip->cache[0].ip = ip;
535         ip->cache[1].ip = ip;
536
537         ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
538         /* ip->save_trunc_off = 0; (already zero) */
539         RB_INIT(&ip->rec_tree);
540         TAILQ_INIT(&ip->target_list);
541
542         ip->ino_data.atime = trans->time;
543         ip->ino_data.mtime = trans->time;
544         ip->ino_data.size = 0;
545         ip->ino_data.nlinks = 0;
546
547         /*
548          * A nohistory designator on the parent directory is inherited by
549          * the child.  We will do this even for pseudo-fs creation... the
550          * sysad can turn it off.
551          */
552         if (dip) {
553                 ip->ino_data.uflags = dip->ino_data.uflags &
554                                       (SF_NOHISTORY|UF_NOHISTORY|UF_NODUMP);
555         }
556
557         ip->ino_leaf.base.btype = HAMMER_BTREE_TYPE_RECORD;
558         ip->ino_leaf.base.localization = ip->obj_localization +
559                                          HAMMER_LOCALIZE_INODE;
560         ip->ino_leaf.base.obj_id = ip->obj_id;
561         ip->ino_leaf.base.key = 0;
562         ip->ino_leaf.base.create_tid = 0;
563         ip->ino_leaf.base.delete_tid = 0;
564         ip->ino_leaf.base.rec_type = HAMMER_RECTYPE_INODE;
565         ip->ino_leaf.base.obj_type = hammer_get_obj_type(vap->va_type);
566
567         ip->ino_data.obj_type = ip->ino_leaf.base.obj_type;
568         ip->ino_data.version = HAMMER_INODE_DATA_VERSION;
569         ip->ino_data.mode = vap->va_mode;
570         ip->ino_data.ctime = trans->time;
571
572         /*
573          * Setup the ".." pointer.  This only needs to be done for directories
574          * but we do it for all objects as a recovery aid.
575          */
576         if (dip)
577                 ip->ino_data.parent_obj_id = dip->ino_leaf.base.obj_id;
578 #if 0
579         /*
580          * The parent_obj_localization field only applies to pseudo-fs roots.
581          * XXX this is no longer applicable, PFSs are no longer directly
582          * tied into the parent's directory structure.
583          */
584         if (ip->ino_data.obj_type == HAMMER_OBJTYPE_DIRECTORY &&
585             ip->obj_id == HAMMER_OBJID_ROOT) {
586                 ip->ino_data.ext.obj.parent_obj_localization = 
587                                                 dip->obj_localization;
588         }
589 #endif
590
591         switch(ip->ino_leaf.base.obj_type) {
592         case HAMMER_OBJTYPE_CDEV:
593         case HAMMER_OBJTYPE_BDEV:
594                 ip->ino_data.rmajor = vap->va_rmajor;
595                 ip->ino_data.rminor = vap->va_rminor;
596                 break;
597         default:
598                 break;
599         }
600
601         /*
602          * Calculate default uid/gid and overwrite with information from
603          * the vap.
604          */
605         if (dip) {
606                 xuid = hammer_to_unix_xid(&dip->ino_data.uid);
607                 xuid = vop_helper_create_uid(hmp->mp, dip->ino_data.mode,
608                                              xuid, cred, &vap->va_mode);
609         } else {
610                 xuid = 0;
611         }
612         ip->ino_data.mode = vap->va_mode;
613
614         if (vap->va_vaflags & VA_UID_UUID_VALID)
615                 ip->ino_data.uid = vap->va_uid_uuid;
616         else if (vap->va_uid != (uid_t)VNOVAL)
617                 hammer_guid_to_uuid(&ip->ino_data.uid, vap->va_uid);
618         else
619                 hammer_guid_to_uuid(&ip->ino_data.uid, xuid);
620
621         if (vap->va_vaflags & VA_GID_UUID_VALID)
622                 ip->ino_data.gid = vap->va_gid_uuid;
623         else if (vap->va_gid != (gid_t)VNOVAL)
624                 hammer_guid_to_uuid(&ip->ino_data.gid, vap->va_gid);
625         else if (dip)
626                 ip->ino_data.gid = dip->ino_data.gid;
627
628         hammer_ref(&ip->lock);
629
630         if (pfsm) {
631                 ip->pfsm = pfsm;
632                 hammer_ref(&pfsm->lock);
633                 error = 0;
634         } else if (dip->obj_localization == ip->obj_localization) {
635                 ip->pfsm = dip->pfsm;
636                 hammer_ref(&ip->pfsm->lock);
637                 error = 0;
638         } else {
639                 ip->pfsm = hammer_load_pseudofs(trans,
640                                                 ip->obj_localization,
641                                                 &error);
642                 error = 0;      /* ignore ENOENT */
643         }
644
645         if (error) {
646                 hammer_free_inode(ip);
647                 ip = NULL;
648         } else if (RB_INSERT(hammer_ino_rb_tree, &hmp->rb_inos_root, ip)) {
649                 panic("hammer_create_inode: duplicate obj_id %llx", ip->obj_id);
650                 /* not reached */
651                 hammer_free_inode(ip);
652         }
653         *ipp = ip;
654         return(error);
655 }
656
657 /*
658  * Final cleanup / freeing of an inode structure
659  */
660 static void
661 hammer_free_inode(hammer_inode_t ip)
662 {
663         KKASSERT(ip->lock.refs == 1);
664         hammer_uncache_node(&ip->cache[0]);
665         hammer_uncache_node(&ip->cache[1]);
666         hammer_inode_wakereclaims(ip);
667         if (ip->objid_cache)
668                 hammer_clear_objid(ip);
669         --hammer_count_inodes;
670         --ip->hmp->count_inodes;
671         if (ip->pfsm) {
672                 hammer_rel_pseudofs(ip->hmp, ip->pfsm);
673                 ip->pfsm = NULL;
674         }
675         kfree(ip, M_HAMMER);
676         ip = NULL;
677 }
678
679 /*
680  * Retrieve pseudo-fs data.  NULL will never be returned.
681  *
682  * If an error occurs *errorp will be set and a default template is returned,
683  * otherwise *errorp is set to 0.  Typically when an error occurs it will
684  * be ENOENT.
685  */
686 hammer_pseudofs_inmem_t
687 hammer_load_pseudofs(hammer_transaction_t trans,
688                      u_int32_t localization, int *errorp)
689 {
690         hammer_mount_t hmp = trans->hmp;
691         hammer_inode_t ip;
692         hammer_pseudofs_inmem_t pfsm;
693         struct hammer_cursor cursor;
694         int bytes;
695
696 retry:
697         pfsm = RB_LOOKUP(hammer_pfs_rb_tree, &hmp->rb_pfsm_root, localization);
698         if (pfsm) {
699                 hammer_ref(&pfsm->lock);
700                 *errorp = 0;
701                 return(pfsm);
702         }
703
704         /*
705          * PFS records are stored in the root inode (not the PFS root inode,
706          * but the real root).  Avoid an infinite recursion if loading
707          * the PFS for the real root.
708          */
709         if (localization) {
710                 ip = hammer_get_inode(trans, NULL, HAMMER_OBJID_ROOT,
711                                       HAMMER_MAX_TID,
712                                       HAMMER_DEF_LOCALIZATION, 0, errorp);
713         } else {
714                 ip = NULL;
715         }
716
717         pfsm = kmalloc(sizeof(*pfsm), M_HAMMER, M_WAITOK | M_ZERO);
718         pfsm->localization = localization;
719         pfsm->pfsd.unique_uuid = trans->rootvol->ondisk->vol_fsid;
720         pfsm->pfsd.shared_uuid = pfsm->pfsd.unique_uuid;
721
722         hammer_init_cursor(trans, &cursor, (ip ? &ip->cache[1] : NULL), ip);
723         cursor.key_beg.localization = HAMMER_DEF_LOCALIZATION +
724                                       HAMMER_LOCALIZE_MISC;
725         cursor.key_beg.obj_id = HAMMER_OBJID_ROOT;
726         cursor.key_beg.create_tid = 0;
727         cursor.key_beg.delete_tid = 0;
728         cursor.key_beg.rec_type = HAMMER_RECTYPE_PFS;
729         cursor.key_beg.obj_type = 0;
730         cursor.key_beg.key = localization;
731         cursor.asof = HAMMER_MAX_TID;
732         cursor.flags |= HAMMER_CURSOR_ASOF;
733
734         if (ip)
735                 *errorp = hammer_ip_lookup(&cursor);
736         else
737                 *errorp = hammer_btree_lookup(&cursor);
738         if (*errorp == 0) {
739                 *errorp = hammer_ip_resolve_data(&cursor);
740                 if (*errorp == 0) {
741                         if (cursor.data->pfsd.mirror_flags &
742                             HAMMER_PFSD_DELETED) {
743                                 *errorp = ENOENT;
744                         } else {
745                                 bytes = cursor.leaf->data_len;
746                                 if (bytes > sizeof(pfsm->pfsd))
747                                         bytes = sizeof(pfsm->pfsd);
748                                 bcopy(cursor.data, &pfsm->pfsd, bytes);
749                         }
750                 }
751         }
752         hammer_done_cursor(&cursor);
753
754         pfsm->fsid_udev = hammer_fsid_to_udev(&pfsm->pfsd.shared_uuid);
755         hammer_ref(&pfsm->lock);
756         if (ip)
757                 hammer_rel_inode(ip, 0);
758         if (RB_INSERT(hammer_pfs_rb_tree, &hmp->rb_pfsm_root, pfsm)) {
759                 kfree(pfsm, M_HAMMER);
760                 goto retry;
761         }
762         return(pfsm);
763 }
764
765 /*
766  * Store pseudo-fs data.  The backend will automatically delete any prior
767  * on-disk pseudo-fs data but we have to delete in-memory versions.
768  */
769 int
770 hammer_save_pseudofs(hammer_transaction_t trans, hammer_pseudofs_inmem_t pfsm)
771 {
772         struct hammer_cursor cursor;
773         hammer_record_t record;
774         hammer_inode_t ip;
775         int error;
776
777         ip = hammer_get_inode(trans, NULL, HAMMER_OBJID_ROOT, HAMMER_MAX_TID,
778                               HAMMER_DEF_LOCALIZATION, 0, &error);
779 retry:
780         pfsm->fsid_udev = hammer_fsid_to_udev(&pfsm->pfsd.shared_uuid);
781         hammer_init_cursor(trans, &cursor, &ip->cache[1], ip);
782         cursor.key_beg.localization = ip->obj_localization +
783                                       HAMMER_LOCALIZE_MISC;
784         cursor.key_beg.obj_id = HAMMER_OBJID_ROOT;
785         cursor.key_beg.create_tid = 0;
786         cursor.key_beg.delete_tid = 0;
787         cursor.key_beg.rec_type = HAMMER_RECTYPE_PFS;
788         cursor.key_beg.obj_type = 0;
789         cursor.key_beg.key = pfsm->localization;
790         cursor.asof = HAMMER_MAX_TID;
791         cursor.flags |= HAMMER_CURSOR_ASOF;
792
793         error = hammer_ip_lookup(&cursor);
794         if (error == 0 && hammer_cursor_inmem(&cursor)) {
795                 record = cursor.iprec;
796                 if (record->flags & HAMMER_RECF_INTERLOCK_BE) {
797                         KKASSERT(cursor.deadlk_rec == NULL);
798                         hammer_ref(&record->lock);
799                         cursor.deadlk_rec = record;
800                         error = EDEADLK;
801                 } else {
802                         record->flags |= HAMMER_RECF_DELETED_FE;
803                         error = 0;
804                 }
805         }
806         if (error == 0 || error == ENOENT) {
807                 record = hammer_alloc_mem_record(ip, sizeof(pfsm->pfsd));
808                 record->type = HAMMER_MEM_RECORD_GENERAL;
809
810                 record->leaf.base.localization = ip->obj_localization +
811                                                  HAMMER_LOCALIZE_MISC;
812                 record->leaf.base.rec_type = HAMMER_RECTYPE_PFS;
813                 record->leaf.base.key = pfsm->localization;
814                 record->leaf.data_len = sizeof(pfsm->pfsd);
815                 bcopy(&pfsm->pfsd, record->data, sizeof(pfsm->pfsd));
816                 error = hammer_ip_add_record(trans, record);
817         }
818         hammer_done_cursor(&cursor);
819         if (error == EDEADLK)
820                 goto retry;
821         hammer_rel_inode(ip, 0);
822         return(error);
823 }
824
825 /*
826  * Create a root directory for a PFS if one does not alredy exist.
827  *
828  * The PFS root stands alone so we must also bump the nlinks count
829  * to prevent it from being destroyed on release.
830  */
831 int
832 hammer_mkroot_pseudofs(hammer_transaction_t trans, struct ucred *cred,
833                        hammer_pseudofs_inmem_t pfsm)
834 {
835         hammer_inode_t ip;
836         struct vattr vap;
837         int error;
838
839         ip = hammer_get_inode(trans, NULL, HAMMER_OBJID_ROOT, HAMMER_MAX_TID,
840                               pfsm->localization, 0, &error);
841         if (ip == NULL) {
842                 vattr_null(&vap);
843                 vap.va_mode = 0755;
844                 vap.va_type = VDIR;
845                 error = hammer_create_inode(trans, &vap, cred, NULL, pfsm, &ip);
846                 if (error == 0) {
847                         ++ip->ino_data.nlinks;
848                         hammer_modify_inode(ip, HAMMER_INODE_DDIRTY);
849                 }
850         }
851         if (ip)
852                 hammer_rel_inode(ip, 0);
853         return(error);
854 }
855
856 /*
857  * Unload any vnodes & inodes associated with a PFS, return ENOTEMPTY
858  * if we are unable to disassociate all the inodes.
859  */
860 static
861 int
862 hammer_unload_pseudofs_callback(hammer_inode_t ip, void *data)
863 {
864         int res;
865
866         hammer_ref(&ip->lock);
867         if (ip->lock.refs == 2 && ip->vp)
868                 vclean_unlocked(ip->vp);
869         if (ip->lock.refs == 1 && ip->vp == NULL)
870                 res = 0;
871         else
872                 res = -1;       /* stop, someone is using the inode */
873         hammer_rel_inode(ip, 0);
874         return(res);
875 }
876
877 int
878 hammer_unload_pseudofs(hammer_transaction_t trans, u_int32_t localization)
879 {
880         int res;
881         int try;
882
883         for (try = res = 0; try < 4; ++try) {
884                 res = hammer_ino_rb_tree_RB_SCAN(&trans->hmp->rb_inos_root,
885                                            hammer_inode_pfs_cmp,
886                                            hammer_unload_pseudofs_callback,
887                                            &localization);
888                 if (res == 0 && try > 1)
889                         break;
890                 hammer_flusher_sync(trans->hmp);
891         }
892         if (res != 0)
893                 res = ENOTEMPTY;
894         return(res);
895 }
896
897
898 /*
899  * Release a reference on a PFS
900  */
901 void
902 hammer_rel_pseudofs(hammer_mount_t hmp, hammer_pseudofs_inmem_t pfsm)
903 {
904         hammer_unref(&pfsm->lock);
905         if (pfsm->lock.refs == 0) {
906                 RB_REMOVE(hammer_pfs_rb_tree, &hmp->rb_pfsm_root, pfsm);
907                 kfree(pfsm, M_HAMMER);
908         }
909 }
910
911 /*
912  * Called by hammer_sync_inode().
913  */
914 static int
915 hammer_update_inode(hammer_cursor_t cursor, hammer_inode_t ip)
916 {
917         hammer_transaction_t trans = cursor->trans;
918         hammer_record_t record;
919         int error;
920         int redirty;
921
922 retry:
923         error = 0;
924
925         /*
926          * If the inode has a presence on-disk then locate it and mark
927          * it deleted, setting DELONDISK.
928          *
929          * The record may or may not be physically deleted, depending on
930          * the retention policy.
931          */
932         if ((ip->flags & (HAMMER_INODE_ONDISK|HAMMER_INODE_DELONDISK)) ==
933             HAMMER_INODE_ONDISK) {
934                 hammer_normalize_cursor(cursor);
935                 cursor->key_beg.localization = ip->obj_localization + 
936                                                HAMMER_LOCALIZE_INODE;
937                 cursor->key_beg.obj_id = ip->obj_id;
938                 cursor->key_beg.key = 0;
939                 cursor->key_beg.create_tid = 0;
940                 cursor->key_beg.delete_tid = 0;
941                 cursor->key_beg.rec_type = HAMMER_RECTYPE_INODE;
942                 cursor->key_beg.obj_type = 0;
943                 cursor->asof = ip->obj_asof;
944                 cursor->flags &= ~HAMMER_CURSOR_INITMASK;
945                 cursor->flags |= HAMMER_CURSOR_GET_LEAF | HAMMER_CURSOR_ASOF;
946                 cursor->flags |= HAMMER_CURSOR_BACKEND;
947
948                 error = hammer_btree_lookup(cursor);
949                 if (hammer_debug_inode)
950                         kprintf("IPDEL %p %08x %d", ip, ip->flags, error);
951                 if (error) {
952                         kprintf("error %d\n", error);
953                         Debugger("hammer_update_inode");
954                 }
955
956                 if (error == 0) {
957                         error = hammer_ip_delete_record(cursor, ip, trans->tid);
958                         if (hammer_debug_inode)
959                                 kprintf(" error %d\n", error);
960                         if (error && error != EDEADLK) {
961                                 kprintf("error %d\n", error);
962                                 Debugger("hammer_update_inode2");
963                         }
964                         if (error == 0) {
965                                 ip->flags |= HAMMER_INODE_DELONDISK;
966                         }
967                         if (cursor->node)
968                                 hammer_cache_node(&ip->cache[0], cursor->node);
969                 }
970                 if (error == EDEADLK) {
971                         hammer_done_cursor(cursor);
972                         error = hammer_init_cursor(trans, cursor,
973                                                    &ip->cache[0], ip);
974                         if (hammer_debug_inode)
975                                 kprintf("IPDED %p %d\n", ip, error);
976                         if (error == 0)
977                                 goto retry;
978                 }
979         }
980
981         /*
982          * Ok, write out the initial record or a new record (after deleting
983          * the old one), unless the DELETED flag is set.  This routine will
984          * clear DELONDISK if it writes out a record.
985          *
986          * Update our inode statistics if this is the first application of
987          * the inode on-disk.
988          */
989         if (error == 0 && (ip->flags & HAMMER_INODE_DELETED) == 0) {
990                 /*
991                  * Generate a record and write it to the media.  We clean-up
992                  * the state before releasing so we do not have to set-up
993                  * a flush_group.
994                  */
995                 record = hammer_alloc_mem_record(ip, 0);
996                 record->type = HAMMER_MEM_RECORD_INODE;
997                 record->flush_state = HAMMER_FST_FLUSH;
998                 record->leaf = ip->sync_ino_leaf;
999                 record->leaf.base.create_tid = trans->tid;
1000                 record->leaf.data_len = sizeof(ip->sync_ino_data);
1001                 record->leaf.create_ts = trans->time32;
1002                 record->data = (void *)&ip->sync_ino_data;
1003                 record->flags |= HAMMER_RECF_INTERLOCK_BE;
1004
1005                 /*
1006                  * If this flag is set we cannot sync the new file size
1007                  * because we haven't finished related truncations.  The
1008                  * inode will be flushed in another flush group to finish
1009                  * the job.
1010                  */
1011                 if ((ip->flags & HAMMER_INODE_WOULDBLOCK) &&
1012                     ip->sync_ino_data.size != ip->ino_data.size) {
1013                         redirty = 1;
1014                         ip->sync_ino_data.size = ip->ino_data.size;
1015                 } else {
1016                         redirty = 0;
1017                 }
1018
1019                 for (;;) {
1020                         error = hammer_ip_sync_record_cursor(cursor, record);
1021                         if (hammer_debug_inode)
1022                                 kprintf("GENREC %p rec %08x %d\n",      
1023                                         ip, record->flags, error);
1024                         if (error != EDEADLK)
1025                                 break;
1026                         hammer_done_cursor(cursor);
1027                         error = hammer_init_cursor(trans, cursor,
1028                                                    &ip->cache[0], ip);
1029                         if (hammer_debug_inode)
1030                                 kprintf("GENREC reinit %d\n", error);
1031                         if (error)
1032                                 break;
1033                 }
1034                 if (error) {
1035                         kprintf("error %d\n", error);
1036                         Debugger("hammer_update_inode3");
1037                 }
1038
1039                 /*
1040                  * The record isn't managed by the inode's record tree,
1041                  * destroy it whether we succeed or fail.
1042                  */
1043                 record->flags &= ~HAMMER_RECF_INTERLOCK_BE;
1044                 record->flags |= HAMMER_RECF_DELETED_FE | HAMMER_RECF_COMMITTED;
1045                 record->flush_state = HAMMER_FST_IDLE;
1046                 hammer_rel_mem_record(record);
1047
1048                 /*
1049                  * Finish up.
1050                  */
1051                 if (error == 0) {
1052                         if (hammer_debug_inode)
1053                                 kprintf("CLEANDELOND %p %08x\n", ip, ip->flags);
1054                         ip->sync_flags &= ~(HAMMER_INODE_DDIRTY |
1055                                             HAMMER_INODE_ATIME |
1056                                             HAMMER_INODE_MTIME);
1057                         ip->flags &= ~HAMMER_INODE_DELONDISK;
1058                         if (redirty)
1059                                 ip->sync_flags |= HAMMER_INODE_DDIRTY;
1060
1061                         /*
1062                          * Root volume count of inodes
1063                          */
1064                         hammer_sync_lock_sh(trans);
1065                         if ((ip->flags & HAMMER_INODE_ONDISK) == 0) {
1066                                 hammer_modify_volume_field(trans,
1067                                                            trans->rootvol,
1068                                                            vol0_stat_inodes);
1069                                 ++ip->hmp->rootvol->ondisk->vol0_stat_inodes;
1070                                 hammer_modify_volume_done(trans->rootvol);
1071                                 ip->flags |= HAMMER_INODE_ONDISK;
1072                                 if (hammer_debug_inode)
1073                                         kprintf("NOWONDISK %p\n", ip);
1074                         }
1075                         hammer_sync_unlock(trans);
1076                 }
1077         }
1078
1079         /*
1080          * If the inode has been destroyed, clean out any left-over flags
1081          * that may have been set by the frontend.
1082          */
1083         if (error == 0 && (ip->flags & HAMMER_INODE_DELETED)) { 
1084                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY |
1085                                     HAMMER_INODE_ATIME |
1086                                     HAMMER_INODE_MTIME);
1087         }
1088         return(error);
1089 }
1090
1091 /*
1092  * Update only the itimes fields.
1093  *
1094  * ATIME can be updated without generating any UNDO.  MTIME is updated
1095  * with UNDO so it is guaranteed to be synchronized properly in case of
1096  * a crash.
1097  *
1098  * Neither field is included in the B-Tree leaf element's CRC, which is how
1099  * we can get away with updating ATIME the way we do.
1100  */
1101 static int
1102 hammer_update_itimes(hammer_cursor_t cursor, hammer_inode_t ip)
1103 {
1104         hammer_transaction_t trans = cursor->trans;
1105         int error;
1106
1107 retry:
1108         if ((ip->flags & (HAMMER_INODE_ONDISK|HAMMER_INODE_DELONDISK)) !=
1109             HAMMER_INODE_ONDISK) {
1110                 return(0);
1111         }
1112
1113         hammer_normalize_cursor(cursor);
1114         cursor->key_beg.localization = ip->obj_localization + 
1115                                        HAMMER_LOCALIZE_INODE;
1116         cursor->key_beg.obj_id = ip->obj_id;
1117         cursor->key_beg.key = 0;
1118         cursor->key_beg.create_tid = 0;
1119         cursor->key_beg.delete_tid = 0;
1120         cursor->key_beg.rec_type = HAMMER_RECTYPE_INODE;
1121         cursor->key_beg.obj_type = 0;
1122         cursor->asof = ip->obj_asof;
1123         cursor->flags &= ~HAMMER_CURSOR_INITMASK;
1124         cursor->flags |= HAMMER_CURSOR_ASOF;
1125         cursor->flags |= HAMMER_CURSOR_GET_LEAF;
1126         cursor->flags |= HAMMER_CURSOR_GET_DATA;
1127         cursor->flags |= HAMMER_CURSOR_BACKEND;
1128
1129         error = hammer_btree_lookup(cursor);
1130         if (error) {
1131                 kprintf("error %d\n", error);
1132                 Debugger("hammer_update_itimes1");
1133         }
1134         if (error == 0) {
1135                 hammer_cache_node(&ip->cache[0], cursor->node);
1136                 if (ip->sync_flags & HAMMER_INODE_MTIME) {
1137                         /*
1138                          * Updating MTIME requires an UNDO.  Just cover
1139                          * both atime and mtime.
1140                          */
1141                         hammer_sync_lock_sh(trans);
1142                         hammer_modify_buffer(trans, cursor->data_buffer,
1143                                      HAMMER_ITIMES_BASE(&cursor->data->inode),
1144                                      HAMMER_ITIMES_BYTES);
1145                         cursor->data->inode.atime = ip->sync_ino_data.atime;
1146                         cursor->data->inode.mtime = ip->sync_ino_data.mtime;
1147                         hammer_modify_buffer_done(cursor->data_buffer);
1148                         hammer_sync_unlock(trans);
1149                 } else if (ip->sync_flags & HAMMER_INODE_ATIME) {
1150                         /*
1151                          * Updating atime only can be done in-place with
1152                          * no UNDO.
1153                          */
1154                         hammer_sync_lock_sh(trans);
1155                         hammer_modify_buffer(trans, cursor->data_buffer,
1156                                              NULL, 0);
1157                         cursor->data->inode.atime = ip->sync_ino_data.atime;
1158                         hammer_modify_buffer_done(cursor->data_buffer);
1159                         hammer_sync_unlock(trans);
1160                 }
1161                 ip->sync_flags &= ~(HAMMER_INODE_ATIME | HAMMER_INODE_MTIME);
1162         }
1163         if (error == EDEADLK) {
1164                 hammer_done_cursor(cursor);
1165                 error = hammer_init_cursor(trans, cursor,
1166                                            &ip->cache[0], ip);
1167                 if (error == 0)
1168                         goto retry;
1169         }
1170         return(error);
1171 }
1172
1173 /*
1174  * Release a reference on an inode, flush as requested.
1175  *
1176  * On the last reference we queue the inode to the flusher for its final
1177  * disposition.
1178  */
1179 void
1180 hammer_rel_inode(struct hammer_inode *ip, int flush)
1181 {
1182         hammer_mount_t hmp = ip->hmp;
1183
1184         /*
1185          * Handle disposition when dropping the last ref.
1186          */
1187         for (;;) {
1188                 if (ip->lock.refs == 1) {
1189                         /*
1190                          * Determine whether on-disk action is needed for
1191                          * the inode's final disposition.
1192                          */
1193                         KKASSERT(ip->vp == NULL);
1194                         hammer_inode_unloadable_check(ip, 0);
1195                         if (ip->flags & HAMMER_INODE_MODMASK) {
1196                                 if (hmp->rsv_inodes > desiredvnodes) {
1197                                         hammer_flush_inode(ip,
1198                                                            HAMMER_FLUSH_SIGNAL);
1199                                 } else {
1200                                         hammer_flush_inode(ip, 0);
1201                                 }
1202                         } else if (ip->lock.refs == 1) {
1203                                 hammer_unload_inode(ip);
1204                                 break;
1205                         }
1206                 } else {
1207                         if (flush)
1208                                 hammer_flush_inode(ip, 0);
1209
1210                         /*
1211                          * The inode still has multiple refs, try to drop
1212                          * one ref.
1213                          */
1214                         KKASSERT(ip->lock.refs >= 1);
1215                         if (ip->lock.refs > 1) {
1216                                 hammer_unref(&ip->lock);
1217                                 break;
1218                         }
1219                 }
1220         }
1221 }
1222
1223 /*
1224  * Unload and destroy the specified inode.  Must be called with one remaining
1225  * reference.  The reference is disposed of.
1226  *
1227  * This can only be called in the context of the flusher.
1228  */
1229 static int
1230 hammer_unload_inode(struct hammer_inode *ip)
1231 {
1232         hammer_mount_t hmp = ip->hmp;
1233
1234         KASSERT(ip->lock.refs == 1,
1235                 ("hammer_unload_inode: %d refs\n", ip->lock.refs));
1236         KKASSERT(ip->vp == NULL);
1237         KKASSERT(ip->flush_state == HAMMER_FST_IDLE);
1238         KKASSERT(ip->cursor_ip_refs == 0);
1239         KKASSERT(ip->lock.lockcount == 0);
1240         KKASSERT((ip->flags & HAMMER_INODE_MODMASK) == 0);
1241
1242         KKASSERT(RB_EMPTY(&ip->rec_tree));
1243         KKASSERT(TAILQ_EMPTY(&ip->target_list));
1244
1245         RB_REMOVE(hammer_ino_rb_tree, &hmp->rb_inos_root, ip);
1246
1247         hammer_free_inode(ip);
1248         return(0);
1249 }
1250
1251 /*
1252  * Called on mount -u when switching from RW to RO or vise-versa.  Adjust
1253  * the read-only flag for cached inodes.
1254  *
1255  * This routine is called from a RB_SCAN().
1256  */
1257 int
1258 hammer_reload_inode(hammer_inode_t ip, void *arg __unused)
1259 {
1260         hammer_mount_t hmp = ip->hmp;
1261
1262         if (hmp->ronly || hmp->asof != HAMMER_MAX_TID)
1263                 ip->flags |= HAMMER_INODE_RO;
1264         else
1265                 ip->flags &= ~HAMMER_INODE_RO;
1266         return(0);
1267 }
1268
1269 /*
1270  * A transaction has modified an inode, requiring updates as specified by
1271  * the passed flags.
1272  *
1273  * HAMMER_INODE_DDIRTY: Inode data has been updated
1274  * HAMMER_INODE_XDIRTY: Dirty in-memory records
1275  * HAMMER_INODE_BUFS:   Dirty buffer cache buffers
1276  * HAMMER_INODE_DELETED: Inode record/data must be deleted
1277  * HAMMER_INODE_ATIME/MTIME: mtime/atime has been updated
1278  */
1279 void
1280 hammer_modify_inode(hammer_inode_t ip, int flags)
1281 {
1282         KKASSERT(ip->hmp->ronly == 0 ||
1283                   (flags & (HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY | 
1284                             HAMMER_INODE_BUFS | HAMMER_INODE_DELETED |
1285                             HAMMER_INODE_ATIME | HAMMER_INODE_MTIME)) == 0);
1286         if ((ip->flags & HAMMER_INODE_RSV_INODES) == 0) {
1287                 ip->flags |= HAMMER_INODE_RSV_INODES;
1288                 ++ip->hmp->rsv_inodes;
1289         }
1290
1291         ip->flags |= flags;
1292 }
1293
1294 /*
1295  * Request that an inode be flushed.  This whole mess cannot block and may
1296  * recurse (if not synchronous).  Once requested HAMMER will attempt to
1297  * actively flush the inode until the flush can be done.
1298  *
1299  * The inode may already be flushing, or may be in a setup state.  We can
1300  * place the inode in a flushing state if it is currently idle and flag it
1301  * to reflush if it is currently flushing.
1302  *
1303  * If the HAMMER_FLUSH_SYNCHRONOUS flag is specified we will attempt to
1304  * flush the indoe synchronously using the caller's context.
1305  */
1306 void
1307 hammer_flush_inode(hammer_inode_t ip, int flags)
1308 {
1309         hammer_mount_t hmp;
1310         hammer_flush_group_t flg;
1311         int good;
1312
1313         /*
1314          * next_flush_group is the first flush group we can place the inode
1315          * in.  It may be NULL.  If it becomes full we append a new flush
1316          * group and make that the next_flush_group.
1317          */
1318         hmp = ip->hmp;
1319         while ((flg = hmp->next_flush_group) != NULL) {
1320                 KKASSERT(flg->running == 0);
1321                 if (flg->total_count + flg->refs <= ip->hmp->undo_rec_limit)
1322                         break;
1323                 hmp->next_flush_group = TAILQ_NEXT(flg, flush_entry);
1324                 hammer_flusher_async(ip->hmp, flg);
1325         }
1326         if (flg == NULL) {
1327                 flg = kmalloc(sizeof(*flg), M_HAMMER, M_WAITOK|M_ZERO);
1328                 hmp->next_flush_group = flg;
1329                 TAILQ_INIT(&flg->flush_list);
1330                 TAILQ_INSERT_TAIL(&hmp->flush_group_list, flg, flush_entry);
1331         }
1332
1333         /*
1334          * Trivial 'nothing to flush' case.  If the inode is in a SETUP
1335          * state we have to put it back into an IDLE state so we can
1336          * drop the extra ref.
1337          *
1338          * If we have a parent dependancy we must still fall through
1339          * so we can run it.
1340          */
1341         if ((ip->flags & HAMMER_INODE_MODMASK) == 0) {
1342                 if (ip->flush_state == HAMMER_FST_SETUP &&
1343                     TAILQ_EMPTY(&ip->target_list)) {
1344                         ip->flush_state = HAMMER_FST_IDLE;
1345                         hammer_rel_inode(ip, 0);
1346                 }
1347                 if (ip->flush_state == HAMMER_FST_IDLE)
1348                         return;
1349         }
1350
1351         /*
1352          * Our flush action will depend on the current state.
1353          */
1354         switch(ip->flush_state) {
1355         case HAMMER_FST_IDLE:
1356                 /*
1357                  * We have no dependancies and can flush immediately.  Some
1358                  * our children may not be flushable so we have to re-test
1359                  * with that additional knowledge.
1360                  */
1361                 hammer_flush_inode_core(ip, flg, flags);
1362                 break;
1363         case HAMMER_FST_SETUP:
1364                 /*
1365                  * Recurse upwards through dependancies via target_list
1366                  * and start their flusher actions going if possible.
1367                  *
1368                  * 'good' is our connectivity.  -1 means we have none and
1369                  * can't flush, 0 means there weren't any dependancies, and
1370                  * 1 means we have good connectivity.
1371                  */
1372                 good = hammer_setup_parent_inodes(ip, flg);
1373
1374                 if (good >= 0) {
1375                         /*
1376                          * We can continue if good >= 0.  Determine how 
1377                          * many records under our inode can be flushed (and
1378                          * mark them).
1379                          */
1380                         hammer_flush_inode_core(ip, flg, flags);
1381                 } else {
1382                         /*
1383                          * parent has no connectivity, tell it to flush
1384                          * us as soon as it does.
1385                          */
1386                         ip->flags |= HAMMER_INODE_CONN_DOWN;
1387                         if (flags & HAMMER_FLUSH_SIGNAL) {
1388                                 ip->flags |= HAMMER_INODE_RESIGNAL;
1389                                 hammer_flusher_async(ip->hmp, flg);
1390                         }
1391                 }
1392                 break;
1393         case HAMMER_FST_FLUSH:
1394                 /*
1395                  * We are already flushing, flag the inode to reflush
1396                  * if needed after it completes its current flush.
1397                  */
1398                 if ((ip->flags & HAMMER_INODE_REFLUSH) == 0)
1399                         ip->flags |= HAMMER_INODE_REFLUSH;
1400                 if (flags & HAMMER_FLUSH_SIGNAL) {
1401                         ip->flags |= HAMMER_INODE_RESIGNAL;
1402                         hammer_flusher_async(ip->hmp, flg);
1403                 }
1404                 break;
1405         }
1406 }
1407
1408 /*
1409  * Scan ip->target_list, which is a list of records owned by PARENTS to our
1410  * ip which reference our ip.
1411  *
1412  * XXX This is a huge mess of recursive code, but not one bit of it blocks
1413  *     so for now do not ref/deref the structures.  Note that if we use the
1414  *     ref/rel code later, the rel CAN block.
1415  */
1416 static int
1417 hammer_setup_parent_inodes(hammer_inode_t ip, hammer_flush_group_t flg)
1418 {
1419         hammer_record_t depend;
1420         int good;
1421         int r;
1422
1423         good = 0;
1424         TAILQ_FOREACH(depend, &ip->target_list, target_entry) {
1425                 r = hammer_setup_parent_inodes_helper(depend, flg);
1426                 KKASSERT(depend->target_ip == ip);
1427                 if (r < 0 && good == 0)
1428                         good = -1;
1429                 if (r > 0)
1430                         good = 1;
1431         }
1432         return(good);
1433 }
1434
1435 /*
1436  * This helper function takes a record representing the dependancy between
1437  * the parent inode and child inode.
1438  *
1439  * record->ip           = parent inode
1440  * record->target_ip    = child inode
1441  * 
1442  * We are asked to recurse upwards and convert the record from SETUP
1443  * to FLUSH if possible.
1444  *
1445  * Return 1 if the record gives us connectivity
1446  *
1447  * Return 0 if the record is not relevant 
1448  *
1449  * Return -1 if we can't resolve the dependancy and there is no connectivity.
1450  */
1451 static int
1452 hammer_setup_parent_inodes_helper(hammer_record_t record,
1453                                   hammer_flush_group_t flg)
1454 {
1455         hammer_mount_t hmp;
1456         hammer_inode_t pip;
1457         int good;
1458
1459         KKASSERT(record->flush_state != HAMMER_FST_IDLE);
1460         pip = record->ip;
1461         hmp = pip->hmp;
1462
1463         /*
1464          * If the record is already flushing, is it in our flush group?
1465          *
1466          * If it is in our flush group but it is a general record or a 
1467          * delete-on-disk, it does not improve our connectivity (return 0),
1468          * and if the target inode is not trying to destroy itself we can't
1469          * allow the operation yet anyway (the second return -1).
1470          */
1471         if (record->flush_state == HAMMER_FST_FLUSH) {
1472                 /*
1473                  * If not in our flush group ask the parent to reflush
1474                  * us as soon as possible.
1475                  */
1476                 if (record->flush_group != flg) {
1477                         pip->flags |= HAMMER_INODE_REFLUSH;
1478                         record->target_ip->flags |= HAMMER_INODE_CONN_DOWN;
1479                         return(-1);
1480                 }
1481
1482                 /*
1483                  * If in our flush group everything is already set up,
1484                  * just return whether the record will improve our
1485                  * visibility or not.
1486                  */
1487                 if (record->type == HAMMER_MEM_RECORD_ADD)
1488                         return(1);
1489                 return(0);
1490         }
1491
1492         /*
1493          * It must be a setup record.  Try to resolve the setup dependancies
1494          * by recursing upwards so we can place ip on the flush list.
1495          */
1496         KKASSERT(record->flush_state == HAMMER_FST_SETUP);
1497
1498         good = hammer_setup_parent_inodes(pip, flg);
1499
1500         /*
1501          * If good < 0 the parent has no connectivity and we cannot safely
1502          * flush the directory entry, which also means we can't flush our
1503          * ip.  Flag the parent and us for downward recursion once the
1504          * parent's connectivity is resolved.
1505          */
1506         if (good < 0) {
1507                 /* pip->flags |= HAMMER_INODE_CONN_DOWN; set by recursion */
1508                 record->target_ip->flags |= HAMMER_INODE_CONN_DOWN;
1509                 return(good);
1510         }
1511
1512         /*
1513          * We are go, place the parent inode in a flushing state so we can
1514          * place its record in a flushing state.  Note that the parent
1515          * may already be flushing.  The record must be in the same flush
1516          * group as the parent.
1517          */
1518         if (pip->flush_state != HAMMER_FST_FLUSH)
1519                 hammer_flush_inode_core(pip, flg, HAMMER_FLUSH_RECURSION);
1520         KKASSERT(pip->flush_state == HAMMER_FST_FLUSH);
1521         KKASSERT(record->flush_state == HAMMER_FST_SETUP);
1522
1523 #if 0
1524         if (record->type == HAMMER_MEM_RECORD_DEL &&
1525             (record->target_ip->flags & (HAMMER_INODE_DELETED|HAMMER_INODE_DELONDISK)) == 0) {
1526                 /*
1527                  * Regardless of flushing state we cannot sync this path if the
1528                  * record represents a delete-on-disk but the target inode
1529                  * is not ready to sync its own deletion.
1530                  *
1531                  * XXX need to count effective nlinks to determine whether
1532                  * the flush is ok, otherwise removing a hardlink will
1533                  * just leave the DEL record to rot.
1534                  */
1535                 record->target_ip->flags |= HAMMER_INODE_REFLUSH;
1536                 return(-1);
1537         } else
1538 #endif
1539         if (pip->flush_group == flg) {
1540                 /*
1541                  * If the parent is in the same flush group as us we can
1542                  * just set the record to a flushing state and we are
1543                  * done.
1544                  */
1545                 record->flush_state = HAMMER_FST_FLUSH;
1546                 record->flush_group = flg;
1547                 ++record->flush_group->refs;
1548                 hammer_ref(&record->lock);
1549
1550                 /*
1551                  * A general directory-add contributes to our visibility.
1552                  *
1553                  * Otherwise it is probably a directory-delete or 
1554                  * delete-on-disk record and does not contribute to our
1555                  * visbility (but we can still flush it).
1556                  */
1557                 if (record->type == HAMMER_MEM_RECORD_ADD)
1558                         return(1);
1559                 return(0);
1560         } else {
1561                 /*
1562                  * If the parent is not in our flush group we cannot
1563                  * flush this record yet, there is no visibility.
1564                  * We tell the parent to reflush and mark ourselves
1565                  * so the parent knows it should flush us too.
1566                  */
1567                 pip->flags |= HAMMER_INODE_REFLUSH;
1568                 record->target_ip->flags |= HAMMER_INODE_CONN_DOWN;
1569                 return(-1);
1570         }
1571 }
1572
1573 /*
1574  * This is the core routine placing an inode into the FST_FLUSH state.
1575  */
1576 static void
1577 hammer_flush_inode_core(hammer_inode_t ip, hammer_flush_group_t flg, int flags)
1578 {
1579         int go_count;
1580
1581         /*
1582          * Set flush state and prevent the flusher from cycling into
1583          * the next flush group.  Do not place the ip on the list yet.
1584          * Inodes not in the idle state get an extra reference.
1585          */
1586         KKASSERT(ip->flush_state != HAMMER_FST_FLUSH);
1587         if (ip->flush_state == HAMMER_FST_IDLE)
1588                 hammer_ref(&ip->lock);
1589         ip->flush_state = HAMMER_FST_FLUSH;
1590         ip->flush_group = flg;
1591         ++ip->hmp->flusher.group_lock;
1592         ++ip->hmp->count_iqueued;
1593         ++hammer_count_iqueued;
1594         ++flg->total_count;
1595
1596         /*
1597          * We need to be able to vfsync/truncate from the backend.
1598          */
1599         KKASSERT((ip->flags & HAMMER_INODE_VHELD) == 0);
1600         if (ip->vp && (ip->vp->v_flag & VINACTIVE) == 0) {
1601                 ip->flags |= HAMMER_INODE_VHELD;
1602                 vref(ip->vp);
1603         }
1604
1605         /*
1606          * Figure out how many in-memory records we can actually flush
1607          * (not including inode meta-data, buffers, etc).
1608          */
1609         if (flags & HAMMER_FLUSH_RECURSION) {
1610                 /*
1611                  * If this is a upwards recursion we do not want to
1612                  * recurse down again!
1613                  */
1614                 go_count = 1;
1615         } else if (ip->flags & HAMMER_INODE_WOULDBLOCK) {
1616                 /*
1617                  * No new records are added if we must complete a flush
1618                  * from a previous cycle, but we do have to move the records
1619                  * from the previous cycle to the current one.
1620                  */
1621 #if 0
1622                 go_count = RB_SCAN(hammer_rec_rb_tree, &ip->rec_tree, NULL,
1623                                    hammer_syncgrp_child_callback, NULL);
1624 #endif
1625                 go_count = 1;
1626         } else {
1627                 /*
1628                  * Normal flush, scan records and bring them into the flush.
1629                  * Directory adds and deletes are usually skipped (they are
1630                  * grouped with the related inode rather then with the
1631                  * directory).
1632                  *
1633                  * go_count can be negative, which means the scan aborted
1634                  * due to the flush group being over-full and we should
1635                  * flush what we have.
1636                  */
1637                 go_count = RB_SCAN(hammer_rec_rb_tree, &ip->rec_tree, NULL,
1638                                    hammer_setup_child_callback, NULL);
1639         }
1640
1641         /*
1642          * This is a more involved test that includes go_count.  If we
1643          * can't flush, flag the inode and return.  If go_count is 0 we
1644          * were are unable to flush any records in our rec_tree and
1645          * must ignore the XDIRTY flag.
1646          */
1647         if (go_count == 0) {
1648                 if ((ip->flags & HAMMER_INODE_MODMASK_NOXDIRTY) == 0) {
1649                         ip->flags |= HAMMER_INODE_REFLUSH;
1650
1651                         --ip->hmp->count_iqueued;
1652                         --hammer_count_iqueued;
1653
1654                         ip->flush_state = HAMMER_FST_SETUP;
1655                         ip->flush_group = NULL;
1656                         if (ip->flags & HAMMER_INODE_VHELD) {
1657                                 ip->flags &= ~HAMMER_INODE_VHELD;
1658                                 vrele(ip->vp);
1659                         }
1660                         if (flags & HAMMER_FLUSH_SIGNAL) {
1661                                 ip->flags |= HAMMER_INODE_RESIGNAL;
1662                                 hammer_flusher_async(ip->hmp, flg);
1663                         }
1664                         if (--ip->hmp->flusher.group_lock == 0)
1665                                 wakeup(&ip->hmp->flusher.group_lock);
1666                         return;
1667                 }
1668         }
1669
1670         /*
1671          * Snapshot the state of the inode for the backend flusher.
1672          *
1673          * We continue to retain save_trunc_off even when all truncations
1674          * have been resolved as an optimization to determine if we can
1675          * skip the B-Tree lookup for overwrite deletions.
1676          *
1677          * NOTE: The DELETING flag is a mod flag, but it is also sticky,
1678          * and stays in ip->flags.  Once set, it stays set until the
1679          * inode is destroyed.
1680          *
1681          * NOTE: If a truncation from a previous flush cycle had to be
1682          * continued into this one, the TRUNCATED flag will still be
1683          * set in sync_flags as will WOULDBLOCK.  When this occurs
1684          * we CANNOT safely integrate a new truncation from the front-end
1685          * because there may be data records in-memory assigned a flush
1686          * state from the previous cycle that are supposed to be flushed
1687          * before the next frontend truncation.
1688          */
1689         if ((ip->flags & (HAMMER_INODE_TRUNCATED | HAMMER_INODE_WOULDBLOCK)) ==
1690             HAMMER_INODE_TRUNCATED) {
1691                 KKASSERT((ip->sync_flags & HAMMER_INODE_TRUNCATED) == 0);
1692                 ip->sync_trunc_off = ip->trunc_off;
1693                 ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
1694                 ip->flags &= ~HAMMER_INODE_TRUNCATED;
1695                 ip->sync_flags |= HAMMER_INODE_TRUNCATED;
1696
1697                 /*
1698                  * The save_trunc_off used to cache whether the B-Tree
1699                  * holds any records past that point is not used until
1700                  * after the truncation has succeeded, so we can safely
1701                  * set it now.
1702                  */
1703                 if (ip->save_trunc_off > ip->sync_trunc_off)
1704                         ip->save_trunc_off = ip->sync_trunc_off;
1705         }
1706         ip->sync_flags |= (ip->flags & HAMMER_INODE_MODMASK &
1707                            ~HAMMER_INODE_TRUNCATED);
1708         ip->sync_ino_leaf = ip->ino_leaf;
1709         ip->sync_ino_data = ip->ino_data;
1710         ip->flags &= ~HAMMER_INODE_MODMASK | HAMMER_INODE_TRUNCATED;
1711 #ifdef DEBUG_TRUNCATE
1712         if ((ip->sync_flags & HAMMER_INODE_TRUNCATED) && ip == HammerTruncIp)
1713                 kprintf("truncateS %016llx\n", ip->sync_trunc_off);
1714 #endif
1715
1716         /*
1717          * The flusher list inherits our inode and reference.
1718          */
1719         KKASSERT(flg->running == 0);
1720         TAILQ_INSERT_TAIL(&flg->flush_list, ip, flush_entry);
1721         if (--ip->hmp->flusher.group_lock == 0)
1722                 wakeup(&ip->hmp->flusher.group_lock);
1723
1724         if (flags & HAMMER_FLUSH_SIGNAL) {
1725                 hammer_flusher_async(ip->hmp, flg);
1726         }
1727 }
1728
1729 /*
1730  * Callback for scan of ip->rec_tree.  Try to include each record in our
1731  * flush.  ip->flush_group has been set but the inode has not yet been
1732  * moved into a flushing state.
1733  *
1734  * If we get stuck on a record we have to set HAMMER_INODE_REFLUSH on
1735  * both inodes.
1736  *
1737  * We return 1 for any record placed or found in FST_FLUSH, which prevents
1738  * the caller from shortcutting the flush.
1739  */
1740 static int
1741 hammer_setup_child_callback(hammer_record_t rec, void *data)
1742 {
1743         hammer_flush_group_t flg;
1744         hammer_inode_t target_ip;
1745         hammer_inode_t ip;
1746         int r;
1747
1748         /*
1749          * Deleted records are ignored.  Note that the flush detects deleted
1750          * front-end records at multiple points to deal with races.  This is
1751          * just the first line of defense.  The only time DELETED_FE cannot
1752          * be set is when HAMMER_RECF_INTERLOCK_BE is set. 
1753          *
1754          * Don't get confused between record deletion and, say, directory
1755          * entry deletion.  The deletion of a directory entry that is on
1756          * the media has nothing to do with the record deletion flags.
1757          */
1758         if (rec->flags & (HAMMER_RECF_DELETED_FE|HAMMER_RECF_DELETED_BE)) {
1759                 if (rec->flush_state == HAMMER_FST_FLUSH) {
1760                         KKASSERT(rec->flush_group == rec->ip->flush_group);
1761                         r = 1;
1762                 } else {
1763                         r = 0;
1764                 }
1765                 return(r);
1766         }
1767
1768         /*
1769          * If the record is in an idle state it has no dependancies and
1770          * can be flushed.
1771          */
1772         ip = rec->ip;
1773         flg = ip->flush_group;
1774         r = 0;
1775
1776         switch(rec->flush_state) {
1777         case HAMMER_FST_IDLE:
1778                 /*
1779                  * The record has no setup dependancy, we can flush it.
1780                  */
1781                 KKASSERT(rec->target_ip == NULL);
1782                 rec->flush_state = HAMMER_FST_FLUSH;
1783                 rec->flush_group = flg;
1784                 ++flg->refs;
1785                 hammer_ref(&rec->lock);
1786                 r = 1;
1787                 break;
1788         case HAMMER_FST_SETUP:
1789                 /*
1790                  * The record has a setup dependancy.  These are typically
1791                  * directory entry adds and deletes.  Such entries will be
1792                  * flushed when their inodes are flushed so we do not
1793                  * usually have to add them to the flush here.  However,
1794                  * if the target_ip has set HAMMER_INODE_CONN_DOWN then
1795                  * it is asking us to flush this record (and it).
1796                  */
1797                 target_ip = rec->target_ip;
1798                 KKASSERT(target_ip != NULL);
1799                 KKASSERT(target_ip->flush_state != HAMMER_FST_IDLE);
1800
1801                 /*
1802                  * If the target IP is already flushing in our group
1803                  * we are golden, otherwise make sure the target
1804                  * reflushes.
1805                  */
1806                 if (target_ip->flush_state == HAMMER_FST_FLUSH) {
1807                         if (target_ip->flush_group == flg) {
1808                                 rec->flush_state = HAMMER_FST_FLUSH;
1809                                 rec->flush_group = flg;
1810                                 ++flg->refs;
1811                                 hammer_ref(&rec->lock);
1812                                 r = 1;
1813                         } else {
1814                                 target_ip->flags |= HAMMER_INODE_REFLUSH;
1815                         }
1816                         break;
1817                 } 
1818
1819                 /*
1820                  * Target IP is not yet flushing.  This can get complex
1821                  * because we have to be careful about the recursion.
1822                  *
1823                  * Directories create an issue for us in that if a flush
1824                  * of a directory is requested the expectation is to flush
1825                  * any pending directory entries, but this will cause the
1826                  * related inodes to recursively flush as well.  We can't
1827                  * really defer the operation so just get as many as we
1828                  * can and
1829                  */
1830 #if 0
1831                 if ((target_ip->flags & HAMMER_INODE_RECLAIM) == 0 &&
1832                     (target_ip->flags & HAMMER_INODE_CONN_DOWN) == 0) {
1833                         /*
1834                          * We aren't reclaiming and the target ip was not
1835                          * previously prevented from flushing due to this
1836                          * record dependancy.  Do not flush this record.
1837                          */
1838                         /*r = 0;*/
1839                 } else
1840 #endif
1841                 if (flg->total_count + flg->refs >
1842                            ip->hmp->undo_rec_limit) {
1843                         /*
1844                          * Our flush group is over-full and we risk blowing
1845                          * out the UNDO FIFO.  Stop the scan, flush what we
1846                          * have, then reflush the directory.
1847                          *
1848                          * The directory may be forced through multiple
1849                          * flush groups before it can be completely
1850                          * flushed.
1851                          */
1852                         ip->flags |= HAMMER_INODE_REFLUSH;
1853                         ip->flags |= HAMMER_INODE_RESIGNAL;
1854                         r = -1;
1855                 } else if (rec->type == HAMMER_MEM_RECORD_ADD) {
1856                         /*
1857                          * If the target IP is not flushing we can force
1858                          * it to flush, even if it is unable to write out
1859                          * any of its own records we have at least one in
1860                          * hand that we CAN deal with.
1861                          */
1862                         rec->flush_state = HAMMER_FST_FLUSH;
1863                         rec->flush_group = flg;
1864                         ++flg->refs;
1865                         hammer_ref(&rec->lock);
1866                         hammer_flush_inode_core(target_ip, flg,
1867                                                 HAMMER_FLUSH_RECURSION);
1868                         r = 1;
1869                 } else {
1870                         /*
1871                          * General or delete-on-disk record.
1872                          *
1873                          * XXX this needs help.  If a delete-on-disk we could
1874                          * disconnect the target.  If the target has its own
1875                          * dependancies they really need to be flushed.
1876                          *
1877                          * XXX
1878                          */
1879                         rec->flush_state = HAMMER_FST_FLUSH;
1880                         rec->flush_group = flg;
1881                         ++flg->refs;
1882                         hammer_ref(&rec->lock);
1883                         hammer_flush_inode_core(target_ip, flg,
1884                                                 HAMMER_FLUSH_RECURSION);
1885                         r = 1;
1886                 }
1887                 break;
1888         case HAMMER_FST_FLUSH:
1889                 /* 
1890                  * If the WOULDBLOCK flag is set records may have been left
1891                  * over from a previous flush attempt.  The flush group will
1892                  * have been left intact - we are probably reflushing it
1893                  * now.
1894                  */
1895                 KKASSERT(rec->flush_group == flg);
1896                 r = 1;
1897                 break;
1898         }
1899         return(r);
1900 }
1901
1902 #if 0
1903 /*
1904  * This version just moves records already in a flush state to the new
1905  * flush group and that is it.
1906  */
1907 static int
1908 hammer_syncgrp_child_callback(hammer_record_t rec, void *data)
1909 {
1910         hammer_inode_t ip = rec->ip;
1911
1912         switch(rec->flush_state) {
1913         case HAMMER_FST_FLUSH:
1914                 KKASSERT(rec->flush_group == ip->flush_group);
1915                 break;
1916         default:
1917                 break;
1918         }
1919         return(0);
1920 }
1921 #endif
1922
1923 /*
1924  * Wait for a previously queued flush to complete.
1925  */
1926 void
1927 hammer_wait_inode(hammer_inode_t ip)
1928 {
1929         hammer_flush_group_t flg;
1930
1931         flg = NULL;
1932         if (ip->flush_state == HAMMER_FST_SETUP) {
1933                 hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
1934         }
1935         while (ip->flush_state != HAMMER_FST_IDLE) {
1936                 ip->flags |= HAMMER_INODE_FLUSHW;
1937                 tsleep(&ip->flags, 0, "hmrwin", 0);
1938         }
1939 }
1940
1941 /*
1942  * Called by the backend code when a flush has been completed.
1943  * The inode has already been removed from the flush list.
1944  *
1945  * A pipelined flush can occur, in which case we must re-enter the
1946  * inode on the list and re-copy its fields.
1947  */
1948 void
1949 hammer_flush_inode_done(hammer_inode_t ip)
1950 {
1951         hammer_mount_t hmp;
1952         int dorel;
1953
1954         KKASSERT(ip->flush_state == HAMMER_FST_FLUSH);
1955
1956         hmp = ip->hmp;
1957
1958         /*
1959          * Merge left-over flags back into the frontend and fix the state.
1960          * Incomplete truncations are retained by the backend.
1961          */
1962         ip->flags |= ip->sync_flags & ~HAMMER_INODE_TRUNCATED;
1963         ip->sync_flags &= HAMMER_INODE_TRUNCATED;
1964
1965         /*
1966          * The backend may have adjusted nlinks, so if the adjusted nlinks
1967          * does not match the fronttend set the frontend's RDIRTY flag again.
1968          */
1969         if (ip->ino_data.nlinks != ip->sync_ino_data.nlinks)
1970                 ip->flags |= HAMMER_INODE_DDIRTY;
1971
1972         /*
1973          * Fix up the dirty buffer status.
1974          */
1975         if (ip->vp && RB_ROOT(&ip->vp->v_rbdirty_tree)) {
1976                 ip->flags |= HAMMER_INODE_BUFS;
1977         }
1978
1979         /*
1980          * Re-set the XDIRTY flag if some of the inode's in-memory records
1981          * could not be flushed.
1982          */
1983         KKASSERT((RB_EMPTY(&ip->rec_tree) &&
1984                   (ip->flags & HAMMER_INODE_XDIRTY) == 0) ||
1985                  (!RB_EMPTY(&ip->rec_tree) &&
1986                   (ip->flags & HAMMER_INODE_XDIRTY) != 0));
1987
1988         /*
1989          * Do not lose track of inodes which no longer have vnode
1990          * assocations, otherwise they may never get flushed again.
1991          */
1992         if ((ip->flags & HAMMER_INODE_MODMASK) && ip->vp == NULL)
1993                 ip->flags |= HAMMER_INODE_REFLUSH;
1994
1995         /*
1996          * Adjust the flush state.
1997          */
1998         if (ip->flags & HAMMER_INODE_WOULDBLOCK) {
1999                 /*
2000                  * We were unable to flush out all our records, leave the
2001                  * inode in a flush state and in the current flush group.
2002                  *
2003                  * This occurs if the UNDO block gets too full
2004                  * or there is too much dirty meta-data and allows the
2005                  * flusher to finalize the UNDO block and then re-flush.
2006                  */
2007                 ip->flags &= ~HAMMER_INODE_WOULDBLOCK;
2008                 dorel = 0;
2009         } else {
2010                 /*
2011                  * Remove from the flush_group
2012                  */
2013                 TAILQ_REMOVE(&ip->flush_group->flush_list, ip, flush_entry);
2014                 ip->flush_group = NULL;
2015
2016                 /*
2017                  * Clean up the vnode ref and tracking counts.
2018                  */
2019                 if (ip->flags & HAMMER_INODE_VHELD) {
2020                         ip->flags &= ~HAMMER_INODE_VHELD;
2021                         vrele(ip->vp);
2022                 }
2023                 --hmp->count_iqueued;
2024                 --hammer_count_iqueued;
2025
2026                 /*
2027                  * And adjust the state.
2028                  */
2029                 if (TAILQ_EMPTY(&ip->target_list) && RB_EMPTY(&ip->rec_tree)) {
2030                         ip->flush_state = HAMMER_FST_IDLE;
2031                         dorel = 1;
2032                 } else {
2033                         ip->flush_state = HAMMER_FST_SETUP;
2034                         dorel = 0;
2035                 }
2036
2037                 /*
2038                  * If the frontend is waiting for a flush to complete,
2039                  * wake it up.
2040                  */
2041                 if (ip->flags & HAMMER_INODE_FLUSHW) {
2042                         ip->flags &= ~HAMMER_INODE_FLUSHW;
2043                         wakeup(&ip->flags);
2044                 }
2045         }
2046
2047         /*
2048          * If the frontend made more changes and requested another flush,
2049          * then try to get it running.
2050          */
2051         if (ip->flags & HAMMER_INODE_REFLUSH) {
2052                 ip->flags &= ~HAMMER_INODE_REFLUSH;
2053                 if (ip->flags & HAMMER_INODE_RESIGNAL) {
2054                         ip->flags &= ~HAMMER_INODE_RESIGNAL;
2055                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
2056                 } else {
2057                         hammer_flush_inode(ip, 0);
2058                 }
2059         }
2060
2061         /*
2062          * If we have no parent dependancies we can clear CONN_DOWN
2063          */
2064         if (TAILQ_EMPTY(&ip->target_list))
2065                 ip->flags &= ~HAMMER_INODE_CONN_DOWN;
2066
2067         /*
2068          * If the inode is now clean drop the space reservation.
2069          */
2070         if ((ip->flags & HAMMER_INODE_MODMASK) == 0 &&
2071             (ip->flags & HAMMER_INODE_RSV_INODES)) {
2072                 ip->flags &= ~HAMMER_INODE_RSV_INODES;
2073                 --hmp->rsv_inodes;
2074         }
2075
2076         if (dorel)
2077                 hammer_rel_inode(ip, 0);
2078 }
2079
2080 /*
2081  * Called from hammer_sync_inode() to synchronize in-memory records
2082  * to the media.
2083  */
2084 static int
2085 hammer_sync_record_callback(hammer_record_t record, void *data)
2086 {
2087         hammer_cursor_t cursor = data;
2088         hammer_transaction_t trans = cursor->trans;
2089         hammer_mount_t hmp = trans->hmp;
2090         int error;
2091
2092         /*
2093          * Skip records that do not belong to the current flush.
2094          */
2095         ++hammer_stats_record_iterations;
2096         if (record->flush_state != HAMMER_FST_FLUSH)
2097                 return(0);
2098
2099 #if 1
2100         if (record->flush_group != record->ip->flush_group) {
2101                 kprintf("sync_record %p ip %p bad flush group %p %p\n", record, record->ip, record->flush_group ,record->ip->flush_group);
2102                 Debugger("blah2");
2103                 return(0);
2104         }
2105 #endif
2106         KKASSERT(record->flush_group == record->ip->flush_group);
2107
2108         /*
2109          * Interlock the record using the BE flag.  Once BE is set the
2110          * frontend cannot change the state of FE.
2111          *
2112          * NOTE: If FE is set prior to us setting BE we still sync the
2113          * record out, but the flush completion code converts it to 
2114          * a delete-on-disk record instead of destroying it.
2115          */
2116         KKASSERT((record->flags & HAMMER_RECF_INTERLOCK_BE) == 0);
2117         record->flags |= HAMMER_RECF_INTERLOCK_BE;
2118
2119         /*
2120          * The backend may have already disposed of the record.
2121          */
2122         if (record->flags & HAMMER_RECF_DELETED_BE) {
2123                 error = 0;
2124                 goto done;
2125         }
2126
2127         /*
2128          * If the whole inode is being deleting all on-disk records will
2129          * be deleted very soon, we can't sync any new records to disk
2130          * because they will be deleted in the same transaction they were
2131          * created in (delete_tid == create_tid), which will assert.
2132          *
2133          * XXX There may be a case with RECORD_ADD with DELETED_FE set
2134          * that we currently panic on.
2135          */
2136         if (record->ip->sync_flags & HAMMER_INODE_DELETING) {
2137                 switch(record->type) {
2138                 case HAMMER_MEM_RECORD_DATA:
2139                         /*
2140                          * We don't have to do anything, if the record was
2141                          * committed the space will have been accounted for
2142                          * in the blockmap.
2143                          */
2144                         /* fall through */
2145                 case HAMMER_MEM_RECORD_GENERAL:
2146                         record->flags |= HAMMER_RECF_DELETED_FE;
2147                         record->flags |= HAMMER_RECF_DELETED_BE;
2148                         error = 0;
2149                         goto done;
2150                 case HAMMER_MEM_RECORD_ADD:
2151                         panic("hammer_sync_record_callback: illegal add "
2152                               "during inode deletion record %p", record);
2153                         break; /* NOT REACHED */
2154                 case HAMMER_MEM_RECORD_INODE:
2155                         panic("hammer_sync_record_callback: attempt to "
2156                               "sync inode record %p?", record);
2157                         break; /* NOT REACHED */
2158                 case HAMMER_MEM_RECORD_DEL:
2159                         /* 
2160                          * Follow through and issue the on-disk deletion
2161                          */
2162                         break;
2163                 }
2164         }
2165
2166         /*
2167          * If DELETED_FE is set special handling is needed for directory
2168          * entries.  Dependant pieces related to the directory entry may
2169          * have already been synced to disk.  If this occurs we have to
2170          * sync the directory entry and then change the in-memory record
2171          * from an ADD to a DELETE to cover the fact that it's been
2172          * deleted by the frontend.
2173          *
2174          * A directory delete covering record (MEM_RECORD_DEL) can never
2175          * be deleted by the frontend.
2176          *
2177          * Any other record type (aka DATA) can be deleted by the frontend.
2178          * XXX At the moment the flusher must skip it because there may
2179          * be another data record in the flush group for the same block,
2180          * meaning that some frontend data changes can leak into the backend's
2181          * synchronization point.
2182          */
2183         if (record->flags & HAMMER_RECF_DELETED_FE) {
2184                 if (record->type == HAMMER_MEM_RECORD_ADD) {
2185                         record->flags |= HAMMER_RECF_CONVERT_DELETE;
2186                 } else {
2187                         KKASSERT(record->type != HAMMER_MEM_RECORD_DEL);
2188                         record->flags |= HAMMER_RECF_DELETED_BE;
2189                         error = 0;
2190                         goto done;
2191                 }
2192         }
2193
2194         /*
2195          * Assign the create_tid for new records.  Deletions already
2196          * have the record's entire key properly set up.
2197          */
2198         if (record->type != HAMMER_MEM_RECORD_DEL)
2199                 record->leaf.base.create_tid = trans->tid;
2200                 record->leaf.create_ts = trans->time32;
2201         for (;;) {
2202                 error = hammer_ip_sync_record_cursor(cursor, record);
2203                 if (error != EDEADLK)
2204                         break;
2205                 hammer_done_cursor(cursor);
2206                 error = hammer_init_cursor(trans, cursor, &record->ip->cache[0],
2207                                            record->ip);
2208                 if (error)
2209                         break;
2210         }
2211         record->flags &= ~HAMMER_RECF_CONVERT_DELETE;
2212
2213         if (error) {
2214                 error = -error;
2215                 if (error != -ENOSPC) {
2216                         kprintf("hammer_sync_record_callback: sync failed rec "
2217                                 "%p, error %d\n", record, error);
2218                         Debugger("sync failed rec");
2219                 }
2220         }
2221 done:
2222         hammer_flush_record_done(record, error);
2223
2224         /*
2225          * Do partial finalization if we have built up too many dirty
2226          * buffers.  Otherwise a buffer cache deadlock can occur when
2227          * doing things like creating tens of thousands of tiny files.
2228          *
2229          * We must release our cursor lock to avoid a 3-way deadlock
2230          * due to the exclusive sync lock the finalizer must get.
2231          */
2232         if (hammer_flusher_meta_limit(hmp)) {
2233                 hammer_unlock_cursor(cursor, 0);
2234                 hammer_flusher_finalize(trans, 0);
2235                 hammer_lock_cursor(cursor, 0);
2236         }
2237
2238         return(error);
2239 }
2240
2241 /*
2242  * XXX error handling
2243  */
2244 int
2245 hammer_sync_inode(hammer_transaction_t trans, hammer_inode_t ip)
2246 {
2247         struct hammer_cursor cursor;
2248         hammer_node_t tmp_node;
2249         hammer_record_t depend;
2250         hammer_record_t next;
2251         int error, tmp_error;
2252         u_int64_t nlinks;
2253
2254         if ((ip->sync_flags & HAMMER_INODE_MODMASK) == 0)
2255                 return(0);
2256
2257         error = hammer_init_cursor(trans, &cursor, &ip->cache[1], ip);
2258         if (error)
2259                 goto done;
2260
2261         /*
2262          * Any directory records referencing this inode which are not in
2263          * our current flush group must adjust our nlink count for the
2264          * purposes of synchronization to disk.
2265          *
2266          * Records which are in our flush group can be unlinked from our
2267          * inode now, potentially allowing the inode to be physically
2268          * deleted.
2269          *
2270          * This cannot block.
2271          */
2272         nlinks = ip->ino_data.nlinks;
2273         next = TAILQ_FIRST(&ip->target_list);
2274         while ((depend = next) != NULL) {
2275                 next = TAILQ_NEXT(depend, target_entry);
2276                 if (depend->flush_state == HAMMER_FST_FLUSH &&
2277                     depend->flush_group == ip->flush_group) {
2278                         /*
2279                          * If this is an ADD that was deleted by the frontend
2280                          * the frontend nlinks count will have already been
2281                          * decremented, but the backend is going to sync its
2282                          * directory entry and must account for it.  The
2283                          * record will be converted to a delete-on-disk when
2284                          * it gets synced.
2285                          *
2286                          * If the ADD was not deleted by the frontend we
2287                          * can remove the dependancy from our target_list.
2288                          */
2289                         if (depend->flags & HAMMER_RECF_DELETED_FE) {
2290                                 ++nlinks;
2291                         } else {
2292                                 TAILQ_REMOVE(&ip->target_list, depend,
2293                                              target_entry);
2294                                 depend->target_ip = NULL;
2295                         }
2296                 } else if ((depend->flags & HAMMER_RECF_DELETED_FE) == 0) {
2297                         /*
2298                          * Not part of our flush group
2299                          */
2300                         KKASSERT((depend->flags & HAMMER_RECF_DELETED_BE) == 0);
2301                         switch(depend->type) {
2302                         case HAMMER_MEM_RECORD_ADD:
2303                                 --nlinks;
2304                                 break;
2305                         case HAMMER_MEM_RECORD_DEL:
2306                                 ++nlinks;
2307                                 break;
2308                         default:
2309                                 break;
2310                         }
2311                 }
2312         }
2313
2314         /*
2315          * Set dirty if we had to modify the link count.
2316          */
2317         if (ip->sync_ino_data.nlinks != nlinks) {
2318                 KKASSERT((int64_t)nlinks >= 0);
2319                 ip->sync_ino_data.nlinks = nlinks;
2320                 ip->sync_flags |= HAMMER_INODE_DDIRTY;
2321         }
2322
2323         /*
2324          * If there is a trunction queued destroy any data past the (aligned)
2325          * truncation point.  Userland will have dealt with the buffer
2326          * containing the truncation point for us.
2327          *
2328          * We don't flush pending frontend data buffers until after we've
2329          * dealt with the truncation.
2330          */
2331         if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
2332                 /*
2333                  * Interlock trunc_off.  The VOP front-end may continue to
2334                  * make adjustments to it while we are blocked.
2335                  */
2336                 off_t trunc_off;
2337                 off_t aligned_trunc_off;
2338                 int blkmask;
2339
2340                 trunc_off = ip->sync_trunc_off;
2341                 blkmask = hammer_blocksize(trunc_off) - 1;
2342                 aligned_trunc_off = (trunc_off + blkmask) & ~(int64_t)blkmask;
2343
2344                 /*
2345                  * Delete any whole blocks on-media.  The front-end has
2346                  * already cleaned out any partial block and made it
2347                  * pending.  The front-end may have updated trunc_off
2348                  * while we were blocked so we only use sync_trunc_off.
2349                  *
2350                  * This operation can blow out the buffer cache, EWOULDBLOCK
2351                  * means we were unable to complete the deletion.  The
2352                  * deletion will update sync_trunc_off in that case.
2353                  */
2354                 error = hammer_ip_delete_range(&cursor, ip,
2355                                                 aligned_trunc_off,
2356                                                 0x7FFFFFFFFFFFFFFFLL, 2);
2357                 if (error == EWOULDBLOCK) {
2358                         ip->flags |= HAMMER_INODE_WOULDBLOCK;
2359                         error = 0;
2360                         goto defer_buffer_flush;
2361                 }
2362
2363                 if (error)
2364                         Debugger("hammer_ip_delete_range errored");
2365
2366                 /*
2367                  * Clear the truncation flag on the backend after we have
2368                  * complete the deletions.  Backend data is now good again
2369                  * (including new records we are about to sync, below).
2370                  *
2371                  * Leave sync_trunc_off intact.  As we write additional
2372                  * records the backend will update sync_trunc_off.  This
2373                  * tells the backend whether it can skip the overwrite
2374                  * test.  This should work properly even when the backend
2375                  * writes full blocks where the truncation point straddles
2376                  * the block because the comparison is against the base
2377                  * offset of the record.
2378                  */
2379                 ip->sync_flags &= ~HAMMER_INODE_TRUNCATED;
2380                 /* ip->sync_trunc_off = 0x7FFFFFFFFFFFFFFFLL; */
2381         } else {
2382                 error = 0;
2383         }
2384
2385         /*
2386          * Now sync related records.  These will typically be directory
2387          * entries, records tracking direct-writes, or delete-on-disk records.
2388          */
2389         if (error == 0) {
2390                 tmp_error = RB_SCAN(hammer_rec_rb_tree, &ip->rec_tree, NULL,
2391                                     hammer_sync_record_callback, &cursor);
2392                 if (tmp_error < 0)
2393                         tmp_error = -error;
2394                 if (tmp_error)
2395                         error = tmp_error;
2396         }
2397         hammer_cache_node(&ip->cache[1], cursor.node);
2398
2399         /*
2400          * Re-seek for inode update, assuming our cache hasn't been ripped
2401          * out from under us.
2402          */
2403         if (error == 0) {
2404                 tmp_node = hammer_ref_node_safe(ip->hmp, &ip->cache[0], &error);
2405                 if (tmp_node) {
2406                         hammer_cursor_downgrade(&cursor);
2407                         hammer_lock_sh(&tmp_node->lock);
2408                         if ((tmp_node->flags & HAMMER_NODE_DELETED) == 0)
2409                                 hammer_cursor_seek(&cursor, tmp_node, 0);
2410                         hammer_unlock(&tmp_node->lock);
2411                         hammer_rel_node(tmp_node);
2412                 }
2413                 error = 0;
2414         }
2415
2416         /*
2417          * If we are deleting the inode the frontend had better not have
2418          * any active references on elements making up the inode.
2419          *
2420          * The call to hammer_ip_delete_clean() cleans up auxillary records
2421          * but not DB or DATA records.  Those must have already been deleted
2422          * by the normal truncation mechanic.
2423          */
2424         if (error == 0 && ip->sync_ino_data.nlinks == 0 &&
2425                 RB_EMPTY(&ip->rec_tree)  &&
2426             (ip->sync_flags & HAMMER_INODE_DELETING) &&
2427             (ip->flags & HAMMER_INODE_DELETED) == 0) {
2428                 int count1 = 0;
2429
2430                 error = hammer_ip_delete_clean(&cursor, ip, &count1);
2431                 if (error == 0) {
2432                         ip->flags |= HAMMER_INODE_DELETED;
2433                         ip->sync_flags &= ~HAMMER_INODE_DELETING;
2434                         ip->sync_flags &= ~HAMMER_INODE_TRUNCATED;
2435                         KKASSERT(RB_EMPTY(&ip->rec_tree));
2436
2437                         /*
2438                          * Set delete_tid in both the frontend and backend
2439                          * copy of the inode record.  The DELETED flag handles
2440                          * this, do not set RDIRTY.
2441                          */
2442                         ip->ino_leaf.base.delete_tid = trans->tid;
2443                         ip->sync_ino_leaf.base.delete_tid = trans->tid;
2444                         ip->ino_leaf.delete_ts = trans->time32;
2445                         ip->sync_ino_leaf.delete_ts = trans->time32;
2446
2447
2448                         /*
2449                          * Adjust the inode count in the volume header
2450                          */
2451                         hammer_sync_lock_sh(trans);
2452                         if (ip->flags & HAMMER_INODE_ONDISK) {
2453                                 hammer_modify_volume_field(trans,
2454                                                            trans->rootvol,
2455                                                            vol0_stat_inodes);
2456                                 --ip->hmp->rootvol->ondisk->vol0_stat_inodes;
2457                                 hammer_modify_volume_done(trans->rootvol);
2458                         }
2459                         hammer_sync_unlock(trans);
2460                 } else {
2461                         Debugger("hammer_ip_delete_clean errored");
2462                 }
2463         }
2464
2465         ip->sync_flags &= ~HAMMER_INODE_BUFS;
2466
2467         if (error)
2468                 Debugger("RB_SCAN errored");
2469
2470 defer_buffer_flush:
2471         /*
2472          * Now update the inode's on-disk inode-data and/or on-disk record.
2473          * DELETED and ONDISK are managed only in ip->flags.
2474          *
2475          * In the case of a defered buffer flush we still update the on-disk
2476          * inode to satisfy visibility requirements if there happen to be
2477          * directory dependancies.
2478          */
2479         switch(ip->flags & (HAMMER_INODE_DELETED | HAMMER_INODE_ONDISK)) {
2480         case HAMMER_INODE_DELETED|HAMMER_INODE_ONDISK:
2481                 /*
2482                  * If deleted and on-disk, don't set any additional flags.
2483                  * the delete flag takes care of things.
2484                  *
2485                  * Clear flags which may have been set by the frontend.
2486                  */
2487                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
2488                                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME |
2489                                     HAMMER_INODE_DELETING);
2490                 break;
2491         case HAMMER_INODE_DELETED:
2492                 /*
2493                  * Take care of the case where a deleted inode was never
2494                  * flushed to the disk in the first place.
2495                  *
2496                  * Clear flags which may have been set by the frontend.
2497                  */
2498                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
2499                                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME |
2500                                     HAMMER_INODE_DELETING);
2501                 while (RB_ROOT(&ip->rec_tree)) {
2502                         hammer_record_t record = RB_ROOT(&ip->rec_tree);
2503                         hammer_ref(&record->lock);
2504                         KKASSERT(record->lock.refs == 1);
2505                         record->flags |= HAMMER_RECF_DELETED_FE;
2506                         record->flags |= HAMMER_RECF_DELETED_BE;
2507                         hammer_rel_mem_record(record);
2508                 }
2509                 break;
2510         case HAMMER_INODE_ONDISK:
2511                 /*
2512                  * If already on-disk, do not set any additional flags.
2513                  */
2514                 break;
2515         default:
2516                 /*
2517                  * If not on-disk and not deleted, set DDIRTY to force
2518                  * an initial record to be written.
2519                  *
2520                  * Also set the create_tid in both the frontend and backend
2521                  * copy of the inode record.
2522                  */
2523                 ip->ino_leaf.base.create_tid = trans->tid;
2524                 ip->ino_leaf.create_ts = trans->time32;
2525                 ip->sync_ino_leaf.base.create_tid = trans->tid;
2526                 ip->sync_ino_leaf.create_ts = trans->time32;
2527                 ip->sync_flags |= HAMMER_INODE_DDIRTY;
2528                 break;
2529         }
2530
2531         /*
2532          * If RDIRTY or DDIRTY is set, write out a new record.  If the inode
2533          * is already on-disk the old record is marked as deleted.
2534          *
2535          * If DELETED is set hammer_update_inode() will delete the existing
2536          * record without writing out a new one.
2537          *
2538          * If *ONLY* the ITIMES flag is set we can update the record in-place.
2539          */
2540         if (ip->flags & HAMMER_INODE_DELETED) {
2541                 error = hammer_update_inode(&cursor, ip);
2542         } else 
2543         if ((ip->sync_flags & HAMMER_INODE_DDIRTY) == 0 &&
2544             (ip->sync_flags & (HAMMER_INODE_ATIME | HAMMER_INODE_MTIME))) {
2545                 error = hammer_update_itimes(&cursor, ip);
2546         } else
2547         if (ip->sync_flags & (HAMMER_INODE_DDIRTY | HAMMER_INODE_ATIME | HAMMER_INODE_MTIME)) {
2548                 error = hammer_update_inode(&cursor, ip);
2549         }
2550         if (error)
2551                 Debugger("hammer_update_itimes/inode errored");
2552 done:
2553         /*
2554          * Save the TID we used to sync the inode with to make sure we
2555          * do not improperly reuse it.
2556          */
2557         hammer_done_cursor(&cursor);
2558         return(error);
2559 }
2560
2561 /*
2562  * This routine is called when the OS is no longer actively referencing
2563  * the inode (but might still be keeping it cached), or when releasing
2564  * the last reference to an inode.
2565  *
2566  * At this point if the inode's nlinks count is zero we want to destroy
2567  * it, which may mean destroying it on-media too.
2568  */
2569 void
2570 hammer_inode_unloadable_check(hammer_inode_t ip, int getvp)
2571 {
2572         struct vnode *vp;
2573
2574         /*
2575          * Set the DELETING flag when the link count drops to 0 and the
2576          * OS no longer has any opens on the inode.
2577          *
2578          * The backend will clear DELETING (a mod flag) and set DELETED
2579          * (a state flag) when it is actually able to perform the
2580          * operation.
2581          */
2582         if (ip->ino_data.nlinks == 0 &&
2583             (ip->flags & (HAMMER_INODE_DELETING|HAMMER_INODE_DELETED)) == 0) {
2584                 ip->flags |= HAMMER_INODE_DELETING;
2585                 ip->flags |= HAMMER_INODE_TRUNCATED;
2586                 ip->trunc_off = 0;
2587                 vp = NULL;
2588                 if (getvp) {
2589                         if (hammer_get_vnode(ip, &vp) != 0)
2590                                 return;
2591                 }
2592
2593                 /*
2594                  * Final cleanup
2595                  */
2596                 if (ip->vp) {
2597                         vtruncbuf(ip->vp, 0, HAMMER_BUFSIZE);
2598                         vnode_pager_setsize(ip->vp, 0);
2599                 }
2600                 if (getvp) {
2601                         vput(vp);
2602                 }
2603         }
2604 }
2605
2606 /*
2607  * After potentially resolving a dependancy the inode is tested
2608  * to determine whether it needs to be reflushed.
2609  */
2610 void
2611 hammer_test_inode(hammer_inode_t ip)
2612 {
2613         if (ip->flags & HAMMER_INODE_REFLUSH) {
2614                 ip->flags &= ~HAMMER_INODE_REFLUSH;
2615                 hammer_ref(&ip->lock);
2616                 if (ip->flags & HAMMER_INODE_RESIGNAL) {
2617                         ip->flags &= ~HAMMER_INODE_RESIGNAL;
2618                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
2619                 } else {
2620                         hammer_flush_inode(ip, 0);
2621                 }
2622                 hammer_rel_inode(ip, 0);
2623         }
2624 }
2625
2626 /*
2627  * Clear the RECLAIM flag on an inode.  This occurs when the inode is
2628  * reassociated with a vp or just before it gets freed.
2629  *
2630  * Wakeup one thread blocked waiting on reclaims to complete.  Note that
2631  * the inode the thread is waiting on behalf of is a different inode then
2632  * the inode we are called with.  This is to create a pipeline.
2633  */
2634 static void
2635 hammer_inode_wakereclaims(hammer_inode_t ip)
2636 {
2637         struct hammer_reclaim *reclaim;
2638         hammer_mount_t hmp = ip->hmp;
2639
2640         if ((ip->flags & HAMMER_INODE_RECLAIM) == 0)
2641                 return;
2642
2643         --hammer_count_reclaiming;
2644         --hmp->inode_reclaims;
2645         ip->flags &= ~HAMMER_INODE_RECLAIM;
2646
2647         if ((reclaim = TAILQ_FIRST(&hmp->reclaim_list)) != NULL) {
2648                 TAILQ_REMOVE(&hmp->reclaim_list, reclaim, entry);
2649                 reclaim->okydoky = 1;
2650                 wakeup(reclaim);
2651         }
2652 }
2653
2654 /*
2655  * Setup our reclaim pipeline.  We only let so many detached (and dirty)
2656  * inodes build up before we start blocking.
2657  *
2658  * When we block we don't care *which* inode has finished reclaiming,
2659  * as lone as one does.  This is somewhat heuristical... we also put a
2660  * cap on how long we are willing to wait.
2661  */
2662 void
2663 hammer_inode_waitreclaims(hammer_mount_t hmp)
2664 {
2665         struct hammer_reclaim reclaim;
2666         int delay;
2667
2668         if (hmp->inode_reclaims > HAMMER_RECLAIM_WAIT) {
2669                 reclaim.okydoky = 0;
2670                 TAILQ_INSERT_TAIL(&hmp->reclaim_list,
2671                                   &reclaim, entry);
2672         } else {
2673                 reclaim.okydoky = 1;
2674         }
2675
2676         if (reclaim.okydoky == 0) {
2677                 delay = (hmp->inode_reclaims - HAMMER_RECLAIM_WAIT) * hz /
2678                         HAMMER_RECLAIM_WAIT;
2679                 if (delay >= 0)
2680                         tsleep(&reclaim, 0, "hmrrcm", delay + 1);
2681                 if (reclaim.okydoky == 0)
2682                         TAILQ_REMOVE(&hmp->reclaim_list, &reclaim, entry);
2683         }
2684 }
2685