HAMMER 12/many - buffer cache sync, buffer cache interactions, misc fixes.
[dragonfly.git] / sys / vfs / hammer / hammer_ondisk.c
1 /*
2  * Copyright (c) 2007 The DragonFly Project.  All rights reserved.
3  * 
4  * This code is derived from software contributed to The DragonFly Project
5  * by Matthew Dillon <dillon@backplane.com>
6  * 
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  * 
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  * 
34  * $DragonFly: src/sys/vfs/hammer/hammer_ondisk.c,v 1.13 2007/12/30 08:49:20 dillon Exp $
35  */
36 /*
37  * Manage HAMMER's on-disk structures.  These routines are primarily
38  * responsible for interfacing with the kernel's I/O subsystem and for
39  * managing in-memory structures.
40  */
41
42 #include "hammer.h"
43 #include <sys/fcntl.h>
44 #include <sys/nlookup.h>
45 #include <sys/buf.h>
46 #include <sys/buf2.h>
47
48 static void hammer_free_volume(hammer_volume_t volume);
49 static int hammer_load_volume(hammer_volume_t volume);
50 static int hammer_load_supercl(hammer_supercl_t supercl, int isnew);
51 static int hammer_load_cluster(hammer_cluster_t cluster, int isnew);
52 static int hammer_load_buffer(hammer_buffer_t buffer, u_int64_t buf_type);
53 static void hammer_remove_node_clist(hammer_buffer_t buffer,
54                         hammer_node_t node);
55 static void initbuffer(hammer_alist_t live, hammer_fsbuf_head_t head,
56                         u_int64_t type);
57 static void alloc_new_buffer(hammer_cluster_t cluster,
58                         hammer_alist_t live, u_int64_t type, int32_t nelements,
59                         int32_t start,
60                         int *errorp, struct hammer_buffer **bufferp);
61 #if 0
62 static void readhammerbuf(hammer_volume_t vol, void *data,
63                         int64_t offset);
64 static void writehammerbuf(hammer_volume_t vol, const void *data,
65                         int64_t offset);
66 #endif
67 static int64_t calculate_cluster_offset(hammer_volume_t vol, int32_t clu_no);
68 static int64_t calculate_supercl_offset(hammer_volume_t vol, int32_t scl_no);
69 static int32_t hammer_alloc_master(hammer_cluster_t cluster, int nblks,
70                         int32_t start, int isfwd);
71 static void hammer_adjust_stats(hammer_cluster_t cluster,
72                         u_int64_t buf_type, int nblks);
73
74 struct hammer_alist_config Buf_alist_config;
75 struct hammer_alist_config Vol_normal_alist_config;
76 struct hammer_alist_config Vol_super_alist_config;
77 struct hammer_alist_config Supercl_alist_config;
78 struct hammer_alist_config Clu_master_alist_config;
79 struct hammer_alist_config Clu_slave_alist_config;
80
81 /*
82  * Red-Black tree support for various structures
83  */
84 static int
85 hammer_ino_rb_compare(hammer_inode_t ip1, hammer_inode_t ip2)
86 {
87         if (ip1->obj_id < ip2->obj_id)
88                 return(-1);
89         if (ip1->obj_id > ip2->obj_id)
90                 return(1);
91         if (ip1->obj_asof < ip2->obj_asof)
92                 return(-1);
93         if (ip1->obj_asof > ip2->obj_asof)
94                 return(1);
95         return(0);
96 }
97
98 static int
99 hammer_inode_info_cmp(hammer_inode_info_t info, hammer_inode_t ip)
100 {
101         if (info->obj_id < ip->obj_id)
102                 return(-1);
103         if (info->obj_id > ip->obj_id)
104                 return(1);
105         if (info->obj_asof < ip->obj_asof)
106                 return(-1);
107         if (info->obj_asof > ip->obj_asof)
108                 return(1);
109         return(0);
110 }
111
112 static int
113 hammer_vol_rb_compare(hammer_volume_t vol1, hammer_volume_t vol2)
114 {
115         if (vol1->vol_no < vol2->vol_no)
116                 return(-1);
117         if (vol1->vol_no > vol2->vol_no)
118                 return(1);
119         return(0);
120 }
121
122 static int
123 hammer_scl_rb_compare(hammer_supercl_t cl1, hammer_supercl_t cl2)
124 {
125         if (cl1->scl_no < cl2->scl_no)
126                 return(-1);
127         if (cl1->scl_no > cl2->scl_no)
128                 return(1);
129         return(0);
130 }
131
132 static int
133 hammer_clu_rb_compare(hammer_cluster_t cl1, hammer_cluster_t cl2)
134 {
135         if (cl1->clu_no < cl2->clu_no)
136                 return(-1);
137         if (cl1->clu_no > cl2->clu_no)
138                 return(1);
139         return(0);
140 }
141
142 static int
143 hammer_buf_rb_compare(hammer_buffer_t buf1, hammer_buffer_t buf2)
144 {
145         if (buf1->buf_no < buf2->buf_no)
146                 return(-1);
147         if (buf1->buf_no > buf2->buf_no)
148                 return(1);
149         return(0);
150 }
151
152 static int
153 hammer_nod_rb_compare(hammer_node_t node1, hammer_node_t node2)
154 {
155         if (node1->node_offset < node2->node_offset)
156                 return(-1);
157         if (node1->node_offset > node2->node_offset)
158                 return(1);
159         return(0);
160 }
161
162 /*
163  * Note: The lookup function for hammer_ino_rb_tree winds up being named
164  * hammer_ino_rb_tree_RB_LOOKUP_INFO(root, info).  The other lookup
165  * functions are normal, e.g. hammer_clu_rb_tree_RB_LOOKUP(root, clu_no).
166  */
167 RB_GENERATE(hammer_ino_rb_tree, hammer_inode, rb_node, hammer_ino_rb_compare);
168 RB_GENERATE_XLOOKUP(hammer_ino_rb_tree, INFO, hammer_inode, rb_node,
169                 hammer_inode_info_cmp, hammer_inode_info_t);
170 RB_GENERATE2(hammer_vol_rb_tree, hammer_volume, rb_node,
171              hammer_vol_rb_compare, int32_t, vol_no);
172 RB_GENERATE2(hammer_scl_rb_tree, hammer_supercl, rb_node,
173              hammer_scl_rb_compare, int32_t, scl_no);
174 RB_GENERATE2(hammer_clu_rb_tree, hammer_cluster, rb_node,
175              hammer_clu_rb_compare, int32_t, clu_no);
176 RB_GENERATE2(hammer_buf_rb_tree, hammer_buffer, rb_node,
177              hammer_buf_rb_compare, int32_t, buf_no);
178 RB_GENERATE2(hammer_nod_rb_tree, hammer_node, rb_node,
179              hammer_nod_rb_compare, int32_t, node_offset);
180
181 /************************************************************************
182  *                              VOLUMES                                 *
183  ************************************************************************
184  *
185  * Load a HAMMER volume by name.  Returns 0 on success or a positive error
186  * code on failure.  Volumes must be loaded at mount time, get_volume() will
187  * not load a new volume.
188  *
189  * Calls made to hammer_load_volume() or single-threaded
190  */
191 int
192 hammer_install_volume(struct hammer_mount *hmp, const char *volname)
193 {
194         struct mount *mp;
195         hammer_volume_t volume;
196         struct hammer_volume_ondisk *ondisk;
197         struct nlookupdata nd;
198         struct buf *bp = NULL;
199         int error;
200         int ronly;
201
202         mp = hmp->mp;
203         ronly = ((mp->mnt_flag & MNT_RDONLY) ? 1 : 0);
204
205         /*
206          * Allocate a volume structure
207          */
208         volume = kmalloc(sizeof(*volume), M_HAMMER, M_WAITOK|M_ZERO);
209         volume->vol_name = kstrdup(volname, M_HAMMER);
210         volume->hmp = hmp;
211         volume->io.type = HAMMER_STRUCTURE_VOLUME;
212         volume->io.offset = 0LL;
213
214         /*
215          * Get the device vnode
216          */
217         error = nlookup_init(&nd, volume->vol_name, UIO_SYSSPACE, NLC_FOLLOW);
218         if (error == 0)
219                 error = nlookup(&nd);
220         if (error == 0)
221                 error = cache_vref(&nd.nl_nch, nd.nl_cred, &volume->devvp);
222         nlookup_done(&nd);
223         if (error == 0) {
224                 vn_isdisk(volume->devvp, &error);
225         }
226         if (error == 0) {
227                 vn_lock(volume->devvp, LK_EXCLUSIVE | LK_RETRY);
228                 error = VOP_OPEN(volume->devvp, (ronly ? FREAD : FREAD|FWRITE),
229                                  FSCRED, NULL);
230                 vn_unlock(volume->devvp);
231         }
232         if (error) {
233                 hammer_free_volume(volume);
234                 return(error);
235         }
236
237         /*
238          * Extract the volume number from the volume header and do various
239          * sanity checks.
240          */
241         error = bread(volume->devvp, 0LL, HAMMER_BUFSIZE, &bp);
242         if (error)
243                 goto late_failure;
244         ondisk = (void *)bp->b_data;
245         if (ondisk->head.buf_type != HAMMER_FSBUF_VOLUME) {
246                 kprintf("hammer_mount: volume %s has an invalid header\n",
247                         volume->vol_name);
248                 error = EFTYPE;
249                 goto late_failure;
250         }
251         volume->vol_no = ondisk->vol_no;
252         volume->cluster_base = ondisk->vol_clo_beg;
253         volume->vol_clsize = ondisk->vol_clsize;
254         volume->vol_flags = ondisk->vol_flags;
255         volume->nblocks = ondisk->vol_nblocks; 
256         RB_INIT(&volume->rb_clus_root);
257         RB_INIT(&volume->rb_scls_root);
258
259         hmp->mp->mnt_stat.f_blocks += volume->nblocks;
260
261         if (RB_EMPTY(&hmp->rb_vols_root)) {
262                 hmp->fsid = ondisk->vol_fsid;
263         } else if (bcmp(&hmp->fsid, &ondisk->vol_fsid, sizeof(uuid_t))) {
264                 kprintf("hammer_mount: volume %s's fsid does not match "
265                         "other volumes\n", volume->vol_name);
266                 error = EFTYPE;
267                 goto late_failure;
268         }
269
270         /*
271          * Insert the volume structure into the red-black tree.
272          */
273         if (RB_INSERT(hammer_vol_rb_tree, &hmp->rb_vols_root, volume)) {
274                 kprintf("hammer_mount: volume %s has a duplicate vol_no %d\n",
275                         volume->vol_name, volume->vol_no);
276                 error = EEXIST;
277         }
278
279         /*
280          * Set the root volume and load the root cluster.  HAMMER special
281          * cases rootvol and rootcl and will not deallocate the structures.
282          * We do not hold a ref because this would prevent related I/O
283          * from being flushed.
284          */
285         if (error == 0 && ondisk->vol_rootvol == ondisk->vol_no) {
286                 hmp->rootvol = volume;
287                 hmp->rootcl = hammer_get_cluster(volume,
288                                                  ondisk->vol0_root_clu_no,
289                                                  &error, 0);
290                 hammer_rel_cluster(hmp->rootcl, 0);
291                 hmp->fsid_udev = dev2udev(vn_todev(volume->devvp));
292         }
293 late_failure:
294         if (bp)
295                 brelse(bp);
296         if (error) {
297                 /*vinvalbuf(volume->devvp, V_SAVE, 0, 0);*/
298                 VOP_CLOSE(volume->devvp, ronly ? FREAD : FREAD|FWRITE);
299                 hammer_free_volume(volume);
300         }
301         return (error);
302 }
303
304 /*
305  * Unload and free a HAMMER volume.  Must return >= 0 to continue scan
306  * so returns -1 on failure.
307  */
308 int
309 hammer_unload_volume(hammer_volume_t volume, void *data __unused)
310 {
311         struct hammer_mount *hmp = volume->hmp;
312         hammer_cluster_t rootcl;
313         int ronly = ((hmp->mp->mnt_flag & MNT_RDONLY) ? 1 : 0);
314
315         /*
316          * Sync clusters, sync volume
317          */
318
319         hmp->mp->mnt_stat.f_blocks -= volume->nblocks;
320
321         /*
322          * Clean up the root cluster, which is held unlocked in the root
323          * volume.
324          */
325         if (hmp->rootvol == volume) {
326                 if ((rootcl = hmp->rootcl) != NULL)
327                         hmp->rootcl = NULL;
328                 hmp->rootvol = NULL;
329         }
330
331         /*
332          * Unload clusters and super-clusters.  Unloading a super-cluster
333          * also unloads related clusters, but the filesystem may not be
334          * using super-clusters so unload clusters anyway.
335          */
336         RB_SCAN(hammer_clu_rb_tree, &volume->rb_clus_root, NULL,
337                         hammer_unload_cluster, NULL);
338         RB_SCAN(hammer_scl_rb_tree, &volume->rb_scls_root, NULL,
339                         hammer_unload_supercl, NULL);
340
341         /*
342          * Release our buffer and flush anything left in the buffer cache.
343          */
344         hammer_io_release(&volume->io, 1);
345
346         /*
347          * There should be no references on the volume, no clusters, and
348          * no super-clusters.
349          */
350         KKASSERT(volume->io.lock.refs == 0);
351         KKASSERT(RB_EMPTY(&volume->rb_clus_root));
352         KKASSERT(RB_EMPTY(&volume->rb_scls_root));
353
354         volume->ondisk = NULL;
355         if (volume->devvp) {
356                 if (ronly) {
357                         vinvalbuf(volume->devvp, 0, 0, 0);
358                         VOP_CLOSE(volume->devvp, FREAD);
359                 } else {
360                         vinvalbuf(volume->devvp, V_SAVE, 0, 0);
361                         VOP_CLOSE(volume->devvp, FREAD|FWRITE);
362                 }
363         }
364
365         /*
366          * Destroy the structure
367          */
368         RB_REMOVE(hammer_vol_rb_tree, &hmp->rb_vols_root, volume);
369         hammer_free_volume(volume);
370         return(0);
371 }
372
373 static
374 void
375 hammer_free_volume(hammer_volume_t volume)
376 {
377         if (volume->vol_name) {
378                 kfree(volume->vol_name, M_HAMMER);
379                 volume->vol_name = NULL;
380         }
381         if (volume->devvp) {
382                 vrele(volume->devvp);
383                 volume->devvp = NULL;
384         }
385         kfree(volume, M_HAMMER);
386 }
387
388 /*
389  * Get a HAMMER volume.  The volume must already exist.
390  */
391 hammer_volume_t
392 hammer_get_volume(struct hammer_mount *hmp, int32_t vol_no, int *errorp)
393 {
394         struct hammer_volume *volume;
395
396         /*
397          * Locate the volume structure
398          */
399         volume = RB_LOOKUP(hammer_vol_rb_tree, &hmp->rb_vols_root, vol_no);
400         if (volume == NULL) {
401                 *errorp = ENOENT;
402                 return(NULL);
403         }
404         hammer_ref(&volume->io.lock);
405
406         /*
407          * Deal with on-disk info
408          */
409         if (volume->ondisk == NULL) {
410                 *errorp = hammer_load_volume(volume);
411                 if (*errorp) {
412                         hammer_rel_volume(volume, 1);
413                         volume = NULL;
414                 }
415         } else {
416                 *errorp = 0;
417         }
418         return(volume);
419 }
420
421 int
422 hammer_ref_volume(hammer_volume_t volume)
423 {
424         int error;
425
426         hammer_ref(&volume->io.lock);
427
428         /*
429          * Deal with on-disk info
430          */
431         if (volume->ondisk == NULL) {
432                 error = hammer_load_volume(volume);
433                 if (error)
434                         hammer_rel_volume(volume, 1);
435         } else {
436                 error = 0;
437         }
438         return (error);
439 }
440
441 hammer_volume_t
442 hammer_get_root_volume(struct hammer_mount *hmp, int *errorp)
443 {
444         hammer_volume_t volume;
445
446         volume = hmp->rootvol;
447         KKASSERT(volume != NULL);
448         hammer_ref(&volume->io.lock);
449
450         /*
451          * Deal with on-disk info
452          */
453         if (volume->ondisk == NULL) {
454                 *errorp = hammer_load_volume(volume);
455                 if (*errorp) {
456                         hammer_rel_volume(volume, 1);
457                         volume = NULL;
458                 }
459         } else {
460                 *errorp = 0;
461         }
462         return (volume);
463 }
464
465 /*
466  * Load a volume's on-disk information.  The volume must be referenced and
467  * not locked.  We temporarily acquire an exclusive lock to interlock
468  * against releases or multiple get's.
469  */
470 static int
471 hammer_load_volume(hammer_volume_t volume)
472 {
473         struct hammer_volume_ondisk *ondisk;
474         int error;
475
476         hammer_lock_ex(&volume->io.lock);
477         if (volume->ondisk == NULL) {
478                 error = hammer_io_read(volume->devvp, &volume->io);
479                 if (error) {
480                         hammer_unlock(&volume->io.lock);
481                         return (error);
482                 }
483                 volume->ondisk = ondisk = (void *)volume->io.bp->b_data;
484
485                 /*
486                  * Configure the volume's A-lists.  These are used to
487                  * allocate clusters.
488                  */
489                 if (volume->vol_flags & HAMMER_VOLF_USINGSUPERCL) {
490                         volume->alist.config = &Vol_super_alist_config;
491                         volume->alist.meta = ondisk->vol_almeta.super;
492                         volume->alist.info = volume;
493                 } else {
494                         volume->alist.config = &Vol_normal_alist_config;
495                         volume->alist.meta = ondisk->vol_almeta.normal;
496                         volume->alist.info = NULL;
497                 }
498         } else {
499                 error = 0;
500         }
501         hammer_unlock(&volume->io.lock);
502         return(0);
503 }
504
505 /*
506  * Release a volume.  Call hammer_io_release on the last reference.  We have
507  * to acquire an exclusive lock to interlock against volume->ondisk tests
508  * in hammer_load_volume(), and hammer_io_release() also expects an exclusive
509  * lock to be held.
510  *
511  * Volumes are not unloaded from memory during normal operation.
512  */
513 void
514 hammer_rel_volume(hammer_volume_t volume, int flush)
515 {
516         if (volume->io.lock.refs == 1) {
517                 hammer_lock_ex(&volume->io.lock);
518                 if (volume->io.lock.refs == 1) {
519                         volume->ondisk = NULL;
520                         hammer_io_release(&volume->io, flush);
521                 }
522                 hammer_unlock(&volume->io.lock);
523         }
524         hammer_unref(&volume->io.lock);
525 }
526
527 /************************************************************************
528  *                              SUPER-CLUSTERS                          *
529  ************************************************************************
530  *
531  * Manage super-clusters.  Note that a supercl holds a reference to its
532  * associated volume.
533  */
534 hammer_supercl_t
535 hammer_get_supercl(hammer_volume_t volume, int32_t scl_no,
536                    int *errorp, int isnew)
537 {
538         hammer_supercl_t supercl;
539
540         /*
541          * Locate and lock the super-cluster structure, creating one
542          * if necessary.
543          */
544 again:
545         supercl = RB_LOOKUP(hammer_scl_rb_tree, &volume->rb_scls_root, scl_no);
546         if (supercl == NULL) {
547                 supercl = kmalloc(sizeof(*supercl), M_HAMMER, M_WAITOK|M_ZERO);
548                 supercl->scl_no = scl_no;
549                 supercl->volume = volume;
550                 supercl->io.offset = calculate_supercl_offset(volume, scl_no);
551                 supercl->io.type = HAMMER_STRUCTURE_SUPERCL;
552                 hammer_ref(&supercl->io.lock);
553
554                 /*
555                  * Insert the cluster into the RB tree and handle late
556                  * collisions.
557                  */
558                 if (RB_INSERT(hammer_scl_rb_tree, &volume->rb_scls_root, supercl)) {
559                         hammer_unref(&supercl->io.lock);
560                         kfree(supercl, M_HAMMER);
561                         goto again;
562                 }
563                 hammer_ref(&volume->io.lock);
564         } else {
565                 hammer_ref(&supercl->io.lock);
566         }
567
568         /*
569          * Deal with on-disk info
570          */
571         if (supercl->ondisk == NULL || isnew) {
572                 *errorp = hammer_load_supercl(supercl, isnew);
573                 if (*errorp) {
574                         hammer_rel_supercl(supercl, 1);
575                         supercl = NULL;
576                 }
577         } else {
578                 *errorp = 0;
579         }
580         return(supercl);
581 }
582
583 static int
584 hammer_load_supercl(hammer_supercl_t supercl, int isnew)
585 {
586         struct hammer_supercl_ondisk *ondisk;
587         hammer_volume_t volume = supercl->volume;
588         int error;
589
590         hammer_lock_ex(&supercl->io.lock);
591         if (supercl->ondisk == NULL) {
592                 if (isnew)
593                         error = hammer_io_new(volume->devvp, &supercl->io);
594                 else
595                         error = hammer_io_read(volume->devvp, &supercl->io);
596                 if (error) {
597                         hammer_unlock(&supercl->io.lock);
598                         return (error);
599                 }
600                 supercl->ondisk = ondisk = (void *)supercl->io.bp->b_data;
601
602                 supercl->alist.config = &Supercl_alist_config;
603                 supercl->alist.meta = ondisk->scl_meta;
604                 supercl->alist.info = NULL;
605         } else if (isnew) {
606                 error = hammer_io_new(volume->devvp, &supercl->io);
607         } else {
608                 error = 0;
609         }
610         if (error == 0 && isnew) {
611                 /*
612                  * If this is a new super-cluster we have to initialize
613                  * various ondisk structural elements.  The caller is
614                  * responsible for the remainder.
615                  */
616                 struct hammer_alist_live dummy;
617
618                 ondisk = supercl->ondisk;
619                 dummy.config = &Buf_alist_config;
620                 dummy.meta = ondisk->head.buf_almeta;
621                 dummy.info = NULL;
622                 initbuffer(&dummy, &ondisk->head, HAMMER_FSBUF_SUPERCL);
623                 hammer_alist_init(&supercl->alist);
624         }
625         hammer_unlock(&supercl->io.lock);
626         return (error);
627 }
628
629 /*
630  * NOTE: Called from RB_SCAN, must return >= 0 for scan to continue.
631  */
632 int
633 hammer_unload_supercl(hammer_supercl_t supercl, void *data __unused)
634 {
635         KKASSERT(supercl->io.lock.refs == 0);
636         hammer_ref(&supercl->io.lock);
637         hammer_rel_supercl(supercl, 1);
638         return(0);
639 }
640
641 /*
642  * Release a super-cluster.  We have to deal with several places where
643  * another thread can ref the super-cluster.
644  *
645  * Only destroy the structure itself if the related buffer cache buffer
646  * was disassociated from it.  This ties the management of the structure
647  * to the buffer cache subsystem.
648  */
649 void
650 hammer_rel_supercl(hammer_supercl_t supercl, int flush)
651 {
652         hammer_volume_t volume;
653
654         if (supercl->io.lock.refs == 1) {
655                 hammer_lock_ex(&supercl->io.lock);
656                 if (supercl->io.lock.refs == 1) {
657                         hammer_io_release(&supercl->io, flush);
658                         if (supercl->io.bp == NULL &&
659                             supercl->io.lock.refs == 1) {
660                                 volume = supercl->volume;
661                                 RB_REMOVE(hammer_scl_rb_tree,
662                                           &volume->rb_scls_root, supercl);
663                                 supercl->volume = NULL; /* sanity */
664                                 kfree(supercl, M_HAMMER);
665                                 hammer_rel_volume(volume, 0);
666                                 return;
667                         }
668                 }
669                 hammer_unlock(&supercl->io.lock);
670         }
671         hammer_unref(&supercl->io.lock);
672 }
673
674 /************************************************************************
675  *                              CLUSTERS                                *
676  ************************************************************************
677  *
678  */
679 hammer_cluster_t
680 hammer_get_cluster(hammer_volume_t volume, int32_t clu_no,
681                    int *errorp, int isnew)
682 {
683         hammer_cluster_t cluster;
684
685 again:
686         cluster = RB_LOOKUP(hammer_clu_rb_tree, &volume->rb_clus_root, clu_no);
687         if (cluster == NULL) {
688                 cluster = kmalloc(sizeof(*cluster), M_HAMMER, M_WAITOK|M_ZERO);
689                 cluster->clu_no = clu_no;
690                 cluster->volume = volume;
691                 cluster->io.offset = calculate_cluster_offset(volume, clu_no);
692                 cluster->state = HAMMER_CLUSTER_IDLE;
693                 RB_INIT(&cluster->rb_bufs_root);
694                 RB_INIT(&cluster->rb_nods_root);
695                 cluster->io.type = HAMMER_STRUCTURE_CLUSTER;
696                 hammer_ref(&cluster->io.lock);
697
698                 /*
699                  * Insert the cluster into the RB tree and handle late
700                  * collisions.
701                  */
702                 if (RB_INSERT(hammer_clu_rb_tree, &volume->rb_clus_root, cluster)) {
703                         hammer_unref(&cluster->io.lock);
704                         kfree(cluster, M_HAMMER);
705                         goto again;
706                 }
707                 hammer_ref(&volume->io.lock);
708         } else {
709                 hammer_ref(&cluster->io.lock);
710         }
711
712         /*
713          * Deal with on-disk info
714          */
715         if (cluster->ondisk == NULL || isnew) {
716                 *errorp = hammer_load_cluster(cluster, isnew);
717                 if (*errorp) {
718                         hammer_rel_cluster(cluster, 1);
719                         cluster = NULL;
720                 }
721         } else {
722                 *errorp = 0;
723         }
724         return (cluster);
725 }
726
727 hammer_cluster_t
728 hammer_get_root_cluster(struct hammer_mount *hmp, int *errorp)
729 {
730         hammer_cluster_t cluster;
731
732         cluster = hmp->rootcl;
733         KKASSERT(cluster != NULL);
734         hammer_ref(&cluster->io.lock);
735
736         /*
737          * Deal with on-disk info
738          */
739         if (cluster->ondisk == NULL) {
740                 *errorp = hammer_load_cluster(cluster, 0);
741                 if (*errorp) {
742                         hammer_rel_cluster(cluster, 1);
743                         cluster = NULL;
744                 }
745         } else {
746                 *errorp = 0;
747         }
748         return (cluster);
749 }
750
751 static
752 int
753 hammer_load_cluster(hammer_cluster_t cluster, int isnew)
754 {
755         hammer_volume_t volume = cluster->volume;
756         struct hammer_cluster_ondisk *ondisk;
757         int error;
758
759         /*
760          * Load the cluster's on-disk info
761          */
762         hammer_lock_ex(&cluster->io.lock);
763         if (cluster->ondisk == NULL) {
764                 if (isnew)
765                         error = hammer_io_new(volume->devvp, &cluster->io);
766                 else
767                         error = hammer_io_read(volume->devvp, &cluster->io);
768                 if (error) {
769                         hammer_unlock(&cluster->io.lock);
770                         return (error);
771                 }
772                 cluster->ondisk = ondisk = (void *)cluster->io.bp->b_data;
773
774                 cluster->alist_master.config = &Clu_master_alist_config;
775                 cluster->alist_master.meta = ondisk->clu_master_meta;
776                 cluster->alist_btree.config = &Clu_slave_alist_config;
777                 cluster->alist_btree.meta = ondisk->clu_btree_meta;
778                 cluster->alist_btree.info = cluster;
779                 cluster->alist_record.config = &Clu_slave_alist_config;
780                 cluster->alist_record.meta = ondisk->clu_record_meta;
781                 cluster->alist_record.info = cluster;
782                 cluster->alist_mdata.config = &Clu_slave_alist_config;
783                 cluster->alist_mdata.meta = ondisk->clu_mdata_meta;
784                 cluster->alist_mdata.info = cluster;
785
786                 if (isnew == 0) {
787                         cluster->clu_btree_beg = ondisk->clu_btree_beg;
788                         cluster->clu_btree_end = ondisk->clu_btree_end;
789                 }
790         } else if (isnew) {
791                 error = hammer_io_new(volume->devvp, &cluster->io);
792         } else {
793                 error = 0;
794         }
795         if (error == 0 && isnew) {
796                 /*
797                  * If this is a new cluster we have to initialize
798                  * various ondisk structural elements.  The caller is
799                  * responsible for the remainder.
800                  */
801                 struct hammer_alist_live dummy;
802                 hammer_node_t croot;
803                 hammer_volume_ondisk_t voldisk;
804                 int32_t nbuffers;
805
806                 hammer_modify_cluster(cluster);
807                 ondisk = cluster->ondisk;
808                 voldisk = volume->ondisk;
809
810                 dummy.config = &Buf_alist_config;
811                 dummy.meta = ondisk->head.buf_almeta;
812                 dummy.info = NULL;
813                 initbuffer(&dummy, &ondisk->head, HAMMER_FSBUF_CLUSTER);
814
815                 hammer_alist_init(&cluster->alist_master);
816                 hammer_alist_init(&cluster->alist_btree);
817                 hammer_alist_init(&cluster->alist_record);
818                 hammer_alist_init(&cluster->alist_mdata);
819
820                 ondisk->vol_fsid = voldisk->vol_fsid;
821                 ondisk->vol_fstype = voldisk->vol_fstype;
822                 ondisk->clu_gen = 1;
823                 ondisk->clu_id = 0;     /* XXX */
824                 ondisk->clu_no = cluster->clu_no;
825                 ondisk->clu_flags = 0;
826                 ondisk->clu_start = HAMMER_BUFSIZE;
827                 KKASSERT(voldisk->vol_clo_end > cluster->io.offset);
828                 if (voldisk->vol_clo_end - cluster->io.offset >
829                     voldisk->vol_clsize) {
830                         ondisk->clu_limit = voldisk->vol_clsize;
831                 } else {
832                         ondisk->clu_limit = (int32_t)(voldisk->vol_clo_end -
833                                                       cluster->io.offset);
834                 }
835                 nbuffers = ondisk->clu_limit / HAMMER_BUFSIZE;
836                 hammer_alist_free(&cluster->alist_master, 1, nbuffers - 1);
837                 ondisk->idx_data = 1 * HAMMER_FSBUF_MAXBLKS;
838                 ondisk->idx_index = 0 * HAMMER_FSBUF_MAXBLKS;
839                 ondisk->idx_record = nbuffers * HAMMER_FSBUF_MAXBLKS;
840
841                 /*
842                  * Initialize the B-Tree.  We don't know what the caller
843                  * intends to do with the cluster so make sure it causes
844                  * an assertion if the caller makes no changes.
845                  */
846                 ondisk->clu_btree_parent_vol_no = -2;
847                 ondisk->clu_btree_parent_clu_no = -2;
848                 ondisk->clu_btree_parent_offset = -2;
849                 ondisk->clu_btree_parent_clu_gen = -2;
850                 hammer_modify_cluster_done(cluster);
851
852                 croot = hammer_alloc_btree(cluster, &error);
853                 if (error == 0) {
854                         hammer_modify_node(croot);
855                         bzero(croot->ondisk, sizeof(*croot->ondisk));
856                         croot->ondisk->count = 0;
857                         croot->ondisk->type = HAMMER_BTREE_TYPE_LEAF;
858                         hammer_modify_node_done(croot);
859                         hammer_modify_cluster(cluster);
860                         ondisk->clu_btree_root = croot->node_offset;
861                         hammer_modify_cluster_done(cluster);
862                         hammer_rel_node(croot);
863                 }
864         }
865         hammer_unlock(&cluster->io.lock);
866         return (error);
867 }
868
869 /*
870  * NOTE: Called from RB_SCAN, must return >= 0 for scan to continue.
871  */
872 int
873 hammer_unload_cluster(hammer_cluster_t cluster, void *data __unused)
874 {
875         hammer_ref(&cluster->io.lock);
876         RB_SCAN(hammer_buf_rb_tree, &cluster->rb_bufs_root, NULL,
877                 hammer_unload_buffer, NULL);
878         KKASSERT(cluster->io.lock.refs == 1);
879         hammer_rel_cluster(cluster, 1);
880         return(0);
881 }
882
883 /*
884  * Reference a cluster that is either already referenced or via a specially
885  * handled pointer (aka rootcl).
886  */
887 int
888 hammer_ref_cluster(hammer_cluster_t cluster)
889 {
890         int error;
891
892         KKASSERT(cluster != NULL);
893         hammer_ref(&cluster->io.lock);
894
895         /*
896          * Deal with on-disk info
897          */
898         if (cluster->ondisk == NULL) {
899                 error = hammer_load_cluster(cluster, 0);
900                 if (error)
901                         hammer_rel_cluster(cluster, 1);
902         } else {
903                 error = 0;
904         }
905         return(error);
906 }
907
908 /*
909  * Release a cluster.  We have to deal with several places where
910  * another thread can ref the cluster.
911  *
912  * Only destroy the structure itself if the related buffer cache buffer
913  * was disassociated from it.  This ties the management of the structure
914  * to the buffer cache subsystem.
915  */
916 void
917 hammer_rel_cluster(hammer_cluster_t cluster, int flush)
918 {
919         hammer_node_t node;
920         hammer_volume_t volume;
921
922         if (cluster->io.lock.refs == 1) {
923                 hammer_lock_ex(&cluster->io.lock);
924                 if (cluster->io.lock.refs == 1) {
925                         /*
926                          * Release the I/O.  If we or the kernel wants to
927                          * flush, this will release the bp.  Otherwise the
928                          * bp may be written and flushed passively by the
929                          * kernel later on.
930                          */
931                         hammer_io_release(&cluster->io, flush);
932
933                         /*
934                          * The B-Tree node cache is not counted in the
935                          * cluster's reference count.  Clean out the
936                          * cache.
937                          *
938                          * If the cluster acquires a new reference while we
939                          * are trying to clean it out, abort the cleaning.
940                          * 
941                          * Any actively referenced nodes will reference the
942                          * related buffer and cluster, so a ref count check
943                          * should be sufficient.
944                          */
945                         while (cluster->io.bp == NULL &&
946                                cluster->io.lock.refs == 1 &&
947                                (node = RB_ROOT(&cluster->rb_nods_root)) != NULL
948                         ) {
949                                 KKASSERT(node->lock.refs == 0);
950                                 hammer_flush_node(node);
951                         }
952
953                         /*
954                          * Final cleanup
955                          */
956                         if (cluster != cluster->volume->hmp->rootcl &&
957                             cluster->io.bp == NULL &&
958                             cluster->io.lock.refs == 1 &&
959                             RB_EMPTY(&cluster->rb_nods_root)) {
960                                 KKASSERT(RB_EMPTY(&cluster->rb_bufs_root));
961                                 volume = cluster->volume;
962                                 RB_REMOVE(hammer_clu_rb_tree,
963                                           &volume->rb_clus_root, cluster);
964                                 cluster->volume = NULL; /* sanity */
965                                 kfree(cluster, M_HAMMER);
966                                 hammer_rel_volume(volume, 0);
967                                 return;
968                         }
969                 }
970                 hammer_unlock(&cluster->io.lock);
971         }
972         hammer_unref(&cluster->io.lock);
973 }
974
975 /************************************************************************
976  *                              BUFFERS                                 *
977  ************************************************************************
978  *
979  * Manage buffers.  Note that a buffer holds a reference to its associated
980  * cluster, and its cluster will hold a reference to the cluster's volume.
981  *
982  * A non-zero buf_type indicates that a new buffer should be created and
983  * zero'd.
984  */
985 hammer_buffer_t
986 hammer_get_buffer(hammer_cluster_t cluster, int32_t buf_no,
987                   u_int64_t buf_type, int *errorp)
988 {
989         hammer_buffer_t buffer;
990
991         /*
992          * Find the buffer.  Note that buffer 0 corresponds to the cluster
993          * header and should never be requested.
994          */
995         KKASSERT(buf_no >= cluster->ondisk->clu_start / HAMMER_BUFSIZE &&
996                  buf_no < cluster->ondisk->clu_limit / HAMMER_BUFSIZE);
997
998         /*
999          * Locate and lock the buffer structure, creating one if necessary.
1000          */
1001 again:
1002         buffer = RB_LOOKUP(hammer_buf_rb_tree, &cluster->rb_bufs_root, buf_no);
1003         if (buffer == NULL) {
1004                 buffer = kmalloc(sizeof(*cluster), M_HAMMER, M_WAITOK|M_ZERO);
1005                 buffer->buf_no = buf_no;
1006                 buffer->cluster = cluster;
1007                 buffer->volume = cluster->volume;
1008                 buffer->io.offset = cluster->io.offset +
1009                                     (buf_no * HAMMER_BUFSIZE);
1010                 buffer->io.type = HAMMER_STRUCTURE_BUFFER;
1011                 TAILQ_INIT(&buffer->clist);
1012                 hammer_ref(&buffer->io.lock);
1013
1014                 /*
1015                  * Insert the cluster into the RB tree and handle late
1016                  * collisions.
1017                  */
1018                 if (RB_INSERT(hammer_buf_rb_tree, &cluster->rb_bufs_root, buffer)) {
1019                         hammer_unref(&buffer->io.lock);
1020                         kfree(buffer, M_HAMMER);
1021                         goto again;
1022                 }
1023                 hammer_ref(&cluster->io.lock);
1024         } else {
1025                 hammer_ref(&buffer->io.lock);
1026         }
1027
1028         /*
1029          * Deal with on-disk info
1030          */
1031         if (buffer->ondisk == NULL || buf_type) {
1032                 *errorp = hammer_load_buffer(buffer, buf_type);
1033                 if (*errorp) {
1034                         hammer_rel_buffer(buffer, 1);
1035                         buffer = NULL;
1036                 }
1037         } else {
1038                 *errorp = 0;
1039         }
1040         return(buffer);
1041 }
1042
1043 static int
1044 hammer_load_buffer(hammer_buffer_t buffer, u_int64_t buf_type)
1045 {
1046         hammer_volume_t volume;
1047         hammer_fsbuf_ondisk_t ondisk;
1048         int error;
1049
1050         /*
1051          * Load the buffer's on-disk info
1052          */
1053         volume = buffer->volume;
1054         hammer_lock_ex(&buffer->io.lock);
1055         if (buffer->ondisk == NULL) {
1056                 if (buf_type) {
1057                         error = hammer_io_new(volume->devvp, &buffer->io);
1058                 } else {
1059                         error = hammer_io_read(volume->devvp, &buffer->io);
1060                 }
1061                 if (error) {
1062                         hammer_unlock(&buffer->io.lock);
1063                         return (error);
1064                 }
1065                 buffer->ondisk = ondisk = (void *)buffer->io.bp->b_data;
1066                 buffer->alist.config = &Buf_alist_config;
1067                 buffer->alist.meta = ondisk->head.buf_almeta;
1068                 buffer->buf_type = ondisk->head.buf_type;
1069         } else if (buf_type) {
1070                 error = hammer_io_new(volume->devvp, &buffer->io);
1071         } else {
1072                 error = 0;
1073         }
1074         if (error == 0 && buf_type) {
1075                 ondisk = buffer->ondisk;
1076                 initbuffer(&buffer->alist, &ondisk->head, buf_type);
1077                 buffer->buf_type = ondisk->head.buf_type;
1078         }
1079         hammer_unlock(&buffer->io.lock);
1080         return (error);
1081 }
1082
1083 /*
1084  * NOTE: Called from RB_SCAN, must return >= 0 for scan to continue.
1085  */
1086 int
1087 hammer_unload_buffer(hammer_buffer_t buffer, void *data __unused)
1088 {
1089         hammer_ref(&buffer->io.lock);
1090         hammer_flush_buffer_nodes(buffer);
1091         KKASSERT(buffer->io.lock.refs == 1);
1092         hammer_rel_buffer(buffer, 1);
1093         return(0);
1094 }
1095
1096 /*
1097  * Reference a buffer that is either already referenced or via a specially
1098  * handled pointer (aka cursor->buffer).
1099  */
1100 int
1101 hammer_ref_buffer(hammer_buffer_t buffer)
1102 {
1103         int error;
1104
1105         hammer_ref(&buffer->io.lock);
1106         if (buffer->ondisk == NULL) {
1107                 error = hammer_load_buffer(buffer, 0);
1108                 if (error) {
1109                         hammer_rel_buffer(buffer, 1);
1110                         /*
1111                          * NOTE: buffer pointer can become stale after
1112                          * the above release.
1113                          */
1114                 } else {
1115                         KKASSERT(buffer->buf_type ==
1116                                  buffer->ondisk->head.buf_type);
1117                 }
1118         } else {
1119                 error = 0;
1120         }
1121         return(error);
1122 }
1123
1124 /*
1125  * Release a buffer.  We have to deal with several places where
1126  * another thread can ref the buffer.
1127  *
1128  * Only destroy the structure itself if the related buffer cache buffer
1129  * was disassociated from it.  This ties the management of the structure
1130  * to the buffer cache subsystem.  buffer->ondisk determines whether the
1131  * embedded io is referenced or not.
1132  */
1133 void
1134 hammer_rel_buffer(hammer_buffer_t buffer, int flush)
1135 {
1136         hammer_cluster_t cluster;
1137         hammer_node_t node;
1138
1139         if (buffer->io.lock.refs == 1) {
1140                 hammer_lock_ex(&buffer->io.lock);
1141                 if (buffer->io.lock.refs == 1) {
1142                         hammer_io_release(&buffer->io, flush);
1143
1144                         /*
1145                          * Clean out the B-Tree node cache, if any, then
1146                          * clean up the cluster ref and free the buffer.
1147                          *
1148                          * If the buffer acquires a new reference while we
1149                          * are trying to clean it out, abort the cleaning.
1150                          */
1151                         while (buffer->io.bp == NULL &&
1152                                buffer->io.lock.refs == 1 &&
1153                                (node = TAILQ_FIRST(&buffer->clist)) != NULL
1154                         ) {
1155                                 KKASSERT(node->lock.refs == 0);
1156                                 hammer_flush_node(node);
1157                         }
1158                         if (buffer->io.bp == NULL &&
1159                             hammer_islastref(&buffer->io.lock)) {
1160                                 cluster = buffer->cluster;
1161                                 RB_REMOVE(hammer_buf_rb_tree,
1162                                           &cluster->rb_bufs_root, buffer);
1163                                 buffer->cluster = NULL; /* sanity */
1164                                 kfree(buffer, M_HAMMER);
1165                                 hammer_rel_cluster(cluster, 0);
1166                                 return;
1167                         }
1168                 }
1169                 hammer_unlock(&buffer->io.lock);
1170         }
1171         hammer_unref(&buffer->io.lock);
1172 }
1173
1174 /*
1175  * Flush passively cached B-Tree nodes associated with this buffer.
1176  *
1177  * NOTE: The buffer is referenced and locked.
1178  */
1179 void
1180 hammer_flush_buffer_nodes(hammer_buffer_t buffer)
1181 {
1182         hammer_node_t node;
1183
1184         node = TAILQ_FIRST(&buffer->clist);
1185         while (node) {
1186                 buffer->save_scan = TAILQ_NEXT(node, entry);
1187                 if (node->lock.refs == 0)
1188                         hammer_flush_node(node);
1189                 node = buffer->save_scan;
1190         }
1191 }
1192
1193 /************************************************************************
1194  *                              NODES                                   *
1195  ************************************************************************
1196  *
1197  * Manage B-Tree nodes.  B-Tree nodes represent the primary indexing
1198  * method used by the HAMMER filesystem.
1199  *
1200  * Unlike other HAMMER structures, a hammer_node can be PASSIVELY
1201  * associated with its buffer.  It can have an active buffer reference
1202  * even when the node itself has no references.  The node also passively
1203  * associates itself with its cluster without holding any cluster refs.
1204  * The cluster ref is indirectly maintained by the active buffer ref when
1205  * a node is acquired.
1206  *
1207  * A hammer_node can also be passively associated with other HAMMER
1208  * structures, such as inodes, while retaining 0 references.  These
1209  * associations can be cleared backwards using a pointer-to-pointer in
1210  * the hammer_node.
1211  *
1212  * This allows the HAMMER implementation to cache hammer_node's long-term
1213  * and short-cut a great deal of the infrastructure's complexity.  In
1214  * most cases a cached node can be reacquired without having to dip into
1215  * either the buffer or cluster management code.
1216  *
1217  * The caller must pass a referenced cluster on call and will retain
1218  * ownership of the reference on return.  The node will acquire its own
1219  * additional references, if necessary.
1220  */
1221 hammer_node_t
1222 hammer_get_node(hammer_cluster_t cluster, int32_t node_offset, int *errorp)
1223 {
1224         hammer_node_t node;
1225
1226         /*
1227          * Locate the structure, allocating one if necessary.
1228          */
1229 again:
1230         node = RB_LOOKUP(hammer_nod_rb_tree, &cluster->rb_nods_root,
1231                          node_offset);
1232         if (node == NULL) {
1233                 node = kmalloc(sizeof(*node), M_HAMMER, M_WAITOK|M_ZERO);
1234                 node->node_offset = node_offset;
1235                 node->cluster = cluster;
1236                 if (RB_INSERT(hammer_nod_rb_tree, &cluster->rb_nods_root,
1237                               node)) {
1238                         kfree(node, M_HAMMER);
1239                         goto again;
1240                 }
1241         }
1242         *errorp = hammer_ref_node(node);
1243         if (*errorp) {
1244                 /*
1245                  * NOTE: The node pointer may be stale on error return.
1246                  * In fact, its probably been destroyed.
1247                  */
1248                 node = NULL;
1249         }
1250         return(node);
1251 }
1252
1253 /*
1254  * Reference the node to prevent disassociations, then associate and
1255  * load the related buffer.  This routine can also be called to reference
1256  * a node from a cache pointer.
1257  *
1258  * NOTE: Because the caller does not have a ref on the node, the caller's
1259  * node pointer will be stale if an error is returned.  We may also wind
1260  * up clearing the related cache pointers.
1261  *
1262  * NOTE: The cluster is indirectly referenced by our buffer ref.
1263  */
1264 int
1265 hammer_ref_node(hammer_node_t node)
1266 {
1267         hammer_buffer_t buffer;
1268         int32_t buf_no;
1269         int error;
1270
1271         hammer_ref(&node->lock);
1272         error = 0;
1273         if (node->ondisk == NULL) {
1274                 hammer_lock_ex(&node->lock);
1275                 if (node->ondisk == NULL) {
1276                         /*
1277                          * This is a little confusing but the jist is that
1278                          * node->buffer determines whether the node is on
1279                          * the buffer's clist and node->ondisk determines
1280                          * whether the buffer is referenced.
1281                          */
1282                         if ((buffer = node->buffer) != NULL) {
1283                                 error = hammer_ref_buffer(buffer);
1284                         } else {
1285                                 buf_no = node->node_offset / HAMMER_BUFSIZE;
1286                                 buffer = hammer_get_buffer(node->cluster,
1287                                                            buf_no, 0, &error);
1288                                 if (buffer) {
1289                                         KKASSERT(error == 0);
1290                                         TAILQ_INSERT_TAIL(&buffer->clist,
1291                                                           node, entry);
1292                                         node->buffer = buffer;
1293                                 }
1294                         }
1295                         if (error == 0) {
1296                                 node->ondisk = (void *)((char *)buffer->ondisk +
1297                                        (node->node_offset & HAMMER_BUFMASK));
1298                         }
1299                 }
1300                 hammer_unlock(&node->lock);
1301         }
1302         if (error)
1303                 hammer_rel_node(node);
1304         return (error);
1305 }
1306
1307 /*
1308  * Release a hammer_node.  The node retains a passive association with
1309  * its cluster, buffer and caches.
1310  *
1311  * However, to avoid cluttering up kernel memory with tons of B-Tree
1312  * node cache structures we destroy the node if no passive cache or
1313  * (instantiated) buffer references exist.
1314  */
1315 void
1316 hammer_rel_node(hammer_node_t node)
1317 {
1318         hammer_cluster_t cluster;
1319         hammer_buffer_t buffer;
1320
1321         if (hammer_islastref(&node->lock)) {
1322                 cluster = node->cluster;
1323                 /*
1324                  * Clutter control, this case only occurs after a failed
1325                  * load since otherwise ondisk will be non-NULL.
1326                  */
1327                 if (node->cache1 == NULL && node->cache2 == NULL && 
1328                     node->ondisk == NULL) {
1329                         RB_REMOVE(hammer_nod_rb_tree, &cluster->rb_nods_root,
1330                                   node);
1331                         if ((buffer = node->buffer) != NULL) {
1332                                 node->buffer = NULL;
1333                                 hammer_remove_node_clist(buffer, node);
1334                         }
1335                         kfree(node, M_HAMMER);
1336                         return;
1337                 }
1338
1339                 /*
1340                  * node->ondisk determines whether we have a buffer reference
1341                  * to get rid of or not.  Only get rid of the reference if
1342                  * the kernel tried to flush the buffer.
1343                  *
1344                  * NOTE: Once unref'd the node can be physically destroyed,
1345                  * so our node is stale afterwords.
1346                  *
1347                  * This case occurs if the node still has cache references.
1348                  * We could remove the references and free the structure
1349                  * but for now we allow them (and the node structure) to
1350                  * remain intact.
1351                  */
1352                 if (node->ondisk && hammer_io_checkflush(&node->buffer->io)) {
1353                         buffer = node->buffer;
1354                         node->buffer = NULL;
1355                         node->ondisk = NULL;
1356                         hammer_remove_node_clist(buffer, node);
1357                         hammer_unref(&node->lock);
1358                         hammer_rel_buffer(buffer, 0);
1359                 } else {
1360                         hammer_unref(&node->lock);
1361                 }
1362         } else {
1363                 hammer_unref(&node->lock);
1364         }
1365 }
1366
1367 /*
1368  * Cache-and-release a hammer_node.  Kinda like catching and releasing a
1369  * fish, but keeping an eye on him.  The node is passively cached in *cache.
1370  *
1371  * NOTE!  HAMMER may NULL *cache at any time, even after you have
1372  * referenced the node!
1373  */
1374 void
1375 hammer_cache_node(hammer_node_t node, struct hammer_node **cache)
1376 {
1377         if (node->cache1 != cache) {
1378                 if (node->cache2 == cache) {
1379                         struct hammer_node **tmp;
1380                         tmp = node->cache1;
1381                         node->cache1 = node->cache2;
1382                         node->cache2 = tmp;
1383                 } else {
1384                         if (node->cache2)
1385                                 *node->cache2 = NULL;
1386                         node->cache2 = node->cache1;
1387                         node->cache1 = cache;
1388                         *cache = node;
1389                 }
1390         }
1391 }
1392
1393 void
1394 hammer_uncache_node(struct hammer_node **cache)
1395 {
1396         hammer_node_t node;
1397
1398         if ((node = *cache) != NULL) {
1399                 *cache = NULL;
1400                 if (node->cache1 == cache) {
1401                         node->cache1 = node->cache2;
1402                         node->cache2 = NULL;
1403                 } else if (node->cache2 == cache) {
1404                         node->cache2 = NULL;
1405                 } else {
1406                         panic("hammer_uncache_node: missing cache linkage");
1407                 }
1408                 if (node->cache1 == NULL && node->cache2 == NULL &&
1409                     node->lock.refs == 0) {
1410                         hammer_flush_node(node);
1411                 }
1412         }
1413 }
1414
1415 /*
1416  * Remove a node's cache references and destroy the node if it has no
1417  * references.  This is typically called from the buffer handling code.
1418  *
1419  * The node may have an active buffer reference (ondisk != NULL) even
1420  * if the node itself has no references.
1421  *
1422  * Note that a caller iterating through nodes via a buffer must have its
1423  * own reference on the buffer or our hammer_rel_buffer() call below may
1424  * rip it out from under the caller.
1425  */
1426 void
1427 hammer_flush_node(hammer_node_t node)
1428 {
1429         hammer_buffer_t buffer;
1430
1431         if (node->cache1)
1432                 *node->cache1 = NULL;
1433         if (node->cache2)
1434                 *node->cache2 = NULL;
1435         if (node->lock.refs == 0) {
1436                 RB_REMOVE(hammer_nod_rb_tree, &node->cluster->rb_nods_root,
1437                           node);
1438                 if ((buffer = node->buffer) != NULL) {
1439                         node->buffer = NULL;
1440                         hammer_remove_node_clist(buffer, node);
1441                         if (node->ondisk) {
1442                                 node->ondisk = NULL;
1443                                 hammer_rel_buffer(buffer, 0);
1444                         }
1445                 }
1446                 kfree(node, M_HAMMER);
1447         } else {
1448                 kprintf("Cannot flush node: %p\n", node);
1449         }
1450 }
1451
1452 /*
1453  * Remove a node from the buffer's clist.  Adjust save_scan as appropriate.
1454  * This is in its own little routine to properly handle interactions with
1455  * save_scan, so it is possible to block while scanning a buffer's node list.
1456  */
1457 static
1458 void
1459 hammer_remove_node_clist(hammer_buffer_t buffer, hammer_node_t node)
1460 {
1461         if (buffer->save_scan == node)
1462                 buffer->save_scan = TAILQ_NEXT(node, entry);
1463         TAILQ_REMOVE(&buffer->clist, node, entry);
1464 }
1465
1466 /************************************************************************
1467  *                              A-LIST ALLOCATORS                       *
1468  ************************************************************************/
1469
1470 /*
1471  * Allocate HAMMER clusters
1472  */
1473 hammer_cluster_t
1474 hammer_alloc_cluster(hammer_mount_t hmp, hammer_cluster_t cluster_hint,
1475                      int *errorp)
1476 {
1477         hammer_volume_t volume;
1478         hammer_cluster_t cluster;
1479         int32_t clu_no;
1480         int32_t clu_hint;
1481         int32_t vol_beg;
1482         int32_t vol_no;
1483
1484         /*
1485          * Figure out our starting volume and hint.
1486          */
1487         if (cluster_hint) {
1488                 vol_beg = cluster_hint->volume->vol_no;
1489                 clu_hint = cluster_hint->clu_no;
1490         } else {
1491                 vol_beg = hmp->volume_iterator;
1492                 clu_hint = -1;
1493         }
1494
1495         /*
1496          * Loop through volumes looking for a free cluster.  If allocating
1497          * a new cluster relative to an existing cluster try to find a free
1498          * cluster on either side (clu_hint >= 0), otherwise just do a
1499          * forwards iteration.
1500          */
1501         vol_no = vol_beg;
1502         do {
1503                 volume = hammer_get_volume(hmp, vol_no, errorp);
1504                 kprintf("VOLUME %p %d\n", volume, vol_no);
1505                 if (*errorp) {
1506                         clu_no = HAMMER_ALIST_BLOCK_NONE;
1507                         break;
1508                 }
1509                 hammer_modify_volume(volume);
1510                 if (clu_hint == -1) {
1511                         clu_hint = volume->clu_iterator;
1512                         clu_no = hammer_alist_alloc_fwd(&volume->alist, 1,
1513                                                         clu_hint);
1514                         if (clu_no == HAMMER_ALIST_BLOCK_NONE) {
1515                                 clu_no = hammer_alist_alloc_fwd(&volume->alist,
1516                                                                 1, 0);
1517                         }
1518                 } else {
1519                         clu_no = hammer_alist_alloc_fwd(&volume->alist, 1,
1520                                                         clu_hint);
1521                         if (clu_no == HAMMER_ALIST_BLOCK_NONE) {
1522                                 clu_no = hammer_alist_alloc_rev(&volume->alist,
1523                                                                 1, clu_hint);
1524                         }
1525                 }
1526                 hammer_modify_volume_done(volume);
1527                 if (clu_no != HAMMER_ALIST_BLOCK_NONE)
1528                         break;
1529                 hammer_rel_volume(volume, 0);
1530                 volume = NULL;
1531                 *errorp = ENOSPC;
1532                 vol_no = (vol_no + 1) % hmp->nvolumes;
1533                 clu_hint = -1;
1534         } while (vol_no != vol_beg);
1535
1536         /*
1537          * Acquire the cluster.  On success this will force *errorp to 0.
1538          */
1539         if (clu_no != HAMMER_ALIST_BLOCK_NONE) {
1540                 kprintf("ALLOC CLUSTER %d\n", clu_no);
1541                 cluster = hammer_get_cluster(volume, clu_no, errorp, 1);
1542                 volume->clu_iterator = clu_no;
1543                 hammer_rel_volume(volume, 0);
1544         } else {
1545                 cluster = NULL;
1546         }
1547         if (cluster)
1548                 hammer_lock_ex(&cluster->io.lock);
1549         return(cluster);
1550 }
1551
1552 void
1553 hammer_init_cluster(hammer_cluster_t cluster, hammer_base_elm_t left_bound, 
1554                     hammer_base_elm_t right_bound)
1555 {
1556         hammer_cluster_ondisk_t ondisk = cluster->ondisk;
1557
1558         hammer_modify_cluster(cluster);
1559         ondisk->clu_btree_beg = *left_bound;
1560         ondisk->clu_btree_end = *right_bound;
1561         cluster->clu_btree_beg = ondisk->clu_btree_beg;
1562         cluster->clu_btree_end = ondisk->clu_btree_end;
1563         hammer_modify_cluster_done(cluster);
1564 }
1565
1566 /*
1567  * Deallocate a cluster
1568  */
1569 void
1570 hammer_free_cluster(hammer_cluster_t cluster)
1571 {
1572         hammer_modify_cluster(cluster);
1573         hammer_alist_free(&cluster->volume->alist, cluster->clu_no, 1);
1574         hammer_modify_cluster_done(cluster);
1575 }
1576
1577 /*
1578  * Allocate HAMMER elements - btree nodes, data storage, and record elements
1579  *
1580  * The passed *bufferp should be initialized to NULL.  On successive calls
1581  * *bufferp caches the most recent buffer used until put away by the caller.
1582  * Note that previously returned pointers using the cached buffer become
1583  * invalid on successive calls which reuse *bufferp.
1584  *
1585  * All allocations first attempt to use the block found at the specified
1586  * iterator.  If that fails the first available block is used.  If that
1587  * fails a new buffer is allocated and associated with the buffer type
1588  * A-list and the element is allocated out of the new buffer.
1589  */
1590
1591 hammer_node_t
1592 hammer_alloc_btree(hammer_cluster_t cluster, int *errorp)
1593 {
1594         hammer_buffer_t buffer;
1595         hammer_alist_t live;
1596         hammer_node_t node;
1597         int32_t elm_no;
1598         int32_t buf_no;
1599         int32_t node_offset;
1600
1601         /*
1602          * Allocate a B-Tree element
1603          */
1604         hammer_modify_cluster(cluster);
1605         buffer = NULL;
1606         live = &cluster->alist_btree;
1607         elm_no = hammer_alist_alloc_fwd(live, 1, cluster->ondisk->idx_index);
1608         if (elm_no == HAMMER_ALIST_BLOCK_NONE)
1609                 elm_no = hammer_alist_alloc_fwd(live, 1, 0);
1610         if (elm_no == HAMMER_ALIST_BLOCK_NONE) {
1611                 alloc_new_buffer(cluster, live,
1612                                  HAMMER_FSBUF_BTREE, HAMMER_BTREE_NODES,
1613                                  cluster->ondisk->idx_index, errorp, &buffer);
1614                 elm_no = hammer_alist_alloc(live, 1);
1615                 if (elm_no == HAMMER_ALIST_BLOCK_NONE) {
1616                         *errorp = ENOSPC;
1617                         if (buffer)
1618                                 hammer_rel_buffer(buffer, 0);
1619                         hammer_modify_cluster_done(cluster);
1620                         return(NULL);
1621                 }
1622         }
1623         cluster->ondisk->idx_index = elm_no;
1624         KKASSERT((elm_no & HAMMER_FSBUF_BLKMASK) < HAMMER_BTREE_NODES);
1625
1626         /*
1627          * Load and return the B-Tree element
1628          */
1629         buf_no = elm_no / HAMMER_FSBUF_MAXBLKS;
1630         node_offset = buf_no * HAMMER_BUFSIZE +
1631                       offsetof(union hammer_fsbuf_ondisk,
1632                                btree.nodes[elm_no & HAMMER_FSBUF_BLKMASK]);
1633         node = hammer_get_node(cluster, node_offset, errorp);
1634         if (node) {
1635                 hammer_modify_node(node);
1636                 bzero(node->ondisk, sizeof(*node->ondisk));
1637                 hammer_modify_node_done(node);
1638         } else {
1639                 hammer_alist_free(live, elm_no, 1);
1640                 hammer_rel_node(node);
1641                 node = NULL;
1642         }
1643         hammer_modify_cluster_done(cluster);
1644         if (buffer)
1645                 hammer_rel_buffer(buffer, 0);
1646         return(node);
1647 }
1648
1649 void *
1650 hammer_alloc_data(hammer_cluster_t cluster, int32_t bytes,
1651                   int *errorp, struct hammer_buffer **bufferp)
1652 {
1653         hammer_buffer_t buffer;
1654         hammer_alist_t live;
1655         int32_t elm_no;
1656         int32_t buf_no;
1657         int32_t nblks;
1658         void *item;
1659
1660         /*
1661          * Deal with large data blocks.  The blocksize is HAMMER_BUFSIZE
1662          * for these allocations.
1663          */
1664         hammer_modify_cluster(cluster);
1665         if ((bytes & HAMMER_BUFMASK) == 0) {
1666                 nblks = bytes / HAMMER_BUFSIZE;
1667                 /* only one block allowed for now (so buffer can hold it) */
1668                 KKASSERT(nblks == 1);
1669
1670                 buf_no = hammer_alloc_master(cluster, nblks,
1671                                              cluster->ondisk->idx_ldata, 1);
1672                 if (buf_no == HAMMER_ALIST_BLOCK_NONE) {
1673                         *errorp = ENOSPC;
1674                         hammer_modify_cluster_done(cluster);
1675                         return(NULL);
1676                 }
1677                 hammer_adjust_stats(cluster, HAMMER_FSBUF_DATA, nblks);
1678                 cluster->ondisk->idx_ldata = buf_no;
1679                 hammer_modify_cluster_done(cluster);
1680                 buffer = *bufferp;
1681                 *bufferp = hammer_get_buffer(cluster, buf_no, -1, errorp);
1682                 if (buffer)
1683                         hammer_rel_buffer(buffer, 0);
1684                 buffer = *bufferp;
1685                 return(buffer->ondisk);
1686         }
1687
1688         /*
1689          * Allocate a data element.  The block size is HAMMER_DATA_BLKSIZE
1690          * (64 bytes) for these allocations.
1691          */
1692         nblks = (bytes + HAMMER_DATA_BLKMASK) & ~HAMMER_DATA_BLKMASK;
1693         nblks /= HAMMER_DATA_BLKSIZE;
1694         live = &cluster->alist_mdata;
1695         elm_no = hammer_alist_alloc_fwd(live, nblks, cluster->ondisk->idx_data);
1696         if (elm_no == HAMMER_ALIST_BLOCK_NONE)
1697                 elm_no = hammer_alist_alloc_fwd(live, 1, 0);
1698         if (elm_no == HAMMER_ALIST_BLOCK_NONE) {
1699                 alloc_new_buffer(cluster, live,
1700                                  HAMMER_FSBUF_DATA, HAMMER_DATA_NODES,
1701                                  cluster->ondisk->idx_data, errorp, bufferp);
1702                 elm_no = hammer_alist_alloc(live, nblks);
1703                 if (elm_no == HAMMER_ALIST_BLOCK_NONE) {
1704                         *errorp = ENOSPC;
1705                         hammer_modify_cluster_done(cluster);
1706                         return(NULL);
1707                 }
1708         }
1709         cluster->ondisk->idx_index = elm_no;
1710         hammer_modify_cluster_done(cluster);
1711
1712         /*
1713          * Load and return the B-Tree element
1714          */
1715         buf_no = elm_no / HAMMER_FSBUF_MAXBLKS;
1716         buffer = *bufferp;
1717         if (buffer == NULL || buffer->cluster != cluster ||
1718             buffer->buf_no != buf_no) {
1719                 if (buffer)
1720                         hammer_rel_buffer(buffer, 0);
1721                 buffer = hammer_get_buffer(cluster, buf_no, 0, errorp);
1722                 *bufferp = buffer;
1723         }
1724         KKASSERT(buffer->ondisk->head.buf_type == HAMMER_FSBUF_DATA);
1725         KKASSERT((elm_no & HAMMER_FSBUF_BLKMASK) < HAMMER_DATA_NODES);
1726         hammer_modify_buffer(buffer);
1727         item = &buffer->ondisk->data.data[elm_no & HAMMER_FSBUF_BLKMASK];
1728         bzero(item, nblks * HAMMER_DATA_BLKSIZE);
1729         hammer_modify_buffer_done(buffer);
1730         *errorp = 0;
1731         return(item);
1732 }
1733
1734 void *
1735 hammer_alloc_record(hammer_cluster_t cluster,
1736                     int *errorp, struct hammer_buffer **bufferp)
1737 {
1738         hammer_buffer_t buffer;
1739         hammer_alist_t live;
1740         int32_t elm_no;
1741         int32_t buf_no;
1742         void *item;
1743
1744         /*
1745          * Allocate a record element
1746          */
1747         hammer_modify_cluster(cluster);
1748         live = &cluster->alist_record;
1749         elm_no = hammer_alist_alloc_rev(live, 1, cluster->ondisk->idx_record);
1750         if (elm_no == HAMMER_ALIST_BLOCK_NONE)
1751                 elm_no = hammer_alist_alloc_rev(live, 1,HAMMER_ALIST_BLOCK_MAX);
1752         if (elm_no == HAMMER_ALIST_BLOCK_NONE) {
1753                 alloc_new_buffer(cluster, live,
1754                                  HAMMER_FSBUF_RECORDS, HAMMER_RECORD_NODES,
1755                                  cluster->ondisk->idx_record, errorp, bufferp);
1756                 elm_no = hammer_alist_alloc_rev(live, 1,HAMMER_ALIST_BLOCK_MAX);
1757                 kprintf("hammer_alloc_record elm again %08x\n", elm_no);
1758                 if (elm_no == HAMMER_ALIST_BLOCK_NONE) {
1759                         *errorp = ENOSPC;
1760                         hammer_modify_cluster_done(cluster);
1761                         return(NULL);
1762                 }
1763         }
1764         cluster->ondisk->idx_record = elm_no;
1765         hammer_modify_cluster_done(cluster);
1766
1767         /*
1768          * Load and return the record element
1769          */
1770         buf_no = elm_no / HAMMER_FSBUF_MAXBLKS;
1771         buffer = *bufferp;
1772         if (buffer == NULL || buffer->cluster != cluster ||
1773             buffer->buf_no != buf_no) {
1774                 if (buffer)
1775                         hammer_rel_buffer(buffer, 0);
1776                 buffer = hammer_get_buffer(cluster, buf_no, 0, errorp);
1777                 *bufferp = buffer;
1778         }
1779         KKASSERT(buffer->ondisk->head.buf_type == HAMMER_FSBUF_RECORDS);
1780         KKASSERT((elm_no & HAMMER_FSBUF_BLKMASK) < HAMMER_RECORD_NODES);
1781         hammer_modify_buffer(buffer);
1782         item = &buffer->ondisk->record.recs[elm_no & HAMMER_FSBUF_BLKMASK];
1783         bzero(item, sizeof(union hammer_record_ondisk));
1784         hammer_modify_buffer_done(buffer);
1785         *errorp = 0;
1786         return(item);
1787 }
1788
1789 void
1790 hammer_free_data_ptr(hammer_buffer_t buffer, void *data, int bytes)
1791 {
1792         int32_t elm_no;
1793         int32_t nblks;
1794         hammer_alist_t live;
1795
1796         hammer_modify_cluster(buffer->cluster);
1797         if ((bytes & HAMMER_BUFMASK) == 0) {
1798                 nblks = bytes / HAMMER_BUFSIZE;
1799                 KKASSERT(nblks == 1 && data == (void *)buffer->ondisk);
1800                 hammer_alist_free(&buffer->cluster->alist_master,
1801                                   buffer->buf_no, nblks);
1802                 hammer_adjust_stats(buffer->cluster, HAMMER_FSBUF_DATA, -nblks);
1803                 hammer_modify_cluster_done(buffer->cluster);
1804                 return;
1805         }
1806
1807         elm_no = ((char *)data - (char *)buffer->ondisk->data.data) /
1808                  HAMMER_DATA_BLKSIZE;
1809         KKASSERT(elm_no >= 0 && elm_no < HAMMER_DATA_NODES);
1810         elm_no += buffer->buf_no * HAMMER_FSBUF_MAXBLKS;
1811         nblks = (bytes + HAMMER_DATA_BLKMASK) & ~HAMMER_DATA_BLKMASK;
1812         nblks /= HAMMER_DATA_BLKSIZE;
1813         live = &buffer->cluster->alist_mdata;
1814         hammer_alist_free(live, elm_no, nblks);
1815         hammer_modify_cluster_done(buffer->cluster);
1816 }
1817
1818 void
1819 hammer_free_record_ptr(hammer_buffer_t buffer, union hammer_record_ondisk *rec)
1820 {
1821         int32_t elm_no;
1822         hammer_alist_t live;
1823
1824         hammer_modify_cluster(buffer->cluster);
1825         elm_no = rec - &buffer->ondisk->record.recs[0];
1826         KKASSERT(elm_no >= 0 && elm_no < HAMMER_BTREE_NODES);
1827         elm_no += buffer->buf_no * HAMMER_FSBUF_MAXBLKS;
1828         live = &buffer->cluster->alist_record;
1829         hammer_alist_free(live, elm_no, 1);
1830         hammer_modify_cluster_done(buffer->cluster);
1831 }
1832
1833 void
1834 hammer_free_btree(hammer_cluster_t cluster, int32_t bclu_offset)
1835 {
1836         const int32_t blksize = sizeof(struct hammer_node_ondisk);
1837         int32_t fsbuf_offset = bclu_offset & HAMMER_BUFMASK;
1838         hammer_alist_t live;
1839         int32_t elm_no;
1840
1841         hammer_modify_cluster(cluster);
1842         elm_no = bclu_offset / HAMMER_BUFSIZE * HAMMER_FSBUF_MAXBLKS;
1843         fsbuf_offset -= offsetof(union hammer_fsbuf_ondisk, btree.nodes[0]);
1844         live = &cluster->alist_btree;
1845         KKASSERT(fsbuf_offset >= 0 && fsbuf_offset % blksize == 0);
1846         elm_no += fsbuf_offset / blksize;
1847         hammer_alist_free(live, elm_no, 1);
1848         hammer_modify_cluster_done(cluster);
1849 }
1850
1851 void
1852 hammer_free_data(hammer_cluster_t cluster, int32_t bclu_offset, int32_t bytes)
1853 {
1854         const int32_t blksize = HAMMER_DATA_BLKSIZE;
1855         int32_t fsbuf_offset = bclu_offset & HAMMER_BUFMASK;
1856         hammer_alist_t live;
1857         int32_t elm_no;
1858         int32_t buf_no;
1859         int32_t nblks;
1860
1861         hammer_modify_cluster(cluster);
1862         if ((bytes & HAMMER_BUFMASK) == 0) {
1863                 nblks = bytes / HAMMER_BUFSIZE;
1864                 KKASSERT(nblks == 1 && (bclu_offset & HAMMER_BUFMASK) == 0);
1865                 buf_no = bclu_offset / HAMMER_BUFSIZE;
1866                 hammer_alist_free(&cluster->alist_master, buf_no, nblks);
1867                 hammer_adjust_stats(cluster, HAMMER_FSBUF_DATA, -nblks);
1868                 hammer_modify_cluster_done(cluster);
1869                 return;
1870         }
1871
1872         elm_no = bclu_offset / HAMMER_BUFSIZE * HAMMER_FSBUF_MAXBLKS;
1873         fsbuf_offset -= offsetof(union hammer_fsbuf_ondisk, data.data[0][0]);
1874         live = &cluster->alist_mdata;
1875         nblks = (bytes + HAMMER_DATA_BLKMASK) & ~HAMMER_DATA_BLKMASK;
1876         nblks /= HAMMER_DATA_BLKSIZE;
1877         KKASSERT(fsbuf_offset >= 0 && fsbuf_offset % blksize == 0);
1878         elm_no += fsbuf_offset / blksize;
1879         hammer_alist_free(live, elm_no, nblks);
1880         hammer_modify_cluster_done(cluster);
1881 }
1882
1883 void
1884 hammer_free_record(hammer_cluster_t cluster, int32_t bclu_offset)
1885 {
1886         const int32_t blksize = sizeof(union hammer_record_ondisk);
1887         int32_t fsbuf_offset = bclu_offset & HAMMER_BUFMASK;
1888         hammer_alist_t live;
1889         int32_t elm_no;
1890
1891         hammer_modify_cluster(cluster);
1892         elm_no = bclu_offset / HAMMER_BUFSIZE * HAMMER_FSBUF_MAXBLKS;
1893         fsbuf_offset -= offsetof(union hammer_fsbuf_ondisk, record.recs[0]);
1894         live = &cluster->alist_record;
1895         KKASSERT(fsbuf_offset >= 0 && fsbuf_offset % blksize == 0);
1896         elm_no += fsbuf_offset / blksize;
1897         hammer_alist_free(live, elm_no, 1);
1898         hammer_modify_cluster_done(cluster);
1899 }
1900
1901
1902 /*
1903  * Allocate a new filesystem buffer and assign it to the specified
1904  * filesystem buffer type.  The new buffer will be added to the
1905  * type-specific A-list and initialized.
1906  */
1907 static void
1908 alloc_new_buffer(hammer_cluster_t cluster, hammer_alist_t live,
1909                  u_int64_t type, int32_t nelements,
1910                  int start, int *errorp, struct hammer_buffer **bufferp)
1911 {
1912         hammer_buffer_t buffer;
1913         int32_t buf_no;
1914         int isfwd;
1915
1916         if (*bufferp)
1917                 hammer_rel_buffer(*bufferp, 0);
1918         *bufferp = NULL;
1919
1920         start = start / HAMMER_FSBUF_MAXBLKS;   /* convert to buf_no */
1921         isfwd = (type != HAMMER_FSBUF_RECORDS);
1922         buf_no = hammer_alloc_master(cluster, 1, start, isfwd);
1923         if (buf_no == HAMMER_ALIST_BLOCK_NONE) {
1924                 *errorp = ENOSPC;
1925                 return;
1926         }
1927
1928         /*
1929          * The new buffer must be initialized (type != 0) regardless of
1930          * whether we already have it cached or not, so don't try to
1931          * optimize the cached buffer check.  Just call hammer_get_buffer().
1932          */
1933         buffer = hammer_get_buffer(cluster, buf_no, type, errorp);
1934         *bufferp = buffer;
1935
1936         /*
1937          * Finally, do a meta-free of the buffer's elements into the
1938          * type-specific A-list and update our statistics to reflect
1939          * the allocation.
1940          */
1941         if (buffer) {
1942                 kprintf("alloc_new_buffer buf_no %d type %016llx nelms %d\n",
1943                         buf_no, type, nelements);
1944                 hammer_modify_buffer(buffer);  /*XXX*/
1945                 hammer_alist_free(live, buf_no * HAMMER_FSBUF_MAXBLKS,
1946                                   nelements);
1947                 hammer_modify_buffer_done(buffer);  /*XXX*/
1948                 hammer_adjust_stats(cluster, type, 1);
1949         }
1950 }
1951
1952 /*
1953  * Sync dirty buffers to the media
1954  */
1955
1956 static int hammer_sync_scan1(struct mount *mp, struct vnode *vp, void *data);
1957 static int hammer_sync_scan2(struct mount *mp, struct vnode *vp, void *data);
1958
1959 int
1960 hammer_sync_hmp(hammer_mount_t hmp, int waitfor)
1961 {
1962         struct hammer_sync_info info;
1963
1964         info.error = 0;
1965         info.waitfor = waitfor;
1966
1967         vmntvnodescan(hmp->mp, VMSC_GETVP|VMSC_NOWAIT,
1968                       hammer_sync_scan1, hammer_sync_scan2, &info);
1969
1970         RB_SCAN(hammer_vol_rb_tree, &hmp->rb_vols_root, NULL,
1971                 hammer_sync_volume, &info);
1972         return(info.error);
1973 }
1974
1975 static int
1976 hammer_sync_scan1(struct mount *mp, struct vnode *vp, void *data)
1977 {
1978         struct hammer_inode *ip;
1979
1980         ip = VTOI(vp);
1981         if (vp->v_type == VNON || ((ip->flags & HAMMER_INODE_MODMASK) == 0 &&
1982             RB_EMPTY(&vp->v_rbdirty_tree))) {
1983                 return(-1);
1984         }
1985         return(0);
1986 }
1987
1988 static int
1989 hammer_sync_scan2(struct mount *mp, struct vnode *vp, void *data)
1990 {
1991         struct hammer_sync_info *info = data;
1992         struct hammer_inode *ip;
1993         int error;
1994
1995         ip = VTOI(vp);
1996         if (vp->v_type == VNON || vp->v_type == VBAD ||
1997             ((ip->flags & HAMMER_INODE_MODMASK) == 0 &&
1998              RB_EMPTY(&vp->v_rbdirty_tree))) {
1999                 return(0);
2000         }
2001         if (vp->v_type != VCHR) {
2002                 error = VOP_FSYNC(vp, info->waitfor);
2003                 if (error)
2004                         info->error = error;
2005         }
2006         return(0);
2007 }
2008
2009 int
2010 hammer_sync_volume(hammer_volume_t volume, void *data)
2011 {
2012         struct hammer_sync_info *info = data;
2013
2014         RB_SCAN(hammer_clu_rb_tree, &volume->rb_clus_root, NULL,
2015                 hammer_sync_cluster, info);
2016         if (hammer_ref_volume(volume) == 0) {
2017                 hammer_io_flush(&volume->io, info);
2018                 hammer_rel_volume(volume, 0);
2019         }
2020         return(0);
2021 }
2022
2023 int
2024 hammer_sync_cluster(hammer_cluster_t cluster, void *data)
2025 {
2026         struct hammer_sync_info *info = data;
2027
2028         RB_SCAN(hammer_buf_rb_tree, &cluster->rb_bufs_root, NULL,
2029                 hammer_sync_buffer, info);
2030         switch(cluster->state) {
2031         case HAMMER_CLUSTER_OPEN:
2032         case HAMMER_CLUSTER_IDLE:
2033                 if (hammer_ref_cluster(cluster) == 0) {
2034                         hammer_io_flush(&cluster->io, info);
2035                         hammer_rel_cluster(cluster, 0);
2036                 }
2037                 break;
2038         default:
2039                 break;
2040         }
2041         return(0);
2042 }
2043
2044 int
2045 hammer_sync_buffer(hammer_buffer_t buffer, void *data)
2046 {
2047         struct hammer_sync_info *info = data;
2048
2049         if (hammer_ref_buffer(buffer) == 0) {
2050                 hammer_lock_ex(&buffer->io.lock);
2051                 hammer_flush_buffer_nodes(buffer);
2052                 hammer_unlock(&buffer->io.lock);
2053                 hammer_io_flush(&buffer->io, info);
2054                 hammer_rel_buffer(buffer, 0);
2055         }
2056         return(0);
2057 }
2058
2059 /*
2060  * Generic buffer initialization
2061  */
2062 static void
2063 initbuffer(hammer_alist_t live, hammer_fsbuf_head_t head, u_int64_t type)
2064 {
2065         head->buf_type = type;
2066         hammer_alist_init(live);
2067 }
2068
2069 /*
2070  * Calculate the cluster's offset in the volume.  This calculation is
2071  * slightly more complex when using superclusters because superclusters
2072  * are grouped in blocks of 16, followed by 16 x N clusters where N
2073  * is the number of clusters a supercluster can manage.
2074  */
2075 static int64_t
2076 calculate_cluster_offset(hammer_volume_t volume, int32_t clu_no)
2077 {
2078         int32_t scl_group;
2079         int64_t scl_group_size;
2080         int64_t off;
2081
2082         if (volume->vol_flags & HAMMER_VOLF_USINGSUPERCL) {
2083                 scl_group = clu_no / HAMMER_VOL_SUPERCLUSTER_GROUP /
2084                             HAMMER_SCL_MAXCLUSTERS;
2085                 scl_group_size = 
2086                             ((int64_t)HAMMER_BUFSIZE *
2087                              HAMMER_VOL_SUPERCLUSTER_GROUP) +
2088                             ((int64_t)HAMMER_VOL_SUPERCLUSTER_GROUP *
2089                              volume->vol_clsize * HAMMER_SCL_MAXCLUSTERS);
2090                 scl_group_size += 
2091                             HAMMER_VOL_SUPERCLUSTER_GROUP * HAMMER_BUFSIZE;
2092
2093                 off = volume->cluster_base +
2094                       scl_group * scl_group_size +
2095                       (HAMMER_BUFSIZE * HAMMER_VOL_SUPERCLUSTER_GROUP) +
2096                       ((int64_t)clu_no % ((int64_t)HAMMER_SCL_MAXCLUSTERS *
2097                        HAMMER_VOL_SUPERCLUSTER_GROUP))
2098                       * volume->vol_clsize;
2099         } else {
2100                 off = volume->cluster_base +
2101                       (int64_t)clu_no * volume->vol_clsize;
2102         }
2103         return(off);
2104 }
2105
2106 /*
2107  * Calculate a super-cluster's offset in the volume.
2108  */
2109 static int64_t
2110 calculate_supercl_offset(hammer_volume_t volume, int32_t scl_no)
2111 {
2112         int64_t off;
2113         int32_t scl_group;
2114         int64_t scl_group_size;
2115
2116         KKASSERT (volume->vol_flags & HAMMER_VOLF_USINGSUPERCL);
2117         scl_group = scl_no / HAMMER_VOL_SUPERCLUSTER_GROUP;
2118         if (scl_group) {
2119                 scl_group_size = 
2120                             ((int64_t)HAMMER_BUFSIZE *
2121                              HAMMER_VOL_SUPERCLUSTER_GROUP) +
2122                             ((int64_t)HAMMER_VOL_SUPERCLUSTER_GROUP *
2123                              volume->vol_clsize * HAMMER_SCL_MAXCLUSTERS);
2124                 scl_group_size += 
2125                             HAMMER_VOL_SUPERCLUSTER_GROUP * HAMMER_BUFSIZE;
2126                 off = volume->cluster_base + (scl_group * scl_group_size) +
2127                       (scl_no % HAMMER_VOL_SUPERCLUSTER_GROUP) * HAMMER_BUFSIZE;
2128         } else {
2129                 off = volume->cluster_base + (scl_no * HAMMER_BUFSIZE);
2130         }
2131         return(off);
2132 }
2133
2134 /*
2135  *
2136  *
2137  */
2138 static int32_t
2139 hammer_alloc_master(hammer_cluster_t cluster, int nblks,
2140                     int32_t start, int isfwd)
2141 {
2142         int32_t buf_no;
2143
2144         hammer_modify_cluster(cluster);
2145         if (isfwd) {
2146                 buf_no = hammer_alist_alloc_fwd(&cluster->alist_master,
2147                                                 nblks, start);
2148                 if (buf_no == HAMMER_ALIST_BLOCK_NONE) {
2149                         buf_no = hammer_alist_alloc_fwd(&cluster->alist_master,
2150                                                 nblks, 0);
2151                 }
2152         } else {
2153                 buf_no = hammer_alist_alloc_rev(&cluster->alist_master,
2154                                                 nblks, start);
2155                 if (buf_no == HAMMER_ALIST_BLOCK_NONE) {
2156                         buf_no = hammer_alist_alloc_rev(&cluster->alist_master,
2157                                                 nblks, HAMMER_ALIST_BLOCK_MAX);
2158                 }
2159         }
2160         hammer_modify_cluster_done(cluster);
2161
2162         /*
2163          * Recover space from empty record, b-tree, and data a-lists.
2164          */
2165
2166         return(buf_no);
2167 }
2168
2169 /*
2170  * Adjust allocation statistics
2171  */
2172 static void
2173 hammer_adjust_stats(hammer_cluster_t cluster, u_int64_t buf_type, int nblks)
2174 {
2175         hammer_modify_cluster(cluster);
2176         hammer_modify_volume(cluster->volume);
2177         hammer_modify_volume(cluster->volume->hmp->rootvol);
2178
2179         switch(buf_type) {
2180         case HAMMER_FSBUF_BTREE:
2181                 cluster->ondisk->stat_idx_bufs += nblks;
2182                 cluster->volume->ondisk->vol_stat_idx_bufs += nblks;
2183                 cluster->volume->hmp->rootvol->ondisk->vol0_stat_idx_bufs += nblks;
2184                 break;
2185         case HAMMER_FSBUF_DATA:
2186                 cluster->ondisk->stat_data_bufs += nblks;
2187                 cluster->volume->ondisk->vol_stat_data_bufs += nblks;
2188                 cluster->volume->hmp->rootvol->ondisk->vol0_stat_data_bufs += nblks;
2189                 break;
2190         case HAMMER_FSBUF_RECORDS:
2191                 cluster->ondisk->stat_rec_bufs += nblks;
2192                 cluster->volume->ondisk->vol_stat_rec_bufs += nblks;
2193                 cluster->volume->hmp->rootvol->ondisk->vol0_stat_rec_bufs += nblks;
2194                 break;
2195         }
2196         hammer_modify_cluster_done(cluster);
2197         hammer_modify_volume_done(cluster->volume);
2198         hammer_modify_volume_done(cluster->volume->hmp->rootvol);
2199 }
2200
2201 /*
2202  * A-LIST SUPPORT
2203  *
2204  * Setup the parameters for the various A-lists we use in hammer.  The
2205  * supercluster A-list must be chained to the cluster A-list and cluster
2206  * slave A-lists are chained to buffer A-lists.
2207  *
2208  * See hammer_init_alist_config() below.
2209  */
2210
2211 /*
2212  * A-LIST - cluster recursion into a filesystem buffer
2213  */
2214 static int
2215 buffer_alist_init(void *info, int32_t blk, int32_t radix)
2216 {
2217         return(0);
2218 #if 0
2219         hammer_cluster_t cluster = info;
2220         hammer_buffer_t buffer;
2221         int32_t buf_no;
2222         int error = 0;
2223
2224         /*
2225          * Calculate the buffer number, initialize based on the buffer type.
2226          * The buffer has already been allocated so assert that it has been
2227          * initialized.
2228          */
2229         buf_no = blk / HAMMER_FSBUF_MAXBLKS;
2230         buffer = hammer_get_buffer(cluster, buf_no, 0, &error);
2231         if (buffer) {
2232                 hammer_adjust_stats(cluster, buffer->ondisk->head.buf_type, 1);
2233                 hammer_rel_buffer(buffer, 0);
2234         }
2235         return (error);
2236 #endif
2237 }
2238
2239 static int
2240 buffer_alist_destroy(void *info, int32_t blk, int32_t radix)
2241 {
2242         return(0);
2243 #if 0
2244         hammer_cluster_t cluster = info;
2245         hammer_buffer_t buffer;
2246         int32_t buf_no;
2247         int error = 0;
2248
2249         /*
2250          * Calculate the buffer number, initialize based on the buffer type.
2251          * The buffer has already been allocated so assert that it has been
2252          * initialized.
2253          */
2254         buf_no = blk / HAMMER_FSBUF_MAXBLKS;
2255         buffer = hammer_get_buffer(cluster, buf_no, 0, &error);
2256         if (buffer) {
2257                 hammer_adjust_stats(cluster, buffer->ondisk->head.buf_type, -1);
2258                 hammer_rel_buffer(buffer, 0);
2259         }
2260         return (error);
2261 #endif
2262 }
2263
2264 /*
2265  * Note: atblk can be negative and atblk - blk can go negative.
2266  */
2267 static int
2268 buffer_alist_alloc_fwd(void *info, int32_t blk, int32_t radix,
2269                       int32_t count, int32_t atblk, int32_t *fullp)
2270 {
2271         hammer_cluster_t cluster = info;
2272         hammer_buffer_t buffer;
2273         int32_t buf_no;
2274         int32_t r;
2275         int error = 0;
2276
2277         buf_no = blk / HAMMER_FSBUF_MAXBLKS;
2278         buffer = hammer_get_buffer(cluster, buf_no, 0, &error);
2279         if (buffer) {
2280                 KKASSERT(buffer->ondisk->head.buf_type != 0);
2281
2282                 hammer_modify_buffer(buffer);
2283                 r = hammer_alist_alloc_fwd(&buffer->alist, count, atblk - blk);
2284                 if (r != HAMMER_ALIST_BLOCK_NONE)
2285                         r += blk;
2286                 hammer_modify_buffer_done(buffer);
2287                 *fullp = hammer_alist_isfull(&buffer->alist);
2288                 hammer_rel_buffer(buffer, 0);
2289         } else {
2290                 r = HAMMER_ALIST_BLOCK_NONE;
2291         }
2292         return(r);
2293 }
2294
2295 static int
2296 buffer_alist_alloc_rev(void *info, int32_t blk, int32_t radix,
2297                       int32_t count, int32_t atblk, int32_t *fullp)
2298 {
2299         hammer_cluster_t cluster = info;
2300         hammer_buffer_t buffer;
2301         int32_t buf_no;
2302         int32_t r;
2303         int error = 0;
2304
2305         buf_no = blk / HAMMER_FSBUF_MAXBLKS;
2306         buffer = hammer_get_buffer(cluster, buf_no, 0, &error);
2307         if (buffer) {
2308                 KKASSERT(buffer->ondisk->head.buf_type != 0);
2309                 hammer_modify_buffer(buffer);
2310                 r = hammer_alist_alloc_rev(&buffer->alist, count, atblk - blk);
2311                 if (r != HAMMER_ALIST_BLOCK_NONE)
2312                         r += blk;
2313                 hammer_modify_buffer_done(buffer);
2314                 *fullp = hammer_alist_isfull(&buffer->alist);
2315                 hammer_rel_buffer(buffer, 0);
2316         } else {
2317                 r = HAMMER_ALIST_BLOCK_NONE;
2318                 *fullp = 0;
2319         }
2320         return(r);
2321 }
2322
2323 static void
2324 buffer_alist_free(void *info, int32_t blk, int32_t radix,
2325                  int32_t base_blk, int32_t count, int32_t *emptyp)
2326 {
2327         hammer_cluster_t cluster = info;
2328         hammer_buffer_t buffer;
2329         int32_t buf_no;
2330         int error = 0;
2331
2332         buf_no = blk / HAMMER_FSBUF_MAXBLKS;
2333         buffer = hammer_get_buffer(cluster, buf_no, 0, &error);
2334         if (buffer) {
2335                 KKASSERT(buffer->ondisk->head.buf_type != 0);
2336                 hammer_modify_buffer(buffer);
2337                 hammer_alist_free(&buffer->alist, base_blk, count);
2338                 hammer_modify_buffer_done(buffer);
2339                 *emptyp = hammer_alist_isempty(&buffer->alist);
2340                 /* XXX don't bother updating the buffer is completely empty? */
2341                 hammer_rel_buffer(buffer, 0);
2342         } else {
2343                 *emptyp = 0;
2344         }
2345 }
2346
2347 static void
2348 buffer_alist_print(void *info, int32_t blk, int32_t radix, int tab)
2349 {
2350 }
2351
2352 /*
2353  * A-LIST - super-cluster recursion into a cluster and cluster recursion
2354  * into a filesystem buffer.  A-List's are mostly self-contained entities,
2355  * but callbacks must be installed to recurse from one A-List to another.
2356  *
2357  * Implementing these callbacks allows us to operate a multi-layered A-List
2358  * as a single entity.
2359  */
2360 static int
2361 super_alist_init(void *info, int32_t blk, int32_t radix)
2362 {
2363         hammer_volume_t volume = info;
2364         hammer_supercl_t supercl;
2365         int32_t scl_no;
2366         int error = 0;
2367
2368         /*
2369          * Calculate the super-cluster number containing the cluster (blk)
2370          * and obtain the super-cluster buffer.
2371          */
2372         scl_no = blk / HAMMER_SCL_MAXCLUSTERS;
2373         supercl = hammer_get_supercl(volume, scl_no, &error, 1);
2374         if (supercl)
2375                 hammer_rel_supercl(supercl, 0);
2376         return (error);
2377 }
2378
2379 static int
2380 super_alist_destroy(void *info, int32_t blk, int32_t radix)
2381 {
2382         return(0);
2383 }
2384
2385 static int
2386 super_alist_alloc_fwd(void *info, int32_t blk, int32_t radix,
2387                       int32_t count, int32_t atblk, int32_t *fullp)
2388 {
2389         hammer_volume_t volume = info;
2390         hammer_supercl_t supercl;
2391         int32_t scl_no;
2392         int32_t r;
2393         int error = 0;
2394
2395         scl_no = blk / HAMMER_SCL_MAXCLUSTERS;
2396         supercl = hammer_get_supercl(volume, scl_no, &error, 0);
2397         if (supercl) {
2398                 hammer_modify_supercl(supercl);
2399                 r = hammer_alist_alloc_fwd(&supercl->alist, count, atblk - blk);
2400                 if (r != HAMMER_ALIST_BLOCK_NONE)
2401                         r += blk;
2402                 hammer_modify_supercl_done(supercl);
2403                 *fullp = hammer_alist_isfull(&supercl->alist);
2404                 hammer_rel_supercl(supercl, 0);
2405         } else {
2406                 r = HAMMER_ALIST_BLOCK_NONE;
2407                 *fullp = 0;
2408         }
2409         return(r);
2410 }
2411
2412 static int
2413 super_alist_alloc_rev(void *info, int32_t blk, int32_t radix,
2414                       int32_t count, int32_t atblk, int32_t *fullp)
2415 {
2416         hammer_volume_t volume = info;
2417         hammer_supercl_t supercl;
2418         int32_t scl_no;
2419         int32_t r;
2420         int error = 0;
2421
2422         scl_no = blk / HAMMER_SCL_MAXCLUSTERS;
2423         supercl = hammer_get_supercl(volume, scl_no, &error, 0);
2424         if (supercl) {
2425                 hammer_modify_supercl(supercl);
2426                 r = hammer_alist_alloc_rev(&supercl->alist, count, atblk - blk);
2427                 if (r != HAMMER_ALIST_BLOCK_NONE)
2428                         r += blk;
2429                 hammer_modify_supercl_done(supercl);
2430                 *fullp = hammer_alist_isfull(&supercl->alist);
2431                 hammer_rel_supercl(supercl, 0);
2432         } else { 
2433                 r = HAMMER_ALIST_BLOCK_NONE;
2434                 *fullp = 0;
2435         }
2436         return(r);
2437 }
2438
2439 static void
2440 super_alist_free(void *info, int32_t blk, int32_t radix,
2441                  int32_t base_blk, int32_t count, int32_t *emptyp)
2442 {
2443         hammer_volume_t volume = info;
2444         hammer_supercl_t supercl;
2445         int32_t scl_no;
2446         int error = 0;
2447
2448         scl_no = blk / HAMMER_SCL_MAXCLUSTERS;
2449         supercl = hammer_get_supercl(volume, scl_no, &error, 0);
2450         if (supercl) {
2451                 hammer_modify_supercl(supercl);
2452                 hammer_alist_free(&supercl->alist, base_blk, count);
2453                 hammer_modify_supercl_done(supercl);
2454                 *emptyp = hammer_alist_isempty(&supercl->alist);
2455                 hammer_rel_supercl(supercl, 0);
2456         } else {
2457                 *emptyp = 0;
2458         }
2459 }
2460
2461 static void
2462 super_alist_print(void *info, int32_t blk, int32_t radix, int tab)
2463 {
2464 }
2465
2466 void
2467 hammer_init_alist_config(void)
2468 {
2469         hammer_alist_config_t config;
2470
2471         hammer_alist_template(&Buf_alist_config, HAMMER_FSBUF_MAXBLKS,
2472                               1, HAMMER_FSBUF_METAELMS);
2473         hammer_alist_template(&Vol_normal_alist_config, HAMMER_VOL_MAXCLUSTERS,
2474                               1, HAMMER_VOL_METAELMS_1LYR);
2475         hammer_alist_template(&Vol_super_alist_config,
2476                           HAMMER_VOL_MAXSUPERCLUSTERS * HAMMER_SCL_MAXCLUSTERS,
2477                               HAMMER_SCL_MAXCLUSTERS, HAMMER_VOL_METAELMS_2LYR);
2478         hammer_alist_template(&Supercl_alist_config, HAMMER_VOL_MAXCLUSTERS,
2479                               1, HAMMER_SUPERCL_METAELMS);
2480         hammer_alist_template(&Clu_master_alist_config, HAMMER_CLU_MAXBUFFERS,
2481                               1, HAMMER_CLU_MASTER_METAELMS);
2482         hammer_alist_template(&Clu_slave_alist_config,
2483                               HAMMER_CLU_MAXBUFFERS * HAMMER_FSBUF_MAXBLKS,
2484                               HAMMER_FSBUF_MAXBLKS, HAMMER_CLU_SLAVE_METAELMS);
2485
2486         config = &Vol_super_alist_config;
2487         config->bl_radix_init = super_alist_init;
2488         config->bl_radix_destroy = super_alist_destroy;
2489         config->bl_radix_alloc_fwd = super_alist_alloc_fwd;
2490         config->bl_radix_alloc_rev = super_alist_alloc_rev;
2491         config->bl_radix_free = super_alist_free;
2492         config->bl_radix_print = super_alist_print;
2493
2494         config = &Clu_slave_alist_config;
2495         config->bl_radix_init = buffer_alist_init;
2496         config->bl_radix_destroy = buffer_alist_destroy;
2497         config->bl_radix_alloc_fwd = buffer_alist_alloc_fwd;
2498         config->bl_radix_alloc_rev = buffer_alist_alloc_rev;
2499         config->bl_radix_free = buffer_alist_free;
2500         config->bl_radix_print = buffer_alist_print;
2501 }
2502