4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
26 * Copyright (c) 2013, 2019 by Delphix. All rights reserved.
27 * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved.
30 #include <sys/zfs_context.h>
33 #include <sys/dnode.h>
35 #include <sys/range_tree.h>
38 * Range trees are tree-based data structures that can be used to
39 * track free space or generally any space allocation information.
40 * A range tree keeps track of individual segments and automatically
41 * provides facilities such as adjacent extent merging and extent
42 * splitting in response to range add/remove requests.
44 * A range tree starts out completely empty, with no segments in it.
45 * Adding an allocation via range_tree_add to the range tree can either:
46 * 1) create a new extent
47 * 2) extend an adjacent extent
48 * 3) merge two adjacent extents
49 * Conversely, removing an allocation via range_tree_remove can:
50 * 1) completely remove an extent
51 * 2) shorten an extent (if the allocation was near one of its ends)
52 * 3) split an extent into two extents, in effect punching a hole
54 * A range tree is also capable of 'bridging' gaps when adding
55 * allocations. This is useful for cases when close proximity of
56 * allocations is an important detail that needs to be represented
57 * in the range tree. See range_tree_set_gap(). The default behavior
58 * is not to bridge gaps (i.e. the maximum allowed gap size is 0).
60 * In order to traverse a range tree, use either the range_tree_walk()
61 * or range_tree_vacate() functions.
63 * To obtain more accurate information on individual segment
64 * operations that the range tree performs "under the hood", you can
65 * specify a set of callbacks by passing a range_tree_ops_t structure
66 * to the range_tree_create function. Any callbacks that are non-NULL
67 * are then called at the appropriate times.
69 * The range tree code also supports a special variant of range trees
70 * that can bridge small gaps between segments. This kind of tree is used
71 * by the dsl scanning code to group I/Os into mostly sequential chunks to
72 * optimize disk performance. The code here attempts to do this with as
73 * little memory and computational overhead as possible. One limitation of
74 * this implementation is that segments of range trees with gaps can only
75 * support removing complete segments.
79 rs_copy(range_seg_t *src, range_seg_t *dest, range_tree_t *rt)
81 ASSERT3U(rt->rt_type, <=, RANGE_SEG_NUM_TYPES);
83 switch (rt->rt_type) {
85 size = sizeof (range_seg32_t);
88 size = sizeof (range_seg64_t);
91 size = sizeof (range_seg_gap_t);
96 bcopy(src, dest, size);
100 range_tree_stat_verify(range_tree_t *rt)
103 zfs_btree_index_t where;
104 uint64_t hist[RANGE_TREE_HISTOGRAM_SIZE] = { 0 };
107 for (rs = zfs_btree_first(&rt->rt_root, &where); rs != NULL;
108 rs = zfs_btree_next(&rt->rt_root, &where, &where)) {
109 uint64_t size = rs_get_end(rs, rt) - rs_get_start(rs, rt);
110 int idx = highbit64(size) - 1;
113 ASSERT3U(hist[idx], !=, 0);
116 for (i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++) {
117 if (hist[i] != rt->rt_histogram[i]) {
118 zfs_dbgmsg("i=%d, hist=%px, hist=%llu, rt_hist=%llu",
119 i, hist, hist[i], rt->rt_histogram[i]);
121 VERIFY3U(hist[i], ==, rt->rt_histogram[i]);
126 range_tree_stat_incr(range_tree_t *rt, range_seg_t *rs)
128 uint64_t size = rs_get_end(rs, rt) - rs_get_start(rs, rt);
129 int idx = highbit64(size) - 1;
133 sizeof (rt->rt_histogram) / sizeof (*rt->rt_histogram));
135 rt->rt_histogram[idx]++;
136 ASSERT3U(rt->rt_histogram[idx], !=, 0);
140 range_tree_stat_decr(range_tree_t *rt, range_seg_t *rs)
142 uint64_t size = rs_get_end(rs, rt) - rs_get_start(rs, rt);
143 int idx = highbit64(size) - 1;
147 sizeof (rt->rt_histogram) / sizeof (*rt->rt_histogram));
149 ASSERT3U(rt->rt_histogram[idx], !=, 0);
150 rt->rt_histogram[idx]--;
154 range_tree_seg32_compare(const void *x1, const void *x2)
156 const range_seg32_t *r1 = x1;
157 const range_seg32_t *r2 = x2;
159 ASSERT3U(r1->rs_start, <=, r1->rs_end);
160 ASSERT3U(r2->rs_start, <=, r2->rs_end);
162 return ((r1->rs_start >= r2->rs_end) - (r1->rs_end <= r2->rs_start));
166 range_tree_seg64_compare(const void *x1, const void *x2)
168 const range_seg64_t *r1 = x1;
169 const range_seg64_t *r2 = x2;
171 ASSERT3U(r1->rs_start, <=, r1->rs_end);
172 ASSERT3U(r2->rs_start, <=, r2->rs_end);
174 return ((r1->rs_start >= r2->rs_end) - (r1->rs_end <= r2->rs_start));
178 range_tree_seg_gap_compare(const void *x1, const void *x2)
180 const range_seg_gap_t *r1 = x1;
181 const range_seg_gap_t *r2 = x2;
183 ASSERT3U(r1->rs_start, <=, r1->rs_end);
184 ASSERT3U(r2->rs_start, <=, r2->rs_end);
186 return ((r1->rs_start >= r2->rs_end) - (r1->rs_end <= r2->rs_start));
190 range_tree_create_impl(range_tree_ops_t *ops, range_seg_type_t type, void *arg,
191 uint64_t start, uint64_t shift,
192 int (*zfs_btree_compare) (const void *, const void *),
195 range_tree_t *rt = kmem_zalloc(sizeof (range_tree_t), KM_SLEEP);
197 ASSERT3U(shift, <, 64);
198 ASSERT3U(type, <=, RANGE_SEG_NUM_TYPES);
200 int (*compare) (const void *, const void *);
203 size = sizeof (range_seg32_t);
204 compare = range_tree_seg32_compare;
207 size = sizeof (range_seg64_t);
208 compare = range_tree_seg64_compare;
211 size = sizeof (range_seg_gap_t);
212 compare = range_tree_seg_gap_compare;
215 panic("Invalid range seg type %d", type);
217 zfs_btree_create(&rt->rt_root, compare, size);
223 rt->rt_start = start;
224 rt->rt_shift = shift;
225 rt->rt_btree_compare = zfs_btree_compare;
227 if (rt->rt_ops != NULL && rt->rt_ops->rtop_create != NULL)
228 rt->rt_ops->rtop_create(rt, rt->rt_arg);
234 range_tree_create(range_tree_ops_t *ops, range_seg_type_t type,
235 void *arg, uint64_t start, uint64_t shift)
237 return (range_tree_create_impl(ops, type, arg, start, shift, NULL, 0));
241 range_tree_destroy(range_tree_t *rt)
243 VERIFY0(rt->rt_space);
245 if (rt->rt_ops != NULL && rt->rt_ops->rtop_destroy != NULL)
246 rt->rt_ops->rtop_destroy(rt, rt->rt_arg);
248 zfs_btree_destroy(&rt->rt_root);
249 kmem_free(rt, sizeof (*rt));
253 range_tree_adjust_fill(range_tree_t *rt, range_seg_t *rs, int64_t delta)
255 if (delta < 0 && delta * -1 >= rs_get_fill(rs, rt)) {
256 zfs_panic_recover("zfs: attempting to decrease fill to or "
257 "below 0; probable double remove in segment [%llx:%llx]",
258 (longlong_t)rs_get_start(rs, rt),
259 (longlong_t)rs_get_end(rs, rt));
261 if (rs_get_fill(rs, rt) + delta > rs_get_end(rs, rt) -
262 rs_get_start(rs, rt)) {
263 zfs_panic_recover("zfs: attempting to increase fill beyond "
264 "max; probable double add in segment [%llx:%llx]",
265 (longlong_t)rs_get_start(rs, rt),
266 (longlong_t)rs_get_end(rs, rt));
269 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
270 rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);
271 rs_set_fill(rs, rt, rs_get_fill(rs, rt) + delta);
272 if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
273 rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);
277 range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill)
279 range_tree_t *rt = arg;
280 zfs_btree_index_t where;
281 range_seg_t *rs_before, *rs_after, *rs;
282 range_seg_max_t tmp, rsearch;
283 uint64_t end = start + size, gap = rt->rt_gap;
284 uint64_t bridge_size = 0;
285 boolean_t merge_before, merge_after;
287 ASSERT3U(size, !=, 0);
288 ASSERT3U(fill, <=, size);
289 ASSERT3U(start + size, >, start);
291 rs_set_start(&rsearch, rt, start);
292 rs_set_end(&rsearch, rt, end);
293 rs = zfs_btree_find(&rt->rt_root, &rsearch, &where);
296 * If this is a gap-supporting range tree, it is possible that we
297 * are inserting into an existing segment. In this case simply
298 * bump the fill count and call the remove / add callbacks. If the
299 * new range will extend an existing segment, we remove the
300 * existing one, apply the new extent to it and re-insert it using
301 * the normal code paths.
305 zfs_panic_recover("zfs: adding existent segment to "
306 "range tree (offset=%llx size=%llx)",
307 (longlong_t)start, (longlong_t)size);
310 uint64_t rstart = rs_get_start(rs, rt);
311 uint64_t rend = rs_get_end(rs, rt);
312 if (rstart <= start && rend >= end) {
313 range_tree_adjust_fill(rt, rs, fill);
317 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
318 rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);
320 range_tree_stat_decr(rt, rs);
321 rt->rt_space -= rend - rstart;
323 fill += rs_get_fill(rs, rt);
324 start = MIN(start, rstart);
325 end = MAX(end, rend);
328 zfs_btree_remove(&rt->rt_root, rs);
329 range_tree_add_impl(rt, start, size, fill);
333 ASSERT3P(rs, ==, NULL);
336 * Determine whether or not we will have to merge with our neighbors.
337 * If gap != 0, we might need to merge with our neighbors even if we
338 * aren't directly touching.
340 zfs_btree_index_t where_before, where_after;
341 rs_before = zfs_btree_prev(&rt->rt_root, &where, &where_before);
342 rs_after = zfs_btree_next(&rt->rt_root, &where, &where_after);
344 merge_before = (rs_before != NULL && rs_get_end(rs_before, rt) >=
346 merge_after = (rs_after != NULL && rs_get_start(rs_after, rt) <= end +
349 if (merge_before && gap != 0)
350 bridge_size += start - rs_get_end(rs_before, rt);
351 if (merge_after && gap != 0)
352 bridge_size += rs_get_start(rs_after, rt) - end;
354 if (merge_before && merge_after) {
355 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) {
356 rt->rt_ops->rtop_remove(rt, rs_before, rt->rt_arg);
357 rt->rt_ops->rtop_remove(rt, rs_after, rt->rt_arg);
360 range_tree_stat_decr(rt, rs_before);
361 range_tree_stat_decr(rt, rs_after);
363 rs_copy(rs_after, &tmp, rt);
364 uint64_t before_start = rs_get_start_raw(rs_before, rt);
365 uint64_t before_fill = rs_get_fill(rs_before, rt);
366 uint64_t after_fill = rs_get_fill(rs_after, rt);
367 zfs_btree_remove_idx(&rt->rt_root, &where_before);
370 * We have to re-find the node because our old reference is
371 * invalid as soon as we do any mutating btree operations.
373 rs_after = zfs_btree_find(&rt->rt_root, &tmp, &where_after);
374 rs_set_start_raw(rs_after, rt, before_start);
375 rs_set_fill(rs_after, rt, after_fill + before_fill + fill);
377 } else if (merge_before) {
378 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
379 rt->rt_ops->rtop_remove(rt, rs_before, rt->rt_arg);
381 range_tree_stat_decr(rt, rs_before);
383 uint64_t before_fill = rs_get_fill(rs_before, rt);
384 rs_set_end(rs_before, rt, end);
385 rs_set_fill(rs_before, rt, before_fill + fill);
387 } else if (merge_after) {
388 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
389 rt->rt_ops->rtop_remove(rt, rs_after, rt->rt_arg);
391 range_tree_stat_decr(rt, rs_after);
393 uint64_t after_fill = rs_get_fill(rs_after, rt);
394 rs_set_start(rs_after, rt, start);
395 rs_set_fill(rs_after, rt, after_fill + fill);
400 rs_set_start(rs, rt, start);
401 rs_set_end(rs, rt, end);
402 rs_set_fill(rs, rt, fill);
403 zfs_btree_add_idx(&rt->rt_root, rs, &where);
407 ASSERT3U(rs_get_fill(rs, rt), <=, rs_get_end(rs, rt) -
408 rs_get_start(rs, rt));
410 ASSERT3U(rs_get_fill(rs, rt), ==, rs_get_end(rs, rt) -
411 rs_get_start(rs, rt));
414 if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
415 rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);
417 range_tree_stat_incr(rt, rs);
418 rt->rt_space += size + bridge_size;
422 range_tree_add(void *arg, uint64_t start, uint64_t size)
424 range_tree_add_impl(arg, start, size, size);
428 range_tree_remove_impl(range_tree_t *rt, uint64_t start, uint64_t size,
431 zfs_btree_index_t where;
433 range_seg_max_t rsearch, rs_tmp;
434 uint64_t end = start + size;
435 boolean_t left_over, right_over;
437 VERIFY3U(size, !=, 0);
438 VERIFY3U(size, <=, rt->rt_space);
439 if (rt->rt_type == RANGE_SEG64)
440 ASSERT3U(start + size, >, start);
442 rs_set_start(&rsearch, rt, start);
443 rs_set_end(&rsearch, rt, end);
444 rs = zfs_btree_find(&rt->rt_root, &rsearch, &where);
446 /* Make sure we completely overlap with someone */
448 zfs_panic_recover("zfs: removing nonexistent segment from "
449 "range tree (offset=%llx size=%llx)",
450 (longlong_t)start, (longlong_t)size);
455 * Range trees with gap support must only remove complete segments
456 * from the tree. This allows us to maintain accurate fill accounting
457 * and to ensure that bridged sections are not leaked. If we need to
458 * remove less than the full segment, we can only adjust the fill count.
460 if (rt->rt_gap != 0) {
462 if (rs_get_fill(rs, rt) == size) {
463 start = rs_get_start(rs, rt);
464 end = rs_get_end(rs, rt);
467 range_tree_adjust_fill(rt, rs, -size);
470 } else if (rs_get_start(rs, rt) != start ||
471 rs_get_end(rs, rt) != end) {
472 zfs_panic_recover("zfs: freeing partial segment of "
473 "gap tree (offset=%llx size=%llx) of "
474 "(offset=%llx size=%llx)",
475 (longlong_t)start, (longlong_t)size,
476 (longlong_t)rs_get_start(rs, rt),
477 (longlong_t)rs_get_end(rs, rt) - rs_get_start(rs,
483 VERIFY3U(rs_get_start(rs, rt), <=, start);
484 VERIFY3U(rs_get_end(rs, rt), >=, end);
486 left_over = (rs_get_start(rs, rt) != start);
487 right_over = (rs_get_end(rs, rt) != end);
489 range_tree_stat_decr(rt, rs);
491 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
492 rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);
494 if (left_over && right_over) {
495 range_seg_max_t newseg;
496 rs_set_start(&newseg, rt, end);
497 rs_set_end_raw(&newseg, rt, rs_get_end_raw(rs, rt));
498 rs_set_fill(&newseg, rt, rs_get_end(rs, rt) - end);
499 range_tree_stat_incr(rt, &newseg);
501 // This modifies the buffer already inside the range tree
502 rs_set_end(rs, rt, start);
504 rs_copy(rs, &rs_tmp, rt);
505 if (zfs_btree_next(&rt->rt_root, &where, &where) != NULL)
506 zfs_btree_add_idx(&rt->rt_root, &newseg, &where);
508 zfs_btree_add(&rt->rt_root, &newseg);
510 if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
511 rt->rt_ops->rtop_add(rt, &newseg, rt->rt_arg);
512 } else if (left_over) {
513 // This modifies the buffer already inside the range tree
514 rs_set_end(rs, rt, start);
515 rs_copy(rs, &rs_tmp, rt);
516 } else if (right_over) {
517 // This modifies the buffer already inside the range tree
518 rs_set_start(rs, rt, end);
519 rs_copy(rs, &rs_tmp, rt);
521 zfs_btree_remove_idx(&rt->rt_root, &where);
527 * The fill of the leftover segment will always be equal to
528 * the size, since we do not support removing partial segments
529 * of range trees with gaps.
531 rs_set_fill_raw(rs, rt, rs_get_end_raw(rs, rt) -
532 rs_get_start_raw(rs, rt));
533 range_tree_stat_incr(rt, &rs_tmp);
535 if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
536 rt->rt_ops->rtop_add(rt, &rs_tmp, rt->rt_arg);
539 rt->rt_space -= size;
543 range_tree_remove(void *arg, uint64_t start, uint64_t size)
545 range_tree_remove_impl(arg, start, size, B_FALSE);
549 range_tree_remove_fill(range_tree_t *rt, uint64_t start, uint64_t size)
551 range_tree_remove_impl(rt, start, size, B_TRUE);
555 range_tree_resize_segment(range_tree_t *rt, range_seg_t *rs,
556 uint64_t newstart, uint64_t newsize)
558 int64_t delta = newsize - (rs_get_end(rs, rt) - rs_get_start(rs, rt));
560 range_tree_stat_decr(rt, rs);
561 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
562 rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);
564 rs_set_start(rs, rt, newstart);
565 rs_set_end(rs, rt, newstart + newsize);
567 range_tree_stat_incr(rt, rs);
568 if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
569 rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);
571 rt->rt_space += delta;
575 range_tree_find_impl(range_tree_t *rt, uint64_t start, uint64_t size)
577 range_seg_max_t rsearch;
578 uint64_t end = start + size;
582 rs_set_start(&rsearch, rt, start);
583 rs_set_end(&rsearch, rt, end);
584 return (zfs_btree_find(&rt->rt_root, &rsearch, NULL));
588 range_tree_find(range_tree_t *rt, uint64_t start, uint64_t size)
590 if (rt->rt_type == RANGE_SEG64)
591 ASSERT3U(start + size, >, start);
593 range_seg_t *rs = range_tree_find_impl(rt, start, size);
594 if (rs != NULL && rs_get_start(rs, rt) <= start &&
595 rs_get_end(rs, rt) >= start + size) {
602 range_tree_verify_not_present(range_tree_t *rt, uint64_t off, uint64_t size)
604 range_seg_t *rs = range_tree_find(rt, off, size);
606 panic("segment already in tree; rs=%p", (void *)rs);
610 range_tree_contains(range_tree_t *rt, uint64_t start, uint64_t size)
612 return (range_tree_find(rt, start, size) != NULL);
616 * Returns the first subset of the given range which overlaps with the range
617 * tree. Returns true if there is a segment in the range, and false if there
621 range_tree_find_in(range_tree_t *rt, uint64_t start, uint64_t size,
622 uint64_t *ostart, uint64_t *osize)
624 if (rt->rt_type == RANGE_SEG64)
625 ASSERT3U(start + size, >, start);
627 range_seg_max_t rsearch;
628 rs_set_start(&rsearch, rt, start);
629 rs_set_end_raw(&rsearch, rt, rs_get_start_raw(&rsearch, rt) + 1);
631 zfs_btree_index_t where;
632 range_seg_t *rs = zfs_btree_find(&rt->rt_root, &rsearch, &where);
635 *osize = MIN(size, rs_get_end(rs, rt) - start);
639 rs = zfs_btree_next(&rt->rt_root, &where, &where);
640 if (rs == NULL || rs_get_start(rs, rt) > start + size)
643 *ostart = rs_get_start(rs, rt);
644 *osize = MIN(start + size, rs_get_end(rs, rt)) -
645 rs_get_start(rs, rt);
650 * Ensure that this range is not in the tree, regardless of whether
651 * it is currently in the tree.
654 range_tree_clear(range_tree_t *rt, uint64_t start, uint64_t size)
661 if (rt->rt_type == RANGE_SEG64)
662 ASSERT3U(start + size, >, start);
664 while ((rs = range_tree_find_impl(rt, start, size)) != NULL) {
665 uint64_t free_start = MAX(rs_get_start(rs, rt), start);
666 uint64_t free_end = MIN(rs_get_end(rs, rt), start + size);
667 range_tree_remove(rt, free_start, free_end - free_start);
672 range_tree_swap(range_tree_t **rtsrc, range_tree_t **rtdst)
676 ASSERT0(range_tree_space(*rtdst));
677 ASSERT0(zfs_btree_numnodes(&(*rtdst)->rt_root));
685 range_tree_vacate(range_tree_t *rt, range_tree_func_t *func, void *arg)
687 if (rt->rt_ops != NULL && rt->rt_ops->rtop_vacate != NULL)
688 rt->rt_ops->rtop_vacate(rt, rt->rt_arg);
692 zfs_btree_index_t *cookie = NULL;
694 while ((rs = zfs_btree_destroy_nodes(&rt->rt_root, &cookie)) !=
696 func(arg, rs_get_start(rs, rt), rs_get_end(rs, rt) -
697 rs_get_start(rs, rt));
700 zfs_btree_clear(&rt->rt_root);
703 bzero(rt->rt_histogram, sizeof (rt->rt_histogram));
708 range_tree_walk(range_tree_t *rt, range_tree_func_t *func, void *arg)
710 zfs_btree_index_t where;
711 for (range_seg_t *rs = zfs_btree_first(&rt->rt_root, &where);
712 rs != NULL; rs = zfs_btree_next(&rt->rt_root, &where, &where)) {
713 func(arg, rs_get_start(rs, rt), rs_get_end(rs, rt) -
714 rs_get_start(rs, rt));
719 range_tree_first(range_tree_t *rt)
721 return (zfs_btree_first(&rt->rt_root, NULL));
725 range_tree_space(range_tree_t *rt)
727 return (rt->rt_space);
731 range_tree_numsegs(range_tree_t *rt)
733 return ((rt == NULL) ? 0 : zfs_btree_numnodes(&rt->rt_root));
737 range_tree_is_empty(range_tree_t *rt)
740 return (range_tree_space(rt) == 0);
745 rt_btree_create(range_tree_t *rt, void *arg)
747 zfs_btree_t *size_tree = arg;
750 switch (rt->rt_type) {
752 size = sizeof (range_seg32_t);
755 size = sizeof (range_seg64_t);
758 size = sizeof (range_seg_gap_t);
761 panic("Invalid range seg type %d", rt->rt_type);
763 zfs_btree_create(size_tree, rt->rt_btree_compare, size);
768 rt_btree_destroy(range_tree_t *rt, void *arg)
770 zfs_btree_t *size_tree = arg;
771 ASSERT0(zfs_btree_numnodes(size_tree));
773 zfs_btree_destroy(size_tree);
778 rt_btree_add(range_tree_t *rt, range_seg_t *rs, void *arg)
780 zfs_btree_t *size_tree = arg;
782 zfs_btree_add(size_tree, rs);
787 rt_btree_remove(range_tree_t *rt, range_seg_t *rs, void *arg)
789 zfs_btree_t *size_tree = arg;
791 zfs_btree_remove(size_tree, rs);
796 rt_btree_vacate(range_tree_t *rt, void *arg)
798 zfs_btree_t *size_tree = arg;
799 zfs_btree_clear(size_tree);
800 zfs_btree_destroy(size_tree);
802 rt_btree_create(rt, arg);
805 range_tree_ops_t rt_btree_ops = {
806 .rtop_create = rt_btree_create,
807 .rtop_destroy = rt_btree_destroy,
808 .rtop_add = rt_btree_add,
809 .rtop_remove = rt_btree_remove,
810 .rtop_vacate = rt_btree_vacate
814 * Remove any overlapping ranges between the given segment [start, end)
815 * from removefrom. Add non-overlapping leftovers to addto.
818 range_tree_remove_xor_add_segment(uint64_t start, uint64_t end,
819 range_tree_t *removefrom, range_tree_t *addto)
821 zfs_btree_index_t where;
822 range_seg_max_t starting_rs;
823 rs_set_start(&starting_rs, removefrom, start);
824 rs_set_end_raw(&starting_rs, removefrom, rs_get_start_raw(&starting_rs,
827 range_seg_t *curr = zfs_btree_find(&removefrom->rt_root,
828 &starting_rs, &where);
831 curr = zfs_btree_next(&removefrom->rt_root, &where, &where);
834 for (; curr != NULL; curr = next) {
837 VERIFY3U(start, <, end);
839 /* there is no overlap */
840 if (end <= rs_get_start(curr, removefrom)) {
841 range_tree_add(addto, start, end - start);
845 uint64_t overlap_start = MAX(rs_get_start(curr, removefrom),
847 uint64_t overlap_end = MIN(rs_get_end(curr, removefrom),
849 uint64_t overlap_size = overlap_end - overlap_start;
850 ASSERT3S(overlap_size, >, 0);
852 rs_copy(curr, &rs, removefrom);
854 range_tree_remove(removefrom, overlap_start, overlap_size);
856 if (start < overlap_start)
857 range_tree_add(addto, start, overlap_start - start);
860 next = zfs_btree_find(&removefrom->rt_root, &rs, &where);
862 * If we find something here, we only removed part of the
863 * curr segment. Either there's some left at the end
864 * because we've reached the end of the range we're removing,
865 * or there's some left at the start because we started
866 * partway through the range. Either way, we continue with
867 * the loop. If it's the former, we'll return at the start of
868 * the loop, and if it's the latter we'll see if there is more
872 ASSERT(start == end || start == rs_get_end(&rs,
876 next = zfs_btree_next(&removefrom->rt_root, &where, &where);
878 VERIFY3P(curr, ==, NULL);
881 VERIFY3U(start, <, end);
882 range_tree_add(addto, start, end - start);
884 VERIFY3U(start, ==, end);
889 * For each entry in rt, if it exists in removefrom, remove it
890 * from removefrom. Otherwise, add it to addto.
893 range_tree_remove_xor_add(range_tree_t *rt, range_tree_t *removefrom,
896 zfs_btree_index_t where;
897 for (range_seg_t *rs = zfs_btree_first(&rt->rt_root, &where); rs;
898 rs = zfs_btree_next(&rt->rt_root, &where, &where)) {
899 range_tree_remove_xor_add_segment(rs_get_start(rs, rt),
900 rs_get_end(rs, rt), removefrom, addto);
905 range_tree_min(range_tree_t *rt)
907 range_seg_t *rs = zfs_btree_first(&rt->rt_root, NULL);
908 return (rs != NULL ? rs_get_start(rs, rt) : 0);
912 range_tree_max(range_tree_t *rt)
914 range_seg_t *rs = zfs_btree_last(&rt->rt_root, NULL);
915 return (rs != NULL ? rs_get_end(rs, rt) : 0);
919 range_tree_span(range_tree_t *rt)
921 return (range_tree_max(rt) - range_tree_min(rt));