From e8969ef0d6a64bedcbd3bb00c18f66f5f8ce46e5 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Sat, 31 May 2008 18:45:04 +0000 Subject: [PATCH] HAMMER Utilities: Add the 'hammer softprune' command. Add a new hammer pruning command called 'hammer softprune'. This command is much simpler to use then the 'hammer prune' command. You simply specify a directory containing softlinks to HAMMER snapshots, typically in the form: "/@@0x<16-char-transaction_id>". The command will scan the directory non-recursively, collect all the softlinks, extract the transaction ids, and prune the HAMMER filesystem to contain only those snapshots. In addition, information created before the snapshot softlink with the lowest transaction id is destroyed and information created after the softlink with the highest transaction id is retained (remains fine-grained). This gives the administrator an easy way to maintain official snapshots while at the same time retaining our 'undo' capability by leaving recent modifications intact. A simple cron job or script coupled with the use of the 'hammer synctid' can be used to create a snapshot softlink every so often, and older snapshots can be cleaned out or thinned simply by removing the associated softlinks, and then re-running 'hammer softprune' on the directory containing the softlinks. Unlike the 'hammer prune' command, the softprune command does not require the time ranges for snapshots to be well-ordered. --- sbin/hammer/Makefile | 6 +- sbin/hammer/cmd_prune.c | 4 +- sbin/hammer/cmd_softprune.c | 362 ++++++++++++++++++++++++++++++++++++ sbin/hammer/hammer.8 | 43 ++++- sbin/hammer/hammer.c | 7 +- sbin/hammer/hammer.h | 4 +- 6 files changed, 419 insertions(+), 7 deletions(-) create mode 100644 sbin/hammer/cmd_softprune.c diff --git a/sbin/hammer/Makefile b/sbin/hammer/Makefile index a9519b0360..47dea014ac 100644 --- a/sbin/hammer/Makefile +++ b/sbin/hammer/Makefile @@ -1,10 +1,10 @@ # -# $DragonFly: src/sbin/hammer/Makefile,v 1.11 2008/05/13 20:49:34 dillon Exp $ +# $DragonFly: src/sbin/hammer/Makefile,v 1.12 2008/05/31 18:45:04 dillon Exp $ PROG= hammer SRCS= hammer.c ondisk.c blockmap.c cache.c misc.c cycle.c \ - cmd_show.c cmd_prune.c cmd_history.c cmd_blockmap.c \ - cmd_reblock.c cmd_synctid.c + cmd_show.c cmd_softprune.c cmd_prune.c cmd_history.c \ + cmd_blockmap.c cmd_reblock.c cmd_synctid.c MAN= hammer.8 CFLAGS+= -I${.CURDIR}/../../sys -DALIST_NO_DEBUG diff --git a/sbin/hammer/cmd_prune.c b/sbin/hammer/cmd_prune.c index 2d3898999a..6575b52005 100644 --- a/sbin/hammer/cmd_prune.c +++ b/sbin/hammer/cmd_prune.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/hammer/Attic/cmd_prune.c,v 1.11 2008/05/18 01:49:41 dillon Exp $ + * $DragonFly: src/sbin/hammer/Attic/cmd_prune.c,v 1.12 2008/05/31 18:45:04 dillon Exp $ */ #include "hammer.h" @@ -69,6 +69,7 @@ hammer_cmd_prune(char **av, int ac) hammer_tid_t now_tid = (hammer_tid_t)time(NULL) * 1000000000LL; bzero(&prune, sizeof(prune)); + prune.elms = malloc(HAMMER_MAX_PRUNE_ELMS * sizeof(*prune.elms)); prune.nelms = 0; prune.beg_localization = HAMMER_MIN_LOCALIZATION; prune.beg_obj_id = HAMMER_MIN_OBJID; @@ -162,6 +163,7 @@ hammer_prune_load_file(hammer_tid_t now_tid, struct hammer_ioc_prune *prune, continue; if (strcmp(av[0], "prune") != 0) continue; + printf("prune %s\n", av[2]); if (hammer_prune_parse_line(now_tid, prune, filesystem, av + 1, ac - 1) < 0) { errx(1, "Malformed prune directive in %s line %d\n", diff --git a/sbin/hammer/cmd_softprune.c b/sbin/hammer/cmd_softprune.c new file mode 100644 index 0000000000..a5cfaf8111 --- /dev/null +++ b/sbin/hammer/cmd_softprune.c @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2008 The DragonFly Project. All rights reserved. + * + * This code is derived from software contributed to The DragonFly Project + * by Matthew Dillon + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of The DragonFly Project nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific, prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $DragonFly: src/sbin/hammer/cmd_softprune.c,v 1.1 2008/05/31 18:45:04 dillon Exp $ + */ + +#include "hammer.h" + +struct softprune { + struct softprune *next; + struct statfs fs; + char *filesystem; + struct hammer_ioc_prune prune; + int maxelms; +}; + +static void softprune_usage(int code); +static void hammer_softprune_scandir(struct softprune **basep, + struct hammer_ioc_prune *template, + const char *dirname); +static void hammer_softprune_addentry(struct softprune **basep, + struct hammer_ioc_prune *template, + const char *dirpath, char *linkbuf, char *tidptr); +static void hammer_softprune_finalize(struct softprune *scan); + +/* + * softprune + */ +void +hammer_cmd_softprune(char **av, int ac) +{ + struct hammer_ioc_prune template; + struct softprune *base, *scan; + int fd; + int rcode; + + base = NULL; + rcode = 0; + + bzero(&template, sizeof(template)); + template.beg_localization = HAMMER_MIN_LOCALIZATION; + template.beg_obj_id = HAMMER_MIN_OBJID; + template.end_localization = HAMMER_MAX_LOCALIZATION; + template.end_obj_id = HAMMER_MAX_OBJID; + hammer_get_cycle(&template.end_obj_id, &template.end_localization); + template.stat_oldest_tid = HAMMER_MAX_TID; + + /* + * For now just allow one directory + */ + if (ac != 1) + softprune_usage(1); + + /* + * Scan the softlink directories + */ + while (ac) { + hammer_softprune_scandir(&base, &template, *av); + ++av; + --ac; + } + + /* + * XXX future (need to store separate cycles for each filesystem) + */ + if (base == NULL) { + fprintf(stderr, "No snapshot softlinks found\n"); + exit(1); + } + if (base->next) { + fprintf(stderr, "Currently only one HAMMER filesystem may " + "be specified in the softlink scan\n"); + exit(1); + } + + /* + * Issue the prunes + */ + for (scan = base; scan; scan = scan->next) { + hammer_softprune_finalize(scan); + printf("Prune %s: %d snapshots\n", + scan->filesystem, scan->prune.nelms); + if (scan->prune.nelms == 0) + continue; + fd = open(scan->filesystem, O_RDONLY); + if (fd < 0) { + warn("Unable to open %s", scan->filesystem); + rcode = 1; + continue; + } + + if (ioctl(fd, HAMMERIOC_PRUNE, &scan->prune) < 0) { + printf("Prune %s failed: %s\n", + scan->filesystem, strerror(errno)); + rcode = 2; + } else if (scan->prune.head.flags & HAMMER_IOC_HEAD_INTR) { + printf("Prune %s interrupted by timer at " + "%016llx %04x\n", + scan->filesystem, + scan->prune.cur_obj_id, + scan->prune.cur_localization); + if (CyclePath) { + hammer_set_cycle(scan->prune.cur_obj_id, + scan->prune.cur_localization); + } + rcode = 0; + } else { + if (CyclePath) + hammer_reset_cycle(); + printf("Prune %s succeeded\n", scan->filesystem); + } + printf("Pruned %lld/%lld records (%lld directory entries) " + "and %lld bytes\n", + scan->prune.stat_rawrecords, + scan->prune.stat_scanrecords, + scan->prune.stat_dirrecords, + scan->prune.stat_bytes + ); + close(fd); + } + if (rcode) + exit(rcode); +} + +/* + * Scan a directory for softlinks representing snapshots and build + * associated softprune structures. + */ +static void +hammer_softprune_scandir(struct softprune **basep, + struct hammer_ioc_prune *template, + const char *dirname) +{ + struct stat st; + struct dirent *den; + DIR *dir; + char *path; + int len; + char *linkbuf; + char *ptr; + + path = NULL; + linkbuf = malloc(MAXPATHLEN); + + if ((dir = opendir(dirname)) == NULL) + err(1, "Cannot open directory %s", dirname); + while ((den = readdir(dir)) != NULL) { + if (strcmp(den->d_name, ".") == 0) + continue; + if (strcmp(den->d_name, "..") == 0) + continue; + if (path) + free(path); + asprintf(&path, "%s/%s", dirname, den->d_name); + if (lstat(path, &st) < 0) + continue; + if (!S_ISLNK(st.st_mode)) + continue; + if ((len = readlink(path, linkbuf, MAXPATHLEN - 1)) < 0) + continue; + linkbuf[len] = 0; + if ((ptr = strrchr(linkbuf, '@')) && + ptr > linkbuf && ptr[-1] == '@') { + hammer_softprune_addentry(basep, template, + dirname, linkbuf, ptr - 1); + } + } + free(linkbuf); + if (path) + free(path); +} + +/* + * Add the softlink to the appropriate softprune structure, creating a new + * if necessary. + */ +static void +hammer_softprune_addentry(struct softprune **basep, + struct hammer_ioc_prune *template, + const char *dirpath, char *linkbuf, char *tidptr) +{ + struct hammer_ioc_prune_elm *elm; + struct softprune *scan; + struct statfs fs; + char *fspath; + + if (linkbuf[0] == '/') { + asprintf(&fspath, "%*.*s", + (tidptr - linkbuf), (tidptr - linkbuf), linkbuf); + } else { + asprintf(&fspath, "%s/%*.*s", dirpath, + (tidptr - linkbuf), (tidptr - linkbuf), linkbuf); + } + if (statfs(fspath, &fs) < 0) { + free(fspath); + return; + } + + /* + * Locate the filesystem in an existing softprune structure + */ + for (scan = *basep; scan; scan = scan->next) { + if (bcmp(&fs.f_fsid, &scan->fs.f_fsid, sizeof(fs.f_fsid)) != 0) + continue; + if (strcmp(fs.f_mntonname, scan->fs.f_mntonname) != 0) + continue; + break; + } + + /* + * Create a new softprune structure if necessasry + */ + if (scan == NULL) { + scan = malloc(sizeof(*scan)); + bzero(scan, sizeof(*scan)); + + scan->fs = fs; + scan->filesystem = fspath; + scan->prune = *template; + scan->next = *basep; + scan->maxelms = 32; + scan->prune.elms = malloc(sizeof(*elm) * scan->maxelms); + *basep = scan; + } else { + free(fspath); + } + + /* + * Add the entry (unsorted). Just set the beg_tid, we will sort + * and set the remaining entries later. + * + * Always leave one entry free for our terminator. + */ + if (scan->prune.nelms >= scan->maxelms - 1) { + scan->maxelms = (scan->maxelms * 3 / 2); + scan->prune.elms = realloc(scan->prune.elms, + sizeof(*elm) * scan->maxelms); + } + elm = &scan->prune.elms[scan->prune.nelms]; + elm->beg_tid = strtoull(tidptr + 2, NULL, 0); + elm->end_tid = 0; + elm->mod_tid = 0; + ++scan->prune.nelms; +} + +/* + * Finalize a softprune structure after scanning in its softlinks. + * Sort the elements, remove duplicates, and then fill in end_tid and + * mod_tid. + * + * The array must end up in descending order. + */ +static int +hammer_softprune_qsort_cmp(const void *arg1, const void *arg2) +{ + const struct hammer_ioc_prune_elm *elm1 = arg1; + const struct hammer_ioc_prune_elm *elm2 = arg2; + + if (elm1->beg_tid < elm2->beg_tid) + return(1); + if (elm1->beg_tid > elm2->beg_tid) + return(-1); + return(0); +} + +static void +hammer_softprune_finalize(struct softprune *scan) +{ + struct hammer_ioc_prune_elm *elm; + int i; + + /* + * Don't do anything if there are no elements. + */ + if (scan->prune.nelms == 0) + return; + + /* + * Sort the elements in descending order, remove duplicates, and + * fill in any missing bits. + */ + qsort(scan->prune.elms, scan->prune.nelms, sizeof(*elm), + hammer_softprune_qsort_cmp); + + for (i = 0; i < scan->prune.nelms; ++i) { + elm = &scan->prune.elms[i]; + if (i == 0) { + /* + * First (highest TID) (also last if only one element) + */ + elm->end_tid = HAMMER_MAX_TID; + } else if (elm[0].beg_tid == elm[-1].beg_tid) { + /* + * Remove duplicate + */ + --scan->prune.nelms; + if (i != scan->prune.nelms) { + bcopy(elm + 1, elm, + (scan->prune.nelms - i) * sizeof(*elm)); + } + --i; + continue; + } else { + /* + * Middle or last. + */ + elm->end_tid = elm[-1].beg_tid; + } + elm->mod_tid = elm->end_tid - elm->beg_tid; + } + + /* + * Add a final element to prune everything from transaction id + * 0 to the lowest transaction id (aka last so far). + */ + assert(scan->prune.nelms < scan->maxelms); + elm = &scan->prune.elms[scan->prune.nelms++]; + elm->beg_tid = 0; + elm->end_tid = elm[-1].beg_tid; + elm->mod_tid = elm->end_tid - elm->beg_tid; +} + +static +void +softprune_usage(int code) +{ + fprintf(stderr, "Badly formed command, use:\n"); + fprintf(stderr, "hammer softprune directory-containing-softlinks\n"); + exit(code); +} + + diff --git a/sbin/hammer/hammer.8 b/sbin/hammer/hammer.8 index ca850f5e53..6aec284876 100644 --- a/sbin/hammer/hammer.8 +++ b/sbin/hammer/hammer.8 @@ -30,7 +30,7 @@ .\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.\" $DragonFly: src/sbin/hammer/hammer.8,v 1.17 2008/05/18 01:49:41 dillon Exp $ +.\" $DragonFly: src/sbin/hammer/hammer.8,v 1.18 2008/05/31 18:45:04 dillon Exp $ .Dd December 31, 2007 .Dt HAMMER 8 .Os @@ -138,6 +138,47 @@ output as 0. .It Ar namekey32 Ar filename Generate the top 32 bits of a HAMMER 64 bit directory hash for the specified file name. +.It Ar softprune Ar softlink-dir +Prune the filesystem based on previously created snapshot softlinks. This +is a more convenient method of pruning then the fixed timestamp ranges used +in the +.Ar prune +command. +The target directory is expected to contain softlinks pointing to +snapshots of the filesystems you wish to retain. The directory is scanned +non-recursively and the mount points and transaction ids stored in the +softlinks are extracted and sorted. +The filesystem is then explicitly pruned according to what is found. +Cleaning out portions of the filesystem is as simple as removing a softlink +and then running the +.Ar softprune +command. +.Pp +As a safety measure pruning only occurs if one or more softlinks are found +containing the @@ snapshot id extension. +.Pp +Currently the scanned softlink directory must contain softlinks pointing +to a single HAMMER mount. The softlinks may specify absolute or relative +paths. Softlinks must use 20-character (@@0x%016llx) transaction ids, +as might be returned from 'hammer synctid '. +.Pp +Example, lets say your snapshot directory contains the following links: +.Pp +.Li "lrwxr-xr-x 1 root wheel 29 May 31 17:57 snap1 -> /usr/obj/@@0x10d2cd05b7270d16" +.Pp +.Li "lrwxr-xr-x 1 root wheel 29 May 31 17:58 snap2 -> /usr/obj/@@0x10d2cd13f3fde98f" +.Pp +.Li "lrwxr-xr-x 1 root wheel 29 May 31 17:59 snap3 -> /usr/obj/@@0x10d2cd222adee364" +.Pp +If you were to run the softprune command on this directory, then the HAMMER +/usr/obj mount will be pruned to retain the above three snapshots. +In addition, modifications made to the filesystem older then the oldest +snapshot will be destroyed and potentially fine-grained modifications made +to the filesystem more recently then the most recent snapshot will be +retained. +.Pp +If you then delete the snap2 softlink and rerun the softprune command, +modifications pertaining to that snapshot would be destroyed. .It Ar prune Ar filesystem Ar from Ar #{smhdMy} Ar to Ar #{smhdMy} Ar every Ar #{smhdMy} .It Ar prune Ar filesystem Ar from Ar #{smhdMy} Ar everything .It Ar prune Ar filesystem Ar everything diff --git a/sbin/hammer/hammer.c b/sbin/hammer/hammer.c index 812eee7a06..9ef610d9ec 100644 --- a/sbin/hammer/hammer.c +++ b/sbin/hammer/hammer.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/hammer/hammer.c,v 1.19 2008/05/18 01:49:41 dillon Exp $ + * $DragonFly: src/sbin/hammer/hammer.c,v 1.20 2008/05/31 18:45:04 dillon Exp $ */ #include "hammer.h" @@ -170,6 +170,10 @@ main(int ac, char **av) hammer_cmd_prune(av + 1, ac - 1); exit(0); } + if (strcmp(av[0], "softprune") == 0) { + hammer_cmd_softprune(av + 1, ac - 1); + exit(0); + } if (strncmp(av[0], "history", 7) == 0) { hammer_cmd_history(av[0] + 7, av + 1, ac - 1); @@ -320,6 +324,7 @@ usage(int exit_code) "hammer [-x] now[64]\n" "hammer [-t timeout] [-c cyclefile] ....\n" "hammer stamp[64]