From 6a6e350f070fda1b42c5f10e052becd78ae7c90c Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Sat, 20 Sep 2008 04:23:21 +0000 Subject: [PATCH] Add the 'hammer cleanup' command. This is a meta-command which will process specified HAMMER filesystem or, if none specified, locate all mounted HAMMER filesystems and run snapshot, pruning, and reblocking commands a appropriate. If the /snapshots directory or /snapshots/config file does not exist, they will be created. --- sbin/hammer/Makefile | 5 +- sbin/hammer/cmd_cleanup.c | 571 ++++++++++++++++++++++++++++++++++++++ sbin/hammer/hammer.8 | 38 ++- sbin/hammer/hammer.c | 8 +- sbin/hammer/hammer.h | 3 +- 5 files changed, 620 insertions(+), 5 deletions(-) create mode 100644 sbin/hammer/cmd_cleanup.c diff --git a/sbin/hammer/Makefile b/sbin/hammer/Makefile index d1d24dd338..de658cfd75 100644 --- a/sbin/hammer/Makefile +++ b/sbin/hammer/Makefile @@ -1,11 +1,12 @@ # -# $DragonFly: src/sbin/hammer/Makefile,v 1.17 2008/06/26 04:07:57 dillon Exp $ +# $DragonFly: src/sbin/hammer/Makefile,v 1.18 2008/09/20 04:23:21 dillon Exp $ PROG= hammer SRCS= hammer.c ondisk.c blockmap.c cache.c misc.c cycle.c \ cmd_show.c cmd_softprune.c cmd_history.c \ cmd_blockmap.c cmd_reblock.c cmd_synctid.c cmd_stats.c \ - cmd_pseudofs.c cmd_snapshot.c cmd_mirror.c cmd_status.c + cmd_pseudofs.c cmd_snapshot.c cmd_mirror.c cmd_status.c \ + cmd_cleanup.c MAN= hammer.8 CFLAGS+= -I${.CURDIR}/../../sys -DALIST_NO_DEBUG diff --git a/sbin/hammer/cmd_cleanup.c b/sbin/hammer/cmd_cleanup.c new file mode 100644 index 0000000000..21ea025de1 --- /dev/null +++ b/sbin/hammer/cmd_cleanup.c @@ -0,0 +1,571 @@ +/* + * Copyright (c) 2008 The DragonFly Project. All rights reserved. + * + * This code is derived from software contributed to The DragonFly Project + * by Matthew Dillon + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of The DragonFly Project nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific, prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $DragonFly: src/sbin/hammer/cmd_cleanup.c,v 1.1 2008/09/20 04:23:21 dillon Exp $ + */ +/* + * Clean up a specific HAMMER filesystem or all HAMMER filesystems. + * + * Each filesystem is expected to have a /snapshots directory. + * No cleanup will be performed on any filesystem that does not. If + * no filesystems are specified the 'df' program is run and any HAMMER + * or null-mounted hammer PFS's are extracted. + * + * The snapshots directory may contain a config file called 'config'. If + * no config file is present one will be created with the following + * defaults: + * + * snapshots 1d 60d (0d 60d for /tmp, /var/tmp, /usr/obj) + * prune 1d 5m + * reblock 1d 5m + * recopy 30d 5m + * + * All hammer commands create and maintain cycle files in the snapshots + * directory. + */ + +#include "hammer.h" + +static void do_cleanup(const char *path); +static int strtosecs(char *ptr); +static const char *dividing_slash(const char *path); +static int check_period(const char *snapshots_path, const char *cmd, int arg1, + time_t *savep); +static void save_period(const char *snapshots_path, const char *cmd, + time_t savet); + +static int cleanup_snapshots(const char *path, const char *snapshots_path, + int arg1, int arg2); +static int cleanup_prune(const char *path, const char *snapshots_path, + int arg1, int arg2); +static int cleanup_reblock(const char *path, const char *snapshots_path, + int arg1, int arg2); +static int cleanup_recopy(const char *path, const char *snapshots_path, + int arg1, int arg2); + +static void runcmd(int *resp, const char *ctl, ...); + +#define WS " \t\r\n" +#define MAXPFS 65536 +#define DIDBITS (sizeof(int) * 8) + +static int DidPFS[MAXPFS/DIDBITS]; + +void +hammer_cmd_cleanup(char **av, int ac) +{ + FILE *fp; + char *ptr; + char *path; + char buf[256]; + + tzset(); + if (ac == 0) { + fp = popen("df -t hammer,null", "r"); + if (fp == NULL) + errx(1, "hammer cleanup: 'df' failed"); + while (fgets(buf, sizeof(buf), fp) != NULL) { + ptr = strtok(buf, WS); + if (ptr && strcmp(ptr, "Filesystem") == 0) + continue; + if (ptr) + ptr = strtok(NULL, WS); + if (ptr) + ptr = strtok(NULL, WS); + if (ptr) + ptr = strtok(NULL, WS); + if (ptr) + ptr = strtok(NULL, WS); + if (ptr) { + path = strtok(NULL, WS); + if (path) + do_cleanup(path); + } + } + fclose(fp); + } else { + while (ac) { + do_cleanup(*av); + --ac; + ++av; + } + } +} + +static +void +do_cleanup(const char *path) +{ + struct hammer_ioc_pseudofs_rw pfs; + union hammer_ioc_mrecord_any mrec_tmp; + char *snapshots_path; + char *config_path; + struct stat st; + char *cmd; + char *ptr; + int arg1; + int arg2; + time_t savet; + char buf[256]; + FILE *fp; + int fd; + int r; + + bzero(&pfs, sizeof(pfs)); + bzero(&mrec_tmp, sizeof(mrec_tmp)); + pfs.ondisk = &mrec_tmp.pfs.pfsd; + pfs.bytes = sizeof(mrec_tmp.pfs.pfsd); + pfs.pfs_id = -1; + + printf("cleanup %-20s -", path); + fd = open(path, O_RDONLY); + if (fd < 0) { + printf(" unable to access directory: %s\n", strerror(errno)); + return; + } + if (ioctl(fd, HAMMERIOC_GET_PSEUDOFS, &pfs) != 0) { + printf(" not a HAMMER filesystem: %s\n", strerror(errno)); + return; + } + close(fd); + if (pfs.version != HAMMER_IOC_PSEUDOFS_VERSION) { + printf(" unrecognized HAMMER version\n"); + return; + } + + /* + * Make sure we have not already handled this PFS. Several nullfs + * mounts might alias the same PFS. + */ + if (pfs.pfs_id < 0 || pfs.pfs_id >= MAXPFS) { + printf(" pfs_id %d illegal\n", pfs.pfs_id); + return; + } + + if (DidPFS[pfs.pfs_id / DIDBITS] & (1 << (pfs.pfs_id % DIDBITS))) { + printf(" pfs_id %d already handled\n", pfs.pfs_id); + return; + } + DidPFS[pfs.pfs_id / DIDBITS] |= (1 << (pfs.pfs_id % DIDBITS)); + + /* + * Create a snapshot directory if necessary, and a config file if + * necessary. + */ + asprintf(&snapshots_path, "%s%ssnapshots", path, dividing_slash(path)); + if (stat(snapshots_path, &st) < 0) { + if (mkdir(snapshots_path, 0755) != 0) { + free(snapshots_path); + printf(" unable to create snapshot dir: %s\n", + strerror(errno)); + return; + } + } + asprintf(&config_path, "%s/config", snapshots_path); + if ((fp = fopen(config_path, "r")) == NULL) { + fp = fopen(config_path, "w"); + if (fp == NULL) { + printf(" cannot create %s: %s\n", + config_path, strerror(errno)); + return; + } + if (strcmp(path, "/tmp") == 0 || + strcmp(path, "/var/tmp") == 0 || + strcmp(path, "/usr/obj") == 0) { + fprintf(fp, "snapshots 0d 60d\n"); + } else { + fprintf(fp, "snapshots 1d 60d\n"); + } + fprintf(fp, + "prune 1d 5m\n" + "reblock 1d 5m\n" + "recopy 30d 10m\n"); + fclose(fp); + fp = fopen(config_path, "r"); + } + if (fp == NULL) { + printf(" cannot access %s: %s\n", + config_path, strerror(errno)); + return; + } + + printf(" processing PFS #%d\n", pfs.pfs_id); + + /* + * Process the config file + */ + while (fgets(buf, sizeof(buf), fp) != NULL) { + cmd = strtok(buf, WS); + arg1 = 0; + arg2 = 0; + if ((ptr = strtok(NULL, WS)) != NULL) { + arg1 = strtosecs(ptr); + if ((ptr = strtok(NULL, WS)) != NULL) + arg2 = strtosecs(ptr); + } + + printf("%20s - ", cmd); + fflush(stdout); + + if (arg1 == 0) { + printf("disabled\n"); + continue; + } + + r = 1; + if (strcmp(cmd, "snapshots") == 0) { + if (check_period(snapshots_path, cmd, arg1, &savet)) { + printf("run\n"); + r = cleanup_snapshots(path, snapshots_path, + arg1, arg2); + } else { + printf("skip\n"); + } + } else if (strcmp(cmd, "prune") == 0) { + if (check_period(snapshots_path, cmd, arg1, &savet)) { + r = cleanup_prune(path, snapshots_path, + arg1, arg2); + } else { + printf("skip\n"); + } + } else if (strcmp(cmd, "reblock") == 0) { + if (check_period(snapshots_path, cmd, arg1, &savet)) { + printf("run"); + fflush(stdout); + if (VerboseOpt) + printf("\n"); + r = cleanup_reblock(path, snapshots_path, + arg1, arg2); + } else { + printf("skip\n"); + } + } else if (strcmp(cmd, "recopy") == 0) { + if (check_period(snapshots_path, cmd, arg1, &savet)) { + printf("run"); + fflush(stdout); + if (VerboseOpt) + printf("\n"); + r = cleanup_recopy(path, snapshots_path, + arg1, arg2); + } else { + printf("skip\n"); + } + } else { + printf("unknown directive\n"); + r = 1; + } + if (r == 0) + save_period(snapshots_path, cmd, savet); + } + fclose(fp); + usleep(1000); +} + +static +int +strtosecs(char *ptr) +{ + int val; + + val = strtol(ptr, &ptr, 0); + switch(*ptr) { + case 'd': + val *= 24; + /* fall through */ + case 'h': + val *= 60; + /* fall through */ + case 'm': + val *= 60; + /* fall through */ + case 's': + break; + default: + errx(1, "illegal suffix converting %s\n", ptr); + break; + } + return(val); +} + +static const char * +dividing_slash(const char *path) +{ + int len = strlen(path); + if (len && path[len-1] == '/') + return(""); + else + return("/"); +} + +/* + * Check whether the desired period has elapsed since the last successful + * run. The run may take a while and cross a boundary so we remember the + * current time_t so we can save it later on. + * + * Periods in minutes, hours, or days are assumed to have been crossed + * if the local time crosses a minute, hour, or day boundary regardless + * of how close the last operation actually was. + */ +static int +check_period(const char *snapshots_path, const char *cmd, int arg1, + time_t *savep) +{ + char *check_path; + struct tm tp1; + struct tm tp2; + FILE *fp; + time_t baset, lastt; + char buf[256]; + + time(savep); + localtime_r(savep, &tp1); + + /* + * Retrieve the start time of the last successful operation. + */ + asprintf(&check_path, "%s/.%s.period", snapshots_path, cmd); + fp = fopen(check_path, "r"); + free(check_path); + if (fp == NULL) + return(1); + if (fgets(buf, sizeof(buf), fp) == NULL) { + fclose(fp); + return(1); + } + fclose(fp); + + lastt = strtol(buf, NULL, 0); + localtime_r(&lastt, &tp2); + + /* + * Normalize the times. e.g. if asked to do something on a 1-day + * interval the operation will be performed as soon as the day + * turns over relative to the previous operation, even if the previous + * operation ran a few seconds ago just before midnight. + */ + if (arg1 % 60 == 0) { + tp1.tm_sec = 0; + tp2.tm_sec = 0; + } + if (arg1 % (60 * 60) == 0) { + tp1.tm_min = 0; + tp2.tm_min = 0; + } + if (arg1 % (24 * 60 * 60) == 0) { + tp1.tm_hour = 0; + tp2.tm_hour = 0; + } + + baset = mktime(&tp1); + lastt = mktime(&tp2); + +#if 0 + printf("%lld vs %lld\n", (long long)(baset - lastt), (long long)arg1); +#endif + + if ((int)(baset - lastt) >= arg1) + return(1); + return(0); +} + +/* + * Store the start time of the last successful operation. + */ +static void +save_period(const char *snapshots_path, const char *cmd, + time_t savet) +{ + char *ocheck_path; + char *ncheck_path; + FILE *fp; + + asprintf(&ocheck_path, "%s/.%s.period", snapshots_path, cmd); + asprintf(&ncheck_path, "%s/.%s.period.new", snapshots_path, cmd); + fp = fopen(ncheck_path, "w"); + fprintf(fp, "0x%08llx\n", (long long)savet); + if (fclose(fp) == 0) + rename(ncheck_path, ocheck_path); + remove(ncheck_path); +} + +/* + * Issue a snapshot. + */ +static int +cleanup_snapshots(const char *path __unused, const char *snapshots_path, + int arg1 __unused, int arg2 __unused) +{ + int r; + + runcmd(&r, "hammer snapshot %s", snapshots_path); + return(r); +} + +static int +cleanup_prune(const char *path __unused, const char *snapshots_path, + int arg1 __unused, int arg2) +{ + if (arg2) { + runcmd(NULL, "hammer -c %s/.prune.cycle -t %d prune %s", + snapshots_path, arg2, snapshots_path); + } else { + runcmd(NULL, "hammer prune %s", snapshots_path); + } + return(0); +} + +static int +cleanup_reblock(const char *path, const char *snapshots_path, + int arg1 __unused, int arg2) +{ + if (VerboseOpt == 0) { + printf("."); + fflush(stdout); + } + runcmd(NULL, + "hammer -c %s/.reblock-1.cycle -t %d reblock-btree %s 95", + snapshots_path, arg2, path); + if (VerboseOpt == 0) { + printf("."); + fflush(stdout); + } + runcmd(NULL, + "hammer -c %s/.reblock-2.cycle -t %d reblock-inodes %s 95", + snapshots_path, arg2, path); + if (VerboseOpt == 0) { + printf("."); + fflush(stdout); + } + runcmd(NULL, + "hammer -c %s/.reblock-3.cycle -t %d reblock-data %s 95", + snapshots_path, arg2, path); + if (VerboseOpt == 0) + printf("\n"); + return(0); +} + +static int +cleanup_recopy(const char *path, const char *snapshots_path, + int arg1 __unused, int arg2) +{ + if (VerboseOpt == 0) { + printf("."); + fflush(stdout); + } + runcmd(NULL, + "hammer -c %s/.recopy-1.cycle -t %d reblock-btree %s", + snapshots_path, arg2, path); + if (VerboseOpt == 0) { + printf("."); + fflush(stdout); + } + runcmd(NULL, + "hammer -c %s/.recopy-2.cycle -t %d reblock-inodes %s", + snapshots_path, arg2, path); + if (VerboseOpt == 0) { + printf("."); + fflush(stdout); + } + runcmd(NULL, + "hammer -c %s/.recopy-3.cycle -t %d reblock-data %s", + snapshots_path, arg2, path); + if (VerboseOpt == 0) + printf("\n"); + return(0); +} + +static +void +runcmd(int *resp, const char *ctl, ...) +{ + va_list va; + char *cmd; + char *arg; + char **av; + int n; + int nmax; + int res; + pid_t pid; + + /* + * Generate the command + */ + va_start(va, ctl); + vasprintf(&cmd, ctl, va); + va_end(va); + if (VerboseOpt) + printf(" %s\n", cmd); + + /* + * Break us down into arguments. We do not just use system() here + * because it blocks SIGINT and friends. + */ + n = 0; + nmax = 16; + av = malloc(sizeof(char *) * nmax); + + for (arg = strtok(cmd, WS); arg; arg = strtok(NULL, WS)) { + if (n == nmax) { + nmax += 16; + av = realloc(av, sizeof(char *) * nmax); + } + av[n++] = arg; + } + + /* + * Run the command. + */ + if ((pid = fork()) == 0) { + if (VerboseOpt < 2) { + int fd = open("/dev/null", O_RDWR); + dup2(fd, 1); + close(fd); + } + execvp(av[0], av); + _exit(127); + } else if (pid < 0) { + res = 127; + } else { + int status; + while (waitpid(pid, &status, 0) != pid) + ; + res = WEXITSTATUS(status); + } + + free(cmd); + free(av); + if (resp) + *resp = res; +} + + diff --git a/sbin/hammer/hammer.8 b/sbin/hammer/hammer.8 index 002ee43e06..8332ea5d3c 100644 --- a/sbin/hammer/hammer.8 +++ b/sbin/hammer/hammer.8 @@ -30,7 +30,7 @@ .\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.\" $DragonFly: src/sbin/hammer/hammer.8,v 1.46 2008/08/21 23:28:43 thomas Exp $ +.\" $DragonFly: src/sbin/hammer/hammer.8,v 1.47 2008/09/20 04:23:21 dillon Exp $ .Dd July 27, 2008 .Dt HAMMER 8 .Os @@ -181,6 +181,42 @@ Generate the top 32 bits of a .Nm HAMMER 64 bit directory hash for the specified file name. +.\" ==== cleanup ==== +.It Ar cleanup Op Ar filesystem +This is a meta-command which executes snapshot, pruning, and reblocking +commands on the specified HAMMER filesystem. +If no filesystem is specified this command will scan all HAMMER and nullfs +mounts, extract PFS id's, and clean-up each PFS found. +.Pp +This command will access a +.Ar snapshots +subdirectory and a +.Ar snapshots/config +file for each filesystem, creating them if necessary. +The default configuration file will create a daily snapshot, do a daily +pruning and reblocking run with a fragmentation level of 95%, and +a monthly unconditional reblocking run. +.Pp +All operations are limited to 5 minutes per function by default. +Reblocking runs are broken down into three separate functions and +take 15 minutes. +Also note that this directive will disable snapshots on +.Pa /tmp , +.Pa /var/tmp , +and +.Pa /usr/obj +by default. +.Pp +The defaults may be adjusted by modifying the config file. +The pruning and reblocking commands automatically maintain a cyclefile +for incremental operation. +If you ^C the program the cyclefile will be updated, but a sub-command +may continue to run in the background for a few seconds until the HAMMER +ioctl detects the interrupt. +.Pp +Work on this command is still in progress. +Expected additions: Limit the number of snapshots to 60 days by default +and remove snapshots as the filesystem becomes full. .\" ==== prune ==== .It Ar prune Ar softlink-dir Prune the file system based on previously created snapshot softlinks. diff --git a/sbin/hammer/hammer.c b/sbin/hammer/hammer.c index 5be575bdf7..2f9fcc2909 100644 --- a/sbin/hammer/hammer.c +++ b/sbin/hammer/hammer.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/hammer/hammer.c,v 1.38 2008/08/21 23:28:43 thomas Exp $ + * $DragonFly: src/sbin/hammer/hammer.c,v 1.39 2008/09/20 04:23:21 dillon Exp $ */ #include "hammer.h" @@ -187,6 +187,10 @@ main(int ac, char **av) hammer_cmd_softprune(av + 1, ac - 1, 0); exit(0); } + if (strcmp(av[0], "cleanup") == 0) { + hammer_cmd_cleanup(av + 1, ac - 1); + exit(0); + } if (strcmp(av[0], "prune-everything") == 0) { hammer_cmd_softprune(av + 1, ac - 1, 1); exit(0); @@ -209,6 +213,7 @@ main(int ac, char **av) exit(0); } if (strncmp(av[0], "reblock", 7) == 0) { + signal(SIGINT, sigalrm); if (strcmp(av[0], "reblock") == 0) hammer_cmd_reblock(av + 1, ac - 1, -1); else if (strcmp(av[0], "reblock-btree") == 0) @@ -300,6 +305,7 @@ usage(int exit_code) fprintf(stderr, "hammer -h\n" "hammer [-v] [-t seconds] [-c cyclefile] command [argument ...]\n" + "hammer cleanup []\n" "hammer synctid [quick]\n" "hammer namekey[32] \n" "hammer prune \n" diff --git a/sbin/hammer/hammer.h b/sbin/hammer/hammer.h index 8f446c483f..3c83dbc562 100644 --- a/sbin/hammer/hammer.h +++ b/sbin/hammer/hammer.h @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/hammer/hammer.h,v 1.24 2008/07/31 06:01:32 dillon Exp $ + * $DragonFly: src/sbin/hammer/hammer.h,v 1.25 2008/09/20 04:23:21 dillon Exp $ */ #include @@ -91,6 +91,7 @@ void hammer_cmd_pseudofs_upgrade(char **av, int ac); void hammer_cmd_pseudofs_downgrade(char **av, int ac); void hammer_cmd_status(char **av, int ac); void hammer_cmd_snapshot(char **av, int ac); +void hammer_cmd_cleanup(char **av, int ac); void hammer_get_cycle(hammer_base_elm_t base, hammer_tid_t *tidp); void hammer_set_cycle(hammer_base_elm_t base, hammer_tid_t tid); -- 2.41.0