hammer utility - Add force support to cleanup
[dragonfly.git] / sbin / hammer / cmd_cleanup.c
index 0cd1f09..8342684 100644 (file)
  * $DragonFly: src/sbin/hammer/cmd_cleanup.c,v 1.6 2008/10/07 22:28:41 thomas Exp $
  */
 /*
- * Clean up a specific HAMMER filesystem or all HAMMER filesystems.
+ * Clean up specific HAMMER filesystems or all HAMMER filesystems.
  *
- * Each filesystem is expected to have a <mount>/snapshots directory.
- * No cleanup will be performed on any filesystem that does not.  If
- * no filesystems are specified the 'df' program is run and any HAMMER
- * or null-mounted hammer PFS's are extracted.
+ * If no filesystems are specified any HAMMER- or null-mounted hammer PFS's
+ * are cleaned.
  *
- * The snapshots directory may contain a config file called 'config'.  If
- * no config file is present one will be created with the following
+ * Each HAMMER filesystem may contain a configuration file.  If no
+ * configuration file is present one will be created with the following
  * defaults:
  *
  *     snapshots 1d 60d        (0d 0d for /tmp, /var/tmp, /usr/obj)
  *     prune     1d 5m
+ *     rebalance 1d 5m
  *     reblock   1d 5m
- *     recopy    30d 5m
+ *     recopy    30d 10m
  *
  * All hammer commands create and maintain cycle files in the snapshots
  * directory.
+ *
+ * For HAMMER version 2- the configuration file is a named 'config' in
+ * the snapshots directory, which defaults to <pfs>/snapshots.
+ * For HAMMER version 3+ the configuration file is saved in filesystem
+ * meta-data. The snapshots directory defaults to /var/hammer/<pfs>
+ * (/var/hammer/root for root mount).
  */
 
 #include "hammer.h"
@@ -62,28 +67,41 @@ struct didpfs {
 };
 
 static void do_cleanup(const char *path);
+static void config_init(const char *path, struct hammer_ioc_config *config);
+static void migrate_config(FILE *fp, struct hammer_ioc_config *config);
+static void migrate_snapshots(int fd, const char *snapshots_path);
+static void migrate_one_snapshot(int fd, const char *fpath,
+                       struct hammer_ioc_snapshot *snapshot);
 static int strtosecs(char *ptr);
 static const char *dividing_slash(const char *path);
 static int check_period(const char *snapshots_path, const char *cmd, int arg1,
                        time_t *savep);
 static void save_period(const char *snapshots_path, const char *cmd,
                        time_t savet);
-static int check_softlinks(const char *snapshots_path);
-static void cleanup_softlinks(const char *path, const char *snapshots_path,
-                       int arg2, char *arg3);
+static int check_softlinks(int fd, int new_config, const char *snapshots_path);
+static void cleanup_softlinks(int fd, int new_config,
+                       const char *snapshots_path, int arg2, char *arg3);
+static void delete_snapshots(int fd, struct hammer_ioc_snapshot *dsnapshot);
 static int check_expired(const char *fpath, int arg2);
 
-static int cleanup_snapshots(const char *path, const char *snapshots_path,
-                             int arg1, int arg2);
+static int create_snapshot(const char *path, const char *snapshots_path);
+static int cleanup_rebalance(const char *path, const char *snapshots_path,
+                       int arg1, int arg2);
 static int cleanup_prune(const char *path, const char *snapshots_path,
-                             int arg1, int arg2, int snapshots_disabled);
+                       int arg1, int arg2, int snapshots_disabled);
 static int cleanup_reblock(const char *path, const char *snapshots_path,
-                             int arg1, int arg2);
+                       int arg1, int arg2);
 static int cleanup_recopy(const char *path, const char *snapshots_path,
-                             int arg1, int arg2);
+                       int arg1, int arg2);
 
 static void runcmd(int *resp, const char *ctl, ...);
 
+/*
+ * WARNING: Do not make the SNAPSHOTS_BASE "/var/snapshots" because
+ * it will interfere with the older HAMMER VERS < 3 snapshots directory
+ * for the /var PFS.
+ */
+#define SNAPSHOTS_BASE "/var/hammer"   /* HAMMER VERS >= 3 */
 #define WS     " \t\r\n"
 
 struct didpfs *FirstPFS;
@@ -91,36 +109,33 @@ struct didpfs *FirstPFS;
 void
 hammer_cmd_cleanup(char **av, int ac)
 {
-       FILE *fp;
-       char *fs, *ptr, *path;
-       char buf[256];
+       char *fstype, *fs, *path;
+       struct statfs *stfsbuf;
+       int mntsize, i;
 
        tzset();
        if (ac == 0) {
-               fp = popen("/sbin/mount -t hammer,null", "r");
-               if (fp == NULL)
-                       errx(1, "hammer cleanup: 'mount' failed");
-               while (fgets(buf, sizeof(buf), fp) != NULL) {
-                       fs = strtok(buf, WS);
-                       if (fs == NULL)
-                               continue;
-                       ptr = strtok(NULL, WS);
-                       if (ptr == NULL)
-                               continue;
-                       path = strtok(NULL, WS);
-                       if (path == NULL)
-                               continue;
-                       ptr = strtok(NULL, WS);
-                       if (ptr == NULL)
-                               continue;
-                       if ((strncmp(ptr, "(hammer,", 8) == 0) ||
-                           ((strncmp(ptr, "(null,", 6) == 0) &&
-                            (strstr(fs, "/@@0x") != NULL ||
-                             strstr(fs, "/@@-1") != NULL))) {
-                               do_cleanup(path);
+               mntsize = getmntinfo(&stfsbuf, MNT_NOWAIT);
+               if (mntsize > 0) {
+                       for (i=0; i < mntsize; i++) {
+                               /*
+                                * We will cleanup in the case fstype is hammer.
+                                * If we have null-mounted PFS, we check the
+                                * mount source. If it looks like a PFS, we
+                                * proceed to cleanup also.
+                                */
+                               fstype = stfsbuf[i].f_fstypename;
+                               fs = stfsbuf[i].f_mntfromname;
+                               if ((strcmp(fstype, "hammer") == 0) ||
+                                   ((strcmp(fstype, "null") == 0) &&
+                                    (strstr(fs, "/@@0x") != NULL ||
+                                     strstr(fs, "/@@-1") != NULL))) {
+                                       path = stfsbuf[i].f_mntonname;
+                                       do_cleanup(path);
+                               }
                        }
                }
-               fclose(fp);
+
        } else {
                while (ac) {
                        do_cleanup(*av);
@@ -135,8 +150,10 @@ void
 do_cleanup(const char *path)
 {
        struct hammer_ioc_pseudofs_rw pfs;
+       struct hammer_ioc_config config;
+       struct hammer_ioc_version version;
        union hammer_ioc_mrecord_any mrec_tmp;
-       char *snapshots_path;
+       char *snapshots_path = NULL;
        char *config_path;
        struct stat st;
        char *cmd;
@@ -146,12 +163,17 @@ do_cleanup(const char *path)
        char *arg3;
        time_t savet;
        char buf[256];
-       FILE *fp;
+       char *cbase;
+       char *cptr;
+       FILE *fp = NULL;
        struct didpfs *didpfs;
        int snapshots_disabled = 0;
        int prune_warning = 0;
+       int new_config = 0;
+       int snapshots_from_pfs = 0;
        int fd;
        int r;
+       int found_rebal = 0;
 
        bzero(&pfs, sizeof(pfs));
        bzero(&mrec_tmp, sizeof(mrec_tmp));
@@ -165,16 +187,31 @@ do_cleanup(const char *path)
                printf(" unable to access directory: %s\n", strerror(errno));
                return;
        }
-       if (ioctl(fd, HAMMERIOC_GET_PSEUDOFS, &pfs) != 0) {
+       if (ioctl(fd, HAMMERIOC_GET_PSEUDOFS, &pfs) < 0) {
                printf(" not a HAMMER filesystem: %s\n", strerror(errno));
+               close(fd);
                return;
        }
-       close(fd);
        if (pfs.version != HAMMER_IOC_PSEUDOFS_VERSION) {
                printf(" unrecognized HAMMER version\n");
+               close(fd);
+               return;
+       }
+       bzero(&version, sizeof(version));
+       if (ioctl(fd, HAMMERIOC_GET_VERSION, &version) < 0) {
+               printf(" HAMMER filesystem but couldn't retrieve version!\n");
+               close(fd);
                return;
        }
 
+       bzero(&config, sizeof(config));
+       if (version.cur_version >= 3) {
+               if (ioctl(fd, HAMMERIOC_GET_CONFIG, &config) == 0 &&
+                   config.head.error == 0) {
+                       new_config = 1;
+               }
+       }
+
        /*
         * Make sure we have not already handled this PFS.  Several nullfs
         * mounts might alias the same PFS.
@@ -182,6 +219,7 @@ do_cleanup(const char *path)
        for (didpfs = FirstPFS; didpfs; didpfs = didpfs->next) {
                if (bcmp(&didpfs->uuid, &mrec_tmp.pfs.pfsd.unique_uuid, sizeof(uuid_t)) == 0) {
                        printf(" PFS #%d already handled\n", pfs.pfs_id);
+                       close(fd);
                        return;
                }
        }
@@ -191,61 +229,158 @@ do_cleanup(const char *path)
        didpfs->uuid = mrec_tmp.pfs.pfsd.unique_uuid;
 
        /*
-        * Figure out where the snapshot directory is.
+        * Calculate the old snapshots directory for HAMMER VERSION < 3
+        *
+        * If the directory is explicitly specified in the PFS config
+        * we flag it and will not migrate it later.
         */
        if (mrec_tmp.pfs.pfsd.snapshots[0] == '/') {
                asprintf(&snapshots_path, "%s", mrec_tmp.pfs.pfsd.snapshots);
+               snapshots_from_pfs = 1;
        } else if (mrec_tmp.pfs.pfsd.snapshots[0]) {
                printf(" WARNING: pfs-slave's snapshots dir is not absolute\n");
+               close(fd);
                return;
        } else if (mrec_tmp.pfs.pfsd.mirror_flags & HAMMER_PFSD_SLAVE) {
-               printf(" WARNING: must configure snapshot dir for PFS slave\n");
-               printf("\tWe suggest <fs>/var/slaves/<name> where "
-                      "<fs> is the base HAMMER fs\n");
-               printf("\tcontaining the slave\n");
-               return;
+               if (version.cur_version < 3) {
+                       printf(" WARNING: must configure snapshot dir for PFS slave\n");
+                       printf("\tWe suggest <fs>/var/slaves/<name> where "
+                              "<fs> is the base HAMMER fs\n");
+                       printf("\tcontaining the slave\n");
+                       close(fd);
+                       return;
+               }
        } else {
                asprintf(&snapshots_path,
                         "%s%ssnapshots", path, dividing_slash(path));
        }
 
        /*
-        * Create a snapshot directory if necessary, and a config file if
-        * necessary.
+        * Check for old-style config file
         */
-       if (stat(snapshots_path, &st) < 0) {
-               if (mkdir(snapshots_path, 0755) != 0) {
-                       free(snapshots_path);
-                       printf(" unable to create snapshot dir \"%s\": %s\n",
-                               snapshots_path, strerror(errno));
-                       return;
-               }
+       if (snapshots_path) {
+               asprintf(&config_path, "%s/config", snapshots_path);
+               fp = fopen(config_path, "r");
        }
-       asprintf(&config_path, "%s/config", snapshots_path);
-       if ((fp = fopen(config_path, "r")) == NULL) {
-               fp = fopen(config_path, "w");
-               if (fp == NULL) {
-                       printf(" cannot create %s: %s\n",
-                               config_path, strerror(errno));
-                       return;
+
+       /*
+        * Handle upgrades to hammer version 3, move the config
+        * file into meta-data.
+        *
+        * For the old config read the file into the config structure,
+        * we will parse it out of the config structure regardless.
+        */
+       if (version.cur_version >= 3) {
+               if (fp) {
+                       printf("(migrating) ");
+                       fflush(stdout);
+                       migrate_config(fp, &config);
+                       migrate_snapshots(fd, snapshots_path);
+                       fclose(fp);
+                       if (ioctl(fd, HAMMERIOC_SET_CONFIG, &config) < 0) {
+                               printf(" cannot init meta-data config!\n");
+                               close(fd);
+                               return;
+                       }
+                       remove(config_path);
+               } else if (new_config == 0) {
+                       config_init(path, &config);
+                       if (ioctl(fd, HAMMERIOC_SET_CONFIG, &config) < 0) {
+                               printf(" cannot init meta-data config!\n");
+                               close(fd);
+                               return;
+                       }
+               }
+               new_config = 1;
+       } else {
+               /*
+                * Create missing snapshots directory for HAMMER VERSION < 3
+                */
+               if (stat(snapshots_path, &st) < 0) {
+                       if (mkdir(snapshots_path, 0755) != 0) {
+                               free(snapshots_path);
+                               printf(" unable to create snapshot dir \"%s\": %s\n",
+                                       snapshots_path, strerror(errno));
+                               close(fd);
+                               return;
+                       }
                }
-               if (strcmp(path, "/tmp") == 0 ||
-                   strcmp(path, "/var/tmp") == 0 ||
-                   strcmp(path, "/usr/obj") == 0) {
-                       fprintf(fp, "snapshots 0d 0d\n");
+
+               /*
+                *  Create missing config file for HAMMER VERSION < 3
+                */
+               if (fp == NULL) {
+                       config_init(path, &config);
+                       fp = fopen(config_path, "w");
+                       if (fp) {
+                               fwrite(config.config.text, 1,
+                                       strlen(config.config.text), fp);
+                               fclose(fp);
+                       }
                } else {
-                       fprintf(fp, "snapshots 1d 60d\n");
+                       migrate_config(fp, &config);
+                       fclose(fp);
                }
-               fprintf(fp, 
-                       "prune     1d 5m\n"
-                       "reblock   1d 5m\n"
-                       "recopy    30d 10m\n");
-               fclose(fp);
-               fp = fopen(config_path, "r");
        }
-       if (fp == NULL) {
-               printf(" cannot access %s: %s\n",
-                      config_path, strerror(errno));
+
+       /*
+        * If snapshots_from_pfs is not set we calculate the new snapshots
+        * directory default (in /var) for HAMMER VERSION >= 3 and migrate
+        * the old snapshots directory over.
+        *
+        * People who have set an explicit snapshots directory will have
+        * to migrate the data manually into /var/hammer, or not bother at
+        * all.  People running slaves may wish to migrate it and then
+        * clear the snapshots specification in the PFS config for the
+        * slave.
+        */
+       if (new_config && snapshots_from_pfs == 0) {
+               char *npath;
+
+               assert(path[0] == '/');
+               if (strcmp(path, "/") == 0)
+                       asprintf(&npath, "%s/root", SNAPSHOTS_BASE);
+               else
+                       asprintf(&npath, "%s/%s", SNAPSHOTS_BASE, path + 1);
+               if (snapshots_path) {
+                       if (stat(npath, &st) < 0 && errno == ENOENT) {
+                               if (stat(snapshots_path, &st) < 0 && errno == ENOENT) {
+                                       printf(" HAMMER UPGRADE: Creating snapshots\n"
+                                              "\tCreating snapshots in %s\n",
+                                              npath);
+                                       runcmd(&r, "mkdir -p %s", npath);
+                               } else {
+                                       printf(" HAMMER UPGRADE: Moving snapshots\n"
+                                              "\tMoving snapshots from %s to %s\n",
+                                              snapshots_path, npath);
+                                       runcmd(&r, "mkdir -p %s", npath);
+                                       runcmd(&r, "cpdup %s %s", snapshots_path, npath);
+                                       if (r != 0) {
+                                   printf("Unable to move snapshots directory!\n");
+                                   printf("Please fix this critical error.\n");
+                                   printf("Aborting cleanup of %s\n", path);
+                                               close(fd);
+                                               return;
+                                       }
+                                       runcmd(&r, "rm -rf %s", snapshots_path);
+                               }
+                       }
+                       free(snapshots_path);
+               } else if (stat(npath, &st) < 0 && errno == ENOENT) {
+                       runcmd(&r, "mkdir -p %s", npath);
+               }
+               snapshots_path = npath;
+       }
+
+       /*
+        * Lock the PFS.  fd is the base directory of the mounted PFS.
+        */
+       if (flock(fd, LOCK_EX|LOCK_NB) == -1) {
+               if (errno == EWOULDBLOCK)
+                       printf(" PFS #%d locked by other process\n", pfs.pfs_id);
+               else
+                       printf(" can not lock %s: %s\n", config_path, strerror(errno));
+               close(fd);
                return;
        }
 
@@ -254,8 +389,17 @@ do_cleanup(const char *path)
        /*
         * Process the config file
         */
-       while (fgets(buf, sizeof(buf), fp) != NULL) {
+       cbase = config.config.text;
+
+       while ((cptr = strchr(cbase, '\n')) != NULL) {
+               bcopy(cbase, buf, cptr - cbase);
+               buf[cptr - cbase] = 0;
+               cbase = cptr + 1;
+
                cmd = strtok(buf, WS);
+               if (cmd == NULL || cmd[0] == '#')
+                       continue;
+
                arg1 = 0;
                arg2 = 0;
                arg3 = NULL;
@@ -273,10 +417,13 @@ do_cleanup(const char *path)
                r = 1;
                if (strcmp(cmd, "snapshots") == 0) {
                        if (arg1 == 0) {
-                               if (arg2 && check_softlinks(snapshots_path)) {
+                               if (arg2 &&
+                                   check_softlinks(fd, new_config,
+                                                   snapshots_path)) {
                                        printf("only removing old snapshots\n");
                                        prune_warning = 1;
-                                       cleanup_softlinks(path, snapshots_path,
+                                       cleanup_softlinks(fd, new_config,
+                                                         snapshots_path,
                                                          arg2, arg3);
                                } else {
                                        printf("disabled\n");
@@ -285,10 +432,10 @@ do_cleanup(const char *path)
                        } else
                        if (check_period(snapshots_path, cmd, arg1, &savet)) {
                                printf("run\n");
-                               cleanup_softlinks(path, snapshots_path,
+                               cleanup_softlinks(fd, new_config,
+                                                 snapshots_path,
                                                  arg2, arg3);
-                               r = cleanup_snapshots(path, snapshots_path,
-                                                 arg1, arg2);
+                               r = create_snapshot(path, snapshots_path);
                        } else {
                                printf("skip\n");
                        }
@@ -313,6 +460,18 @@ do_cleanup(const char *path)
                        } else {
                                printf("skip\n");
                        }
+               } else if (strcmp(cmd, "rebalance") == 0) {
+                       found_rebal = 1;
+                       if (check_period(snapshots_path, cmd, arg1, &savet)) {
+                               printf("run");
+                               fflush(stdout);
+                               if (VerboseOpt)
+                                       printf("\n");
+                               r = cleanup_rebalance(path, snapshots_path,
+                                               arg1, arg2);
+                       } else {
+                               printf("skip\n");
+                       }
                } else if (strcmp(cmd, "reblock") == 0) {
                        if (check_period(snapshots_path, cmd, arg1, &savet)) {
                                printf("run");
@@ -342,10 +501,185 @@ do_cleanup(const char *path)
                if (r == 0)
                        save_period(snapshots_path, cmd, savet);
        }
-       fclose(fp);
+
+       /*
+        * Add new rebalance feature if the config doesn't have it.
+        * (old style config only).
+        */
+       if (new_config == 0 && found_rebal == 0) {
+               if ((fp = fopen(config_path, "r+")) != NULL) {
+                       fseek(fp, 0L, 2);
+                       fprintf(fp, "rebalance 1d 5m\n");
+                       fclose(fp);
+               }
+       }
+
+       /*
+        * Cleanup, and delay a little
+        */
+       close(fd);
        usleep(1000);
 }
 
+/*
+ * Initialize new config data (new or old style)
+ */
+static void
+config_init(const char *path, struct hammer_ioc_config *config)
+{
+       const char *snapshots;
+
+       if (strcmp(path, "/tmp") == 0 ||
+           strcmp(path, "/var/tmp") == 0 ||
+           strcmp(path, "/usr/obj") == 0) {
+               snapshots = "snapshots 0d 0d\n";
+       } else {
+               snapshots = "snapshots 1d 60d\n";
+       }
+       bzero(config->config.text, sizeof(config->config.text));
+       snprintf(config->config.text, sizeof(config->config.text) - 1, "%s%s",
+               snapshots,
+               "prune     1d 5m\n"
+               "rebalance 1d 5m\n"
+               "reblock   1d 5m\n"
+               "recopy    30d 10m\n");
+}
+
+/*
+ * Migrate configuration data from the old snapshots/config
+ * file to the new meta-data format.
+ */
+static void
+migrate_config(FILE *fp, struct hammer_ioc_config *config)
+{
+       int n;
+
+       n = fread(config->config.text, 1, sizeof(config->config.text) - 1, fp);
+       if (n >= 0)
+               bzero(config->config.text + n, sizeof(config->config.text) - n);
+}
+
+/*
+ * Migrate snapshot softlinks in the snapshots directory to the
+ * new meta-data format.  The softlinks are left intact, but
+ * this way the pruning code won't lose track of them if you
+ * happen to blow away the snapshots directory.
+ */
+static void
+migrate_snapshots(int fd, const char *snapshots_path)
+{
+       struct hammer_ioc_snapshot snapshot;
+       struct dirent *den;
+       struct stat st;
+       DIR *dir;
+       char *fpath;
+
+       bzero(&snapshot, sizeof(snapshot));
+
+       if ((dir = opendir(snapshots_path)) != NULL) {
+               while ((den = readdir(dir)) != NULL) {
+                       if (den->d_name[0] == '.')
+                               continue;
+                       asprintf(&fpath, "%s/%s", snapshots_path, den->d_name);
+                       if (lstat(fpath, &st) == 0 && S_ISLNK(st.st_mode)) {
+                               migrate_one_snapshot(fd, fpath, &snapshot);
+                       }
+                       free(fpath);
+               }
+               closedir(dir);
+       }
+       migrate_one_snapshot(fd, NULL, &snapshot);
+
+}
+
+/*
+ * Migrate a single snapshot.  If fpath is NULL the ioctl is flushed,
+ * otherwise it is flushed when it fills up.
+ */
+static void
+migrate_one_snapshot(int fd, const char *fpath,
+                    struct hammer_ioc_snapshot *snapshot)
+{
+       if (fpath) {
+               struct hammer_snapshot_data *snap;
+               struct tm tm;
+               time_t t;
+               int year;
+               int month;
+               int day = 0;
+               int hour = 0;
+               int minute = 0;
+               int r;
+               char linkbuf[1024];
+               const char *ptr;
+               hammer_tid_t tid;
+
+               t = (time_t)-1;
+               tid = (hammer_tid_t)(int64_t)-1;
+
+               /* fpath may contain directory components */
+               if ((ptr = strrchr(fpath, '/')) != NULL)
+                       ++ptr;
+               else
+                       ptr = fpath;
+               while (*ptr && *ptr != '-' && *ptr != '.')
+                       ++ptr;
+               if (*ptr)
+                       ++ptr;
+               r = sscanf(ptr, "%4d%2d%2d-%2d%2d",
+                          &year, &month, &day, &hour, &minute);
+
+               if (r >= 3) {
+                       bzero(&tm, sizeof(tm));
+                       tm.tm_isdst = -1;
+                       tm.tm_min = minute;
+                       tm.tm_hour = hour;
+                       tm.tm_mday = day;
+                       tm.tm_mon = month - 1;
+                       tm.tm_year = year - 1900;
+                       t = mktime(&tm);
+               }
+               bzero(linkbuf, sizeof(linkbuf));
+               if (readlink(fpath, linkbuf, sizeof(linkbuf) - 1) > 0 &&
+                   (ptr = strrchr(linkbuf, '@')) != NULL &&
+                   ptr > linkbuf && ptr[-1] == '@') {
+                       tid = strtoull(ptr + 1, NULL, 16);
+               }
+               if (t != (time_t)-1 && tid != (hammer_tid_t)(int64_t)-1) {
+                       snap = &snapshot->snaps[snapshot->count];
+                       bzero(snap, sizeof(*snap));
+                       snap->tid = tid;
+                       snap->ts = (u_int64_t)t * 1000000ULL;
+                       snprintf(snap->label, sizeof(snap->label),
+                                "migrated");
+                       ++snapshot->count;
+               } else {
+                       printf("    non-canonical snapshot softlink: %s->%s\n",
+                              fpath, linkbuf);
+               }
+       }
+
+       if ((fpath == NULL && snapshot->count) ||
+           snapshot->count == HAMMER_SNAPS_PER_IOCTL) {
+               printf(" (%d snapshots)", snapshot->count);
+again:
+               if (ioctl(fd, HAMMERIOC_ADD_SNAPSHOT, snapshot) < 0) {
+                       printf("    Ioctl to migrate snapshots failed: %s\n",
+                              strerror(errno));
+               } else if (snapshot->head.error == EALREADY) {
+                       ++snapshot->index;
+                       goto again;
+               } else if (snapshot->head.error) {
+                       printf("    Ioctl to migrate snapshots failed: %s\n",
+                              strerror(snapshot->head.error));
+               }
+               printf("index %d\n", snapshot->index);
+               snapshot->index = 0;
+               snapshot->count = 0;
+               snapshot->head.error = 0;
+       }
+}
+
 static
 int
 strtosecs(char *ptr)
@@ -390,6 +724,8 @@ dividing_slash(const char *path)
  * Periods in minutes, hours, or days are assumed to have been crossed
  * if the local time crosses a minute, hour, or day boundary regardless
  * of how close the last operation actually was.
+ *
+ * If ForceOpt is set always return true.
  */
 static int
 check_period(const char *snapshots_path, const char *cmd, int arg1,
@@ -406,6 +742,12 @@ check_period(const char *snapshots_path, const char *cmd, int arg1,
        localtime_r(savep, &tp1);
 
        /*
+        * Force run if -F
+        */
+       if (ForceOpt)
+               return(1);
+
+       /*
         * Retrieve the start time of the last successful operation.
         */
        asprintf(&check_path, "%s/.%s.period", snapshots_path, cmd);
@@ -482,7 +824,7 @@ save_period(const char *snapshots_path, const char *cmd,
  * Simply count the number of softlinks in the snapshots dir
  */
 static int
-check_softlinks(const char *snapshots_path)
+check_softlinks(int fd, int new_config, const char *snapshots_path)
 {
        struct dirent *den;
        struct stat st;
@@ -490,6 +832,9 @@ check_softlinks(const char *snapshots_path)
        char *fpath;
        int res = 0;
 
+       /*
+        * Old-style softlink-based snapshots
+        */
        if ((dir = opendir(snapshots_path)) != NULL) {
                while ((den = readdir(dir)) != NULL) {
                        if (den->d_name[0] == '.')
@@ -501,15 +846,33 @@ check_softlinks(const char *snapshots_path)
                }
                closedir(dir);
        }
-       return(res);
+
+       /*
+        * New-style snapshots are stored as filesystem meta-data,
+        * count those too.
+        */
+       if (new_config) {
+               struct hammer_ioc_snapshot snapshot;
+
+               bzero(&snapshot, sizeof(snapshot));
+               do {
+                       if (ioctl(fd, HAMMERIOC_GET_SNAPSHOT, &snapshot) < 0) {
+                               err(2, "hammer cleanup: check_softlink "
+                                       "snapshot error");
+                               /* not reached */
+                       }
+                       res += snapshot.count;
+               } while (snapshot.head.error == 0 && snapshot.count);
+       }
+       return (res);
 }
 
 /*
  * Clean up expired softlinks in the snapshots dir
  */
 static void
-cleanup_softlinks(const char *path __unused, const char *snapshots_path,
-                 int arg2, char *arg3)
+cleanup_softlinks(int fd, int new_config,
+                 const char *snapshots_path, int arg2, char *arg3)
 {
        struct dirent *den;
        struct stat st;
@@ -540,6 +903,78 @@ cleanup_softlinks(const char *path __unused, const char *snapshots_path,
                }
                closedir(dir);
        }
+
+       /*
+        * New-style snapshots are stored as filesystem meta-data,
+        * count those too.
+        */
+       if (new_config) {
+               struct hammer_ioc_snapshot snapshot;
+               struct hammer_ioc_snapshot dsnapshot;
+               struct hammer_snapshot_data *snap;
+               struct tm *tp;
+               time_t t;
+               time_t dt;
+               char snapts[32];
+               u_int32_t i;
+
+               bzero(&snapshot, sizeof(snapshot));
+               bzero(&dsnapshot, sizeof(dsnapshot));
+               do {
+                       if (ioctl(fd, HAMMERIOC_GET_SNAPSHOT, &snapshot) < 0) {
+                               err(2, "hammer cleanup: check_softlink "
+                                       "snapshot error");
+                               /* not reached */
+                       }
+                       for (i = 0; i < snapshot.count; ++i) {
+                               snap = &snapshot.snaps[i];
+                               t = snap->ts / 1000000ULL;
+                               dt = time(NULL) - t;
+                               if ((int)dt > arg2 || snap->tid == 0) {
+                                       dsnapshot.snaps[dsnapshot.count++] =
+                                               *snap;
+                               }
+                               if ((int)dt > arg2 && VerboseOpt) {
+                                       tp = localtime(&t);
+                                       strftime(snapts, sizeof(snapts),
+                                                "%Y-%m-%d %H:%M:%S %Z", tp);
+                                       printf("    expire 0x%016jx %s %s\n",
+                                              (uintmax_t)snap->tid,
+                                              snapts,
+                                              snap->label);
+                               }
+                               if (dsnapshot.count == HAMMER_SNAPS_PER_IOCTL)
+                                       delete_snapshots(fd, &dsnapshot);
+                       }
+               } while (snapshot.head.error == 0 && snapshot.count);
+
+               if (dsnapshot.count)
+                       delete_snapshots(fd, &dsnapshot);
+       }
+}
+
+static void
+delete_snapshots(int fd, struct hammer_ioc_snapshot *dsnapshot)
+{
+       for (;;) {
+               if (ioctl(fd, HAMMERIOC_DEL_SNAPSHOT, dsnapshot) < 0) {
+                       printf("    Ioctl to delete snapshots failed: %s\n",
+                              strerror(errno));
+                       break;
+               }
+               if (dsnapshot->head.error) {
+                       printf("    Ioctl to delete snapshots failed at "
+                              "snap=%016jx: %s\n",
+                              dsnapshot->snaps[dsnapshot->index].tid,
+                              strerror(dsnapshot->head.error));
+                       if (++dsnapshot->index < dsnapshot->count)
+                               continue;
+               }
+               break;
+       }
+       dsnapshot->index = 0;
+       dsnapshot->count = 0;
+       dsnapshot->head.error = 0;
 }
 
 /*
@@ -589,8 +1024,7 @@ check_expired(const char *fpath, int arg2)
  * Issue a snapshot.
  */
 static int
-cleanup_snapshots(const char *path __unused, const char *snapshots_path,
-                 int arg1 __unused, int arg2 __unused)
+create_snapshot(const char *path, const char *snapshots_path)
 {
        int r;
 
@@ -621,6 +1055,27 @@ cleanup_prune(const char *path __unused, const char *snapshots_path,
 }
 
 static int
+cleanup_rebalance(const char *path, const char *snapshots_path,
+                 int arg1 __unused, int arg2)
+{
+       if (VerboseOpt == 0) {
+               printf(".");
+               fflush(stdout);
+       }
+
+       runcmd(NULL,
+              "hammer -c %s/.rebalance.cycle -t %d rebalance %s",
+              snapshots_path, arg2, path);
+       if (VerboseOpt == 0) {
+               printf(".");
+               fflush(stdout);
+       }
+       if (VerboseOpt == 0)
+               printf("\n");
+       return(0);
+}
+
+static int
 cleanup_reblock(const char *path, const char *snapshots_path,
                  int arg1 __unused, int arg2)
 {
@@ -782,5 +1237,3 @@ runcmd(int *resp, const char *ctl, ...)
        if (resp)
                *resp = res;
 }
-
-