Implement periodic hammer2 snapshots.
authorDaniel Fojt <df@neosystem.org>
Mon, 8 Jun 2020 07:38:49 +0000 (09:38 +0200)
committerDaniel Fojt <df@neosystem.org>
Mon, 8 Jun 2020 07:42:21 +0000 (09:42 +0200)
- add new periodic(8) script 220.snapshot-hammer2 to daily interval
- symlink the script also into weekly and monthly intervals
- extend default periodic.conf(5) with variables to configure the script
- describe new variables in periodic.conf.5 manpage
- mention the functionality also in hammer2.8 manpage

Note: "daily" variables are used by the script as defaults, so it's
      possible to set custom cron entry for different periodic invocation
      of the script, to create for example hourly snapshots

Discussed with, reviewed and tested by: Matthew Dillon

etc/Makefile
etc/defaults/periodic.conf
etc/periodic/daily/220.snapshot-hammer2 [new file with mode: 0644]
etc/periodic/daily/Makefile
sbin/hammer2/hammer2.8
share/man/man5/periodic.conf.5

index c06d47b..e2cdcc4 100644 (file)
@@ -170,6 +170,8 @@ upgrade_base:       upgrade_check preupgrade remove-obsolete-files
        cd ${UPGRADE_SRCDIR}/defaults; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 444 \
            ${DEFAULTS} ${DESTDIR}/etc/defaults
        cd ${UPGRADE_SRCDIR}/periodic; ${MAKE} install
        cd ${UPGRADE_SRCDIR}/defaults; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 444 \
            ${DEFAULTS} ${DESTDIR}/etc/defaults
        cd ${UPGRADE_SRCDIR}/periodic; ${MAKE} install
+       ${LN} -fs /etc/periodic/daily/220.snapshot-hammer2 ${DESTDIR}/etc/periodic/weekly/220.snapshot-hammer2
+       ${LN} -fs /etc/periodic/daily/220.snapshot-hammer2 ${DESTDIR}/etc/periodic/monthly/220.snapshot-hammer2
        mkdir -p ${DESTDIR}/etc/rc.d
        cd ${UPGRADE_SRCDIR}/rc.d; ${MAKE} install
        cd ${UPGRADE_SRCDIR}/devd; ${MAKE} install
        mkdir -p ${DESTDIR}/etc/rc.d
        cd ${UPGRADE_SRCDIR}/rc.d; ${MAKE} install
        cd ${UPGRADE_SRCDIR}/devd; ${MAKE} install
index e69fdf6..48d9a14 100644 (file)
@@ -87,6 +87,23 @@ daily_backup_passwd_enable="YES"                     # Backup passwd & group
 # 210.backup-aliases
 daily_backup_aliases_enable="YES"                      # Backup mail aliases
 
 # 210.backup-aliases
 daily_backup_aliases_enable="YES"                      # Backup mail aliases
 
+# 220.snapshot-hammer2
+daily_snapshot_hammer2_enable="NO"                     # HAMMER2 snapshots
+daily_snapshot_hammer2_tag="daily"                     # snapshot tag
+daily_snapshot_hammer2_keep="auto"                     # snapshots history
+daily_snapshot_hammer2_dirs="auto"                     # directories to snap
+daily_snapshot_hammer2_capacity=90                     # space usage threshold
+weekly_snapshot_hammer2_enable="NO"
+weekly_snapshot_hammer2_tag="weekly"
+weekly_snapshot_hammer2_keep="auto"
+weekly_snapshot_hammer2_dirs="auto"
+weekly_snapshot_hammer2_capacity=90
+monthly_snapshot_hammer2_enable="NO"
+monthly_snapshot_hammer2_tag="monthly"
+monthly_snapshot_hammer2_keep="auto"
+monthly_snapshot_hammer2_dirs="auto"
+monthly_snapshot_hammer2_capacity=90
+
 # 300.calendar
 daily_calendar_enable="NO"                             # Run calendar -a
 
 # 300.calendar
 daily_calendar_enable="NO"                             # Run calendar -a
 
diff --git a/etc/periodic/daily/220.snapshot-hammer2 b/etc/periodic/daily/220.snapshot-hammer2
new file mode 100644 (file)
index 0000000..ff01210
--- /dev/null
@@ -0,0 +1,80 @@
+#!/bin/sh
+#
+# create snapshots of hammer2 directories
+
+if [ -r /etc/defaults/periodic.conf ]
+then
+    . /etc/defaults/periodic.conf
+    source_periodic_confs
+fi
+
+case "${PERIODIC}" in
+       weekly)
+       snap_dirs=${weekly_snapshot_hammer2_dirs:-"auto"}
+       snap_capacity=${weekly_snapshot_hammer2_capacity:-"90"}
+       snap_keep=${weekly_snapshot_hammer2_keep:-"auto"}
+       snap_keep_auto=6
+       snap_tag=${weekly_snapshot_hammer2_tag:-"weekly"}
+       snap_enable="${weekly_snapshot_hammer2_enable}"
+       ;;
+       monthly)
+       snap_dirs=${monthly_snapshot_hammer2_dirs:-"auto"}
+       snap_capacity=${monthly_snapshot_hammer2_capacity:-"90"}
+       snap_keep=${monthly_snapshot_hammer2_keep:-"auto"}
+       snap_keep_auto=12
+       snap_tag=${monthly_snapshot_hammer2_tag:-"monthly"}
+       snap_enable="${monthly_snapshot_hammer2_enable}"
+       ;;
+       *)
+       snap_dirs=${daily_snapshot_hammer2_dirs:-"auto"}
+       snap_capacity=${daily_snapshot_hammer2_capacity:-"90"}
+       snap_keep=${daily_snapshot_hammer2_keep:-"auto"}
+       snap_keep_auto=15
+       snap_tag=${daily_snapshot_hammer2_tag:-"daily"}
+       snap_enable="${daily_snapshot_hammer2_enable}"
+       ;;
+esac
+
+rc=0
+
+case "${snap_enable}" in
+       [Yy][Ee][Ss])
+       echo ""
+       echo "HAMMER2 snapshots:"
+
+       if [ "${snap_dirs}" = "auto" ]; then
+               snap_dirs="$(mount -v -t hammer2 | awk '{ print $3; }')"
+       fi
+
+       for dir in ${snap_dirs}; do
+               echo -n "${dir}: "
+               if [ ! -d "${dir}" ]; then
+                       echo "omitted, not a directory"
+                       continue
+               fi
+               _df=$(df ${dir} | grep -oE "[0-9]+%")
+               _pcnt=${_df%%%}
+               if [ ${_pcnt} -gt ${snap_capacity} ]; then
+                       echo "omitted, not enough free space"
+                       continue
+               fi
+               hammer2 snapshot ${dir} ${dir}.${snap_tag}.$(date +%Y%m%d.%H:%M) \
+                       || rc=1
+               _keep=${snap_keep}
+               if [ "${_keep}" = "auto" ]; then
+                       # adjust number of snapshots dynamically, according to free space
+                       _keep=$(echo "scale=2;inc=((${snap_capacity}-${_pcnt})/50* \
+                               ${snap_keep_auto});scale=0;${snap_keep_auto}+inc/1" | bc)
+               fi
+               for _stale in $(hammer2 -s ${dir} pfs-list | grep -v "^Type" | \
+                               grep -oE "${dir}\.${snap_tag}\..*$" | \
+                               sort -r | tail -n+${_keep}); do
+                       hammer2 -s ${dir} pfs-delete ${_stale} || rc=1
+               done
+               echo "${dir}: $(hammer2 -s ${dir} pfs-list | grep -v "^Type" | \
+                               grep -coE "${dir}\.${snap_tag}\..*$") snapshot(s)"
+       done
+       ;;
+esac
+
+exit $rc
index ccc68ea..5504cf0 100644 (file)
@@ -10,6 +10,7 @@ FILES=        100.clean-disks \
        161.clean-hammer2 \
        200.backup-passwd \
        210.backup-aliases \
        161.clean-hammer2 \
        200.backup-passwd \
        210.backup-aliases \
+       220.snapshot-hammer2 \
        300.calendar \
        310.accounting \
        330.news \
        300.calendar \
        310.accounting \
        330.news \
index 00bf94e..f3a7cdb 100644 (file)
@@ -30,7 +30,7 @@
 .\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
 .\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.Dd September 29, 2019
+.Dd June 8, 2020
 .Dt HAMMER2 8
 .Os
 .Sh NAME
 .Dt HAMMER2 8
 .Os
 .Sh NAME
@@ -357,6 +357,12 @@ Snapshots are effectively separate from the cluster they came from
 and can be used as a starting point for a new cluster.
 So unless you build a new cluster from the snapshot, it will stay local
 to the machine it was made on.
 and can be used as a starting point for a new cluster.
 So unless you build a new cluster from the snapshot, it will stay local
 to the machine it was made on.
+.Pp
+Snapshots can be maintained automatically with
+.Xr periodic 8 .
+See
+.Xr periodic.conf 5
+for details of enabling and configuring the functionality.
 .\" ==== snapshot-debug ====
 .It Cm snapshot-debug Ar path Op label
 Snapshot without filesystem sync.
 .\" ==== snapshot-debug ====
 .It Cm snapshot-debug Ar path Op label
 Snapshot without filesystem sync.
@@ -680,7 +686,8 @@ TODO.
 .Xr mount_null 8 ,
 .Xr newfs_hammer2 8 ,
 .Xr swapcache 8 ,
 .Xr mount_null 8 ,
 .Xr newfs_hammer2 8 ,
 .Xr swapcache 8 ,
-.Xr sysctl 8
+.Xr sysctl 8 ,
+.Xr periodic.conf 5
 .Sh HISTORY
 The
 .Nm
 .Sh HISTORY
 The
 .Nm
index 81f97d6..26a6b7e 100644 (file)
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD: head/share/man/man5/periodic.conf.5 323550 2017-09-13 16:35:16Z gordon $
 .\"
 .\"
 .\" $FreeBSD: head/share/man/man5/periodic.conf.5 323550 2017-09-13 16:35:16Z gordon $
 .\"
-.Dd April 21, 2019
+.Dd June 8, 2020
 .Dt PERIODIC.CONF 5
 .Os
 .Sh NAME
 .Dt PERIODIC.CONF 5
 .Os
 .Sh NAME
@@ -368,6 +368,30 @@ Set to
 .Dq Li YES
 if you want to run
 .Pa /etc/news.expire .
 .Dq Li YES
 if you want to run
 .Pa /etc/news.expire .
+.It Va daily_snapshot_hammer2_capacity
+.Pq Vt num
+Storage usage threshold, in percents. Snapshots won't be created once
+used capacity exceeds this limit. Default is 90.
+.It Va daily_snapshot_hammer2_dirs
+.Pq Vt str
+Space-separated list of directories on HAMMER2 filesystem(s) to snapshot.
+Default is (a special keyword) "auto" which means snapshots will be created
+for all currently mounted HAMMER2 volumes.
+.It Va daily_snapshot_hammer2_enable
+.Pq Vt bool
+Set to
+.Dq Li YES
+if you want to create daily snapshots of directories on HAMMER2 filesystem(s).
+.It Va daily_snapshot_hammer2_keep
+.Pq Vt num or "auto"
+Maximum number of daily snapshots to keep for each configured HAMMER2 directory.
+If set to "auto", 15 is used as the initial value, but more snapshots can be
+kept actually, according to free storage capacity.
+.It Va daily_snapshot_hammer2_tag
+.Pq Vt str
+Tag to be used for daily snapshots labels. Default is "daily".
+The actual label is then composed adhering to this pattern:
+<path>.<flag>.<year><month><day>.<hour>:<minute>
 .It Va daily_status_disks_enable
 .Pq Vt bool
 Set to
 .It Va daily_status_disks_enable
 .Pq Vt bool
 Set to
@@ -567,6 +591,29 @@ An orphaned file is one with an invalid owner or group.
 A list of directories under which orphaned files are searched for.
 This would usually be set to
 .Pa / .
 A list of directories under which orphaned files are searched for.
 This would usually be set to
 .Pa / .
+.It Va weekly_snapshot_hammer2_capacity
+.Pq Vt num
+Weekly counterpart of
+.Va daily_snapshot_hammer2_capacity .
+.It Va weekly_snapshot_hammer2_dirs
+.Pq Vt str
+Weekly counterpart of
+.Va daily_snapshot_hammer2_dirs .
+.It Va weekly_snapshot_hammer2_enable
+.Pq Vt bool
+Set to
+.Dq Li YES
+if you want to create weekly snapshots of directories on HAMMER2 filesystem(s).
+.It Va weekly_snapshot_hammer2_keep
+.Pq Vt num or "auto"
+Weekly counterpart of
+.Va daily_snapshot_hammer2_keep .
+If set to "auto", 6 is used as the initial value.
+.It Va weekly_snapshot_hammer2_tag
+.Pq Vt str
+Weekly counterpart of
+.Va daily_snapshot_hammer2_tag .
+Default is "weekly".
 .It Va weekly_status_security_enable
 .Pq Vt bool
 Weekly counterpart of
 .It Va weekly_status_security_enable
 .Pq Vt bool
 Weekly counterpart of
@@ -596,6 +643,29 @@ Set to
 if you want to do login accounting using the
 .Xr ac 8
 command.
 if you want to do login accounting using the
 .Xr ac 8
 command.
+.It Va monthly_snapshot_hammer2_capacity
+.Pq Vt num
+Monthly counterpart of
+.Va daily_snapshot_hammer2_capacity .
+.It Va monthly_snapshot_hammer2_dirs
+.Pq Vt str
+Monthly counterpart of
+.Va daily_snapshot_hammer2_dirs .
+.It Va monthly_snapshot_hammer2_enable
+.Pq Vt bool
+Set to
+.Dq Li YES
+if you want to create monthly snapshots of directories on HAMMER2 filesystem(s).
+.It Va monthly_snapshot_hammer2_keep
+.Pq Vt num or "auto"
+Monthly counterpart of
+.Va daily_snapshot_hammer2_keep .
+If set to "auto", 12 is used as the initial value.
+.It Va monthly_snapshot_hammer2_tag
+.Pq Vt str
+Monthly counterpart of
+.Va daily_snapshot_hammer2_tag .
+Default is "monthly".
 .It Va monthly_status_security_enable
 .Pq Vt bool
 Monthly counterpart of
 .It Va monthly_status_security_enable
 .Pq Vt bool
 Monthly counterpart of