#!/usr/bin/env bash read -rd '' AWK_PREFILTER <<'EOF' /(\/|^)[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{6}$/ { a[$0] } END { n = asorti(a) for (i = n; i >= 1; --i) print a[i] } EOF read -rd '' AWK_FILTER <<'EOF' function flr(n, s) { return int(n / s) * s } function mstart(t) { return mktime(strftime("%Y %m 01 00 00 00", t, 1), 1) } BEGIN { n = split("h d w M", arr) for (i = 1; i <= n; ++i) { b = arr[i] if (!match(opt_retain, "([*0-9]+)(\\^?)" b, md)) continue ret[b] = md[1] newest_in[b] = (md[2] == "^") buckets[++n_buckets] = b } split("sun mon tue wed thu fri sat", arr) for (i in arr) arr2[arr[i]] = i - 1 wshift = (11 - arr2[opt_week_start])*24*60*60 } NR == 1 { ++keep[$0] } { t = mktime(gensub(/(.*\/|^)(.+)-(..)-(..)-(..)(..)(..)$/, "\\2 \\3 \\4 \\5 \\6 \\7", 1), 1) if (t < 0) { ++keep[$0] next } bt["h"] = flr(t, 60*60) bt["d"] = flr(t, 24*60*60) bt["w"] = flr(t + wshift, 7*24*60*60) - wshift bt["M"] = mstart(t) for (i = 1; i <= n_buckets; ++i) { b = buckets[i] if (b == "M" && "w" in ret) { if (newest_in["w"]) { if (mstart(bt["w"] + 6*24*60*60) > bt["M"]) next } else if (bt["w"] < bt["M"]) { next } } if (newest_in[b] && bt[b] == last[b, b] && (i == 1 || bt[buckets[i-1]] != last[b, buckets[i-1]])) next if ((b, bt[b]) in bkeep || ret[b] == "*" || bc[b]++ < ret[b]) bkeep[b, bt[b]] = $0 for (j in buckets) last[b, buckets[j]] = bt[buckets[j]] } } END { for (i in bkeep) ++keep[bkeep[i]] n = asorti(keep) for (i = 1; i <= n; ++i) print keep[i] } EOF err() { echo "${0##*/}: error: $*" >&2 } run() { echo "* $*" if ! "$@"; then err "command failed: $*" return 1 fi } current() { local str="${1}" interval="${2}" local s=0 val d while [[ "${interval}" =~ ([0-9]+)(.?) ]]; do val="${BASH_REMATCH[1]}" case "${BASH_REMATCH[2]}" in d) ((s += val * 24*60*60)) ;; h) ((s += val * 60*60)) ;; m) ((s += val * 60)) ;; *) ((s += val)) ;; esac interval="${interval#*${BASH_REMATCH[0]}}" done ((s < 1)) && return 1 d="${str:0:10} ${str:11:2}:${str:13:2}:${str:15:2}" (($(date -u +%s) / s == $(date -ud "${d}" +%s) / s)) } exists() { local path="${1}" local res if [[ "${path}" == rclone:* ]]; then rclone lsf -- "${path#rclone:}" &>/dev/null res="$?" [[ "${res}" -eq 0 ]] && return 0 [[ "${res}" -eq 3 ]] && return 1 err "rclone lsf failed: ${path#rclone:}" return 2 fi [[ -e "${path}" ]] } filter() { local -n __src="${1}" __dest="${2}" local __dest_name="${2}" __label="${3}" __script="${4}" shift 4 mapfile -td $'\0' "${__dest_name}" < <( printf '%s\0' "${__src[@]}" | awk -v 'RS=\0' -v 'ORS=\0' \ "${@}" -- "${__script}") wait "$!" if [[ "$?" -ne 0 ]]; then err "${__label} failed" return 1 fi if [[ ${#__src[@]} -gt 0 ]]; then exists "${__dest[0]}" && return 0 [[ "$?" -eq 1 ]] && \ err "${__label} returned empty or invalid list" return 1 fi } populate() { local -n __arr="${1}" local __arr_name="${1}" __dir="${2}" if [[ "${__dir}" == rclone:* ]]; then mapfile -t "${__arr_name}" < \ <(rclone lsf --dir-slash=false -- "${__dir#rclone:}" \ 2>/dev/null) wait "$!" if [[ "$?" -ne 0 && "$?" -ne 3 ]]; then err "rclone lsf failed: ${__dir#rclone:}" return 1 fi __arr=( "${__arr[@]/#/${__dir}/}" ) return fi mapfile -td $'\0' "${__arr_name}" < \ <(shopt -s nullglob; printf '%s\0' "${__dir}/"*) } copy() { local src="${1}" dest="${2}" link_dest="${3}" local path res r_dest r_link_dest for path in "${dest}" "${dest}.tmp"; do exists "${path}" res="$?" if [[ "${res}" -ne 1 ]]; then [[ "${res}" -eq 0 ]] && \ err "destination already exists: ${path}" return 1 fi done if [[ "${dest}" == rclone:* ]]; then r_dest="${dest}.tmp" r_link_dest="${link_dest}" if [[ -d "${src}" ]]; then # emulates rsync src trailing slash behaviour r_dest+="/${src##*/}" r_link_dest+="${r_link_dest:+/${src##*/}}" fi run rclone copy --sftp-copy-is-hardlink \ ${r_link_dest:+"--copy-dest=${r_link_dest#rclone:}"} \ -- "${src}" "${r_dest#rclone:}" || return 1 run rclone move -- "${dest#rclone:}.tmp" "${dest#rclone:}" return fi run rsync -ac --mkpath \ ${link_dest:+"--link-dest=${link_dest}"} \ -- "${src}" "${dest}.tmp/" || return 1 run mv -T -- "${dest}.tmp" "${dest}" || return 1 } prune() { local -n __snaps="${1}" __keep="${2}" local __i __k for ((__i = ${#__snaps[@]} - 1, __k = 0; __i >= 0; --__i)); do if [[ "${__snaps[__i]}" == "${__keep[__k]}" ]]; then ((++__k)) else if [[ "${__snaps[__i]}" == rclone:* ]]; then run rclone purge -- "${__snaps[__i]#rclone:}" else run rm -rf -- "${__snaps[__i]}" fi fi done } backup() { local src_path="${1}" dest_dir="${2}" local retain="${3}" interval="${4}" week_start="${5}" local dest_path snaps keep if [[ ! -e "${src_path}" ]]; then err "source does not exist: ${src_path}" return 1 fi populate snaps "${dest_dir}" || return 1 filter snaps snaps 'pre-snapshot awk prefilter' "${AWK_PREFILTER}" \ || return 1 [[ -n "${snaps}" ]] && current "${snaps[0]##*/}" "${interval}" \ && return dest_path="${dest_dir}/$(date -u '+%Y-%m-%d-%H%M%S')" copy "${src_path}" "${dest_path}" "${snaps[0]}" || return 1 if [[ ! "${retain}" =~ ^( *([0-9]+|\*)\^?[hdwM] *)+$ ]]; then err "invalid \$retain setting: ${retain}" return 1 fi snaps+=("${dest_path}") filter snaps snaps 'post-snapshot awk prefilter' "${AWK_PREFILTER}" \ || return 1 filter snaps keep 'awk filter' "${AWK_FILTER}" \ -v "opt_retain=${retain}" -v "opt_week_start=${week_start}" \ || return 1 prune snaps keep } declare -A def_opts=( ['retain']='*h' # e.g. 24h7d4w12M or 24^h10d*M ['interval']='0' ['week_start']='mon' ) declare -A opts declare -A seen while IFS='=' read -r f1 f2; do [[ -z "${f1}" ]] || [[ "${f1}" == '#'* ]] && continue if [[ "${f1}" =~ ^\[(.*)]$ ]]; then base_path="${BASH_REMATCH[1]}" declare -A opts="$(declare -p def_opts | sed '1s/[^=]*=//')" elif [[ "${f1}" == '$'* ]]; then opt="${f1:1}" [[ "${f2}" == '-' ]] && opts["${opt}"]= || opts["${opt}"]="${f2:-"${def_opts["${opt}"]}"}" else dest="${base_path}/${f1}" printf '[%d] [%s] %s => %s%s\n' "$((++dcount))" \ "${opts['retain']}" "${f2}" "${dest}" dest_rp="$(realpath -m -- "${dest}")" if [[ -v "seen["${dest_rp}"]" ]]; then err "destination already seen: ${dest}" ((++errors)) continue fi seen["${dest_rp}"]= backup "${f2}" "${dest}" \ "${opts['retain']}" \ "${opts['interval']}" \ "${opts['week_start']}" \ || ((++errors)) fi done < "${1}" ((!errors))