I’m trying to find a good method of making periodic, incremental backups. I assume that the most minimal approach would be to have a Cronjob run rsync
periodically, but I’m curious what other solutions may exist.
I’m interested in both command-line, and GUI solutions.
I made my own bash script that uses rsync. I stopped using Github so here’s a paste lol.
I define the backups like this, first item is source, other items on that line are it’s exclusions.
#!/usr/bin/ssh-agent /bin/bash # chronicle.sh # Get absolute directory chronicle.sh is in REAL_PATH=`(cd $(dirname "$0"); pwd)` # Defaults BACKUP_DEF_FILE="${REAL_PATH}/backup.conf" CONF_FILE="${REAL_PATH}/chronicle.conf" FAIL_IF_PRE_FAILS='0' FIXPERMS='true' FORCE='false' LOG_DIR='/var/log/chronicle' LOG_PREFIX='chronicle' NAME='backup' PID_FILE='~/chronicle/chronicle.pid' RSYNC_OPTS="-qRrltH --perms --delete --delete-excluded" SSH_KEYFILE="${HOME}/.ssh/id_rsa" TIMESTAMP='date +%Y%m%d-%T' # Set PID file for root user [ $EUID = 0 ] && PID_FILE='/var/run/chronicle.pid' # Print an error message and exit ERROUT () { TS="$(TS)" echo "$TS $LOG_PREFIX (error): $1" echo "$TS $LOG_PREFIX (error): Backup failed" rm -f "$PID_FILE" exit 1 } # Usage output USAGE () { cat << EOF USAGE chronicle.sh [ OPTIONS ] OPTIONS -f path configuration file (default: chronicle.conf) -F force overwrite incomplete backup -h display this help EOF exit 0 } # Timestamp TS () { if echo $TIMESTAMP | grep tai64n &>/dev/null then echo "" | eval $TIMESTAMP else eval $TIMESTAMP fi } # Logger function # First positional parameter is message severity (notice|warn|error) # The log message can be the second positional parameter, stdin, or a HERE string LOG () { local TS="$(TS)" # local input="" msg_type="$1" # if [[ -p /dev/stdin ]]; then # msg="$(cat -)" # else shift msg="${@}" # fi echo "$TS chronicle ("$msg_type"): $msg" } # Logger function # First positional parameter is message severity (notice|warn|error) # The log message canbe stdin or a HERE string LOGPIPE () { local TS="$(TS)" msg_type="$1" msg="$(cat -)" echo "$TS chronicle ("$msg_type"): $msg" } # Process Options while getopts ":d:f:Fmh" options; do case $options in d ) BACKUP_DEF_FILE="$OPTARG" ;; f ) CONF_FILE="$OPTARG" ;; F ) FORCE='true' ;; m ) FIXPERMS='false' ;; h ) USAGE; exit 0 ;; * ) USAGE; exit 1 ;; esac done # Ensure a configuration file is found if [ "x${CONF_FILE}" = 'x' ] then ERROUT "Cannot find configuration file $CONF_FILE" fi # Read the config file . "$CONF_FILE" # Set the owner and mode for backup files if [ $FIXPERMS = 'true' ]; then #FIXVAR="--chown=${SSH_USER}:${SSH_USER} --chmod=D770,F660" FIXVAR="--usermap=*:${SSH_USER} --groupmap=*:${SSH_USER} --chmod=D770,F660" fi # Set up logging if [ "${LOG_DIR}x" = 'x' ]; then ERROUT "(error): ${LOG_DIR} not specified" fi mkdir -p "$LOG_DIR" LOGFILE="${LOG_DIR}/chronicle.log" # Make all output go to the log file exec >> $LOGFILE 2>&1 # Ensure a backup definitions file is found if [ "x${BACKUP_DEF_FILE}" = 'x' ] then ERROUT "Cannot find backup definitions file $BACKUP_DEF_FILE" fi # Check for essential variables VARS='BACKUP_SERVER SSH_USER BACKUP_DIR BACKUP_QTY NAME TIMESTAMP' for var in $VARS; do if [ ${var}x = x ]; then ERROUT "${var} not specified" fi done LOG notice "Backup started, keeping $BACKUP_QTY snapshots with name \"$NAME\"" # Export variables for use with external scripts export SSH_USER RSYNC_USER BACKUP_SERVER BACKUP_DIR LOG_DIR NAME REAL_PATH # Check for PID if [ -e "$PID_FILE" ] then LOG error "$PID_FILE exists" LOG error 'Backup failed' exit 1 fi # Write PID touch "$PID_FILE" # Add key to SSH agent ssh-add "$SSH_KEYFILE" 2>&1 | LOGPIPE notice - # enhance script readability CONN="${SSH_USER}@${BACKUP_SERVER}" # Make sure the SSH server is available if ! ssh $CONN echo -n '' then ERROUT "$BACKUP_SERVER is unreachable" fi # Fail if ${NAME}.0.tmp is found on the backup server. if ssh ${CONN} [ -e "${BACKUP_DIR}/${NAME}.0.tmp" ] && [ "$FORCE" = 'false' ] then ERROUT "${NAME}.0.tmp exists, ensure backup data is in order on the server" fi # Try to create the destination directory if it does not already exist if ssh $CONN [ ! -d $BACKUP_DIR ] then if ssh $CONN mkdir -p "$BACKUP_DIR" ssh $CONN chown ${SSH_USER}:${SSH_USER} "$BACKUP_DIR" then : else ERROUT "Cannot create $BACKUP_DIR" fi fi # Create metadata directory ssh $CONN mkdir -p "$BACKUP_DIR/chronicle_metadata" #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= # PRE_COMMAND if [ -n "$PRE_COMMAND" ] then LOG notice "Running ${PRE_COMMAND}" if $PRE_COMMAND then LOG notice "${PRE_COMMAND} complete" else LOG error "Execution of ${PRE_COMMAND} was not successful" if [ "$FAIL_IF_PRE_FAILS" -eq 1 ]; then ERROUT 'Command specified by PRE_COMMAND failed and FAIL_IF_PRE_FAILS enabled' fi fi fi #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= # Backup # Make a hard link copy of backup.0 to rsync with if [ $FORCE = 'false' ]; then ssh $CONN "[ -d ${BACKUP_DIR}/${NAME}.0 ] && cp -al ${BACKUP_DIR}/${NAME}.0 ${BACKUP_DIR}/${NAME}.0.tmp" fi while read -u 9 l; do # Skip commented lines if [[ "$l" =~ ^#.* ]]; then continue fi if [[ $l = '/*'* ]]; then LOG warn "$SOURCE is not an absolute path" continue fi # Reduce whitespace to one tab line=$(echo $l | tr -s [:space:] '\t') # get the source SOURCE=$(echo "$line" | cut -f1) # get the exclusions EXCLUSIONS=$(echo "$line" | cut -f2-) # Format exclusions for the rsync command unset exclude_line if [ ! "$EXCLUSIONS" = '' ]; then for each in $EXCLUSIONS; do exclude_line="$exclude_line--exclude $each " done fi LOG notice "Using SSH transport for $SOURCE" # get directory metadata PERMS="$(getfacl -pR "$SOURCE")" # Copy metadata ssh $CONN mkdir -p ${BACKUP_DIR}/chronicle_metadata/${SOURCE} echo "$PERMS" | ssh $CONN -T "cat > ${BACKUP_DIR}/chronicle_metadata/${SOURCE}/metadata" LOG debug "rsync $RSYNC_OPTS $exclude_line "$FIXVAR" "$SOURCE" \ "${SSH_USER}"@"$BACKUP_SERVER":"${BACKUP_DIR}/${NAME}.0.tmp"" rsync $RSYNC_OPTS $exclude_line $FIXVAR "$SOURCE" \ "${SSH_USER}"@"$BACKUP_SERVER":"${BACKUP_DIR}/${NAME}.0.tmp" done 9< "${BACKUP_DEF_FILE}" #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= # Try to see if the backup succeeded if ssh $CONN [ ! -d "${BACKUP_DIR}/${NAME}.0.tmp" ] then ERROUT "${BACKUP_DIR}/${NAME}.0.tmp not found, no new backup created" fi # Test for empty temp directory if ssh $CONN [ ! -z "$(ls -A ${BACKUP_DIR}/${NAME}.0.tmp 2>/dev/null)" ] then ERROUT "No new backup created" fi #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= # Rotate # Number of oldest backup X=`expr $BACKUP_QTY - 1` LOG notice 'Rotating previous backups' # keep oldest directory temporarily in case rotation fails ssh $CONN [ -d "${BACKUP_DIR}/${NAME}.${X}" ] && \ ssh $CONN mv "${BACKUP_DIR}/${NAME}.${X}" "${BACKUP_DIR}/${NAME}.${X}.tmp" # Rotate previous backups until [ $X -eq -1 ]; do Y=$X X=`expr $X - 1` ssh $CONN [ -d "${BACKUP_DIR}/${NAME}.${X}" ] && \ ssh $CONN mv "${BACKUP_DIR}/${NAME}.${X}" "${BACKUP_DIR}/${NAME}.${Y}" [ $X -eq 0 ] && break done # Create "backup.0" directory ssh $CONN mkdir -p "${BACKUP_DIR}/${NAME}.0" # Get individual items in "backup.0.tmp" directory into "backup.0" # so that items removed from backup definitions rotate out while read -u 9 l; do # Skip commented lines if [[ "$l" =~ ^#.* ]]; then continue fi # Skip invalid sources that are not an absolute path" if [[ $l = '/*'* ]]; then continue fi # Reduce multiple tabs to one line=$(echo $l | tr -s [:space:] '\t') source=$(echo "$line" | cut -f1) source_basedir="$(dirname $source)" ssh $CONN mkdir -p "${BACKUP_DIR}/${NAME}.0/${source_basedir}" LOG debug "ssh $CONN cp -al "${BACKUP_DIR}/${NAME}.0.tmp${source}" "${BACKUP_DIR}/${NAME}.0${source_basedir}"" ssh $CONN cp -al "${BACKUP_DIR}/${NAME}.0.tmp${source}" "${BACKUP_DIR}/${NAME}.0${source_basedir}" done 9< "${BACKUP_DEF_FILE}" # Remove oldest backup X=`expr $BACKUP_QTY - 1` # Number of oldest backup ssh $CONN rm -Rf "${BACKUP_DIR}/${NAME}.${X}.tmp" # Set time stamp on backup directory ssh $CONN touch -m "${BACKUP_DIR}/${NAME}.0" # Delete temp copy of backup ssh $CONN rm -Rf "${BACKUP_DIR}/${NAME}.0.tmp" #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= # Post Command if [ ! "${POST_COMMAND}x" = 'x' ] then LOG notice "Running ${POST_COMMAND}" if $POST_COMMAND then LOG notice "${POST_COMMAND} complete" else LOG warning "${POST_COMMAND} complete with errors" fi fi # Delete PID file rm -f "$PID_FILE" # Log success message LOG notice 'Backup completed successfully'