I’m trying to find a good method of making periodic, incremental backups. I assume that the most minimal approach would be to have a Cronjob run rsync periodically, but I’m curious what other solutions may exist.

I’m interested in both command-line, and GUI solutions.

  • kool_newt@lemm.ee
    link
    fedilink
    English
    arrow-up
    2
    ·
    edit-2
    1 year ago

    I made my own bash script that uses rsync. I stopped using Github so here’s a paste lol.

    I define the backups like this, first item is source, other items on that line are it’s exclusions.

    /home/shared
    /home/jamie     tmp/ dj_music/ Car_Music_USB
    /home/jamie_work
    
    
    #!/usr/bin/ssh-agent /bin/bash
    
    # chronicle.sh
    
    
    
    # Get absolute directory chronicle.sh is in
    REAL_PATH=`(cd $(dirname "$0"); pwd)`
    
    # Defaults
    BACKUP_DEF_FILE="${REAL_PATH}/backup.conf"
    CONF_FILE="${REAL_PATH}/chronicle.conf"
    FAIL_IF_PRE_FAILS='0'
    FIXPERMS='true'
    FORCE='false'
    LOG_DIR='/var/log/chronicle'
    LOG_PREFIX='chronicle'
    NAME='backup'
    PID_FILE='~/chronicle/chronicle.pid'
    RSYNC_OPTS="-qRrltH --perms --delete --delete-excluded"
    SSH_KEYFILE="${HOME}/.ssh/id_rsa"
    TIMESTAMP='date +%Y%m%d-%T'
    
    # Set PID file for root user
    [ $EUID = 0 ] && PID_FILE='/var/run/chronicle.pid'
    
    
    # Print an error message and exit
    ERROUT () {
        TS="$(TS)"
        echo "$TS $LOG_PREFIX (error): $1"
        echo "$TS $LOG_PREFIX (error): Backup failed"
        rm -f "$PID_FILE"
        exit 1
    }
    
    
    # Usage output
    USAGE () {
    cat << EOF
    USAGE chronicle.sh [ OPTIONS ]
    
    OPTIONS
        -f path   configuration file (default: chronicle.conf)
        -F        force overwrite incomplete backup
        -h        display this help
    EOF
    exit 0
    }
    
    
    # Timestamp
    TS ()
    {
        if
            echo $TIMESTAMP | grep tai64n &>/dev/null
        then
            echo "" | eval $TIMESTAMP
        else
            eval $TIMESTAMP
        fi
    }
    
    
    # Logger function
    # First positional parameter is message severity (notice|warn|error)
    # The log message can be the second positional parameter, stdin, or a HERE string
    LOG () {
        local TS="$(TS)"
        # local input=""
    
        msg_type="$1"
    
        # if [[ -p /dev/stdin ]]; then
        #     msg="$(cat -)"
        # else
            shift
            msg="${@}"
        # fi
        echo "$TS chronicle ("$msg_type"): $msg"
    }
    
    # Logger function
    # First positional parameter is message severity (notice|warn|error)
    # The log message canbe stdin or a HERE string
    LOGPIPE () {
        local TS="$(TS)"
        msg_type="$1"
        msg="$(cat -)"
        echo "$TS chronicle ("$msg_type"): $msg"
    }
    
    # Process Options
    while
        getopts ":d:f:Fmh" options; do
            case $options in
                d ) BACKUP_DEF_FILE="$OPTARG" ;;
                f ) CONF_FILE="$OPTARG" ;;
                F ) FORCE='true' ;;
                m ) FIXPERMS='false' ;;
                h ) USAGE; exit 0 ;;
                * ) USAGE; exit 1 ;;
        esac
    done
    
    
    # Ensure a configuration file is found
    if
        [ "x${CONF_FILE}" = 'x' ]
    then
        ERROUT "Cannot find configuration file $CONF_FILE"
    fi
    
    # Read the config file
    . "$CONF_FILE"
    
    
    # Set the owner and mode for backup files
    if [ $FIXPERMS = 'true' ]; then
    #FIXVAR="--chown=${SSH_USER}:${SSH_USER} --chmod=D770,F660"
    FIXVAR="--usermap=*:${SSH_USER} --groupmap=*:${SSH_USER} --chmod=D770,F660"
    fi
    
    
    # Set up logging
    
    if [ "${LOG_DIR}x" = 'x' ]; then
        ERROUT "(error): ${LOG_DIR} not specified"
    fi
    
    mkdir -p "$LOG_DIR"
    LOGFILE="${LOG_DIR}/chronicle.log"
    
    # Make all output go to the log file
    exec >> $LOGFILE 2>&1
    
    
    # Ensure a backup definitions file is found
    if
        [ "x${BACKUP_DEF_FILE}" = 'x' ]
    then
        ERROUT "Cannot find backup definitions file $BACKUP_DEF_FILE"
    fi
    
    
    # Check for essential variables
    VARS='BACKUP_SERVER SSH_USER BACKUP_DIR BACKUP_QTY NAME TIMESTAMP'
    for var in $VARS; do
        if [ ${var}x = x ]; then
            ERROUT "${var} not specified"
        fi
    done
    
    
    LOG notice "Backup started, keeping $BACKUP_QTY snapshots with name \"$NAME\""
    
    
    # Export variables for use with external scripts
    export SSH_USER RSYNC_USER BACKUP_SERVER BACKUP_DIR LOG_DIR NAME REAL_PATH
    
    
    # Check for PID
    if
        [ -e "$PID_FILE" ]
    then
        LOG error "$PID_FILE exists"
        LOG error 'Backup failed'
        exit 1
    fi
    
    # Write PID
    touch "$PID_FILE"
    
    # Add key to SSH agent
    ssh-add "$SSH_KEYFILE" 2>&1 | LOGPIPE notice -
    
    # enhance script readability
    CONN="${SSH_USER}@${BACKUP_SERVER}"
    
    
    # Make sure the SSH server is available
    if
        ! ssh $CONN echo -n ''
    then
        ERROUT "$BACKUP_SERVER is unreachable"
    fi
    
    
    # Fail if ${NAME}.0.tmp is found on the backup server.
    if
        ssh ${CONN} [ -e "${BACKUP_DIR}/${NAME}.0.tmp" ] && [ "$FORCE" = 'false' ]
    then
        ERROUT "${NAME}.0.tmp exists, ensure backup data is in order on the server"
    fi
    
    
    # Try to create the destination directory if it does not already exist
    if
        ssh $CONN [ ! -d $BACKUP_DIR ]
    then
        if
            ssh $CONN mkdir -p "$BACKUP_DIR"
            ssh $CONN chown ${SSH_USER}:${SSH_USER} "$BACKUP_DIR"
        then :
        else
            ERROUT "Cannot create $BACKUP_DIR"
        fi
    fi
    
    # Create metadata directory
    ssh $CONN mkdir -p "$BACKUP_DIR/chronicle_metadata"
    
    
    #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    # PRE_COMMAND
    
    if
        [ -n "$PRE_COMMAND" ]
    then
        LOG notice "Running ${PRE_COMMAND}"
        if
            $PRE_COMMAND
        then
            LOG notice "${PRE_COMMAND} complete"
        else
            LOG error "Execution of ${PRE_COMMAND} was not successful"
            if [ "$FAIL_IF_PRE_FAILS" -eq 1 ]; then
                ERROUT 'Command specified by PRE_COMMAND failed and FAIL_IF_PRE_FAILS enabled'
            fi
        fi
    fi
    
    
    #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    # Backup
    
    # Make a hard link copy of backup.0 to rsync with
    if [ $FORCE = 'false' ]; then
        ssh $CONN "[ -d ${BACKUP_DIR}/${NAME}.0 ] && cp -al ${BACKUP_DIR}/${NAME}.0 ${BACKUP_DIR}/${NAME}.0.tmp"
    fi
    
    
    while read -u 9 l; do
    
        # Skip commented lines
        if [[ "$l" =~ ^#.* ]]; then
        continue
        fi
    
        if [[ $l = '/*'* ]]; then
            LOG warn "$SOURCE is not an absolute path"
            continue
        fi
    
        # Reduce whitespace to one tab
        line=$(echo $l | tr -s [:space:] '\t')
    
        # get the source
        SOURCE=$(echo "$line" | cut -f1)
    
        # get the exclusions
        EXCLUSIONS=$(echo "$line" | cut -f2-)
    
        # Format exclusions for the rsync command
        unset exclude_line
        if [ ! "$EXCLUSIONS" = '' ]; then
            for each in $EXCLUSIONS; do
                exclude_line="$exclude_line--exclude $each "
            done
        fi
    
    
        LOG notice "Using SSH transport for $SOURCE"
    
    
        # get directory metadata
        PERMS="$(getfacl -pR "$SOURCE")"
    
    
        # Copy metadata
        ssh $CONN mkdir -p ${BACKUP_DIR}/chronicle_metadata/${SOURCE}
        echo "$PERMS" | ssh $CONN -T "cat > ${BACKUP_DIR}/chronicle_metadata/${SOURCE}/metadata"
    
    
        LOG debug "rsync $RSYNC_OPTS $exclude_line "$FIXVAR" "$SOURCE" \
        "${SSH_USER}"@"$BACKUP_SERVER":"${BACKUP_DIR}/${NAME}.0.tmp""
    
        rsync $RSYNC_OPTS $exclude_line $FIXVAR "$SOURCE" \
        "${SSH_USER}"@"$BACKUP_SERVER":"${BACKUP_DIR}/${NAME}.0.tmp"
    
    done 9< "${BACKUP_DEF_FILE}"
    
    
    #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    # Try to see if the backup succeeded
    
    if
        ssh $CONN [ ! -d "${BACKUP_DIR}/${NAME}.0.tmp" ]
    then
        ERROUT "${BACKUP_DIR}/${NAME}.0.tmp not found, no new backup created"
    fi
    
    
    # Test for empty temp directory
    if
        ssh $CONN [ ! -z "$(ls -A ${BACKUP_DIR}/${NAME}.0.tmp 2>/dev/null)" ]
    then
        ERROUT "No new backup created"
    fi
    
    #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    # Rotate
    
    # Number of oldest backup
    X=`expr $BACKUP_QTY - 1`
    
    
    LOG notice 'Rotating previous backups'
    
    # keep oldest directory temporarily in case rotation fails
    ssh $CONN [ -d "${BACKUP_DIR}/${NAME}.${X}" ] && \
    ssh $CONN mv "${BACKUP_DIR}/${NAME}.${X}" "${BACKUP_DIR}/${NAME}.${X}.tmp"
    
    
    # Rotate previous backups
    until [ $X -eq -1 ]; do
        Y=$X
        X=`expr $X - 1`
    
        ssh $CONN [ -d "${BACKUP_DIR}/${NAME}.${X}" ] && \
        ssh $CONN mv "${BACKUP_DIR}/${NAME}.${X}" "${BACKUP_DIR}/${NAME}.${Y}"
        [ $X -eq 0 ] && break
    done
    
    # Create "backup.0" directory
    ssh $CONN mkdir -p "${BACKUP_DIR}/${NAME}.0"
    
    
    # Get individual items in "backup.0.tmp" directory into "backup.0"
    # so that items removed from backup definitions rotate out
    while read -u 9 l; do
    
        # Skip commented lines
        if [[ "$l" =~ ^#.* ]]; then
        continue
        fi
    
        # Skip invalid sources that are not an absolute path"
        if [[ $l = '/*'* ]]; then
            continue
        fi
    
        # Reduce multiple tabs to one
        line=$(echo $l | tr -s [:space:] '\t')
    
        source=$(echo "$line" | cut -f1)
    
        source_basedir="$(dirname $source)"
    
        ssh $CONN mkdir -p "${BACKUP_DIR}/${NAME}.0/${source_basedir}"
    
        LOG debug "ssh $CONN cp -al "${BACKUP_DIR}/${NAME}.0.tmp${source}" "${BACKUP_DIR}/${NAME}.0${source_basedir}""
    
        ssh $CONN cp -al "${BACKUP_DIR}/${NAME}.0.tmp${source}" "${BACKUP_DIR}/${NAME}.0${source_basedir}"
    
    done 9< "${BACKUP_DEF_FILE}"
    
    
    # Remove oldest backup
    X=`expr $BACKUP_QTY - 1` # Number of oldest backup
    ssh $CONN rm -Rf "${BACKUP_DIR}/${NAME}.${X}.tmp"
    
    # Set time stamp on backup directory
    ssh $CONN touch -m "${BACKUP_DIR}/${NAME}.0"
    
    # Delete temp copy of backup
    ssh $CONN rm -Rf "${BACKUP_DIR}/${NAME}.0.tmp"
    
    #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    # Post Command
    
    if
        [ ! "${POST_COMMAND}x" = 'x' ]
    then
        LOG notice "Running ${POST_COMMAND}"
        if
            $POST_COMMAND
        then
            LOG notice "${POST_COMMAND} complete"
        else
            LOG warning "${POST_COMMAND} complete with errors"
        fi
    fi
    
    # Delete PID file
    rm -f "$PID_FILE"
    
    # Log success message
    LOG notice 'Backup completed successfully'