#!/bin/bash # ------------------------------------------------------------------ # # Shell program to backup specified directories as gzip'ed tar # archives on certain days on a ftp server, keeping a number of # older archives of the directories there (max is 10) and removing # superfluous ones. Additionally, the list of packages on a Debian # system is archived. # # The name, user and password of the ftp server, the directories to # backup, the days of the week on which a backup is created, the # number of old copies and package lists (if any) to keep can be # specified in three configuration files located in "/etc/backup" # described below. On error (all errors are considered fatal and # will lead this program to abort) mail can be send to an # administrator. # # Samples of the three configuration files 'ftp', 'locations' and # 'config' sourced and parsed by this script are given here: # # ftp: stores the information this script needs to login into remote # FTP server specified here. # # -------------------------------- # HOST="ftp.server.org" # USER=zippy" # PASSWORD="ThePinHead" # -------------------------------- # # locations: specifies which directories to backup on which days of # the week, how the backup archive should be called and # how many old backup archives should be kept on the FTP # server. # # -------------------------------- # # This files specifies the directories to backup, the days-of-week # # setting is a comma separated list of integers from 1 to 7 # # indicating the days of the week (1 corresponding to Monday) that # # directory should be backed up. Name is the basename of the # # filename to use for the resulting archive, leading to filenames # # of the form "Name.YY-MM-DD.tar.gz". The last value specifies # # the number of archives to keep on the ftp server (>0, # # <=10). Additional (older) copies will be deleted! # # # directory days-of-week name copies to keep # /home 1,2,3,4,5,6,7 home 3 # /var/lib 1,2,3,4,5,6,7 var.lib 1 # /var/www 1 var.www 7 # /etc 1,3 etc 9 # /root 1 root 10 # -------------------------------- # # conf: specifies mail address of backup administrator in case of an # error and if (on a Debian system) the list of installed # packages should be backed up, too. # # -------------------------------- # # who to send mail to on error; leave empty to turn off mailing. # MAILTO="email@address.org" # # number of backup package list created on a Debian systems with "dpkg # # --get-selections"; must be a number lower equal to 10. Set to 0 to # # turn off backing up of the package list. # PACKAGES=3 # -------------------------------- # # Copyright 2005-2006, Sebastian Ley # and Thorsten Bonow . # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of the # License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # Description: # # NOTE: You must be the superuser to run this script. # # Usage: # # backup [ -h | --help ] # # Options: # # -h, --help Display this help message and exit. # # Arguments: # # none # # External programs: # # mktemp - make temporary filename (unique) # id - print real and effective UIDs and GIDs # sed - stream editor # basename - strip directory and suffix from filenames # stat - display file or filesystem status # dpkg - package manager for Debian # cat - concatenate files and print on the standard # output # awk - pattern scanning and processing language # tar - the GNU version of the tar archiving utility # gzip - compress or expand files # sort - sort lines of text files # ftp - Internet file transfer program # grep - print lines matching a pattern # mail - send and receive mail # # CAVEATS: # # - No error checking is done # - if MAILTO specifies a valid eMail address # - if HOST specifies a valid ftp server # - if USER specifies a valid user on ftp server # - if PASSWORD is valid for the user on ftp server # # - if there is enough space available on ftp server # - if there is enough space available in the temp dir # # - Current day timestamp for archives is set once at beginning # of this script. If this script is started before midnight # but continues to create backup archives still after # midnight, these too will have the same timestamp as the # archives created before midnight. # # - if PACKAGES option is specified, it is not possible to # create an backup archive with the basename "packages". # # - since we grep explicitly for a year with four digits, this # script is *not* Year 10000 save ;-) # # Revision History: # # 2005-03-10 Initial version # 2005-10-05 New version implementing deletion of backups on # the ftp server # # See ChangeLog for additional changes # # CVS: $Id: backup,v 1.5 2006-08-29 14:01:57 toto Exp $ # # ------------------------------------------------------------------ ##### Preamble ##### # ------------------------------------------------------------------ # Constants # ------------------------------------------------------------------ declare -r PROGNAME=$(basename $0) declare -r VERSION="0.3" declare -r TEMP_DIR=/tmp declare -r CONF_DIR=/etc/backup declare -r CONF_FILE=$CONF_DIR/conf declare -r LOCATIONS_FILE=$CONF_DIR/locations declare -r FTP_CONF_FILE=$CONF_DIR/ftp declare -r PIDFILE_DIR=/var/run declare -r PIDFILE=$PIDFILE_DIR/$PROGNAME.pid declare -r CURDATE=$(date --iso-8601) # don't change the suffixes, because in ftp_delete grep looks for # them and has to specify them as regexp, changing them here will # make everything fall apart later... declare -r BACKUP_SUFFIX=".$CURDATE.tar.gz" declare -r PACKAGE_LIST_SUFFIX=".$CURDATE.gz" declare -i MAX_COPIES=10 ##### Functions ##### # ------------------------------------------------------------------ # Functions # ------------------------------------------------------------------ function clean_up() { # ------------------------------------------------------------------ # Function to remove temporary files and other housekeeping # No arguments # ------------------------------------------------------------------ rm -f -r ${TEMP_DIR1} rm -f ${PIDFILE} return } # end of clean_up function error_exit() { # ------------------------------------------------------------------ # Function for exit due to fatal program error # (All errors are considered fatal!) # Arguments: # 1 (optional) string containing descriptive error message # ------------------------------------------------------------------ echo "${PROGNAME}: ${1:-"Unknown Error."}" >&2 if [ -n "$MAILTO" ]; then echo "${PROGNAME}: ${1:-"Unknown Error."}" | \ mail -s "Backup fault!" ${MAILTO} fi clean_up exit 1 } # end of error_exit function graceful_exit() { # ------------------------------------------------------------------ # Function called for a graceful exit # No arguments # ------------------------------------------------------------------ clean_up exit } # end of graceful_exit function signal_exit() { # ------------------------------------------------------------------ # Function to handle termination signals # Arguments: # 1 (optional) signal_spec # ------------------------------------------------------------------ case $1 in INT) echo "$PROGNAME: Program aborted by user." >&2 clean_up exit ;; TERM) echo "$PROGNAME: Program terminated." >&2 clean_up exit ;; *) error_exit "$PROGNAME: Terminating on unknown signal." ;; esac } # end of signal_exit function make_temp_files() { # ------------------------------------------------------------------ # Function to create temporary file and directory # No arguments # ------------------------------------------------------------------ # Temp dir and file for this script, using paranoid method of # creation to insure that file name is not predictable. This is # for security to avoid "tmp race" attacks. TEMP_DIR1=$(mktemp -d -q "${TEMP_DIR}/${PROGNAME}.$$.XXXXXX") if [ "$TEMP_DIR1" = "" ]; then error_exit "Cannot create temp dir!" fi TEMP_FILE1=$(mktemp -q "${TEMP_DIR1}/${PROGNAME}.$$.XXXXXX") if [ "$TEMP_FILE1" = "" ]; then error_exit "Cannot create temp file!" fi } # end of make_temp_files function usage() { # ------------------------------------------------------------------ # Function to display usage message (does not exit) # No arguments # ------------------------------------------------------------------ echo "Usage: ${PROGNAME} [-h | --help]" } # end of usage function helptext() { # ------------------------------------------------------------------ # Function to display help message for program # No arguments # ------------------------------------------------------------------ cat <0, # <=10). Additional (older) copies will be deleted! # directory days-of-week name copies to keep /home 1,2,3,4,5,6,7 home 3 /var/lib 1,2,3,4,5,6,7 var.lib 1 /var/www 1 var.www 7 /etc 1,3 etc 9 /root 1 root 10 -------------------------------- conf: specifies mail address of backup administrator in case of an error and if (on a Debian system) the list of installed packages should be backed up, too. -------------------------------- # who to send mail to on error; leave empty to turn off mailing. MAILTO="email@address.org" # number of backup package list created on a Debian systems with "dpkg # --get-selections"; must be a number lower equal to 10. Set to 0 to # turn off backing up of the package list. PACKAGES=3 -------------------------------- Copyright 2005, Sebastian Ley and Thorsten Bonow . This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. $(usage) Options: -h, --help Display this help message and exit. Arguments: none NOTE: You must be the superuser to run this script. EOF } # end of helptext function root_check() { # ------------------------------------------------------------------ # Function to check if user is root # No arguments # ------------------------------------------------------------------ if [ "$(id | sed 's/uid=\([0-9]*\).*/\1/')" != "0" ]; then error_exit "You must be the superuser to run this script." fi } # end of root_check function check_config() { # ------------------------------------------------------------------ # Function to check if configuration files are valid, have the # correct permissions etc. # No arguments # ------------------------------------------------------------------ local file echo -n "Checking configuration..." for file in "$LOCATIONS_FILE" "$FTP_CONF_FILE" "$CONF_FILE"; do if [ ! -f $file ]; then error_exit "Missing configuration file '$file'." fi if [ "$(stat -c '%U %a' $file 2>/dev/null)" != "root 600" ]; then error_exit "File '$file' isn't owned by root:root with permissions set to 0600." fi if [ "$file" != "$LOCATIONS_FILE" ]; then source $file || error_exit "Couldn't source file '$file'." fi done for variable in "$HOST" "$USER" "$PASSWORD" "$PACKAGES"; do if [ -z "$variable" ]; then error_exit "Variable '$variable' not set correctly." fi done if [[ $PACKAGES -lt 0 || $PACKAGES -gt $MAX_COPIES ]]; then error_exit "PACKAGES option set to invalid value '$PACKAGES'." fi echo "done." return } # end of check_config function create_tarballs() { # ------------------------------------------------------------------ # Function to create the tar archives from the backup directories # No arguments # ------------------------------------------------------------------ # The 'locations' file is parsed for all data needed to create the # archives; only the number of old copies to keep is error # checked. Tar archives are created from directories schedules for # backup today and their names are added to an archive list. A # corresponding array is filled with the number of old copies to # preserve on the server. The archive list is checked for # duplicates. local directory local days local name local copies local i=1 # parse the locations file while read; do if [ -n "$REPLY" ] && ! echo "$REPLY" | grep -q ^\ *\# ; then directory=$(echo "$REPLY" | awk '{print $1}') days=$(echo "$REPLY" | awk '{print $2}') name=$(echo "$REPLY" | awk '{print $3}')$BACKUP_SUFFIX copies=$(echo "$REPLY" | awk '{print $4}') if [[ $copies -le 0 || $copies -gt $MAX_COPIES ]]; then error_exit "Invalid number of backup copies '$copies' for file '$name'." fi if echo $days | grep -q $(date +%u); then echo -n "Creating tarball for directory '$directory'..." /usr/bin/nice -n 19 tar -c -z -f "$TEMP_DIR1/$name" "$directory" > /dev/null 2>&1 if [ $? -ne 0 ]; then error_exit "Error creating tar archieve '$name'." fi if [ -z "$archive_list" ]; then archive_list="$name" else # check for duplicates echo "$archive_list" | grep $name > /dev/null 2>&1 if [ $? -eq 0 ]; then error_exit "Archive '$name' already backed up." else archive_list="$archive_list $name" fi fi number_of_backup_copies_array[$i]="$copies" i=$[ i + 1 ] echo "done." fi fi done < $LOCATIONS_FILE if [ $? -ne 0 ]; then error_exit "Error reading file '$LOCATIONS_FILE'." fi return } # end of create_tarballs function dump_packages_list() { # ------------------------------------------------------------------ # Function to create a file list of all installed Debian packages # No arguments # ------------------------------------------------------------------ # A list of all installed Debian packages is stuffed into a # gziped archive and it's name is added *up front* into the # archive list; the number of old copies to keep on the FTP server # therefore is added at the beginning of the corresponding array, # too. Again the archive list is checked for duplicates. echo -n "Dumping packages list..." dpkg --get-selections | gzip - > ${TEMP_DIR1}/packages$PACKAGE_LIST_SUFFIX if [ $? -ne 0 ]; then error_exit "Error creating packages list '${TEMP_DIR1}/packages$PACKAGE_LIST_SUFFIX'." fi if [ -z "$archive_list" ]; then archive_list="packages$PACKAGE_LIST_SUFFIX" else # check for duplicates echo "$archive_list" | grep "packages$PACKAGE_LIST_SUFFIX" > /dev/null 2>&1 if [ $? -eq 0 ]; then error_exit "Archive 'packages$PACKAGE_LIST_SUFFIX' already backed up." else # package list always first element in list! archive_list="packages$PACKAGE_LIST_SUFFIX $archive_list" fi fi # number of backup copies of package list alwasy first element of array! number_of_backup_copies_array[0]="$PACKAGES" echo "done." return } # end of dump_packages_list function ftp_remote_command() { # ------------------------------------------------------------------ # Function to execute an ftp command on the ftp server # Arguments: # 1 (required) ftp_command # ------------------------------------------------------------------ # Fatal error if required arguments are missing if [ "$1" = "" ]; then error_exit "ftp_remote_command: missing argument 1" fi ftp -n $HOST > /dev/null 2>&1 </dev/null 2>&1 && \ error_exit "Archive '$archive' already on FTP server." done ftp_remote_command "mput $archive_list" if [ $? -ne 0 ]; then error_exit "Error uploading archives." fi echo "done." return } # end of ftp_upload function ftp_delete() { # ------------------------------------------------------------------ # Function to delete old backup archives on ftp server # No arguments # ------------------------------------------------------------------ # (for archive loop:) Walk through all elements of the archive # list and check if there are old superfluous copies backed up on # the FTP server; these are deleted. # # Before doing anything else, the file list is updated from the # FTP server because hopefully the new backup archives created # today have been successfully uploaded. # # Since the archive list stores the complete names of local # archives backed up today, the time-stamp suffix first has to be # removed in order to compare the remaining basename with files # stored on the FTP server (which have similar suffixes, but with # older dates). # # (for file loop:) The older copies of an archive file are listed # in reverse order (newest first) and counted; if their number is # higher than the maximum number of old copies stored on the FTP # server, the filenames of the surplus files are added to a delete # list. # # After checking the delete list for duplicates, an FTP connection # is established and the files are deleted. # elements of the archive list local archive # archived file stored on the FTP server local file # index variable for the archives stored in archive variable local i # index variable for the archive files stored in files variable local j=1 # update the file list from the FTP server ftp_get_file_list if [ $PACKAGES -ne 0 ]; then # if PACKAGES option is specified, our array has a zeroth # element i=0 else i=1 fi echo -n "Start deleting old backup archives..." for archive in $archive_list; do if [ "$archive" == "packages$PACKAGE_LIST_SUFFIX" ]; then archive=$(basename $archive $PACKAGE_LIST_SUFFIX) else archive=$(basename $archive $BACKUP_SUFFIX) fi # since we check for a year with 4 digits, this is not Year # 10000 save :-) # # the $(cat...) command works on FTP directory listings in # this form, the filename being the ninth element: # -rw----r-- 1 b007210 500 2290 Oct 7 16:03 foo.bar.2005-10-07.gz for file in $(cat $TEMP_FILE1 | awk '{print $9}' | grep "$archive.[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\(.tar\)\{0,1\}.gz" | sort -r); do if [ $j -gt $((${number_of_backup_copies_array[$i]})) ]; then if [ -z "$delete_list" ]; then delete_list="$file" else # check for duplicates echo "$delete_list" | grep $file > /dev/null 2>&1 if [ $? -eq 0 ]; then error_exit "Archive '$file' already selected for deletion." else delete_list="$delete_list $file" fi fi fi j=$[ j + 1 ] done j=1 i=$[ i + 1 ] done ftp_remote_command "mdelete $delete_list" if [ $? -ne 0 ]; then error_exit "Error deleting old archives." fi echo "done." return } # end of ftp_delete function ftp_verify() { # ------------------------------------------------------------------ # Function to verify if all new archives have been uploaded to FTP # server correctly and old archives have been deleted from it # No arguments # ------------------------------------------------------------------ local archive local error_flag="" local missing_file_list local not_deleted_file_list # update the file list from the FTP server ftp_get_file_list echo -n "Start verifying backup transactions..." # check if all archives have been put on FTP server for archive in ${archive_list}; do grep "$archive" $TEMP_FILE1 >/dev/null 2>&1 if [ $? -ne 0 ]; then echo "Error: Could not find archive '$archive' on FTP server!" error_flag="raised" if [ -z "$missing_file_list" ]; then missing_file_list="$archive" else missing_file_list="$missing_file_list $archive" fi fi done # check if all old archives supposed to be deleted actually have # been deleted for archive in ${delete_list}; do grep "$archive" $TEMP_FILE1 >/dev/null 2>&1 if [ $? -eq 0 ]; then echo "Error: Archive '$archive' still found on FTP server!" error_flag="raised" if [ -z "$not_deleted_file_list" ]; then not_deleted_file_list="$archive" else not_deleted_file_list="$not_deleted_file_list $archive" fi fi done if [ -n "$error_flag" ]; then error_exit "Archives '$missing_file_list' not found on FTP server, archives '$not_deleted_file_list' still there." fi echo "done." return } # end of ftp_verify # ------------------------------------------------------------------ # Program starts here # ------------------------------------------------------------------ ##### Initialization And Setup ##### root_check # Trap TERM, HUP, and INT signals and properly exit trap "signal_exit TERM" TERM HUP trap "signal_exit INT" INT ## Create temporary file(s) # make_temp_files ## Create pid file # if [ -f "$PIDFILE" ]; then error_exit "PID file '$PIDFILE' already exists." else echo $$ > $PIDFILE || error_exit "Couldn't create PID file '$PIDFILE'." fi ##### Command Line Processing ##### # simple command line parsing with getopts buildin instead of external # getopt if only -h|--help option is needed if [ "$1" = "--help" ]; then helptext graceful_exit fi while getopts :h OPT; do case $OPT in h) helptext graceful_exit ;; *) usage clean_up exit 1 esac done ##### Main Logic ##### # list of backup archives # "basename1.YYYY-MM-DD.tar.gz basename2.YYYY-MM-DD.tar.gz # basename3.YYYY-MM-DD.tar.gz ..." declare archive_list # array storing the numbers which specify how many copies of the # backed up archives should be kept on the FTP server. The first # number (stored in number_of_backup_copies_array[1]!) corresponds to # the first archive from archive_list. 0 element is reserved for the # package list. declare -a number_of_backup_copies_array # list of files to be removed from remote FTP server (looks like # archive_list, but stores old archives about to be deleted) declare delete_list echo "Starting new backup on $CURDATE." check_config create_tarballs if [ $PACKAGES -ne 0 ]; then dump_packages_list fi ftp_get_file_list ftp_upload ftp_delete ftp_verify echo "Backup finished. Exiting..." graceful_exit # end of backup # Local Variables: # mode: outline-minor # outline-regexp: "\\(function\\)\\|\\(##### \\)" # outline-heading-end-regexp: "\\(() {\n\\)\\|\\( #####\n\\)" # fill-column: 70 # End: