diff --git a/bin/cron/osh-backup-acl-keys.sh b/bin/cron/osh-backup-acl-keys.sh index d1eebdf..c572f8f 100755 --- a/bin/cron/osh-backup-acl-keys.sh +++ b/bin/cron/osh-backup-acl-keys.sh @@ -7,6 +7,13 @@ basedir=$(readlink -f "$(dirname "$0")"/../..) # shellcheck source=lib/shell/functions.inc . "$basedir"/lib/shell/functions.inc +trap "_err 'Unexpected termination!'" EXIT + +exit_fail() { + trap - EXIT + exit 1 +} + if command -v gpg1 >/dev/null 2>&1; then gpgcmd="gpg1" else @@ -23,7 +30,7 @@ fi if [ -z "$config_list" ]; then _err "No configuration loaded, aborting" - exit 1 + exit_fail fi # load the config files only if they're owned by root:root and mode is o-rwx @@ -33,7 +40,7 @@ for file in $config_list; do . "$file" else _err "Configuration file not secure ($file), aborting." - exit 1 + exit_fail fi done @@ -44,12 +51,12 @@ fi if [ -z "$DESTDIR" ] ; then _err "$0: Missing DESTDIR in configuration, aborting." - exit 1 + exit_fail fi if ! echo "$DAYSTOKEEP" | grep -Eq '^[0-9]+$' ; then _err "$0: Invalid specified DAYSTOKEEP value ($DAYSTOKEEP), aborting." - exit 1 + exit_fail fi _log "Starting backup..." @@ -63,22 +70,54 @@ for entry in /root/.gnupg /root/.ssh /var/otp do [ -e "$entry" ] && supp_entries="$supp_entries $entry" done -# SC2086: we don't want to quote $supp_entries, we want it expanded -# shellcheck disable=SC2086 -tar czf "$tarfile" -p --xattrs --acls --one-file-system --numeric-owner \ - --exclude=".encrypt" \ - --exclude="ttyrec" \ - --exclude="*.sqlite" \ - --exclude="*.log" \ - --exclude="*.ttyrec" \ - --exclude="*.ttyrec.*" \ - --exclude="*.gz" \ - --exclude="*.zst" \ - /home/ /etc/passwd /etc/group /etc/shadow /etc/gshadow /etc/bastion /etc/ssh $supp_entries 2>/dev/null; ret=$? -if [ $ret -eq 0 ]; then - _log "File created" -else - _err "Error while creating file (sysret=$ret)" + +maxtries=50 +for try in $(seq 1 $maxtries) +do + # tar may output unimportant warnings to stderr, so we don't want to get noisy + # if it exits with 0: save its stderr in a tmpfile, and cat it to stderr only if it returns != 0 + tarstderr=$(mktemp) + set +e + # SC2086: we don't want to quote $supp_entries, we want it expanded + # shellcheck disable=SC2086 + tar czf "$tarfile" -p --xattrs --acls --one-file-system --numeric-owner \ + --exclude=".encrypt" \ + --exclude="ttyrec" \ + --exclude="*.sqlite" \ + --exclude="*.log" \ + --exclude="*.ttyrec" \ + --exclude="*.gpg" \ + --exclude="*.gz" \ + --exclude="*.zst" \ + /home/ /etc/passwd /etc/group /etc/shadow /etc/gshadow /etc/bastion /etc/ssh $supp_entries 2>"$tarstderr"; ret=$? + set -e + if [ $ret -eq 0 ]; then + _log "File created" + rm -f "$tarstderr" + break + else + # special case: if a file changed while we were reading it, tar fails, in that case: retry + if [ $ret -eq 1 ] && grep -q 'file changed as we read it' "$tarstderr"; then + _log "Transient tar failure (try $try):" + while read -r line; do + _log "tar: $line" + done < "$tarstderr" + rm -f "$tarstderr" + _log "Retrying after $try seconds..." + sleep "$try" + continue + fi + _err "Error while creating file (sysret=$ret)" + while read -r line; do + _err "tar: $line" + done < "$tarstderr" + rm -f "$tarstderr" + exit_fail + fi +done +if [ "$try" = "$maxtries" ]; then + _err "Failed creating tar archive after $maxtries tries!" + exit_fail fi encryption_worked=0 @@ -109,7 +148,9 @@ fi if [ -n "$PUSH_REMOTE" ] && [ "$encryption_worked" = 1 ] && [ -r "$tarfile.gpg" ] ; then _log "Pushing backup file ($tarfile.gpg) remotely..." # shellcheck disable=SC2086 + set +e scp $PUSH_OPTIONS "$tarfile.gpg" "$PUSH_REMOTE"; ret=$? + set -e if [ $ret -eq 0 ]; then _log "Push done" else @@ -122,4 +163,5 @@ _log "Cleaning up old backups..." find "$DESTDIR/" -mindepth 1 -maxdepth 1 -type f -name 'backup-????-??-??.tar.gz' -mtime +"$DAYSTOKEEP" -delete find "$DESTDIR/" -mindepth 1 -maxdepth 1 -type f -name 'backup-????-??-??.tar.gz.gpg' -mtime +"$DAYSTOKEEP" -delete _log "Done" +trap - EXIT exit 0 diff --git a/bin/cron/osh-compress-old-logs.sh b/bin/cron/osh-compress-old-logs.sh index 61ebc35..e753f30 100755 --- a/bin/cron/osh-compress-old-logs.sh +++ b/bin/cron/osh-compress-old-logs.sh @@ -7,6 +7,8 @@ basedir=$(readlink -f "$(dirname "$0")"/../..) LOG_FACILITY=local6 +trap "_err 'Unexpected termination!'" EXIT + _log "Compressing old sqlite databases..." while IFS= read -r -d '' sqlite @@ -34,3 +36,4 @@ if command -v chattr >/dev/null; then fi _log "Done" +trap - EXIT diff --git a/bin/cron/osh-lingering-sessions-reaper.sh b/bin/cron/osh-lingering-sessions-reaper.sh index e95a46d..35ed71f 100755 --- a/bin/cron/osh-lingering-sessions-reaper.sh +++ b/bin/cron/osh-lingering-sessions-reaper.sh @@ -8,6 +8,8 @@ basedir=$(readlink -f "$(dirname "$0")"/../..) LOG_FACILITY=local6 +trap "_err 'Unexpected termination!'" EXIT + _log "Terminating lingering sessions..." tokill='' @@ -45,3 +47,4 @@ if [ -n "$tokill" ]; then fi _log "Done" +trap - EXIT diff --git a/bin/cron/osh-orphaned-homedir.sh b/bin/cron/osh-orphaned-homedir.sh index 9a3cf0f..0e4a477 100755 --- a/bin/cron/osh-orphaned-homedir.sh +++ b/bin/cron/osh-orphaned-homedir.sh @@ -9,6 +9,8 @@ basedir=$(readlink -f "$(dirname "$0")"/../..) LOG_FACILITY=local6 +trap "_err 'Unexpected termination!'" EXIT + _log "Checking orphaned home directories..." while IFS= read -r -d '' dir @@ -49,4 +51,4 @@ do done < <(find /home/ -mindepth 1 -maxdepth 1 -type d -nouser -nogroup -mmin +3 -print0) _log "Done" -exit 0 +trap - EXIT diff --git a/bin/cron/osh-rotate-ttyrec.sh b/bin/cron/osh-rotate-ttyrec.sh index 6430edd..cfd96fb 100755 --- a/bin/cron/osh-rotate-ttyrec.sh +++ b/bin/cron/osh-rotate-ttyrec.sh @@ -8,6 +8,8 @@ basedir=$(readlink -f "$(dirname "$0")"/../..) LOG_FACILITY=local6 +trap "_err 'Unexpected termination!'" EXIT + if [ "$1" = "--big-only" ]; then _log "Rotating big ttyrec files..." tokill='' @@ -34,3 +36,4 @@ else fi fi _log "Done" +trap - EXIT