enh: satellite scripts: better error handling

bin/cron/osh-backup-acl-keys.sh

@@ -7,6 +7,13 @@ basedir=$(readlink -f "$(dirname "$0")"/../..)
 # shellcheck source=lib/shell/functions.inc
 . "$basedir"/lib/shell/functions.inc
 
+trap "_err 'Unexpected termination!'" EXIT
+
+exit_fail() {
+    trap - EXIT
+    exit 1
+}
+
 if command -v gpg1 >/dev/null 2>&1; then
     gpgcmd="gpg1"
 else
@@ -23,7 +30,7 @@ fi
 
 if [ -z "$config_list" ]; then
     _err "No configuration loaded, aborting"
-    exit 1
+    exit_fail
 fi
 
 # load the config files only if they're owned by root:root and mode is o-rwx
@@ -33,7 +40,7 @@ for file in $config_list; do
         . "$file"
     else
         _err "Configuration file not secure ($file), aborting."
-        exit 1
+        exit_fail
     fi
 done
 
@@ -44,12 +51,12 @@ fi
 
 if [ -z "$DESTDIR" ] ; then
     _err "$0: Missing DESTDIR in configuration, aborting."
-    exit 1
+    exit_fail
 fi
 
 if ! echo "$DAYSTOKEEP" | grep -Eq '^[0-9]+$' ; then
     _err "$0: Invalid specified DAYSTOKEEP value ($DAYSTOKEEP), aborting."
-    exit 1
+    exit_fail
 fi
 
 _log "Starting backup..."
@@ -63,22 +70,54 @@ for entry in /root/.gnupg /root/.ssh /var/otp
 do
     [ -e "$entry" ] && supp_entries="$supp_entries $entry"
 done
-# SC2086: we don't want to quote $supp_entries, we want it expanded
-# shellcheck disable=SC2086
-tar czf "$tarfile" -p --xattrs --acls --one-file-system --numeric-owner \
-    --exclude=".encrypt" \
-    --exclude="ttyrec" \
-    --exclude="*.sqlite" \
-    --exclude="*.log" \
-    --exclude="*.ttyrec" \
-    --exclude="*.ttyrec.*" \
-    --exclude="*.gz" \
-    --exclude="*.zst" \
-    /home/ /etc/passwd /etc/group /etc/shadow /etc/gshadow /etc/bastion /etc/ssh $supp_entries 2>/dev/null; ret=$?
-if [ $ret -eq 0 ]; then
-    _log "File created"
-else
-    _err "Error while creating file (sysret=$ret)"
+
+maxtries=50
+for try in $(seq 1 $maxtries)
+do
+    # tar may output unimportant warnings to stderr, so we don't want to get noisy
+    # if it exits with 0: save its stderr in a tmpfile, and cat it to stderr only if it returns != 0
+    tarstderr=$(mktemp)
+    set +e
+    # SC2086: we don't want to quote $supp_entries, we want it expanded
+    # shellcheck disable=SC2086
+    tar czf "$tarfile" -p --xattrs --acls --one-file-system --numeric-owner \
+        --exclude=".encrypt" \
+        --exclude="ttyrec" \
+        --exclude="*.sqlite" \
+        --exclude="*.log" \
+        --exclude="*.ttyrec" \
+        --exclude="*.gpg" \
+        --exclude="*.gz" \
+        --exclude="*.zst" \
+        /home/ /etc/passwd /etc/group /etc/shadow /etc/gshadow /etc/bastion /etc/ssh $supp_entries 2>"$tarstderr"; ret=$?
+    set -e
+    if [ $ret -eq 0 ]; then
+        _log "File created"
+        rm -f "$tarstderr"
+        break
+    else
+        # special case: if a file changed while we were reading it, tar fails, in that case: retry
+        if [ $ret -eq 1 ] && grep -q 'file changed as we read it' "$tarstderr"; then
+            _log "Transient tar failure (try $try):"
+            while read -r line; do
+                _log "tar: $line"
+            done < "$tarstderr"
+            rm -f "$tarstderr"
+            _log "Retrying after $try seconds..."
+            sleep "$try"
+            continue
+        fi
+        _err "Error while creating file (sysret=$ret)"
+        while read -r line; do
+            _err "tar: $line"
+        done < "$tarstderr"
+        rm -f "$tarstderr"
+        exit_fail
+    fi
+done
+if [ "$try" = "$maxtries" ]; then
+    _err "Failed creating tar archive after $maxtries tries!"
+    exit_fail
 fi
 
 encryption_worked=0
@@ -109,7 +148,9 @@ fi
 if [ -n "$PUSH_REMOTE" ] && [ "$encryption_worked" = 1 ] && [ -r "$tarfile.gpg" ] ; then
     _log "Pushing backup file ($tarfile.gpg) remotely..."
     # shellcheck disable=SC2086
+    set +e
     scp $PUSH_OPTIONS "$tarfile.gpg" "$PUSH_REMOTE"; ret=$?
+    set -e
     if [ $ret -eq 0 ]; then
         _log "Push done"
     else
@@ -122,4 +163,5 @@ _log "Cleaning up old backups..."
 find "$DESTDIR/" -mindepth 1 -maxdepth 1 -type f -name 'backup-????-??-??.tar.gz'     -mtime +"$DAYSTOKEEP" -delete
 find "$DESTDIR/" -mindepth 1 -maxdepth 1 -type f -name 'backup-????-??-??.tar.gz.gpg' -mtime +"$DAYSTOKEEP" -delete
 _log "Done"
+trap - EXIT
 exit 0

bin/cron/osh-compress-old-logs.sh

@@ -7,6 +7,8 @@ basedir=$(readlink -f "$(dirname "$0")"/../..)
 
 LOG_FACILITY=local6
 
+trap "_err 'Unexpected termination!'" EXIT
+
 _log "Compressing old sqlite databases..."
 
 while IFS= read -r -d '' sqlite
@@ -34,3 +36,4 @@ if command -v chattr >/dev/null; then
 fi
 
 _log "Done"
+trap - EXIT

bin/cron/osh-lingering-sessions-reaper.sh

@@ -8,6 +8,8 @@ basedir=$(readlink -f "$(dirname "$0")"/../..)
 
 LOG_FACILITY=local6
 
+trap "_err 'Unexpected termination!'" EXIT
+
 _log "Terminating lingering sessions..."
 
 tokill=''
@@ -45,3 +47,4 @@ if [ -n "$tokill" ]; then
 fi
 
 _log "Done"
+trap - EXIT

bin/cron/osh-orphaned-homedir.sh

@@ -9,6 +9,8 @@ basedir=$(readlink -f "$(dirname "$0")"/../..)
 
 LOG_FACILITY=local6
 
+trap "_err 'Unexpected termination!'" EXIT
+
 _log "Checking orphaned home directories..."
 
 while IFS= read -r -d '' dir
@@ -49,4 +51,4 @@ do
 done < <(find /home/ -mindepth 1 -maxdepth 1 -type d -nouser -nogroup -mmin +3 -print0)
 
 _log "Done"
-exit 0
+trap - EXIT

bin/cron/osh-rotate-ttyrec.sh

@@ -8,6 +8,8 @@ basedir=$(readlink -f "$(dirname "$0")"/../..)
 
 LOG_FACILITY=local6
 
+trap "_err 'Unexpected termination!'" EXIT
+
 if [ "$1" = "--big-only" ]; then
     _log "Rotating big ttyrec files..."
     tokill=''
@@ -34,3 +36,4 @@ else
     fi
 fi
 _log "Done"
+trap - EXIT