enh: cron scripts: factorize common code and standardize logging

bin/admin/osh-sync-watcher.sh

@@ -122,6 +122,7 @@ do
         # shellcheck disable=SC2206
         remotehosts=( $remotehostlist )
         remotehostslen=${#remotehosts[@]}
+        nberrs=0
         for i in "${!remotehosts[@]}"
         do
             remote=${remotehosts[i]}
@@ -133,15 +134,26 @@ do
             fi
 
             _log "$remote: [Server $((i+1))/$remotehostslen - Step 1/3] syncing needed data..."
-            rsync -vaA --numeric-ids --delete --filter "merge $rsyncfilterfile" --rsh "$rshcmd -p $remoteport" / "$remoteuser@$remote:/"
-            _log "$remote: [Server $((i+1))/$remotehostslen - Step 1/3] sync ended with return value $?"
+            rsync -vaA --numeric-ids --delete --filter "merge $rsyncfilterfile" --rsh "$rshcmd -p $remoteport" / "$remoteuser@$remote:/"; ret=$?
+            _log "$remote: [Server $((i+1))/$remotehostslen - Step 1/3] sync ended with return value $ret"
+            if [ "$ret" != 0 ]; then (( ++nberrs )); fi
 
             _log "$remote: [Server $((i+1))/$remotehostslen - Step 2/3] syncing lastlog files from master to slave, only if master version is newer..."
-            rsync -vaA --numeric-ids --update --include '/' --include '/home/' --include '/home/*/' --include '/home/*/lastlog' --exclude='*' --rsh "$rshcmd -p $remoteport" / "$remoteuser@$remote:/"
-            _log "$remote: [Server $((i+1))/$remotehostslen - Step 2/3] sync ended with return value $?"
+            rsync -vaA --numeric-ids --update --include '/' --include '/home/' --include '/home/*/' --include '/home/*/lastlog' \
+                --exclude='*' --rsh "$rshcmd -p $remoteport" / "$remoteuser@$remote:/"; ret=$?
+            _log "$remote: [Server $((i+1))/$remotehostslen - Step 2/3] sync ended with return value $ret"
+            if [ "$ret" != 0 ]; then (( ++nberrs )); fi
 
             _log "$remote: [Server $((i+1))/$remotehostslen - Step 3/3] syncing lastlog files from slave to master, only if slave version is newer..."
-            find /home -mindepth 2 -maxdepth 2 -type f -name lastlog | rsync -vaA --numeric-ids --update --prune-empty-dirs --include='/' --include='/home' --include='/home/*/' --include-from=- --exclude='*' --rsh "$rshcmd -p $remoteport" "$remoteuser@$remote:/" /
-            _log "$remote: [Server $((i+1))/$remotehostslen - Step 3/3] sync ended with return value $?"
+            find /home -mindepth 2 -maxdepth 2 -type f -name lastlog | rsync -vaA --numeric-ids --update --prune-empty-dirs --include='/' \
+                --include='/home' --include='/home/*/' --include-from=- --exclude='*' --rsh "$rshcmd -p $remoteport" "$remoteuser@$remote:/" /; ret=$?
+            _log "$remote: [Server $((i+1))/$remotehostslen - Step 3/3] sync ended with return value $ret"
+            if [ "$ret" != 0 ]; then (( ++nberrs )); fi
         done
+
+        if [ "$nberrs" = 0 ]; then
+            _log "All secondaries have been synchronized successfully"
+        else
+            _err "Encountered $nberrs error(s) while synchronizing, see above"
+        fi
 done

bin/cron/osh-backup-acl-keys.sh

@@ -7,11 +7,7 @@ basedir=$(readlink -f "$(dirname "$0")"/../..)
 # shellcheck source=lib/shell/functions.inc
 . "$basedir"/lib/shell/functions.inc
 
-trap "_err 'Unexpected termination!'" EXIT
-
-# setting default values
-LOGFILE=""
-LOG_FACILITY="local6"
+# default config values for this script
 DESTDIR=""
 DAYSTOKEEP="90"
 GPGKEYS=""
@@ -20,33 +16,8 @@ SIGNING_KEY_PASSPHRASE=""
 PUSH_REMOTE=""
 PUSH_OPTIONS=""
 
-# building config files list
-config_list=''
-if [ -f "$BASTION_ETC_DIR/osh-backup-acl-keys.conf" ]; then
-    config_list="$BASTION_ETC_DIR/osh-backup-acl-keys.conf"
-fi
-if [ -d "$BASTION_ETC_DIR/osh-backup-acl-keys.conf.d" ]; then
-    config_list="$config_list $(find "$BASTION_ETC_DIR/osh-backup-acl-keys.conf.d" -mindepth 1 -maxdepth 1 -type f -name "*.conf" | sort)"
-fi
-
-if [ -z "$config_list" ]; then
-    exit_fail "No configuration loaded, aborting"
-fi
-
-# load the config files only if they're owned by root:root and mode is o-rwx
-for file in $config_list; do
-    if check_secure "$file"; then
-        # shellcheck source=etc/bastion/osh-backup-acl-keys.conf.dist
-        . "$file"
-    else
-        exit_fail "Configuration file not secure ($file), aborting."
-    fi
-done
-
-# shellcheck disable=SC2153
-if [ -n "$LOGFILE" ] ; then
-    exec &>> >(tee -a "$LOGFILE")
-fi
+# set error trap, read config, setup logging, exit early if script is disabled, etc.
+script_init osh-backup-acl-keys config_mandatory check_secure
 
 if [ -z "$DESTDIR" ] ; then
     exit_fail "$0: Missing DESTDIR in configuration, aborting."
@@ -187,6 +158,4 @@ _log "Cleaning up old backups..."
 find "$DESTDIR/" -mindepth 1 -maxdepth 1 -type f -name 'backup-????-??-??.tar.gz'     -mtime +"$DAYSTOKEEP" -delete
 find "$DESTDIR/" -mindepth 1 -maxdepth 1 -type f -name 'backup-????-??-??.tar.gz.gpg' -mtime +"$DAYSTOKEEP" -delete
 
-_log "Done"
-trap - EXIT
-exit 0
+exit_success

bin/cron/osh-lingering-sessions-reaper.sh

@@ -6,9 +6,11 @@ basedir=$(readlink -f "$(dirname "$0")"/../..)
 # shellcheck source=lib/shell/functions.inc
 . "$basedir"/lib/shell/functions.inc
 
-LOG_FACILITY=local6
+# default config values for this script
+:
 
-trap "_err 'Unexpected termination!'" EXIT
+# set error trap, read config, setup logging, exit early if script is disabled, etc.
+script_init osh-lingering-sessions-reaper config_optional check_secure_lax
 
 _log "Terminating lingering sessions..."
 
@@ -48,5 +50,4 @@ if [ -n "$tokill" ]; then
     _log "Terminated $nb orphan sshd sessions (pids$tokill)"
 fi
 
-_log "Done"
-trap - EXIT
+exit_success

bin/cron/osh-orphaned-homedir.sh

@@ -7,9 +7,11 @@ basedir=$(readlink -f "$(dirname "$0")"/../..)
 # shellcheck source=lib/shell/functions.inc
 . "$basedir"/lib/shell/functions.inc
 
-LOG_FACILITY=local6
+# default config values for this script
+:
 
-trap "_err 'Unexpected termination!'" EXIT
+# set error trap, read config, setup logging, exit early if script is disabled, etc.
+script_init osh-orphaned-homedir config_optional check_secure_lax
 
 # first, verify that we're not a master instance, if this is the case, do nothing
 set +e
@@ -29,10 +31,10 @@ set -e
 
 case $ret in
     0)   _log "Checking orphaned home directories...";;
-    100) _log "We're a master instance, don't do anything"; trap - EXIT; exit 0;;
-    101) _err "Couldn't load the main bastion configurationg, aborting"; trap - EXIT; exit 1;;
-    102) _err "Invalid main bastion configuration, aborting"; trap - EXIT; exit 1;;
-    *)   _err "Unknown return code ($ret), aborting"; trap - EXIT; exit 1;;
+    100) _log "We're a master instance, don't do anything"; exit_success;;
+    101) exit_fail "Couldn't load the main bastion configurationg, aborting";;
+    102) exit_fail "Invalid main bastion configuration, aborting";;
+    *)   exit_fail "Unknown return code ($ret), aborting";;
 esac
 
 while IFS= read -r -d '' dir
@@ -49,9 +51,7 @@ do
     if [ -n "$user" ] || [ -n "$group" ]; then
 
         # wow, `find' lied to us?!
-        _err "Would have archived $dir, but it seems the user ($uid=$user) or the group ($gid=$group) actually still exists (!), aborting the script"
-        trap - EXIT
-        exit 1
+        exit_fail "Would have archived $dir, but it seems the user ($uid=$user) or the group ($gid=$group) actually still exists (!), aborting the script"
     fi
 
     archive="/home/oldkeeper/orphaned/$(basename "$dir").at-$(date +%s).by-orphaned-homedir-script.tar.gz"
@@ -91,5 +91,4 @@ do
     fi
 done < <(find /home/ -mindepth 1 -maxdepth 1 -type d -nouser -nogroup -mmin +3 -print0)
 
-_log "Done"
-trap - EXIT
+exit_success

bin/cron/osh-piv-grace-reaper.pl

@@ -117,4 +117,4 @@ foreach my $account (%{$fnret->value}) {
     }
 }
 
-_log "Done";
+_log "Done, got " . (OVH::SimpleLog::nb_errors()) . " error(s) and " . (OVH::SimpleLog::nb_warnings()) . " warning(s).";

bin/cron/osh-remove-empty-folders.sh

@@ -12,45 +12,11 @@ basedir=$(readlink -f "$(dirname "$0")"/../..)
 # shellcheck source=lib/shell/functions.inc
 . "$basedir"/lib/shell/functions.inc
 
-trap "_err 'Unexpected termination!'" EXIT
-
-# setting default values
-LOGFILE=""
-LOG_FACILITY="local6"
-ENABLED=1
+# default config values for this script
 MTIME_DAYS=1
 
-# building config files list
-config_list=''
-if [ -f "$BASTION_ETC_DIR/osh-remove-empty-folders.conf" ]; then
-    config_list="$BASTION_ETC_DIR/osh-remove-empty-folders.conf"
-fi
-if [ -d "$BASTION_ETC_DIR/osh-remove-empty-folders.conf.d" ]; then
-    config_list="$config_list $(find "$BASTION_ETC_DIR/osh-remove-empty-folders.conf.d" -mindepth 1 -maxdepth 1 -type f -name "*.conf" | sort)"
-fi
-
-if [ -z "$config_list" ]; then
-    exit_fail "No configuration loaded, aborting"
-fi
-
-# load the config files only if they're owned by root:root and mode is o-rwx
-for file in $config_list; do
-    if check_secure "$file"; then
-        # shellcheck source=etc/bastion/osh-remove-empty-folders.conf.dist
-        . "$file"
-    else
-        exit_fail "Configuration file not secure ($file), aborting."
-    fi
-done
-
-# shellcheck disable=SC2153
-if [ -n "$LOGFILE" ] ; then
-    exec &>> >(tee -a "$LOGFILE")
-fi
-
-if [ "$ENABLED" != 1 ]; then
-    exit_success "Script is disabled"
-fi
+# set error trap, read config, setup logging, exit early if script is disabled, etc.
+script_init osh-remove-empty-folders config_optional check_secure_lax
 
 # first, we list all the directories to get a count
 _log "Counting the number of directories before the cleanup..."
@@ -70,4 +36,4 @@ _log "Finally deleted $((nbdirs_before - nbdirs_after)) directories in this run"
 # note that there is a slight TOCTTOU in the counting, as some external process might actually *add*
 # directories so our count might be slightly wrong, but as this is just for logging sake, this is not an issue
 
-exit_success "Done"
+exit_success

bin/cron/osh-rotate-ttyrec.sh

@@ -6,9 +6,11 @@ basedir=$(readlink -f "$(dirname "$0")"/../..)
 # shellcheck source=lib/shell/functions.inc
 . "$basedir"/lib/shell/functions.inc
 
-LOG_FACILITY=local6
+# default config values for this script
+:
 
-trap "_err 'Unexpected termination!'" EXIT
+# set error trap, read config, setup logging, exit early if script is disabled, etc.
+script_init osh-rotate-ttyrec config_optional check_secure_lax
 
 if [ "$1" = "--big-only" ]; then
     _log "Rotating big ttyrec files..."
@@ -36,5 +38,5 @@ else
         _log "No ttyrec files to rotate"
     fi
 fi
-_log "Done"
-trap - EXIT
+
+exit_success

lib/perl/OVH/SimpleLog.pm

@@ -23,8 +23,8 @@ my $PROGNAME;
 
 # Incremented at each call of _err and _warn, count can be
 # fetched with nb_errors() and nb_warnings()
-my $nb_errors   = 0;
-my $nb_warnings = 0;
+my $NB_ERRORS   = 0;
+my $NB_WARNINGS = 0;
 
 BEGIN {
     # Extract program base name
@@ -65,8 +65,8 @@ sub closeSyslog {
 }
 
 sub _log  { _display('LOG',  @_); return 1; }                    ## no critic (RequireArgUnpacking,ProhibitUnusedPrivateSubroutines)
-sub _warn { _display('WARN', @_); $nb_warnings++; return 1; }    ## no critic (RequireArgUnpacking,ProhibitUnusedPrivateSubroutines)
-sub _err  { _display('ERR',  @_); $nb_errors++; return 1; }      ## no critic (RequireArgUnpacking,ProhibitUnusedPrivateSubroutines)
+sub _warn { _display('WARN', @_); $NB_WARNINGS++; return 1; }    ## no critic (RequireArgUnpacking,ProhibitUnusedPrivateSubroutines)
+sub _err  { _display('ERR',  @_); $NB_ERRORS++; return 1; }      ## no critic (RequireArgUnpacking,ProhibitUnusedPrivateSubroutines)
 
 #   Display a message
 sub _display {
@@ -109,9 +109,11 @@ sub _display {
     # Push to syslog (only if a facility has been defined, which means openlog() has been called)
     if ($FACILITY) {
 
-        $level = lc($level);
-        $level = 'info' if (!grep { $level eq $_ } qw{ warn err });
-        eval { Sys::Syslog::syslog($level, $fullmsg); };
+        # valid levels: DEBUG, INFO, NOTICE, WARNING, ERR, EMERG
+        my $priority = uc($level);
+        $priority = 'WARNING' if $priority eq 'WARN';
+        $priority = 'INFO'    if (!grep { $priority eq $_ } qw{ DEBUG NOTICE WARNING ERR EMERG });
+        eval { Sys::Syslog::syslog($priority, $fullmsg); };
         if ($@) {
             print STDERR "Couldn't syslog, report to administrator ($@)\n";
         }
@@ -120,8 +122,8 @@ sub _display {
     return 1;
 }
 
-sub nb_errors   { return $nb_errors; }
-sub nb_warnings { return $nb_warnings; }
+sub nb_errors   { return $NB_ERRORS; }
+sub nb_warnings { return $NB_WARNINGS; }
 
 END {
     close($LOG_FH) if (defined $LOG_FH);

lib/shell/functions.inc

@@ -327,10 +327,9 @@ get_file_gid_compat()
     stat -c "%g" "$1"
 }
 
-# return true (0) if (perms are o-rwx AND uid = gid = 0)
-check_secure()
+# used by check_secure() and check_secure_lax()
+__check_secure()
 {
-
     # check that perms are o-rwx
     local cmd
     if [ "$OS_FAMILY" = FreeBSD ]; then
@@ -338,7 +337,7 @@ check_secure()
     else
         cmd="stat -c %a"
     fi
-    if ! $cmd "$1" | grep -q '0$'; then
+    if ! $cmd "$1" | grep -q "${2:-0$}"; then
         return 1
     fi
 
@@ -356,6 +355,22 @@ check_secure()
     return 0
 }
 
+# return true (0) if (perms are o-rwx AND uid = gid = 0)
+# useful to test the safeness of a configuration file containing credentials
+check_secure()
+{
+    # file chmod must end in 0 (o-rwx)
+    __check_secure "$1" "0$"
+}
+
+# return true (0) if (perms are o-w AND uid = gid = 0)
+# useful to test the safeness of a configuration file driving a script run by root
+check_secure_lax()
+{
+    # file chmod must end in 0, 1, 4 or 5 (o-w)
+    __check_secure "$1" "[0145]$"
+}
+
 _logtag="$(basename "$0")[$$]"
 __log()
 {
@@ -368,13 +383,17 @@ _log()
 {
     __log info "$*"
 }
+NB_WARN=0
 _warn()
 {
     __log warn "WARN: $*"
+    (( ++NB_WARN ))
 }
+NB_ERR=0
 _err()
 {
     __log err "ERROR: $*" >&2
+    (( ++NB_ERR ))
 }
 
 exit_fail()
@@ -390,7 +409,62 @@ exit_success()
 {
     if [ -n "${1:-}" ]; then
         _log "$1"
+    else
+        _log "Done, got $NB_ERR error(s) and $NB_WARN warning(s)."
     fi
     trap - EXIT
     exit 0
 }
+
+# common func used by all osh-*.sh satellite scripts
+# setup default config vars, load script config,
+# setup logging, and exit early if script is disabled.
+script_init() {
+    trap "_err 'Unexpected termination!'" EXIT
+
+    local script_name="${1:-}"
+    # config_optional or config_mandatory
+    local config_mandatory="${2:-config_optional}"
+    # check_secure or check_secure_lax
+    local config_file_security="${3:-check_secure}"
+
+    # setting default common config values
+    LOGFILE=""
+    LOG_FACILITY="local6"
+    ENABLED=1
+
+    # building config files list
+    local config_list=''
+    if [ -f "$BASTION_ETC_DIR/$script_name.conf" ]; then
+        config_list="$BASTION_ETC_DIR/$script_name.conf"
+    fi
+    if [ -d "$BASTION_ETC_DIR/$script_name.conf.d" ]; then
+        config_list="$config_list $(find "$BASTION_ETC_DIR/$script_name.conf.d" \
+            -mindepth 1 -maxdepth 1 -type f -name "*.conf" | sort)"
+    fi
+
+    if [ "$config_mandatory" != config_optional ] && [ -z "$config_list" ]; then
+        exit_fail "No configuration loaded, aborting"
+    fi
+
+    # load the config files only if they're owned by root:root and mode is o-rwx
+    for file in $config_list; do
+        if [ "$config_file_security" = check_secure ] && check_secure "$file"; then
+            . "$file"
+        elif [ "$config_file_security" = check_secure_lax ] && check_secure_lax "$file"; then
+            . "$file"
+        else
+            exit_fail "Configuration file not secure ($file), aborting."
+        fi
+    done
+
+    # setup logging to a logfile, if enabled
+    # shellcheck disable=SC2153
+    if [ -n "$LOGFILE" ] ; then
+        exec &>> >(tee -a "$LOGFILE")
+    fi
+
+    if [ "$ENABLED" != 1 ]; then
+        exit_success "Script is disabled"
+    fi
+}