enh: tests: refactor the framework for more maintainability

The chain of executions is as is:
- `docker_build_and_run_tests_all.sh`
  - launches several instances of `docker_build_and_run_tests.sh`
    - builds docker images with the `target_role.sh` and `tester_role.sh` entrypoints
      - inside the tester docker, `tester_role.sh` launches `launch_tests_on_instance.sh`
      - the target docker gets tested after setting up accounts, SSH etc.

Previously, these scripts passed options to each other either by a mix of environment
variables and command-line arguments, with some inconsistencies here and there.

Now, `launch_tests_on_instance.sh` supports a lot of command-line options, which can
be specified directly if testing a remote server, or can be passed-through by the calling
script in case of docker tests. `docker_build_and_run_tests.sh` and
`docker_build_and_run_tests_all.sh` also support to passthrough these options down.

.github/workflows/freebsd.yml

@@ -8,7 +8,7 @@ jobs:
   freebsd:
     runs-on: macos-latest
     name: FreeBSD
-    if: contains(github.event.pull_request.labels.*.name, 'tests:full')
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'tests:full') || contains(github.event.pull_request.labels.*.name, 'tests:long') || contains(github.event.pull_request.labels.*.name, 'tests:freebsd') }}
     steps:
     - uses: actions/checkout@v2
     - name: Functional tests under FreeBSD
@@ -26,8 +26,8 @@ jobs:
           /opt/bastion/bin/admin/packages-check.sh -i
           /opt/bastion/bin/admin/install-ttyrec.sh -s
           /opt/bastion/bin/admin/install-yubico-piv-checker.sh -s
-          /opt/bastion/bin/admin/install --new-install --no-wait
+          /opt/bastion/bin/admin/install --new-install
           ssh-keygen -t ed25519 -f id_user
           ssh-keygen -t ed25519 -f id_root
-          NO_SLEEP=1 user_pubkey=$(cat id_user.pub) root_pubkey=$(cat id_root.pub) TARGET_USER=user5000 /opt/bastion/tests/functional/docker/target_role.sh
+          WANT_HTTP_PROXY=0 NO_SLEEP=1 user_pubkey=$(cat id_user.pub) root_pubkey=$(cat id_root.pub) TARGET_USER=user5000 /opt/bastion/tests/functional/docker/target_role.sh
-          HAS_MFA=0 HAS_MFA_PASSWORD=1 HAS_PAMTESTER=1 nocc=1 /opt/bastion/tests/functional/launch_tests_on_instance.sh 127.0.0.1 22 0 user5000 id_user id_root /usr/local/etc/bastion
+          /opt/bastion/tests/functional/launch_tests_on_instance.sh --module=500-http-proxy.sh --has-mfa=0 --has-mfa-password=1 --has-pamtester=1 --skip-consistency-check --remote-etc-bastion=/usr/local/etc/bastion 127.0.0.1 22 0 user5000 id_user id_root

.github/workflows/tests.yml

@@ -12,7 +12,21 @@ jobs:
         steps:
         - uses: actions/checkout@v2
         - name: run tests inside a debian10 docker
-          run: tests/functional/docker/docker_build_and_run_tests.sh debian10
+          run: tests/functional/docker/docker_build_and_run_tests.sh debian10 --skip-consistency-check --no-pause-on-fail
+          env:
+            DOCKER_TTY: false
+
+    tests_long:
+        name: Long
+        strategy:
+            matrix:
+                platform: ['centos7@centos:7.9.2009', 'centos8@centos:8.3.2011', debian9, debian10, debian11, 'opensuse15@opensuse/leap:15.3', ubuntu1604, ubuntu1804, ubuntu2004]
+        runs-on: ubuntu-latest
+        if: contains(github.event.pull_request.labels.*.name, 'tests:long')
+        steps:
+        - uses: actions/checkout@v2
+        - name: run tests inside a ${{ matrix.platform }} docker
+          run: tests/functional/docker/docker_build_and_run_tests.sh ${{ matrix.platform }} --no-pause-on-fail
           env:
             DOCKER_TTY: false
 
@@ -20,12 +34,12 @@ jobs:
         name: Full
         strategy:
             matrix:
-                platform: ['centos7@centos:7.7.1908', 'centos7@centos:7.8.2003', 'centos7@centos:7.9.2009', 'centos8@centos:8.1.1911', 'centos8@centos:8.2.2004', 'centos8@centos:8.3.2011', debian8, debian9, debian10, debian11, opensuse151, opensuse152, ubuntu1404, ubuntu1604, ubuntu1804, ubuntu2004]
+                platform: ['centos7@centos:7.7.1908', 'centos7@centos:7.8.2003', 'centos7@centos:7.9.2009', 'centos8@centos:8.1.1911', 'centos8@centos:8.2.2004', 'centos8@centos:8.3.2011', debian9, debian10, debian11, 'opensuse15@opensuse/leap:15.2', 'opensuse15@opensuse/leap:15.3', ubuntu1604, ubuntu1804, ubuntu2004]
         runs-on: ubuntu-latest
         if: contains(github.event.pull_request.labels.*.name, 'tests:full')
         steps:
         - uses: actions/checkout@v2
         - name: run tests inside a ${{ matrix.platform }} docker
-          run: tests/functional/docker/docker_build_and_run_tests.sh ${{ matrix.platform }}
+          run: tests/functional/docker/docker_build_and_run_tests.sh ${{ matrix.platform }} --no-pause-on-fail
           env:
             DOCKER_TTY: false

docker/Dockerfile.centos7

@@ -19,10 +19,10 @@ COPY . /opt/bastion
 RUN ["/opt/bastion/bin/dev/perl-check.sh"]
 
 # setup ssh/sshd config and setup bastion install
-RUN ["/opt/bastion/bin/admin/install","--new-install","--no-wait"]
+RUN ["/opt/bastion/bin/admin/install","--new-install"]
 
 # start at entrypoint
 ENTRYPOINT /opt/bastion/docker/entrypoint.sh
 
-# TESTENV HAS_ED25519=1 HAS_BLACKLIST=0 HAS_MFA=1 HAS_PAMTESTER=1 HAS_PIV=1
+# TESTOPT --has-mfa=1 --has-pamtester=1 --has-piv=1
 # TESTFROM centos:7.9.2009 centos:7.8.2003 centos:7.7.1908

docker/Dockerfile.centos8

@@ -19,10 +19,10 @@ COPY . /opt/bastion
 RUN ["/opt/bastion/bin/dev/perl-check.sh"]
 
 # setup ssh/sshd config and setup bastion install
-RUN ["/opt/bastion/bin/admin/install","--new-install","--no-wait"]
+RUN ["/opt/bastion/bin/admin/install","--new-install"]
 
 # start at entrypoint
 ENTRYPOINT /opt/bastion/docker/entrypoint.sh
 
-# TESTENV HAS_ED25519=1 HAS_BLACKLIST=0 HAS_MFA=1 HAS_PAMTESTER=1 HAS_PIV=1
+# TESTOPT --has-mfa=1 --has-pamtester=1 --has-piv=1
 # TESTFROM centos:8.3.2011 centos:8.2.2004 centos:8.1.1911

docker/Dockerfile.debian10

@@ -22,9 +22,9 @@ COPY . /opt/bastion
 RUN ["/opt/bastion/bin/dev/perl-check.sh"]
 
 # setup ssh/sshd config and setup bastion install
-RUN ["/opt/bastion/bin/admin/install","--new-install","--no-wait"]
+RUN ["/opt/bastion/bin/admin/install","--new-install"]
 
 # start at entrypoint
 ENTRYPOINT /opt/bastion/docker/entrypoint.sh
 
-# TESTENV HAS_ED25519=1 HAS_BLACKLIST=0 HAS_MFA=1 HAS_PAMTESTER=1 HAS_PIV=1
+# TESTOPT --has-mfa=1 --has-pamtester=1 --has-piv=1

docker/Dockerfile.debian11

@@ -22,9 +22,9 @@ COPY . /opt/bastion
 RUN ["/opt/bastion/bin/dev/perl-check.sh"]
 
 # setup ssh/sshd config and setup bastion install
-RUN ["/opt/bastion/bin/admin/install","--new-install","--no-wait"]
+RUN ["/opt/bastion/bin/admin/install","--new-install"]
 
 # start at entrypoint
 ENTRYPOINT /opt/bastion/docker/entrypoint.sh
 
-# TESTENV HAS_ED25519=1 HAS_BLACKLIST=0 HAS_MFA=1 HAS_PAMTESTER=1 HAS_PIV=1
+# TESTOPT --has-mfa=1 --has-pamtester=1 --has-piv=1

docker/Dockerfile.debian9

@@ -22,9 +22,9 @@ COPY . /opt/bastion
 RUN ["/opt/bastion/bin/dev/perl-check.sh"]
 
 # setup ssh/sshd config and setup bastion install
-RUN ["/opt/bastion/bin/admin/install","--new-install","--no-wait"]
+RUN ["/opt/bastion/bin/admin/install","--new-install"]
 
 # start at entrypoint
 ENTRYPOINT /opt/bastion/docker/entrypoint.sh
 
-# TESTENV HAS_ED25519=1 HAS_BLACKLIST=0 HAS_MFA=1 HAS_PAMTESTER=1 HAS_PIV=1
+# TESTOPT --has-mfa=1 --has-pamtester=1 --has-piv=1

docker/Dockerfile.sandbox

@@ -24,7 +24,7 @@ RUN \
     # accountUidMax & ttyrecGroupIdOffset change: fixes https://github.com/ovh/the-bastion/issues/24 \
     sed -i -re 's/^"accountUidMax":.+/"accountUidMax": 9999,/;s/^"ttyrecGroupIdOffset":.+/"ttyrecGroupIdOffset": 10000,/' /opt/bastion/etc/bastion/bastion.conf.dist && \
     # install the software \
-    /opt/bastion/bin/admin/install --new-install --no-wait
+    /opt/bastion/bin/admin/install --new-install
 
 # We'll expose our port 22
 EXPOSE 22/tcp

docker/Dockerfile.ubuntu1604

@@ -22,9 +22,9 @@ COPY . /opt/bastion
 RUN ["/opt/bastion/bin/dev/perl-check.sh"]
 
 # setup ssh/sshd config and setup bastion install
-RUN ["/opt/bastion/bin/admin/install","--new-install","--no-wait"]
+RUN ["/opt/bastion/bin/admin/install","--new-install"]
 
 # start at entrypoint
 ENTRYPOINT /opt/bastion/docker/entrypoint.sh
 
-# TESTENV HAS_ED25519=1 HAS_BLACKLIST=0 HAS_MFA=1 HAS_PAMTESTER=1 HAS_PIV=1
+# TESTOPT --has-mfa=1 --has-pamtester=1 --has-piv=1

docker/Dockerfile.ubuntu1804

@@ -22,9 +22,9 @@ COPY . /opt/bastion
 RUN ["/opt/bastion/bin/dev/perl-check.sh"]
 
 # setup ssh/sshd config and setup bastion install
-RUN ["/opt/bastion/bin/admin/install","--new-install","--no-wait"]
+RUN ["/opt/bastion/bin/admin/install","--new-install"]
 
 # start at entrypoint
 ENTRYPOINT /opt/bastion/docker/entrypoint.sh
 
-# TESTENV HAS_ED25519=1 HAS_BLACKLIST=0 HAS_MFA=1 HAS_PAMTESTER=1 HAS_PIV=1
+# TESTOPT --has-mfa=1 --has-pamtester=1 --has-piv=1

docker/Dockerfile.ubuntu2004

@@ -22,9 +22,9 @@ COPY . /opt/bastion
 RUN ["/opt/bastion/bin/dev/perl-check.sh"]
 
 # setup ssh/sshd config and setup bastion install
-RUN ["/opt/bastion/bin/admin/install","--new-install","--no-wait"]
+RUN ["/opt/bastion/bin/admin/install","--new-install"]
 
 # start at entrypoint
 ENTRYPOINT /opt/bastion/docker/entrypoint.sh
 
-# TESTENV HAS_ED25519=1 HAS_BLACKLIST=0 HAS_MFA=1 HAS_PAMTESTER=1 HAS_PIV=1
+# TESTOPT --has-mfa=1 --has-pamtester=1 --has-piv=1

lib/shell/colors.inc

@@ -2,10 +2,14 @@
 # shellcheck shell=bash
 # shellcheck disable=SC2034
 
-RED=$(printf "%b" '\033[31m')
+RED=$(printf       "%b" '\033[31m')
-GREEN=$(printf "%b" '\033[32m')
+GREEN=$(printf     "%b" '\033[32m')
-YELLOW=$(printf "%b" '\033[33m')
+YELLOW=$(printf    "%b" '\033[33m')
-BLUE=$(printf "%b" '\033[34m')
+BLUE=$(printf      "%b" '\033[34m')
+MAGENTA=$(printf   "%b" '\033[35m')
+CYAN=$(printf      "%b" '\033[36m')
+LIGHTGRAY=$(printf "%b" '\033[37m')
+DARKGRAY=$(printf      "%b" '\033[90m')
 
 BOLD_CYAN=$(printf "%b" '\033[1;36m')
 

tests/functional/docker/docker_build_and_run_tests.sh

@@ -9,12 +9,14 @@ basedir=$(readlink -f "$(dirname "$0")"/../../..)
 namespace=the-bastion-test
 
 target="$1"
-test_script="$2"
+shift
+
+# all remaining options will be passed as-is on the target docker, through target_role.sh to launch-tests-on-instance.sh
 
 get_supported_targets() {
     local target targets subtarget
     for dockerfile in "$(dirname "$0")"/../../../docker/Dockerfile.*; do
-        if grep -q '^# TESTENV ' "$dockerfile"; then
+        if grep -q '^# TESTOPT ' "$dockerfile"; then
             target=$(basename "$dockerfile")
             target=${target/Dockerfile./}
             # if the file has a TESTFROM entry, then it's actually multiple similar targets
@@ -24,8 +26,9 @@ get_supported_targets() {
                     subtarget="$target@$testfrom"
                     targets="$targets $subtarget"
                 done
+            else
+                targets="$targets $target"
             fi
-            targets="$targets $target"
         fi
     done
     # shellcheck disable=SC2086
@@ -76,23 +79,21 @@ docker build -f "$testenv_dockerfile" -t "$namespace:tester" "$(dirname "$0")"/.
 if [ -n "$subtarget" ]; then
     dockerfiletmp=$(mktemp)
     trap 'rm -f $dockerfiletmp' EXIT
-    sed -re "s/^FROM .+/FROM $subtarget/" "$target_dockerfile" > "$dockerfiletmp"
+    sed -re "s=^FROM .+=FROM $subtarget=" "$target_dockerfile" > "$dockerfiletmp"
     target_dockerfile="$dockerfiletmp"
 fi
 
 # build target
 echo "Building target environment"
 target=$(echo "$target" | sed -re 's/[^a-zA-Z0-9_-]/_/g')
-docker build -f "$target_dockerfile" -t "$namespace:$target" --build-arg "TEST_QUICK=$TEST_QUICK" "$(dirname "$0")"/../../..
+docker build -f "$target_dockerfile" -t "$namespace:$target" "$(dirname "$0")"/../../..
 
 # get the target environment we want from the dockerfile
-varstoadd=''
+testopts="$(grep '^# TESTOPT' "$target_dockerfile" | tail -n1 | cut -c10-)"
 privileged=''
-for var in $(grep '^# TESTENV' "$target_dockerfile" | tail -n1 | sed -re 's/^# TESTENV//')
+if grep -q '^# PRIVILEGED' "$target_dockerfile"; then
-do
+    privileged='--privileged'
-    echo "$var" | grep -Eq '^[A-Z0-9_]+=[01]$' && varstoadd="$varstoadd -e $var "
+fi
-    [ "$var" = "PRIVILEGED=1" ] && privileged='--privileged'
-done
 
 # cleanup the dockerfile temp if applicable
 if [ -n "$subtarget" ]; then
@@ -131,7 +132,6 @@ docker run $privileged \
     -e USER_PUBKEY_B64="$USER_PUBKEY_B64" \
     -e ROOT_PUBKEY_B64="$ROOT_PUBKEY_B64" \
     -e TARGET_USER="user.5000" \
-    -e TEST_QUICK="${TEST_QUICK:-0}" \
     -e WANT_HTTP_PROXY=1 \
     $namespace:"$target"
 docker logs -f "bastion_${target}_target" | sed -u -e 's/^/target: /;s/$/\r/' &
@@ -168,7 +168,7 @@ if [[ -t 1 ]] && [ -z "$DOCKER_TTY" ]; then
 else
     DOCKER_TTY="false"
 fi
-echo "Starting test instance and run tests with --tty=$DOCKER_TTY"
+echo "Starting test instance and run tests with --tty=$DOCKER_TTY (testopts: $testopts, extra params: $*)"
 set +e
 # shellcheck disable=SC2086
 docker run \
@@ -182,10 +182,8 @@ docker run \
     -e TARGET_USER="user.5000" \
     -e USER_PRIVKEY_B64="$USER_PRIVKEY_B64" \
     -e ROOT_PRIVKEY_B64="$ROOT_PRIVKEY_B64" \
-    -e TARGET="$target " \
+    -e EXTRA_OPTIONS="$testopts $*" \
-    -e TEST_SCRIPT="$test_script" \
+    $namespace:tester
-    -e TEST_QUICK="${TEST_QUICK:-0}" \
-    $varstoadd $namespace:tester
 ret=$?
 if [ $ret -ne 0 ]; then
     printf '%b%b%b\n' "$WHITE_ON_RED" "Test instance returned $ret" "$NOC"

tests/functional/docker/docker_build_and_run_tests_all.sh

@@ -12,19 +12,29 @@ targets=$(./docker_build_and_run_tests.sh --list-targets)
 printf '%b%b%b\n' "$WHITE_ON_BLUE" "============================================================" "$NOC"
 printf '%b%b%b\n' "$WHITE_ON_BLUE" "Testing all targets in parallel, ensure you have enough RAM!" "$NOC"
 printf '%b%b%b\n' "$WHITE_ON_BLUE" "============================================================" "$NOC"
-echo "Targets: $targets"
+echo "Targets:"
+echo "$targets"
+echo
+echo "Starting in 5 seconds, you can still CTRL+C in the meantime."
 
 sleep 5
 
+echo "GO!"
+
 tempdir=$(mktemp -d)
-trap 'test -d $tempdir && rm -rf $tempdir' EXIT
+cleanup() {
+	test -d "$tempdir" && rm -rf "$tempdir"
+	docker ps | grep -Eo 'bastion_.*_(target|tester)$' | xargs -r docker kill
+}
+trap 'cleanup' EXIT
 
 for t in $targets
 do
     [ "$t" = "-" ] && continue
     (
-        DOCKER_TTY=false ./docker_build_and_run_tests.sh "$t"
+        friendlyname=$(echo "$t" | sed -re 's/@[^:]+//')
-        echo $? > "$tempdir/$t"
+        DOCKER_TTY=false ./docker_build_and_run_tests.sh "$t" "--log-prefix=$friendlyname $*"
+        echo $? > "$tempdir/$friendlyname"
     ) &
 done
 wait
@@ -36,21 +46,22 @@ nberrors=0
 for t in $targets
 do
     [ "$t" = "-" ] && continue
-    err=$(cat "$tempdir/$t" 2>/dev/null)
+    friendlyname=$(echo "$t" | sed -re 's/@[^:]+//')
-    rm -f "$tempdir/$t"
+    err=$(cat "$tempdir/$friendlyname" 2>/dev/null)
+    rm -f "$tempdir/$friendlyname"
     if [ -z "$err" ]; then
-        printf "%b%23s: tests couldn't run properly%b\\n" "$BLACK_ON_RED" "$t" "$NOC"
+        printf "%b%16s: tests couldn't run properly%b\\n" "$BLACK_ON_RED" "$friendlyname" "$NOC"
         nberrors=$(( nberrors + 1 ))
     elif [ "$err" = 0 ]; then
-        printf "%b%23s: no errors :)%b\\n" "$BLACK_ON_GREEN" "$t" "$NOC"
+        printf "%b%16s: no errors :)%b\\n" "$BLACK_ON_GREEN" "$friendlyname" "$NOC"
     elif [ "$err" = 143 ]; then
-        printf "%b%23s: tests interrupted%b\\n" "$BLACK_ON_RED" "$t" "$NOC"
+        printf "%b%16s: tests interrupted%b\\n" "$BLACK_ON_RED" "$friendlyname" "$NOC"
         nberrors=$(( nberrors + 1 ))
     elif [ "$err" -lt 254 ]; then
-        printf "%b%23s: $err tests failed%b\\n" "$BLACK_ON_RED" "$t" "$NOC"
+        printf "%b%16s: $err tests failed%b\\n" "$BLACK_ON_RED" "$friendlyname" "$NOC"
         nberrors=$(( nberrors + 1 ))
     else
-        printf "%b%23s: $err errors%b\\n" "$BLACK_ON_RED" "$t" "$NOC"
+        printf "%b%16s: $err errors%b\\n" "$BLACK_ON_RED" "$friendlyname" "$NOC"
         nberrors=$(( nberrors + 1 ))
     fi
 done

tests/functional/docker/target_role.sh

@@ -63,7 +63,7 @@ add_user_to_group_compat test-shell_ bastion-nopam
 
 # install a fake ttyrec just so that our connection tests work
 if ! command -v ttyrec >/dev/null; then
-    "$basedir"/bin/admin/install --nothing --no-wait --install-fake-ttyrec
+    "$basedir"/bin/admin/install --nothing --install-fake-ttyrec
 fi
 
 # if we have other specific scripts to run, run them

tests/functional/docker/tester_role.sh

@@ -34,8 +34,9 @@ for i in $(seq 1 $delay); do
     sleep 1
     if echo test | nc -w 1 "$TARGET_IP" "$TARGET_PORT" | grep -q ^SSH-2 ; then
         echo "tester: it's alive, starting tests!"
-        [ "$TEST_QUICK" = 1 ] && export nocc=1
+        # we want EXTRA_OPTIONS to expand
-        "$(dirname "$0")"/../launch_tests_on_instance.sh "$TARGET_IP" "$TARGET_PORT" "${TARGET_PROXY_PORT:-0}" "$TARGET_USER" /root/user.privkey /root/root.privkey; ret=$?
+        # shellcheck disable=SC2086
+        "$(dirname "$0")"/../launch_tests_on_instance.sh ${EXTRA_OPTIONS:-} "$TARGET_IP" "$TARGET_PORT" "${TARGET_PROXY_PORT:-0}" "$TARGET_USER" /root/user.privkey /root/root.privkey; ret=$?
         [ "$ret" -gt 253 ] && ret=253
         exit "$ret"
     elif ! fping -r 1 "$TARGET_IP" >/dev/null 2>&1; then

tests/functional/launch_tests_on_instance.sh

@@ -3,12 +3,107 @@
 # shellcheck disable=SC2086
 # shellcheck disable=SC2016
 # shellcheck disable=SC2046
-set -e
+set -eu
 
 basedir=$(readlink -f "$(dirname "$0")"/../..)
 # shellcheck source=lib/shell/functions.inc
 . "$basedir"/lib/shell/functions.inc
 
+opt_remote_etc_bastion=/etc/bastion
+opt_remote_basedir=$basedir
+opt_skip_consistency_check=0
+opt_no_pause_on_fail=0
+opt_log_prefix=
+opt_module=
+declare -A capabilities=( [ed25519]=1 [blacklist]=0 [mfa]=1 [mfa-password]=0 [pamtester]=1 [piv]=1 )
+
+# set the helptext now to get the proper default values
+help_text=$(cat <<EOF
+
+Usage: $0 [OPTIONS] <IP> <SSH_Port> <HTTP_Proxy_Port_or_Zero> <Remote_Admin_User_Name> <Admin_User_SSH_Key_Path> <Root_SSH_Key_Path>
+
+Test Options:
+    --skip-consistency-check   Speed up tests by skipping the consistency check between every test
+    --no-pause-on-fail         Don't pause when a test fails
+    --log-prefix=X             Prefix all logs by this name
+    --module=X                 Only test this module (specify a filename found in \`functional/tests.d/\`)
+
+Remote OS directory locations:
+    --remote-etc-bastion=X     Override the default remote bastion configuration directory (default: $opt_remote_etc_bastion)
+    --remote-basedir=X         Override the default remote basedir location (default: $opt_remote_basedir)
+
+Specifying features support of the underlying OS of the tested bastion:
+    --has-ed25519=[0|1]        Ed25519 keys are supported (default: ${capabilities[ed25519]})
+    --has-blacklist=[0|1]      Detection of bad SSH keys generated during the Debian OpenSSL debacle of 2006 is supported (default: ${capabilities[blacklist]})
+    --has-mfa=[0|1]            PAM is usable to check passwords and TOTP (default: ${capabilities[mfa]})
+    --has-mfa-password=[0|1]   PAM is usable to check passwords (default: ${capabilities[mfa-password]})
+    --has-pamtester=[0|1]      The \`pamtester\` binary is available, and PAM is usable (default: ${capabilities[pamtester]})
+    --has-piv=[0|1]            The \`yubico-piv-tool\` binary is available (default: ${capabilities[piv]})
+
+EOF
+)
+
+
+usage() {
+    echo "$help_text"
+}
+
+while [ -n "${1:-}" ]
+do
+    optval="${1/*=/}"
+    case "$1" in
+        --remote-etc-bastion=*)
+            opt_remote_etc_bastion="$optval"
+            ;;
+        --remote-basedir=*)
+            opt_remote_basedir="$optval"
+            ;;
+        --skip-consistency-check)
+            opt_skip_consistency_check=1
+            ;;
+        --no-pause-on-fail)
+            opt_no_pause_on_fail=1
+            ;;
+        --log-prefix=*)
+            opt_log_prefix="$optval"
+            ;;
+        --module=*)
+            opt_module="$optval"
+            if [ ! -e "$basedir/tests/functional/tests.d/$optval" ]; then
+                echo "Unknown module specified '$opt_module', supported modules are:"
+                cd "$basedir/tests/functional/tests.d"
+                ls -- ???-*.sh
+                exit 1
+            fi
+            ;;
+        --has-*=*)
+            optname=${1/--has-/}
+            optname=${optname/=*/}
+            capabilities[$optname]=$optval
+            ;;
+        -*)
+            echo "Unknown option: $1"
+            usage
+            exit 1
+            ;;
+        *) break
+            ;;
+    esac
+    shift
+done
+
+if [ -n "${7:-}" ]; then
+    echo "Error: too many parameters"
+    usage
+    exit 1
+fi
+
+if [ -z "${6:-}" ]; then
+    echo "Error: missing parameters"
+    usage
+    exit 1
+fi
+
 remote_ip="$1"
 remote_port="$2"
 # the var below is used in sourced test files
@@ -17,29 +112,6 @@ remote_proxy_port="$3"
 account0="$4"
 user_ssh_key_path="$5"
 root_ssh_key_path="$6"
-osh_etc="$7"
-remote_basedir="$8"
-[ -n "$osh_etc" ] || osh_etc=/etc/bastion
-[ -n "$remote_basedir" ] || remote_basedir="$basedir"
-
-[ -z "$HAS_ED25519"      ] && HAS_ED25519=1
-[ -z "$HAS_BLACKLIST"    ] && HAS_BLACKLIST=0
-[ -z "$HAS_MFA"          ] && HAS_MFA=1
-[ -z "$HAS_MFA_PASSWORD" ] && HAS_MFA_PASSWORD=0
-[ -z "$HAS_PAMTESTER"    ] && HAS_PAMTESTER=1
-[ -z "$HAS_PIV"          ] && HAS_PIV=1
-[ -z "$nocc"             ] && nocc=0
-[ -z "$nowait"           ] && nowait=0
-[ -z "$TARGET"           ] && TARGET=''
-[ -z "$TEST_SCRIPT"      ] && TEST_SCRIPT=''
-
-# die if using an unset var
-set -u
-
-if [ -z "$root_ssh_key_path" ] ; then
-    echo "Usage: $0 <IP> <Port> <HTTP_Proxy_Port_or_zero> <remote_user_name> <user_ssh_key_path> <root_ssh_key_path> [osh_etc] [remote_basedir]"
-    exit 1
-fi
 
 # does ssh work there ?
 server_output=$(echo test | nc -w 1 $remote_ip $remote_port)
@@ -110,7 +182,7 @@ cat >"$mytmpdir/ssh_config" <<EOF
    PasswordAuthentication no
    RequestTTY yes
 EOF
-if [ "$HAS_MFA" = 1 ] || [ "$HAS_MFA_PASSWORD" = 1 ]; then
+if [ "${capabilities[mfa]}" = 1 ] || [ "${capabilities[mfa-password]}" = 1 ]; then
         cat >>"$mytmpdir/ssh_config" <<EOF
            ChallengeResponseAuthentication yes
            KbdInteractiveAuthentication yes
@@ -158,11 +230,14 @@ prefix()
     local elapsed=$(( $(date +%s) - start_time))
     local min=$(( elapsed / 60 ))
     local sec=$(( elapsed - min * 60 ))
+    local prefixfmt="%b"
     update_totalerrors
+
+    [ -n "$opt_log_prefix" ] && prefixfmt="%16b "
     if [ "$totalerrors" = 0 ]; then
-        printf "%b%02dm%02d [noerror]" "$TARGET" "$min" "$sec"
+        printf "${prefixfmt}%02dm%02d %b[--]%b" "$opt_log_prefix" "$min" "$sec" "$DARKGRAY" "$NOC"
     else
-        printf "%b%02dm%02d %b[%d err]%b" "$TARGET" "$min" "$sec" "$RED" "$totalerrors" "$NOC"
+        printf "${prefixfmt}%02dm%02d %b[%d err]%b" "$opt_log_prefix" "$min" "$sec" "$RED" "$totalerrors" "$NOC"
     fi
 }
 
@@ -180,11 +255,11 @@ run()
         cat "$outdir/$basename.log"
         printf "%b%b%b\\n" "$WHITE_ON_BLUE" "[INFO] returned json follows" "$NOC"
         grep "^JSON_OUTPUT=" -- $outdir/$basename.log | cut -d= -f2- | $jq .
-        if [ "$nocc" != 1 ]; then
+        if [ "$opt_skip_consistency_check" != 1 ]; then
             printf "%b%b%b\\n" "$WHITE_ON_BLUE" "[INFO] consistency check follows" "$NOC"
             cat "$outdir/$basename.cc"
         fi
-        if test -t 0 && [ "$nowait" != 1 ]; then
+        if test -t 0 && [ "$opt_no_pause_on_fail" != 1 ]; then
             printf "%b%b%b\\n" "$WHITE_ON_BLUE" "[INFO] press enter to continue" "$NOC"
             read -r _
         fi
@@ -205,7 +280,7 @@ run()
         cp "$1" "$outdir/$basename.script"
     fi
 
-    printf '%b %b*** [%03d/%03d] %b::%b %s(%b)\n' "$(prefix)" "$BOLD_CYAN" "$testno" "$testcount" "$name" "$case" "$NOC" "$*"
+    printf '%b %b*** [%03d/%03d] %b::%b %b(%b)%b\n' "$(prefix)" "$BOLD_CYAN" "$testno" "$testcount" "$name" "$case" "$NOC$DARKGRAY" "$*" "$NOC"
 
     # special case for scp: we need to wait a bit before terminating the test
     sleepafter=0
@@ -227,7 +302,7 @@ run()
     fi
 
     # now run consistency check on the target, unless configured otherwise
-    if [ "$nocc" != 1 ]; then
+    if [ "$opt_skip_consistency_check" != 1 ]; then
         flock "$outdir/$basename.retval" $screen "$outdir/$basename.cc" -D -m -fn -ln $r0 '
                 /opt/bastion/bin/admin/check-consistency.pl ; echo _RETVAL_CC=$?= ;
                 grep -Fw -e warn -e die -e code-warning /var/log/bastion/bastion.log | grep -Fv "'"${code_warn_exclude:-__none__}"'" | sed "s/^/_SYSLOG=/" ;
@@ -403,7 +478,7 @@ nocontain()
 
 configchg()
 {
-    success bastion configchange $r0 perl -pe "$*" -i $osh_etc/bastion.conf
+    success bastion configchange $r0 perl -pe "$*" -i $opt_remote_etc_bastion/bastion.conf
 }
 
 sshclientconfigchg()
@@ -419,13 +494,13 @@ runtests()
 
     # backup the original default configuration on target side
     now=$(date +%s)
-    success bastion backupconfig $r0 "dd if=$osh_etc/bastion.conf of=$osh_etc/bastion.conf.bak.$now"
+    success bastion backupconfig $r0 "dd if=$opt_remote_etc_bastion/bastion.conf of=$opt_remote_etc_bastion/bastion.conf.bak.$now"
 
     grant accountRevokeCommand
 
     for module in "$(dirname $0)"/tests.d/???-*.sh
     do
-        if [ -n "$TEST_SCRIPT" ] && [ "$TEST_SCRIPT" != "$(basename "$module")" ]; then
+        if [ -n "$opt_module" ] && [ "$opt_module" != "$(basename "$module")" ]; then
             echo "### SKIPPING MODULE $(basename $module)"
             continue
         fi
@@ -437,7 +512,7 @@ runtests()
     done
 
     # put the backed up configuration back
-    success bastion restoreconfig $r0 "dd if=$osh_etc/bastion.conf.bak.$now of=$osh_etc/bastion.conf"
+    success bastion restoreconfig $r0 "dd if=$opt_remote_etc_bastion/bastion.conf.bak.$now of=$opt_remote_etc_bastion/bastion.conf"
 }
 
 COUNTONLY=0
@@ -448,7 +523,7 @@ for f in $(find "$basedir/tests/unit/" -mindepth 1 -maxdepth 1 -type f -name "*.
 do
     fbasename=$(basename "$f")
     echo "-> $fbasename"
-    if ! $r0 perl "$remote_basedir/tests/unit/$fbasename"; then
+    if ! $r0 perl "$opt_remote_basedir/tests/unit/$fbasename"; then
         printf "%b%b%b\\n" "$WHITE_ON_RED" "Unit tests failed :(" "$NOC"
         exit 1
     fi

tests/functional/tests.d/305-admin-superowner.sh

@@ -24,7 +24,7 @@ testsuite_admin_superowner()
     json .error_code KO_RESTRICTED_COMMAND .command null
 
     # now set account1 as superowner
-    success admin_superowner set_a1_as_superowner $r0 "\". $remote_basedir/lib/shell/functions.inc; add_user_to_group_compat $account1 osh-superowner\""
+    success admin_superowner set_a1_as_superowner $r0 "\". $opt_remote_basedir/lib/shell/functions.inc; add_user_to_group_compat $account1 osh-superowner\""
     configchg 's=^\\\\x22superOwnerAccounts\\\\x22.+=\\\\x22superOwnerAccounts\\\\x22:[\\\\x22'"$account1"'\\\\x22],='
 
     # account1 now can add/del members
@@ -37,7 +37,7 @@ testsuite_admin_superowner()
     contain OVERRIDE
 
     # now set account1 as admin
-    success admin_superowner set_a1_as_admin $r0 "\". $remote_basedir/lib/shell/functions.inc; add_user_to_group_compat $account1 osh-admin\""
+    success admin_superowner set_a1_as_admin $r0 "\". $opt_remote_basedir/lib/shell/functions.inc; add_user_to_group_compat $account1 osh-admin\""
     configchg 's=^\\\\x22adminAccounts\\\\x22.+=\\\\x22adminAccounts\\\\x22:[\\\\x22'"$account0"'\\\\x22,\\\\x22'"$account1"'\\\\x22],='
 
     # account1 now can add/del aclkeepers
@@ -50,7 +50,7 @@ testsuite_admin_superowner()
     contain OVERRIDE
 
     # now remove superowner grant from a1, the account is still admin so it should inherhit superowner powers
-    success admin_superowner del_a1_as_superowner $r0 "\". $remote_basedir/lib/shell/functions.inc; del_user_from_group_compat $account1 osh-superowner\""
+    success admin_superowner del_a1_as_superowner $r0 "\". $opt_remote_basedir/lib/shell/functions.inc; del_user_from_group_compat $account1 osh-superowner\""
     configchg 's=^\\\\x22superOwnerAccounts\\\\x22.+=\\\\x22superOwnerAccounts\\\\x22:[],='
 
     # account1 can add/del gatekeepers
@@ -63,7 +63,7 @@ testsuite_admin_superowner()
     contain OVERRIDE
 
     # and finally remove admin grant
-    success admin_superowner del_a1_as_admin $r0 "\". $remote_basedir/lib/shell/functions.inc; del_user_from_group_compat $account1 osh-admin\""
+    success admin_superowner del_a1_as_admin $r0 "\". $opt_remote_basedir/lib/shell/functions.inc; del_user_from_group_compat $account1 osh-admin\""
     configchg 's=^\\\\x22adminAccounts\\\\x22.+=\\\\x22adminAccounts\\\\x22:[\\\\x22'"$account0"'\\\\x22],='
 
     # account1 can no longer add members

tests/functional/tests.d/330-selfkeys.sh

@@ -471,7 +471,7 @@ EOS
     [ "$FP_TYPE" = md5 ] && fped="d7:92:5b:77:8b:69:03:cb:e7:5a:11:76:d1:a6:ea:e4"
     fplist="$fp4096 $fp8192 $fp16384 $fpe256 $fpe384 $fpe521"
     script selfAddIngressKey  ed25519   $a1 -osh selfAddIngressKey "<<< \"ssh-ed25519 $b64 test@ed25519\""
-    if [ "$HAS_ED25519" = "1" ] ; then
+    if [ "${capabilities[ed25519]}" = "1" ] ; then
         fplist="$fplist $fped"
         retvalshouldbe 0
         contain "key successfully added"
@@ -506,7 +506,7 @@ EOS
             .value.key.prefix ""
     fi
 
-    if [ "$HAS_BLACKLIST" = 1 ] ; then
+    if [ "${capabilities[blacklist]}" = 1 ] ; then
     script selfAddIngressKey  rsa1024fucked $a1 -osh selfAddIngressKey "<<< \"ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEA90Td1GTx+tYMbsti93lyiyKYelBgaXRrnweoYJXjUFNU93jZ+RmBR8yp5J6mx7jz9ECaMS7Dn49fNQi5uG75+m+DTUgq3bfNv8cygoVC4g3NhzA3e+uA22D+iI53j3Gm9YxaJVOypGXGkOoWnmXZy7FQ4aSBFvgqa81xfnoa+4M= compromised@rsa1024\""
     retvalshouldbe 100
     contain "IT IS VULNERABLE"
@@ -624,7 +624,7 @@ EOS
     revoke accountDelete
 
     # restore default config
-    success bastion configrestore $r0 "dd if=$osh_etc/bastion.conf.bak.$now of=$osh_etc/bastion.conf"
+    success bastion configrestore $r0 "dd if=$opt_remote_etc_bastion/bastion.conf.bak.$now of=$opt_remote_etc_bastion/bastion.conf"
 }
 
 testsuite_selfkeys

tests/functional/tests.d/340-selfaccesses.sh

@@ -153,18 +153,8 @@ testsuite_selfaccesses()
 
     # /forcekey
 
-    # this should work...
-
-    set +e
-    if [ "$COUNTONLY" = 1 ]; then
-        targethostname=dummy
-    else
-        targethostname=$($r0 hostname | tail -n1 | grep -E -o '[a-z0-9._-]+')
-    fi
-    set -e
-
     success ssh shellaccountatlo2_mustwork   $a0 $shellaccount@127.0.0.2 --kbd-interactive -- echo $randomstr
-    contain REGEX "$shellaccount@($targethostname|127.0.0.2|fv-[a-z0-9-]+):22"
+    contain REGEX "$shellaccount@[a-zA-Z0-9._-]+:22"
     contain "allowed ... log on"
     nocontain "Permission denied"
     contain "$randomstr"
@@ -391,7 +381,7 @@ testsuite_selfaccesses()
     # should
 
     success ssh shellaccountatlo2_mustwork226   $a0 $shellaccount@127.0.0.2 -p 226 -- echo $randomstr
-    contain REGEX "$shellaccount@(127.0.0.2|$targethostname|fv-[a-z0-9-]+):226"
+    contain REGEX "$shellaccount@[a-zA-Z0-9._-]+:226"
     contain "allowed ... log on"
     nocontain "Permission denied"
     contain "$randomstr"

tests/functional/tests.d/370-mfa.sh

@@ -52,7 +52,7 @@ testsuite_mfa()
 
     # now try to connect after we have a pass
     run mfa a4_connect_after_pass $a4f --osh groupList
-    if [ "$HAS_MFA" = 1 ] || [ "$HAS_MFA_PASSWORD" = 1 ]; then
+    if [ "${capabilities[mfa]}" = 1 ] || [ "${capabilities[mfa-password]}" = 1 ]; then
         # now we need a password, we don't enter it so it'll timeout (124)
         retvalshouldbe 124
         contain 'Multi-Factor Authentication enabled, an additional authentication factor is required (password).'
@@ -66,7 +66,7 @@ testsuite_mfa()
         json .command groupList .error_code OK_EMPTY
     fi
 
-    if [ "$HAS_PAMTESTER" = 1 ]; then
+    if [ "${capabilities[pamtester]}" = 1 ]; then
         grant groupCreate
 
         success mfa a0_create_g3 $a0 --osh groupCreate --group $group3 --algo rsa --size 4096 --owner $account4
@@ -192,7 +192,7 @@ testsuite_mfa()
 
     # change our password
     a4_password_new="rkw=*Ffyqs23"
-    if [ "$HAS_MFA" = 1 ] || [ "$HAS_MFA_PASSWORD" = 1 ]; then
+    if [ "${capabilities[mfa]}" = 1 ] || [ "${capabilities[mfa-password]}" = 1 ]; then
         script mfa a4_change_pass "echo 'set timeout 30; \
             spawn $a4 --osh selfMFASetupPassword --yes; \
             expect \":\" { sleep 0.2; send \"$a4_password\\n\"; }; \
@@ -224,7 +224,7 @@ testsuite_mfa()
     a4_password="$a4_password_new"
     unset a4_password_new
 
-    if [ "$HAS_MFA" = 1 ] || [ "$HAS_MFA_PASSWORD" = 1 ]; then
+    if [ "${capabilities[mfa]}" = 1 ] || [ "${capabilities[mfa-password]}" = 1 ]; then
         script mfa a4_connect_with_pass "echo 'set timeout 30; \
             spawn $a4 --osh groupList; \
             expect \":\" { sleep 0.2; send \"$a4_password\\n\"; }; \
@@ -246,7 +246,7 @@ testsuite_mfa()
     json .error_code OK .command accountModify .value.mfa_totp_required.error_code OK_NO_CHANGE
 
     # now try to connect with account4
-    if [ "$HAS_MFA" = 1 ] || [ "$HAS_MFA_PASSWORD" = 1 ]; then
+    if [ "${capabilities[mfa]}" = 1 ] || [ "${capabilities[mfa-password]}" = 1 ]; then
         script mfa a4_connect_with_totpreq "echo 'set timeout 30; \
             spawn $a4 --osh groupList; \
             expect \":\" { sleep 0.2; send \"$a4_password\\n\"; }; \
@@ -259,7 +259,7 @@ testsuite_mfa()
     retvalshouldbe 123
     json .error_code KO_MFA_TOTP_SETUP_REQUIRED
 
-    if [ "$HAS_MFA" = 1 ]; then
+    if [ "${capabilities[mfa]}" = 1 ]; then
         # setup totp
         script mfa a4_setup_totp "echo 'set timeout 30; \
             spawn $a4 --osh selfMFASetupTOTP --no-confirm; \

tests/functional/tests.d/390-mfa-realm.sh

@@ -139,7 +139,7 @@ testsuite_mfa_realm()
     revoke groupDelete
 }
 
-if [ "$HAS_MFA" = 1 ] || [ "$HAS_MFA_PASSWORD" = 1 ]; then
+if [ "${capabilities[mfa]}" = 1 ] || [ "${capabilities[mfa-password]}" = 1 ]; then
     testsuite_mfa_realm
 fi
 unset -f testsuite_mfa_realm

tests/functional/tests.d/400-piv.sh

@@ -113,7 +113,7 @@ EOF
     # for this trick, a0 needs to use adminSudo hence needs to be an admin
     configchg 's=^\\\\x22adminAccounts\\\\x22.+=\\\\x22adminAccounts\\\\x22:[\\\\x22'"$account0"'\\\\x22],='
 
-    success root set_a0_as_admin $r0 "\". $remote_basedir/lib/shell/functions.inc; add_user_to_group_compat $account0 osh-admin\""
+    success root set_a0_as_admin $r0 "\". $opt_remote_basedir/lib/shell/functions.inc; add_user_to_group_compat $account0 osh-admin\""
 
     script sudo-selfListIngressKeys a0_sudo_a1_selfaddnonpiv $a0 --osh adminSudo -- --sudo-as $account1 --sudo-cmd selfAddIngressKey -- $js "< $account2key1file.pub"
     retvalshouldbe 0
@@ -148,7 +148,7 @@ EOF
 
     # manually launch the grace reaper (normally done by cron)
     echo "manually launching piv grace reaper..."
-    success root grace_reaper $r0 $remote_basedir/bin/cron/osh-piv-grace-reaper.pl
+    success root grace_reaper $r0 $opt_remote_basedir/bin/cron/osh-piv-grace-reaper.pl
 
     # account1 should no longer be able to connect, as PIV grace expired
     run info a1_noconnect_grace_expired $a1 --osh info
@@ -168,7 +168,7 @@ EOF
     json .command selfDelIngressKey .error_code OK
 
     # remove a0 from admins
-    success root del_a0_as_admin $r0 "\". $remote_basedir/lib/shell/functions.inc; del_user_from_group_compat $account0 osh-admin\""
+    success root del_a0_as_admin $r0 "\". $opt_remote_basedir/lib/shell/functions.inc; del_user_from_group_compat $account0 osh-admin\""
 
     revoke accountListIngressKeys
     revoke accountPIV
@@ -182,9 +182,10 @@ EOF
     revoke accountDelete
 
     # restore default config
-    success bastion configrestore $r0 "dd if=$osh_etc/bastion.conf.bak.$now of=$osh_etc/bastion.conf"
+    success bastion configrestore $r0 "dd if=$opt_remote_etc_bastion/bastion.conf.bak.$now of=$opt_remote_etc_bastion/bastion.conf"
 }
 
-if [ "$HAS_PIV" = 1 ]; then
+if [ "${capabilities[piv]}" = 1 ]; then
     testsuite_piv
 fi
+unset -f testsuite_piv