From 16f42221ca26eb20d7bf5b95ca11e6137afe6faa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lesimple?= Date: Tue, 8 Dec 2020 12:14:22 +0000 Subject: [PATCH 1/4] feat: add LC_BASTION_DETAILS envvar --- bin/shell/osh.pl | 65 +++++++++++++++++++- tests/functional/launch_tests_on_instance.sh | 2 +- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/bin/shell/osh.pl b/bin/shell/osh.pl index 7a409c4..856c5b2 100755 --- a/bin/shell/osh.pl +++ b/bin/shell/osh.pl @@ -12,6 +12,7 @@ use Getopt::Long qw(GetOptionsFromString :config pass_through no_ignore_case); use Sys::Hostname; use POSIX qw(strftime); use Term::ANSIColor; +use JSON; $ENV{'LANG'} = 'C'; $| = 1; @@ -645,6 +646,26 @@ my $hasMfaPasswordBypass = OVH::Bastion::is_user_in_group(account => $sysself my $isMfaTOTPRequired = OVH::Bastion::is_user_in_group(account => $sysself, group => OVH::Bastion::MFA_TOTP_REQUIRED_GROUP); my $hasMfaTOTPBypass = OVH::Bastion::is_user_in_group(account => $sysself, group => OVH::Bastion::MFA_TOTP_BYPASS_GROUP); +# MFA information from a potential ingress realm: +my $remoteMfaValidated = 0; +my $remoteMfaPassword = 0; +my $remoteMfaTOTP = 0; + +# if we're coming from a realm, we're receiving a connection from another bastion, keep all the traces: +my @previous_bastion_details; +if ($realm && $ENV{'LC_BASTION_DETAILS'}) { + my $decoded_details; + eval { $decoded_details = decode_json($ENV{'LC_BASTION_DETAILS'}); }; + if (!$@) { + @previous_bastion_details = @$decoded_details; + + # if the remote bastion did validate MFA, trust it + $remoteMfaValidated = $decoded_details->[0]{'mfa'}{'validated'} ? 1 : 0; + $remoteMfaPassword = $decoded_details->[0]{'mfa'}{'type'}{'password'} ? 1 : 0; + $remoteMfaTOTP = $decoded_details->[0]{'mfa'}{'type'}{'totp'} ? 1 : 0; + } +} + if ($mfaPolicy ne 'disabled' && !grep { $osh_command eq $_ } qw{ selfMFASetupPassword selfMFASetupTOTP help info }) { if (($mfaPolicy eq 'password-required' && !$hasMfaPasswordBypass) || $isMfaPasswordRequired) { @@ -751,6 +772,9 @@ if ($sshAs) { exec(@cmd) or main_exit(OVH::Bastion::EXIT_EXEC_FAILED, "ssh_as_failed", "Couldn't start a session under the account $sshAs ($!)"); } +# This will be filled with details we might want to pass on to the remote machine as a json-encoded envvar +my %bastion_details; + # # First case. We have an OSH command # @@ -895,6 +919,16 @@ if ($osh_command) { $fnret = OVH::Bastion::do_pamtester(self => $self, sysself => $sysself); $fnret or main_exit(OVH::Bastion::EXIT_MFA_FAILED, 'mfa_failed', $fnret->msg); + + # so that the remote server, which can be a bastion in case we're chaining, can enforce its own policy: + $bastion_details{'mfa'} = { + validated => \1, + reason => 'mfa_required_for_plugin', + type => { + password => $isMfaPasswordConfigured ? \1 : \0, + totp => $isMfaTOTPConfigured ? \1 : \0, + } + }; } OVH::Bastion::set_terminal_mode_for_plugin(plugin => $osh_command, action => 'set'); @@ -1192,9 +1226,19 @@ else { } } -# add remoteUser as LC_BASTION to be passed via ssh +# add current account name as LC_BASTION to be passed via ssh $ENV{'LC_BASTION'} = $self; +$bastion_details{'mfa'}{'validated'} //= \0; +$bastion_details{'mfa'}{'type'}{'password'} //= \0; +$bastion_details{'mfa'}{'type'}{'totp'} //= \0; +$bastion_details{'from'} = {addr => $ipfrom, host => $hostfrom, port => $portfrom + 0}; +$bastion_details{'via'} = {addr => $bastionip, host => $bastionhost, port => $bastionport + 0}; +$bastion_details{'to'} = {addr => $ip, host => $hostto, port => $port + 0, user => $user}; +$bastion_details{'account'} = $self; +$bastion_details{'uniqid'} = $log_uniq_id; +$bastion_details{'version'} = $OVH::Bastion::VERSION; + if (!@command) { main_exit OVH::Bastion::EXIT_UNKNOWN_COMMAND, "empty_command", "Found no command to execute!"; } @@ -1303,9 +1347,28 @@ if ($JITMFARequired) { else { $fnret = OVH::Bastion::do_pamtester(self => $self, sysself => $sysself); $fnret or main_exit(OVH::Bastion::EXIT_MFA_FAILED, 'mfa_failed', $fnret->msg); + + # so that the remote server, which can be a bastion in case we're chaining, can enforce its own policy + $bastion_details{'mfa'} = { + validated => \1, + reason => 'mfa_required_for_host', + type => { + password => $isMfaPasswordConfigured ? \1 : \0, + totp => $isMfaTOTPConfigured ? \1 : \0, + } + }; } } +# now that we're about to connect, convert the bastion_details to a json envvar: +my @details_json = (\%bastion_details); + +# if we have data from a previous bastion (due to a realm connection), include it on top: +push @details_json, @previous_bastion_details if @previous_bastion_details; + +# then convert to json: +$ENV{'LC_BASTION_DETAILS'} = encode_json(\@details_json); + # here is a nice hack to drastically improve the memory footprint of an # heavily used bastion. we exec() another script that is way lighter, see # comments in the connect.pl file for more information. diff --git a/tests/functional/launch_tests_on_instance.sh b/tests/functional/launch_tests_on_instance.sh index 46f24d9..2892ec6 100755 --- a/tests/functional/launch_tests_on_instance.sh +++ b/tests/functional/launch_tests_on_instance.sh @@ -101,7 +101,7 @@ revoke() { success prereq revokecmd $a0 --osh accountRevokeCommand --account $ac cat >"$mytmpdir/ssh_config" < Date: Mon, 21 Dec 2020 11:14:18 +0000 Subject: [PATCH 2/4] feat: realms: use remote bastion MFA validation information for local policy enforcement --- bin/shell/osh.pl | 23 +++- tests/functional/tests.d/310-realm.sh | 1 + tests/functional/tests.d/390-mfa-realm.sh | 145 ++++++++++++++++++++++ 3 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 tests/functional/tests.d/390-mfa-realm.sh diff --git a/bin/shell/osh.pl b/bin/shell/osh.pl index 856c5b2..9e1e756 100755 --- a/bin/shell/osh.pl +++ b/bin/shell/osh.pl @@ -672,17 +672,17 @@ if ($mfaPolicy ne 'disabled' && !grep { $osh_command eq $_ } qw{ selfMFASetupPas main_exit(OVH::Bastion::EXIT_MFA_PASSWORD_SETUP_REQUIRED, 'mfa_password_setup_required', "Sorry, but you need to setup the Multi-Factor Authentication before using this bastion, please use the `--osh selfMFASetupPassword' option to do so") - if !$isMfaPasswordConfigured; + if (!$isMfaPasswordConfigured && !$remoteMfaPassword); } if (($mfaPolicy eq 'totp-required' && !$hasMfaTOTPBypass) || $isMfaTOTPRequired) { main_exit(OVH::Bastion::EXIT_MFA_TOTP_SETUP_REQUIRED, 'mfa_totp_setup_required', "Sorry, but you need to setup the Multi-Factor Authentication before using this bastion, please use the `--osh selfMFASetupTOTP' option to do so") - if !$isMfaTOTPConfigured; + if !($isMfaTOTPConfigured && !$remoteMfaTOTP); } - if ($mfaPolicy eq 'any-required' && (!$isMfaPasswordConfigured && !$hasMfaPasswordBypass) && (!$isMfaTOTPConfigured && !$hasMfaTOTPBypass)) { + if ($mfaPolicy eq 'any-required' && (!$isMfaPasswordConfigured && !$hasMfaPasswordBypass) && (!$isMfaTOTPConfigured && !$hasMfaTOTPBypass) && !$remoteMfaValidated) { main_exit(OVH::Bastion::EXIT_MFA_ANY_SETUP_REQUIRED, 'mfa_any_setup_required', "Sorry, but you need to setup the Multi-Factor Authentication before using this bastion, please use either the `--osh selfMFASetupPassword' or the `--osh selfMFASetupTOTP' option, at your discretion, to do so" ); @@ -1306,12 +1306,16 @@ if (!$logret) { # if we have JIT MFA, do it now if ($JITMFARequired) { - my $skipMFA = 0; + my $skipMFA = 0; + my $realmMFA = 0; print "As this is required for this host, entering MFA phase.\n"; if ($JITMFARequired eq 'totp' && !$isMfaTOTPConfigured) { if ($hasMfaTOTPBypass) { $skipMFA = 1; } + elsif ($remoteMfaTOTP && $remoteMfaValidated) { + $realmMFA = 1; + } else { main_exit(OVH::Bastion::EXIT_MFA_TOTP_SETUP_REQUIRED, 'mfa_totp_setup_required', @@ -1322,6 +1326,9 @@ if ($JITMFARequired) { if ($hasMfaPasswordBypass) { $skipMFA = 1; } + elsif ($remoteMfaPassword && $remoteMfaValidated) { + $realmMFA = 1; + } else { main_exit(OVH::Bastion::EXIT_MFA_PASSWORD_SETUP_REQUIRED, 'mfa_password_setup_required', @@ -1334,6 +1341,9 @@ if ($JITMFARequired) { # FIXME: should actually be $hasMFABypassAll (not yet implemented) $skipMFA = 1; } + elsif ($remoteMfaValidated) { + $realmMFA = 1; + } else { main_exit(OVH::Bastion::EXIT_MFA_ANY_SETUP_REQUIRED, 'mfa_any_setup_required', "Sorry, but you need to setup the Multi-Factor Authentication before connecting to this host,\nplease use either the `--osh selfMFASetupPassword' or the `--osh selfMFASetupTOTP' option, at your discretion, to do so" @@ -1342,7 +1352,10 @@ if ($JITMFARequired) { } if ($skipMFA) { - print "... skipping as your account is exempt from MFA\n"; + print "... skipping as your account is exempt from MFA.\n"; + } + elsif ($realmMFA) { + print "... you already validated MFA on the bastion you're coming from.\n"; } else { $fnret = OVH::Bastion::do_pamtester(self => $self, sysself => $sysself); diff --git a/tests/functional/tests.d/310-realm.sh b/tests/functional/tests.d/310-realm.sh index 617f9d2..64749f6 100644 --- a/tests/functional/tests.d/310-realm.sh +++ b/tests/functional/tests.d/310-realm.sh @@ -277,3 +277,4 @@ testsuite_realm() } testsuite_realm +unset -f testsuite_realm diff --git a/tests/functional/tests.d/390-mfa-realm.sh b/tests/functional/tests.d/390-mfa-realm.sh new file mode 100644 index 0000000..9a07f7d --- /dev/null +++ b/tests/functional/tests.d/390-mfa-realm.sh @@ -0,0 +1,145 @@ +# vim: set filetype=sh ts=4 sw=4 sts=4 et: +# shellcheck shell=bash +# shellcheck disable=SC2086,SC2016,SC2046 +# below: convoluted way that forces shellcheck to source our caller +# shellcheck source=tests/functional/launch_tests_on_instance.sh +. "$(dirname "${BASH_SOURCE[0]}")"/dummy + +testsuite_mfa_realm() +{ + local realm_egress_group=realmsuppgrp + local realm_shared_account=supplier42 + grant accountCreate + + # create account4 + success mfarealm a0_create_a4 $a0 --osh accountCreate --always-active --account $account4 --uid $uid4 --public-key "\"$(cat $account4key1file.pub)\"" + json .error_code OK .command accountCreate .value null + + revoke accountCreate + + # now setup a realm + grant groupCreate + + # create realm-egress group on local bastion + success realm create_support_group $a0 --osh groupCreate --group $realm_egress_group --owner $account4 --algo ed25519 + local realm_group_key + realm_group_key=$(get_json | $jq '.value.public_key.line') + + grant realmCreate + + # create shared realm-account on remote bastion + success realm create_shared_account $a0 --osh realmCreate --realm $realm_shared_account --public-key \"$realm_group_key\" --from 0.0.0.0/0 + + revoke realmCreate + + # add remote bastion ip on group of local bastion + success realm add_remote_bastion_to_group $a4 --osh groupAddServer --host 127.0.0.1 --user realm_$realm_shared_account --port 22 --group $realm_egress_group --kbd-interactive + + # attempt inter-realm connection + success realm firstconnect1 $a4 realm_$realm_shared_account@127.0.0.1 --kbd-interactive -- $js --osh info + json .value.account $account4 .value.realm $realm_shared_account + + # create a remote-group on which we'll add the realm user + success mfarealm remote_group_create $a0 --osh groupCreate --group remotegrp --owner $account0 --algo ed25519 + revoke groupCreate + + success mfarealm remote_group_add_server $a0 --osh groupAddServer --group remotegrp --host 127.0.0.5 --port 22 --user nevermind --force + + # try to connect, as a realm user, to 127.0.0.5 through the realm: won't work + run mfarealm realm_user_fail_connect_not_member $a4 realm_$realm_shared_account@127.0.0.1 --kbd-interactive -- $js nevermind@127.0.0.5 + retvalshouldbe 107 + json .error_code KO_ACCESS_DENIED .error_message "Access denied for $realm_shared_account/$account4 to nevermind@127.0.0.5:22" + + # now add the realm user and retry + success mfarealm remote_group_add_user $a0 --osh groupAddMember --group remotegrp --account $realm_shared_account/$account4 + + run mfarealm realm_user_fail_connect_not_member $a4 realm_$realm_shared_account@127.0.0.1 --kbd-interactive -- $js nevermind@127.0.0.5 + retvalshouldbe 255 + contain "group-member of remotegrp" + contain "Permission denied (publickey)" + + # now setup mandatory MFA on the group + success mfarealm remote_group_set_mfa $a0 --osh groupModify --group remotegrp --mfa-required password + + # try to connect won't work + run mfarealm realm_user_fail_connect_no_mfa $a4 realm_$realm_shared_account@127.0.0.1 --kbd-interactive -- $js nevermind@127.0.0.5 + retvalshouldbe 122 + json .error_code KO_MFA_PASSWORD_SETUP_REQUIRED + + # setup our MFA + # setup our password, step1 + run mfa a4_setup_pass_step1of2 $a4f --osh selfMFASetupPassword --yes + retvalshouldbe 124 + contain 'enter this:' + local a4_password_tmp + a4_password_tmp=$(get_stdout | grep -Eo 'enter this: [a-zA-Z0-9_-]+' | sed -e 's/enter this: //') + + # setup our password, step2 + local a4_password='Hfv$!OKiG:(xl>Th8Kv!alz4436BFt~' + script mfa a4_setup_pass_step2of2 "echo 'set timeout 30; \ + spawn $a4 --osh selfMFASetupPassword --yes; \ + expect \":\" { sleep 0.2; send \"$a4_password_tmp\\n\"; }; \ + expect \":\" { sleep 0.2; send \"$a4_password\\n\"; }; \ + expect \":\" { sleep 0.2; send \"$a4_password\\n\"; }; \ + expect eof; \ + lassign [wait] pid spawnid value value; \ + exit \$value' | expect -f -" + retvalshouldbe 0 + unset a4_password_tmp + nocontain 'enter this:' + nocontain 'unchanged' + nocontain 'sorry' + json .command selfMFASetupPassword .error_code OK + + # set account4 as nopam, to only use JIT MFA because that's what we want to test + grant accountModify + + success mfarealm a4_set_nopam $a0 --osh accountModify --account $account4 --pam-auth-bypass yes + json .command accountModify .error_code OK + + revoke accountModify + + # try to connect will still not work because we have MFA but we're asked for it on our first bastion + run mfarealm realm_user_still_fail_connect_no_mfa $a4 realm_$realm_shared_account@127.0.0.1 --kbd-interactive -- $js nevermind@127.0.0.5 + retvalshouldbe 122 + json .error_code KO_MFA_PASSWORD_SETUP_REQUIRED + + # force MFA for the support group + success mfarealm set_mfa_for_support_group $a4 --osh groupModify --group $realm_egress_group --mfa-required password + json .command groupModify .error_code OK + + # try to connect, this one will finally work + script mfarealm a4_connect_success_realm_with_remote_mfa "echo 'set timeout 30; \ + spawn $a4 realm_$realm_shared_account@127.0.0.1 --kbd-interactive -- $js nevermind@127.0.0.5; \ + expect \"word:\" { sleep 0.2; send \"$a4_password\\n\"; }; \ + expect eof; \ + lassign [wait] pid spawnid value value; \ + exit \$value' | expect -f -" + retvalshouldbe 255 + contain "you already validated MFA on the bastion you're coming from" + contain "Permission denied (publickey)" + + # cleanup + grant realmDelete + + success mfarealm realmDelete $a0 --osh realmDelete --realm $realm_shared_account "<<< \"Yes, do as I say and delete $realm_shared_account, kthxbye\"" + + revoke realmDelete + grant accountDelete + + script mfarealm a0_delete_a4 $a0 --osh accountDelete --account $account4 "<<< \"Yes, do as I say and delete $account4, kthxbye\"" + retvalshouldbe 0 + json .command accountDelete .error_code OK + + revoke accountDelete + grant groupDelete + + success mfarealm groupDelete $a0 --osh groupDelete --group $realm_egress_group --no-confirm + + revoke groupDelete +} + +if [ "$HAS_MFA" = 1 ] || [ "$HAS_MFA_PASSWORD" = 1 ]; then + testsuite_mfa_realm +fi +unset -f testsuite_mfa_realm From 2cfde997f3b06246f6f0df1276ae87f279a24e0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lesimple?= Date: Mon, 21 Dec 2020 15:13:09 +0000 Subject: [PATCH 3/4] fix: realmDelete: bad sudoers configuration --- bin/helper/osh-accountDelete | 43 +++++++++++++++++--------- etc/sudoers.d/osh-plugin-accountDelete | 3 +- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/bin/helper/osh-accountDelete b/bin/helper/osh-accountDelete index 1b50218..cf3db54 100755 --- a/bin/helper/osh-accountDelete +++ b/bin/helper/osh-accountDelete @@ -1,7 +1,9 @@ #! /usr/bin/perl -T # vim: set filetype=perl ts=4 sw=4 sts=4 et: # NEEDGROUP osh-accountDelete -# SUDOERS %osh-accountDelete ALL=(root) NOPASSWD:/usr/bin/env perl -T /opt/bastion/bin/helper/osh-accountDelete * +# SUDOERS %osh-accountDelete ALL=(root) NOPASSWD:/usr/bin/env perl -T /opt/bastion/bin/helper/osh-accountDelete --type normal * +# NEEDGROUP osh-realmDelete +# SUDOERS %osh-realmDelete ALL=(root) NOPASSWD:/usr/bin/env perl -T /opt/bastion/bin/helper/osh-accountDelete --type realm * # FILEMODE 0700 # FILEOWN 0 0 @@ -55,20 +57,6 @@ if (!$account || !$type) { #
RIGHTSCHECK -if ($self eq 'root') { - osh_debug "Real root, skipping checks of permissions"; -} -else { - # need to perform another security check - $fnret = OVH::Bastion::is_user_in_group(user => $self, group => "osh-accountDelete"); - if (!$fnret) { - HEXIT('ERR_SECURITY_VIOLATION', msg => "You're not allowed to run this, dear $self"); - } -} - -#PARAMS:TYPE osh_debug("Checking type"); if (not grep { $type eq $_ } qw{ normal realm }) { @@ -77,6 +65,31 @@ if (not grep { $type eq $_ } qw{ normal realm }) { #RIGHTSCHECK +if ($self eq 'root') { + osh_debug "Real root, skipping checks of permissions"; +} +else { + # need to perform another security check + if ($type eq 'normal') { + $fnret = OVH::Bastion::is_user_in_group(user => $self, group => "osh-accountDelete"); + if (!$fnret) { + HEXIT('ERR_SECURITY_VIOLATION', msg => "You're not allowed to run this, dear $self"); + } + } + elsif ($type eq 'realm') { + $fnret = OVH::Bastion::is_user_in_group(user => $self, group => "osh-realmDelete"); + if (!$fnret) { + HEXIT('ERR_SECURITY_VIOLATION', msg => "You're not allowed to run this, dear $self"); + } + } + else { + HEXIT('ERR_INTERNAL'); + } +} + +#PARAMS:ACCOUNT osh_debug("Checking account"); $fnret = OVH::Bastion::is_bastion_account_valid_and_existing(account => $account, accountType => $type); diff --git a/etc/sudoers.d/osh-plugin-accountDelete b/etc/sudoers.d/osh-plugin-accountDelete index feae1dc..1575c53 100644 --- a/etc/sudoers.d/osh-plugin-accountDelete +++ b/etc/sudoers.d/osh-plugin-accountDelete @@ -1 +1,2 @@ -%osh-accountDelete ALL=(root) NOPASSWD:/usr/bin/env perl -T /opt/bastion/bin/helper/osh-accountDelete * +%osh-accountDelete ALL=(root) NOPASSWD:/usr/bin/env perl -T /opt/bastion/bin/helper/osh-accountDelete --type normal * +%osh-realmDelete ALL=(root) NOPASSWD:/usr/bin/env perl -T /opt/bastion/bin/helper/osh-accountDelete --type realm * From 5228c863b03997021bd402e7f6e8a515ad488e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lesimple?= Date: Fri, 25 Dec 2020 16:59:55 +0100 Subject: [PATCH 4/4] chore: tests_all: use proper tempdir --- .../docker/docker_build_and_run_tests_all.sh | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/functional/docker/docker_build_and_run_tests_all.sh b/tests/functional/docker/docker_build_and_run_tests_all.sh index 05fcf55..3e3216d 100755 --- a/tests/functional/docker/docker_build_and_run_tests_all.sh +++ b/tests/functional/docker/docker_build_and_run_tests_all.sh @@ -16,12 +16,15 @@ echo "Targets: $targets" sleep 5 +tempdir=$(mktemp -d) +trap 'test -d $tempdir && rm -rf $tempdir' EXIT + for t in $targets do + [ "$t" = "-" ] && continue ( - rm -f "/tmp/.$t" DOCKER_TTY=false ./docker_build_and_run_tests.sh "$t" - echo $? > "/tmp/.$t" + echo $? > "$tempdir/$t" ) & done wait @@ -32,21 +35,22 @@ nberrors=0 for t in $targets do - err=$(cat "/tmp/.$t" 2>/dev/null) - rm -f "/tmp/.$t" + [ "$t" = "-" ] && continue + err=$(cat "$tempdir/$t" 2>/dev/null) + rm -f "$tempdir/$t" if [ -z "$err" ]; then - printf "%b%15s: tests couldn't run properly%b\\n" "$BLACK_ON_RED" "$t" "$NOC" + printf "%b%23s: tests couldn't run properly%b\\n" "$BLACK_ON_RED" "$t" "$NOC" nberrors=$(( nberrors + 1 )) elif [ "$err" = 0 ]; then - printf "%b%15s: no errors :)%b\\n" "$BLACK_ON_GREEN" "$t" "$NOC" + printf "%b%23s: no errors :)%b\\n" "$BLACK_ON_GREEN" "$t" "$NOC" elif [ "$err" = 143 ]; then - printf "%b%15s: tests interrupted%b\\n" "$BLACK_ON_RED" "$t" "$NOC" + printf "%b%23s: tests interrupted%b\\n" "$BLACK_ON_RED" "$t" "$NOC" nberrors=$(( nberrors + 1 )) elif [ "$err" -lt 254 ]; then - printf "%b%15s: $err tests failed%b\\n" "$BLACK_ON_RED" "$t" "$NOC" + printf "%b%23s: $err tests failed%b\\n" "$BLACK_ON_RED" "$t" "$NOC" nberrors=$(( nberrors + 1 )) else - printf "%b%15s: $err errors%b\\n" "$BLACK_ON_RED" "$t" "$NOC" + printf "%b%23s: $err errors%b\\n" "$BLACK_ON_RED" "$t" "$NOC" nberrors=$(( nberrors + 1 )) fi done