feat: httpproxy: add functional tests

.github/workflows/freebsd.yml

@@ -20,7 +20,7 @@ jobs:
           set -ex
           freebsd-version
           mount -o acls /
-          pkg install -y bash rsync ca_root_nss jq fping screen flock
+          pkg install -y bash rsync ca_root_nss jq fping screen flock curl
           mkdir -p /opt/bastion
           rsync -a . /opt/bastion/
           /opt/bastion/bin/admin/packages-check.sh -i
@@ -30,4 +30,4 @@ jobs:
           ssh-keygen -t ed25519 -f id_user
           ssh-keygen -t ed25519 -f id_root
           NO_SLEEP=1 user_pubkey=$(cat id_user.pub) root_pubkey=$(cat id_root.pub) TARGET_USER=user5000 /opt/bastion/tests/functional/docker/target_role.sh
-          HAS_MFA=0 HAS_MFA_PASSWORD=1 HAS_PAMTESTER=1 nocc=1 /opt/bastion/tests/functional/launch_tests_on_instance.sh 127.0.0.1 22 user5000 id_user id_root /usr/local/etc/bastion
+          HAS_MFA=0 HAS_MFA_PASSWORD=1 HAS_PAMTESTER=1 nocc=1 /opt/bastion/tests/functional/launch_tests_on_instance.sh 127.0.0.1 22 0 user5000 id_user id_root /usr/local/etc/bastion

bin/admin/packages-check.sh

@@ -45,7 +45,7 @@ if echo "$DISTRO_LIKE" | grep -q -w debian; then
     if [ "$(uname -m)" = armv7l ]; then
         wanted_list="$wanted_list wget"
     fi
-    [ "$opt_dev" = 1 ] && wanted_list="$wanted_list libperl-critic-perl perltidy shellcheck"
+    [ "$opt_dev" = 1 ] && wanted_list="$wanted_list libperl-critic-perl perltidy shellcheck openssl"
     if { [ "$LINUX_DISTRO" = debian ] && [ "$DISTRO_VERSION_MAJOR" -lt 9 ]; } ||
        { [ "$LINUX_DISTRO" = ubuntu ] && [ "$DISTRO_VERSION_MAJOR" -le 16 ]; }; then
         wanted_list="$wanted_list openssh-blacklist openssh-blacklist-extra"

docker/Dockerfile.tester

@@ -2,7 +2,7 @@ FROM debian:buster
 LABEL maintainer="stephane.lesimple+bastion@ovhcloud.com"
 
 # install prerequisites
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y jq netcat openssh-client procps bsdutils screen expect shellcheck libperl-critic-perl fping
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y jq netcat openssh-client procps bsdutils screen expect shellcheck libperl-critic-perl fping curl
 
 # add our code
 COPY . /opt/bastion

lib/perl/OVH/Bastion/ProxyHTTP.pm

@@ -436,8 +436,11 @@ sub process_http_request {
     push @cmd, "--allow-downgrade" if $allow_downgrade;
     push @cmd, "--insecure"        if ($self->{'proxy_config'}{'insecure'} && !$enforce_secure);
 
-    foreach my $key (qw{ accept content-type connection }) {
+    # X-Test-* is only used for functional tests, and has to be passed to the remote
-        push @cmd, "--header", $key . ':' . $req_headers->{$key} if (defined $req_headers->{$key});
+    foreach my $key (keys %$req_headers) {
+        if ($key =~ /^x-test-/i || grep { lc($key) eq $_ } qw{ accept content-type connection }) {
+            push @cmd, "--header", $key . ':' . $req_headers->{$key};
+        }
     }
 
     # we don't want the CGI module to parse/modify/interpret the content, so we

tests/functional/docker/docker_build_and_run_tests.sh

@@ -132,6 +132,7 @@ docker run $privileged \
     -e ROOT_PUBKEY_B64="$ROOT_PUBKEY_B64" \
     -e TARGET_USER="user.5000" \
     -e TEST_QUICK="${TEST_QUICK:-0}" \
+    -e WANT_HTTP_PROXY=1 \
     $namespace:"$target"
 docker logs -f "bastion_${target}_target" | sed -u -e 's/^/target: /;s/$/\r/' &
 
@@ -177,6 +178,7 @@ docker run \
     --tty=$DOCKER_TTY \
     -e TARGET_IP="bastion_${target}_target" \
     -e TARGET_PORT=22 \
+    -e TARGET_PROXY_PORT=8443 \
     -e TARGET_USER="user.5000" \
     -e USER_PRIVKEY_B64="$USER_PRIVKEY_B64" \
     -e ROOT_PRIVKEY_B64="$ROOT_PRIVKEY_B64" \

tests/functional/docker/target_role.sh

@@ -125,7 +125,34 @@ if [ -n "$NO_SLEEP" ]; then
     exit 0
 fi
 
+if [ "$WANT_HTTP_PROXY" = 1 ]; then
+
+    # build a self-signed certificate for the http proxy and adjust the config
+    openssl req -x509 -nodes -days 7 -newkey rsa:2048 -keyout /tmp/selfsigned.key -out /tmp/selfsigned.crt -subj "/CN=testcert"
+    chgrp proxyhttp /tmp/selfsigned.key
+    chmod g+r /tmp/selfsigned.key
+    sed -i -re 's="ssl_certificate":.*="ssl_certificate": "/tmp/selfsigned.crt",=' /etc/bastion/osh-http-proxy.conf
+    sed -i -re 's="ssl_key":.*="ssl_key": "/tmp/selfsigned.key",=' /etc/bastion/osh-http-proxy.conf
+    sed -i -re 's="enabled":.+="enabled":true,=' /etc/bastion/osh-http-proxy.conf
+    sed -i -re 's="insecure":.+="insecure":true,=' /etc/bastion/osh-http-proxy.conf
+
+    # ensure the remote daemon is executable
+    chmod 0755 "$basedir"/tests/functional/proxy/remote-daemon
+
+    while : ; do
+        echo "Starting HTTP Proxy and fake remote server"
+        if [ -x /etc/init.d/osh-http-proxy ]; then
+            /etc/init.d/osh-http-proxy start
+        else
+            sudo -n -u proxyhttp -- /opt/bastion/bin/proxy/osh-http-proxy-daemon &
+            disown
+        fi
+        "$basedir"/tests/functional/proxy/remote-daemon
+        sleep 1
+    done
+else
     echo "Now sleeping forever (docker mode)"
     while : ; do
         sleep 3600
     done
+fi

tests/functional/docker/tester_role.sh

@@ -35,7 +35,7 @@ for i in $(seq 1 $delay); do
     if echo test | nc -w 1 "$TARGET_IP" "$TARGET_PORT" | grep -q ^SSH-2 ; then
         echo "tester: it's alive, starting tests!"
         [ "$TEST_QUICK" = 1 ] && export nocc=1
-        "$(dirname "$0")"/../launch_tests_on_instance.sh "$TARGET_IP" "$TARGET_PORT" "$TARGET_USER" /root/user.privkey /root/root.privkey; ret=$?
+        "$(dirname "$0")"/../launch_tests_on_instance.sh "$TARGET_IP" "$TARGET_PORT" "${TARGET_PROXY_PORT:-0}" "$TARGET_USER" /root/user.privkey /root/root.privkey; ret=$?
         [ "$ret" -gt 253 ] && ret=253
         exit "$ret"
     elif ! fping -r 1 "$TARGET_IP" >/dev/null 2>&1; then

tests/functional/launch_tests_on_instance.sh

@@ -11,11 +11,14 @@ basedir=$(readlink -f "$(dirname "$0")"/../..)
 
 remote_ip="$1"
 remote_port="$2"
-account0="$3"
+# the var below is used in sourced test files
-user_ssh_key_path="$4"
+# shellcheck disable=SC2034
-root_ssh_key_path="$5"
+remote_proxy_port="$3"
-osh_etc="$6"
+account0="$4"
-remote_basedir="$7"
+user_ssh_key_path="$5"
+root_ssh_key_path="$6"
+osh_etc="$7"
+remote_basedir="$8"
 [ -n "$osh_etc" ] || osh_etc=/etc/bastion
 [ -n "$remote_basedir" ] || remote_basedir="$basedir"
 
@@ -34,7 +37,7 @@ remote_basedir="$7"
 set -u
 
 if [ -z "$root_ssh_key_path" ] ; then
-    echo "Usage: $0 <IP> <Port> <remote_user_name> <user_ssh_key_path> <root_ssh_key_path>"
+    echo "Usage: $0 <IP> <Port> <HTTP_Proxy_Port_or_zero> <remote_user_name> <user_ssh_key_path> <root_ssh_key_path> [osh_etc] [remote_basedir]"
     exit 1
 fi
 

tests/functional/proxy/remote-daemon

@@ -0,0 +1,59 @@
+#! /usr/bin/perl
+use strict;
+use warnings;
+use base qw{Net::Server::HTTP};
+use CGI;
+use Data::Dumper;
+
+__PACKAGE__->run(
+    port => ["9080", "9443/ssl"],
+    ipv => 4,
+    SSL_key_file => "/tmp/selfsigned.key",
+    SSL_cert_file => "/tmp/selfsigned.crt",
+    max_requests      => 1,
+);
+
+sub process_http_request {
+    my $self = shift;
+
+    my $hasContentType;
+    my $wantedResponseSize = 64;
+
+    my $real_content_type = $ENV{'CONTENT_TYPE'};
+    $ENV{'CONTENT_TYPE'} = 'application/xml';
+    my $content = CGI->new->param('XForms:Model');
+    $ENV{'CONTENT_TYPE'} = $real_content_type;
+
+    foreach my $headerTuple (@{ $self->{'request_info'}{'request_headers'} }) {
+        if ($headerTuple->[0] =~ /^x-test-add-response-header-(.+)/i) {
+            print "$1: ".$headerTuple->[1]."\n";
+            $hasContentType = 1 if lc($1) eq 'content-type';
+        }
+        elsif (lc $headerTuple->[0] eq 'x-test-wanted-response-size') {
+            $wantedResponseSize = $headerTuple->[1];
+        }
+    }
+    print "Content-type: text/plain\n" if !$hasContentType;
+
+    if ($content) {
+        print "Content-Length: ".length($content)."\n\n";
+        print $content;
+    }
+    else {
+        print "Content-Length: ".$wantedResponseSize."\n\n";
+        my @chars = ('0'..'9', 'a'..'z', 'A'..'Z', "\n");
+
+        my $buffer;
+        for (2..$wantedResponseSize) {
+            $buffer .= $chars[rand @chars];
+            if (length($buffer) > 16384) {
+                print $buffer;
+                $buffer = '';
+            }
+        }
+        print $buffer;
+        print "\n";
+    }
+    close(STDOUT);
+    return;
+}

tests/functional/tests.d/500-http-proxy.sh

@@ -0,0 +1,239 @@
+# vim: set filetype=sh ts=4 sw=4 sts=4 et:
+# shellcheck shell=bash
+# shellcheck disable=SC2086,SC2016,SC2046
+# below: convoluted way that forces shellcheck to source our caller
+# shellcheck source=tests/functional/launch_tests_on_instance.sh
+. "$(dirname "${BASH_SOURCE[0]}")"/dummy
+
+testsuite_proxy()
+{
+    # note: we use "curl | cat" to force curl to disable color output, to be grep friendly,
+    # as a --no-color or similar option doesn't seem to exist for curl.
+
+    # check that the proxy is up
+    script 500-http-proxy monitoring "curl -ski https://$remote_ip:$remote_proxy_port/bastion-health-check | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain 'running nominally'
+
+    # and let's go
+    script 500-http-proxy noauth "curl -ski https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain 'HTTP/1.0 401 Authorization required (no auth provided)'
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    contain 'WWW-Authenticate: Basic realm="bastion"'
+    contain 'Content-Type: text/plain'
+    contain 'No authentication provided, and authentication is mandatory'
+
+    script 500-http-proxy bad_auth_format "curl -ski -u test:test https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain 'HTTP/1.0 400 Bad Request (bad login format)'
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    nocontain 'WWW-Authenticate: '
+    contain 'Content-Type: text/plain'
+    contain 'Expected an Authorization line with credentials of the form'
+
+    script 500-http-proxy bad_auth "curl -ski -u test@test@test:test https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain 'HTTP/1.0 403 Access Denied'
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    nocontain 'WWW-Authenticate: '
+    contain 'Content-Type: text/plain'
+    contain 'Incorrect username (test) or password (#REDACTED#, length=4)'
+
+    # create valid credentials
+    success 500-http-proxy generate_proxy_password $a0 --osh selfGenerateProxyPassword --do-it
+    json .command selfGenerateProxyPassword .error_code OK
+    local proxy_password
+    proxy_password=$(get_json | jq -r '.value.password')
+
+    # now try to use these
+    script 500-http-proxy good_auth_bad_host "curl -ski -u '$account0@test@test.invalid:$proxy_password' https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain 'HTTP/1.0 400 Bad Request (host not resolved)'
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    nocontain 'WWW-Authenticate: '
+    contain 'Content-Type: text/plain'
+    contain 'X-Bastion-Remote-IP: test.invalid'
+    contain 'X-Bastion-Request-Length: 0'
+    contain 'X-Bastion-Local-Status: 400'
+    contain 'Content-Type: text/plain'
+    contain "Specified remote host couldn't be resolved through the DNS"
+
+    # change credentials again
+    success 500-http-proxy generate_proxy_password2 $a0 --osh selfGenerateProxyPassword --do-it
+    json .command selfGenerateProxyPassword .error_code OK
+    local proxy_password2
+    proxy_password2=$(get_json | jq -r '.value.password')
+
+    # attempt to use the previous credentials (and fail)
+    script 500-http-proxy bad_auth2 "curl -ski -u test@test@test:test https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain 'HTTP/1.0 403 Access Denied'
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    nocontain 'WWW-Authenticate: '
+    contain 'Content-Type: text/plain'
+    contain 'Incorrect username (test) or password (#REDACTED#, length='
+
+    proxy_password="$proxy_password2"
+
+    script 500-http-proxy good_auth_no_access "curl -ski -u '$account0@test@127.0.0.1:$proxy_password' https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain 'HTTP/1.0 403 Access Denied (access denied to remote)'
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    nocontain 'WWW-Authenticate: '
+    contain 'Content-Type: text/plain'
+    contain 'X-Bastion-Remote-IP: 127.0.0.1'
+    contain 'X-Bastion-Request-Length: 0'
+    contain 'X-Bastion-Auth-Mode: self/default'
+    contain 'X-Bastion-Local-Status: 403'
+    contain 'Content-Type: text/plain'
+    contain "This account doesn't have access to this user@host tuple (Access denied for $account0 to test@127.0.0.1:443)"
+
+    script 500-http-proxy good_auth_no_access_other_port "curl -ski -u '$account0@test@127.0.0.1%9443:$proxy_password' https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain 'HTTP/1.0 403 Access Denied (access denied to remote)'
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    nocontain 'WWW-Authenticate: '
+    contain 'Content-Type: text/plain'
+    contain 'X-Bastion-Remote-IP: 127.0.0.1'
+    contain 'X-Bastion-Request-Length: 0'
+    contain 'X-Bastion-Auth-Mode: self/default'
+    contain 'X-Bastion-Local-Status: 403'
+    contain 'Content-Type: text/plain'
+    contain "This account doesn't have access to this user@host tuple (Access denied for $account0 to test@127.0.0.1:9443)"
+
+    # add ourselves access
+    grant selfAddPersonalAccess
+
+    success 500-http-proxy add_personal_access $a0 --osh selfAddPersonalAccess --host 127.0.0.1 --port 9443 --user test --force
+    json .command selfAddPersonalAccess .error_code OK
+
+    revoke selfAddPersonalAccess
+
+    script 500-http-proxy missing_egress_pwd "curl -ski -u '$account0@test@127.0.0.1%9443:$proxy_password' https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain 'HTTP/1.0 412 Precondition Failed (egress password missing)'
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    nocontain 'WWW-Authenticate: '
+    contain 'Content-Type: text/plain'
+    contain 'X-Bastion-Remote-IP: 127.0.0.1'
+    contain 'X-Bastion-Request-Length: 0'
+    contain 'X-Bastion-Auth-Mode: self/default'
+    contain 'X-Bastion-Local-Status: 412'
+    contain 'Content-Type: text/plain'
+    contain "Unable to find (or read) a password file in context 'self' and name '$account0'"
+
+    # generate an egress password
+    success 500-http-proxy generate_egress_pwd $a0 --osh selfGeneratePassword --do-it
+    json .command selfGeneratePassword .error_code OK .value.account $account0 .value.context account
+
+    # and retry
+    script 500-http-proxy bad_certificate "curl -ski -H 'X-Bastion-Enforce-Secure: 1' -u '$account0@test@127.0.0.1%9443:$proxy_password' https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    # not all versions of LWP add "(certificate verify failed)" at the end of the below error message, so omit it
+    contain "HTTP/1.0 500 Can't connect to 127.0.0.1:9443"
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    nocontain 'WWW-Authenticate: '
+    contain 'Content-Type: text/plain'
+    contain 'X-Bastion-Remote-IP: 127.0.0.1'
+    contain 'X-Bastion-Request-Length: 0'
+    contain 'X-Bastion-Auth-Mode: self/default'
+    contain 'X-Bastion-Local-Status: 200 OK'
+    contain 'Content-Type: text/plain'
+    contain "Can't connect to 127.0.0.1:9443"
+
+    script 500-http-proxy insecure "curl -ski -u '$account0@test@127.0.0.1%9443:$proxy_password' https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain "HTTP/1.0 200 OK"
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    nocontain 'WWW-Authenticate: '
+    contain 'Content-Type: text/plain'
+    contain 'X-Bastion-Remote-IP: 127.0.0.1'
+    contain 'X-Bastion-Request-Length: 0'
+    contain 'X-Bastion-Auth-Mode: self/default'
+    contain 'X-Bastion-Local-Status: 200 OK'
+    contain "X-Bastion-Remote-Client-SSL-Cert-Subject: "
+    contain "X-Bastion-Remote-Client-SSL-Cipher: "
+    contain "X-Bastion-Remote-Client-SSL-Warning: Peer certificate not verified"
+    contain "X-Bastion-Remote-Status: 200"
+    contain "X-Bastion-Remote-Server: Net::Server::HTTP/"
+    contain "X-Bastion-Egress-Timing: "
+    contain "Content-Length: 64"
+
+    # generate 1MB of data
+    script 500-http-proxy one_megabyte "curl -ski -H 'X-Test-Add-Response-Header-Content-Type: application/json' -H 'X-Test-Wanted-Response-Size: 1000000' -u '$account0@test@127.0.0.1%9443:$proxy_password' https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain "HTTP/1.0 200 OK"
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    nocontain 'WWW-Authenticate: '
+    contain 'Content-Type: application/json'
+    contain 'X-Bastion-Remote-IP: 127.0.0.1'
+    contain 'X-Bastion-Request-Length: 0'
+    contain 'X-Bastion-Auth-Mode: self/default'
+    contain 'X-Bastion-Local-Status: 200 OK'
+    contain "X-Bastion-Remote-Client-SSL-Cert-Subject: "
+    contain "X-Bastion-Remote-Client-SSL-Cipher: "
+    contain "X-Bastion-Remote-Client-SSL-Warning: Peer certificate not verified"
+    contain "X-Bastion-Remote-Status: 200"
+    contain "X-Bastion-Remote-Server: Net::Server::HTTP/"
+    contain "X-Bastion-Egress-Timing: "
+    contain "Content-Length: 1000000"
+
+    # use a disallowed verb
+    script 500-http-proxy forbidden_verb "curl -ski -X OPTIONS -u '$account0@test@127.0.0.1%9443:$proxy_password' https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain 'HTTP/1.0 400 Bad Request (method forbidden)'
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    nocontain 'WWW-Authenticate: '
+    contain 'Content-Type: text/plain'
+    contain 'Only GET and POST methods are allowed'
+
+    # post some data
+    script 500-http-proxy post_data "curl -ski -d somedata -u '$account0@test@127.0.0.1%9443:$proxy_password' https://$remote_ip:$remote_proxy_port/test | cat; exit \${PIPESTATUS[0]}"
+    retvalshouldbe 0
+    contain "HTTP/1.0 200 OK"
+    contain 'Server: The Bastion'
+    contain 'X-Bastion-Instance: '
+    contain 'X-Bastion-ReqID: '
+    nocontain 'WWW-Authenticate: '
+    contain 'Content-Type: text/plain'
+    contain 'X-Bastion-Remote-IP: 127.0.0.1'
+    contain 'X-Bastion-Request-Length: 8'
+    contain 'X-Bastion-Auth-Mode: self/default'
+    contain 'X-Bastion-Local-Status: 200 OK'
+    contain "X-Bastion-Remote-Client-SSL-Cert-Subject: "
+    contain "X-Bastion-Remote-Client-SSL-Cipher: "
+    contain "X-Bastion-Remote-Client-SSL-Warning: Peer certificate not verified"
+    contain "X-Bastion-Remote-Status: 200"
+    contain "X-Bastion-Remote-Server: Net::Server::HTTP/"
+    contain "X-Bastion-Egress-Timing: "
+    contain "Content-Length: 8"
+    contain "somedata"
+}
+
+[ "${remote_proxy_port:-0}" != 0 ] && testsuite_proxy
+unset -f testsuite_proxy