#! /usr/bin/env perl # vim: set filetype=perl ts=4 sw=4 sts=4 et: use strict; use warnings; use GnuPG; # pragma optional module use File::Temp; use File::Basename; use File::Find; use File::Path; use Getopt::Long; use Fcntl qw{ :flock }; use File::Basename; use lib dirname(__FILE__) . '/../../lib/perl'; use OVH::Bastion; use OVH::SimpleLog; my %config; my ($dryRun, $configTest, $forceRsync, $noDelete, $encryptOnly, $rsyncOnly, $verbose); local $| = 1; my $isoldversion = ($GnuPG::VERSION ge '0.18') ? 0 : 1; sub test_config { # normalize / define defaults / quick checks $config{'trace'} = $config{'trace'} ? 1 : 0; if (not exists $config{'recipients'}) { _err "config error: recipients must be defined"; return 1; } if (ref $config{'recipients'} ne 'ARRAY') { _err "config error: recipients must be an array of array of GPG key IDs! (layer 1)"; return 1; } if (my @intruders = grep { ref $config{'recipients'}[$_] ne 'ARRAY' } 0 .. $#{$config{'recipients'}}) { local $" = ', '; _err "config error: recipients must be an array of array of GPG key IDs! (layer 2, indexes @intruders)"; return 1; } if ($config{'encrypt_and_move_delay_days'} !~ /^\d+$/) { _err "config error: encrypt_and_move_delay_days is not a positive integer!"; return 1; } if ($config{'rsync_delay_before_remove_days'} !~ /^\d+$/) { _err "config error: rsync_delay_before_remove_days is not a positive integer!"; return 1; } # ok, check if my gpg conf is good my $input = File::Temp->new(UNLINK => 1, TMPDIR => 1); print {$input} time(); close($input); _log "Testing signature with key $config{signing_key}... "; eval { my $gpgtest = GnuPG->new(trace => $config{'trace'}); my $outfile = File::Temp->new(UNLINK => 1, TMPDIR => 1); # first, check we can sign $gpgtest->sign(plaintext => $input . "", output => $outfile . "", "local-user" => $config{signing_key}, passphrase => $config{signing_key_passphrase}); if (not -s $outfile) { die "Couldn't sign with the specified key $config{signing_key}, check your configuration"; } }; if ($@) { if ($@ =~ /BAD_PASSPHRASE/) { _err "Bad passphrase for signing key $config{signing_key}"; return 1; } elsif ($@ =~ /expected NEED_PASSPHRASE/) { _err "Signing key $config{signing_key} was not found"; return 1; } _err "When testing signing key: $@"; return 1; } my %recipients_uniq; foreach my $recipient_list (@{$config{'recipients'}}) { foreach my $recipient (@$recipient_list) { $recipients_uniq{$recipient}++; } } eval { foreach my $recipient (keys %recipients_uniq) { _log "Testing encryption for recipient $recipient... "; my $gpgtest = GnuPG->new(trace => $config{'trace'}); # then, check we can encrypt to each of the recipients my $outfile = File::Temp->new(UNLINK => 1, TMPDIR => 1); my $recipientparam = $isoldversion ? $recipient : [$recipient, $recipient]; $gpgtest->encrypt(plaintext => $input . "", output => $outfile . "", recipient => $recipientparam); if (not -s $outfile) { die "Couldn't encrypt for the specified recipient <$recipient>, check your configuration"; } } }; if ($@) { _err "When testing recipient key: $@"; return 1; } if ($isoldversion and keys %recipients_uniq > 1) { _err "You have an old version of the GnuPG module that doesn't support multiple recipients, sorry."; return 1; } _log "Testing encryption for all recipients + signature... "; eval { my $gpgtest = GnuPG->new(trace => $config{'trace'}); # then, encrypt to all the recipients, sign, and check the signature my $outfile = File::Temp->new(UNLINK => 1, TMPDIR => 1); my $recipientparam = $isoldversion ? (keys %recipients_uniq)[0] : [keys %recipients_uniq]; $gpgtest->encrypt( plaintext => $input . "", output => $outfile . "", recipient => $recipientparam, sign => 1, "local-user" => $config{signing_key}, passphrase => $config{signing_key_passphrase} ); if (not -s $outfile) { die "Couldn't encrypt and sign, check your configuration"; } }; if ($@) { _err "When testing encrypt+sign: $@"; return 1; } _log "Config test passed"; return 0; } sub encrypt_multi { my %params = @_; my $source_file = $params{'source_file'}; my $destination_directory = $params{'destination_directory'}; my $remove_source_on_success = $params{'remove_source_on_success'} || 0; my $outfile = $source_file; $outfile =~ s!^/home/!$destination_directory/!; my $outdir = File::Basename::dirname($outfile); if (!-e $outdir) { _log "Creating $outdir"; $dryRun or File::Path::mkpath(File::Basename::dirname($outfile), 0, oct(700)); } my $layers = scalar(@{$config{'recipients'}}); _log "Encrypting $source_file to $outfile" . ".gpg" x $layers; my $layer = 0; my $current_source_file = $source_file; my $current_destination_file = $outfile . '.gpg'; my $success = 1; foreach my $recipients_array (@{$config{'recipients'}}) { $layer++; _log " ... encrypting $current_source_file to $current_destination_file" if $verbose; my $error = encrypt_once( source_file => $current_source_file, destination_file => $current_destination_file, recipients => ($isoldversion ? $recipients_array->[0] : $recipients_array) ); if ($layer > 1 and $layer <= $layers) { # transient file _log " ... deleting transient file $current_source_file" if $verbose; $dryRun or unlink $current_source_file; } if ($error) { $success = 0; last; } $current_source_file = $current_destination_file; $current_destination_file .= '.gpg'; } if ($success and $remove_source_on_success) { _log " ... removing source file $source_file" if $verbose; $dryRun or unlink $source_file; } return !$success; } sub encrypt_once { my %params = @_; my $source_file = $params{'source_file'}; my $destination_file = $params{'destination_file'}; my $recipients = $params{'recipients'}; if (not -f $source_file and not $dryRun) { _err "encrypt_once: source file $source_file is not a file!"; return 1; } # don't care ... overwrite # TODO check if GnuPG overwrites silently or dies #if (-f $destination_file) #{ # _err "encrypt_once: destination file $destination_file already exists!"; # return 1; #} my $GPG = GnuPG->new(trace => $config{'trace'}); eval { $dryRun or $GPG->encrypt( plaintext => $source_file, output => $destination_file, recipient => $recipients, sign => 1, "local-user" => $config{signing_key}, passphrase => $config{signing_key_passphrase} ); }; if ($@) { _err "encrypt_once: when working on $source_file => $destination_file, got error $@"; return 1; } return 0; # no error } my $openedFiles = undef; sub potentially_work_on_this_file { # file must be a ttyrec file or an osh_http_proxy_ttyrec-ish file my $filetype; $filetype = 'ttyrec' if m{^/home/[^/]+/ttyrec/[^/]+/[^/]+(\.ttyrec(\.zst)?)?$}; $filetype = 'proxylog' if m{^/home/[^/]+/ttyrec/[^/]+/\d+-\d+-\d+\.txt$}; $filetype or return; # must exist and be a file -f or return; my $file = $_; # first, check if we populated $openedFiles as a hashref if (ref $openedFiles ne 'HASH') { $openedFiles = {}; if (open(my $fh_lsof, '-|', "lsof -a -n -c ttyrec -- /home/")) { while (<$fh_lsof>) { chomp; m{\s(/home/[^/]+/ttyrec/\S+)$} and $openedFiles->{$1} = 1; } close($fh_lsof); _log "Found " . (scalar keys %$openedFiles) . " opened ttyrec files we won't touch"; } else { _warn "Error trying to get the list of opened ttyrec files, we might rotate opened files!"; } } # still open ? don't touch if (exists $openedFiles->{$file}) { _log "File $file is still opened by ttyrec, skipping"; return; } # and must be older than encrypt_and_move_delay_days days my $mtime = (stat($file))[9]; if ($mtime > time() - 86400 * $config{'encrypt_and_move_delay_days'}) { _log "File $file is too recent, skipping" if $verbose; return; } # for proxylog, never touch a file that's < 86400 sec old (because we might still write to it) if ($filetype eq 'proxylog' and $mtime > time() - 86400) { _log "File $file is too recent (proxylog), skipping" if $verbose; return; } my $error = encrypt_multi(source_file => $file, destination_directory => $config{'encrypt_and_move_to_directory'}, remove_source_on_success => not $noDelete); if ($error) { _err "Got an error for $file, skipping!"; } return; } sub directory_filter { ## no critic (RequireArgUnpacking) # /home ? check the subdirs if ($File::Find::dir eq '/home') { my @out = (); foreach (@_) { if (-d "/home/$_/ttyrec") { #_log("DBG: filtering /home, $_ is OK"); push @out, $_ if -d "/home/$_/ttyrec"; } else { ; #_log("DBG: filtering /home, $_ is COMPLETELY OUT"); } } return @out; } if ($File::Find::dir =~ m{^/home/[^/]+($|/ttyrec)}) { #_log("DBG: yep ok $File::Find::dir"); return @_; } #_log("DBG: quickill $File::Find::dir"); return (); } sub main { _log "Starting..."; if ( not GetOptions( "dry-run" => \$dryRun, "config-test" => \$configTest, "no-delete" => \$noDelete, "encrypt-only" => \$encryptOnly, "rsync-only" => \$rsyncOnly, "force-rsync" => \$forceRsync, "verbose" => \$verbose, ) ) { _err "Error while parsing command-line options"; return 1; } # we can have CONFIGDIR/osh-encrypt-rsync.conf # but also CONFIGDIR/osh-encrypt-rsync.conf.d/* # later files override the previous ones, item by item my $fnret; my $lockfile; my @configfilelist; if (-f -r OVH::Bastion::main_configuration_directory() . "/osh-encrypt-rsync.conf") { push @configfilelist, OVH::Bastion::main_configuration_directory() . "/osh-encrypt-rsync.conf"; } if (-d -x OVH::Bastion::main_configuration_directory() . "/osh-encrypt-rsync.conf.d") { if (opendir(my $dh, OVH::Bastion::main_configuration_directory() . "/osh-encrypt-rsync.conf.d")) { my @subfiles = map { OVH::Bastion::main_configuration_directory() . "/osh-encrypt-rsync.conf.d/" . $_ } grep { /\.conf$/ } readdir($dh); closedir($dh); push @configfilelist, sort @subfiles; } } if (not @configfilelist) { _err "Error, no config file found!"; return 1; } foreach my $configfile (@configfilelist) { _log "Configuration: loading configfile $configfile..."; $fnret = OVH::Bastion::load_configuration_file( file => $configfile, secure => 1, ); if (not $fnret) { _err "Error while loading configuration from $configfile, aborting (" . $fnret->msg . ")"; return 1; } foreach my $key (keys %{$fnret->value}) { $config{$key} = $fnret->value->{$key}; } # we'll be using our own config file as a handy flock() backend $lockfile = $configfile if not defined $lockfile; } $verbose ||= $config{'verbose'}; # ensure no other copy of myself is already running # except if we are in rsync-only mode (concurrency is then not a problem) my $lockfh; if (not $rsyncOnly) { if (!open($lockfh, '<', $lockfile)) { # flock() needs a file handler _log "Couldn't open config file, aborting"; return 1; } if (!flock($lockfh, LOCK_EX | LOCK_NB)) { _log "Another instance is running, aborting this one!"; return 1; } } # ensure the various config files defined all the keywords we need foreach my $keyword ( qw{ signing_key signing_key_passphrase recipients encrypt_and_move_to_directory encrypt_and_move_delay_days rsync_destination rsync_delay_before_remove_days } ) { next if defined $config{$keyword}; _err "Missing mandatory configuration item '$keyword', aborting"; return 1; } OVH::SimpleLog::setLogFile($config{'logfile'}) if $config{'logfile'}; OVH::SimpleLog::setSyslog($config{'syslog_facility'}) if $config{'syslog_facility'}; if ($forceRsync) { config { 'rsync_delay_days' } = 0; } if (test_config() != 0) { _err "Config test failed, aborting"; return 1; } if ($configTest) { return 0; } if ($dryRun) { _log "Dry-run mode enabled, won't actually encrypt, move or delete files!"; } if (not $rsyncOnly) { _log "Looking for files in /home/ ..."; File::Find::find( { no_chdir => 1, preprocess => \&directory_filter, wanted => \&potentially_work_on_this_file }, "/home/", ); } if (not($encryptOnly || $config{'encrypt_only'}) and $config{'rsync_destination'}) { my @command; my $sysret; if (!-d $config{'encrypt_and_move_to_directory'} && $dryRun) { _log "DRYRUN: source directory doesn't exist, substituting with another one (namely the config directory which we know exists), just to try the rsync in dry-run mode"; $config{'encrypt_and_move_to_directory'} = '/etc/cron.d/'; } if (!-d $config{'encrypt_and_move_to_directory'}) { _log "Nothing to rsync as the rsync source dir doesn't exist"; } else { _log "Now rsyncing files to remote host ..."; @command = qw{ rsync --prune-empty-dirs --one-file-system -a }; push @command, '-v' if $verbose; if ($config{'rsync_rsh'}) { push @command, '--rsh', $config{'rsync_rsh'}; } if ($dryRun) { push @command, '--dry-run'; } push @command, $config{'encrypt_and_move_to_directory'} . '/'; push @command, $config{'rsync_destination'} . '/'; _log "Launching the following command: @command"; $sysret = system(@command); if ($sysret != 0) { _err "Error while rsyncing, stopping here"; return 1; } # now run rsync again BUT only with files having mtime +rsync_delay_before_remove_days AND specifying --remove-source-files # this way only files old enough AND successfully transferred to the other side will be removed if (!$dryRun) { my $prevdir = $ENV{'PWD'}; if (not chdir $config{'encrypt_and_move_to_directory'}) { _err "Error while trying to chdir to " . $config{'encrypt_and_move_to_directory'} . ", aborting"; return 1; } _log "Building a list of rsynced files to potentially delete (older than " . $config{'rsync_delay_before_remove_days'} . " days)"; my $cmdstr = "find . -xdev -type f -name '*.gpg' -mtime +" . ($config{'rsync_delay_before_remove_days'} - 1) . " -print0 | rsync -" . ($verbose ? 'v' : '') . "a "; if ($config{'rsync_rsh'}) { $cmdstr .= "--rsh '" . $config{'rsync_rsh'} . "' "; } if ($dryRun) { $cmdstr .= "--dry-run "; } $cmdstr .= "--remove-source-files --files-from=- --from0 " . $config{'encrypt_and_move_to_directory'} . '/' . " " . $config{'rsync_destination'} . '/'; _log "Launching the following command: $cmdstr"; $sysret = system($cmdstr); if ($sysret != 0) { _err "Error while rsyncing for deletion, stopping here"; return 1; } # remove empty directories _log "Removing now empty directories..."; system("find " . $config{'encrypt_and_move_to_directory'} . " -type d ! -wholename " . $config{'encrypt_and_move_to_directory'} . " -delete 2>/dev/null") ; # errors would be printed for non empty dirs, we don't care chdir $prevdir; } } } _log "Done"; return 0; } exit main();