diff --git a/npbackup/__main__.py b/npbackup/__main__.py index ac4efa5..d271f4a 100644 --- a/npbackup/__main__.py +++ b/npbackup/__main__.py @@ -191,6 +191,17 @@ This is free software, and you are welcome to redistribute it under certain cond action="store_true", help="Run in JSON API mode. Nothing else than JSON will be printed to stdout", ) + parser.add_argument( + "--stdin", + action="store_true", + help="Backup using data from stdin input" + ) + parser.add_argument( + "--stdin-filename", + type=str, + default=None, + help="Alternate filename for stdin, defaults to 'stdin.data'" + ) parser.add_argument( "-v", "--verbose", action="store_true", help="Show verbose output" ) @@ -289,7 +300,14 @@ This is free software, and you are welcome to redistribute it under certain cond "op_args": {}, } - if args.backup: + if args.stdin: + cli_args["operation"] = "backup" + cli_args["op_args"] = { + "force": True, + "read_from_stdin": True, + "stdin_filename": args.stdin_filename if args.stdin_filename else None + } + elif args.backup: cli_args["operation"] = "backup" cli_args["op_args"] = {"force": args.force} elif args.restore: diff --git a/npbackup/core/runner.py b/npbackup/core/runner.py index 216a2dd..b8e5a61 100644 --- a/npbackup/core/runner.py +++ b/npbackup/core/runner.py @@ -823,87 +823,88 @@ class NPBackupRunner: @is_ready @apply_config_to_restic_runner @catch_exceptions - def backup(self, force: bool = False) -> bool: + def backup(self, force: bool = False, read_from_stdin: bool = False, stdin_filename: str = "stdin.data") -> bool: """ Run backup after checking if no recent backup exists, unless force == True """ # Preflight checks - paths = self.repo_config.g("backup_opts.paths") - if not paths: - self.write_logs( - f"No paths to backup defined for repo {self.repo_config.g('name')}.", - level="error", - ) - return False - - # Make sure we convert paths to list if only one path is give - # Also make sure we remove trailing and ending spaces - try: - if not isinstance(paths, list): - paths = [paths] - paths = [path.strip() for path in paths] - for path in paths: - if path == self.repo_config.g("repo_uri"): - self.write_logs( - f"You cannot backup source into it's own path in repo {self.repo_config.g('name')}. No inception allowed !", - level="critical", - ) - return False - except KeyError: - self.write_logs( - f"No backup source given for repo {self.repo_config.g('name')}.", - level="error", - ) - return False - - source_type = self.repo_config.g("backup_opts.source_type") - - # MSWindows does not support one-file-system option - exclude_patterns = self.repo_config.g("backup_opts.exclude_patterns") - if not isinstance(exclude_patterns, list): - exclude_patterns = [exclude_patterns] - - exclude_files = self.repo_config.g("backup_opts.exclude_files") - if not isinstance(exclude_files, list): - exclude_files = [exclude_files] - - excludes_case_ignore = self.repo_config.g("backup_opts.excludes_case_ignore") - exclude_caches = self.repo_config.g("backup_opts.exclude_caches") - - exclude_files_larger_than = self.repo_config.g( - "backup_opts.exclude_files_larger_than" - ) - if exclude_files_larger_than: - if not exclude_files_larger_than[-1] in ( - "k", - "K", - "m", - "M", - "g", - "G", - "t", - "T", - ): + if not read_from_stdin: + paths = self.repo_config.g("backup_opts.paths") + if not paths: self.write_logs( - f"Bogus suffix for exclude_files_larger_than value given: {exclude_files_larger_than}", - level="warning", + f"No paths to backup defined for repo {self.repo_config.g('name')}.", + level="error", ) - exclude_files_larger_than = None + return False + + # Make sure we convert paths to list if only one path is give + # Also make sure we remove trailing and ending spaces try: - float(exclude_files_larger_than[:-1]) - except (ValueError, TypeError): + if not isinstance(paths, list): + paths = [paths] + paths = [path.strip() for path in paths] + for path in paths: + if path == self.repo_config.g("repo_uri"): + self.write_logs( + f"You cannot backup source into it's own path in repo {self.repo_config.g('name')}. No inception allowed !", + level="critical", + ) + return False + except KeyError: self.write_logs( - f"Cannot check whether excludes_files_larger_than is a float: {exclude_files_larger_than}", - level="warning", + f"No backup source given for repo {self.repo_config.g('name')}.", + level="error", ) - exclude_files_larger_than = None + return False - one_file_system = ( - self.repo_config.g("backup_opts.one_file_system") - if os.name != "nt" - else False - ) - use_fs_snapshot = self.repo_config.g("backup_opts.use_fs_snapshot") + source_type = self.repo_config.g("backup_opts.source_type") + + # MSWindows does not support one-file-system option + exclude_patterns = self.repo_config.g("backup_opts.exclude_patterns") + if not isinstance(exclude_patterns, list): + exclude_patterns = [exclude_patterns] + + exclude_files = self.repo_config.g("backup_opts.exclude_files") + if not isinstance(exclude_files, list): + exclude_files = [exclude_files] + + excludes_case_ignore = self.repo_config.g("backup_opts.excludes_case_ignore") + exclude_caches = self.repo_config.g("backup_opts.exclude_caches") + + exclude_files_larger_than = self.repo_config.g( + "backup_opts.exclude_files_larger_than" + ) + if exclude_files_larger_than: + if not exclude_files_larger_than[-1] in ( + "k", + "K", + "m", + "M", + "g", + "G", + "t", + "T", + ): + self.write_logs( + f"Bogus suffix for exclude_files_larger_than value given: {exclude_files_larger_than}", + level="warning", + ) + exclude_files_larger_than = None + try: + float(exclude_files_larger_than[:-1]) + except (ValueError, TypeError): + self.write_logs( + f"Cannot check whether excludes_files_larger_than is a float: {exclude_files_larger_than}", + level="warning", + ) + exclude_files_larger_than = None + + one_file_system = ( + self.repo_config.g("backup_opts.one_file_system") + if os.name != "nt" + else False + ) + use_fs_snapshot = self.repo_config.g("backup_opts.use_fs_snapshot") minimum_backup_size_error = self.repo_config.g( "backup_opts.minimum_backup_size_error" @@ -957,16 +958,19 @@ class NPBackupRunner: self.restic_runner.verbose = self.verbose # Run backup here - if source_type not in ["folder_list", None]: - self.write_logs( - f"Running backup of files in {paths} list to repo {self.repo_config.g('name')}", - level="info", - ) + if not read_from_stdin: + if source_type not in ["folder_list", None]: + self.write_logs( + f"Running backup of files in {paths} list to repo {self.repo_config.g('name')}", + level="info", + ) + else: + self.write_logs( + f"Running backup of {paths} to repo {self.repo_config.g('name')}", + level="info", + ) else: - self.write_logs( - f"Running backup of {paths} to repo {self.repo_config.g('name')}", - level="info", - ) + self.write_logs(f"Running backup of piped stdin data as name {stdin_filename} to repo {self.repo_config.g('name')}", level="info") pre_exec_commands_success = True if pre_exec_commands: @@ -989,19 +993,27 @@ class NPBackupRunner: ) self.restic_runner.dry_run = self.dry_run - result, result_string = self.restic_runner.backup( - paths=paths, - source_type=source_type, - exclude_patterns=exclude_patterns, - exclude_files=exclude_files, - excludes_case_ignore=excludes_case_ignore, - exclude_caches=exclude_caches, - exclude_files_larger_than=exclude_files_larger_than, - one_file_system=one_file_system, - use_fs_snapshot=use_fs_snapshot, - tags=tags, - additional_backup_only_parameters=additional_backup_only_parameters, - ) + if not read_from_stdin: + result, result_string = self.restic_runner.backup( + paths=paths, + source_type=source_type, + exclude_patterns=exclude_patterns, + exclude_files=exclude_files, + excludes_case_ignore=excludes_case_ignore, + exclude_caches=exclude_caches, + exclude_files_larger_than=exclude_files_larger_than, + one_file_system=one_file_system, + use_fs_snapshot=use_fs_snapshot, + tags=tags, + additional_backup_only_parameters=additional_backup_only_parameters, + ) + else: + result, result_string = self.restic_runner.backup( + read_from_stdin=read_from_stdin, + stdin_filename=stdin_filename, + tags=tags, + additional_backup_only_parameters=additional_backup_only_parameters + ) self.write_logs(f"Restic output:\n{result_string}", level="debug") # Extract backup size from result_string diff --git a/npbackup/restic_wrapper/__init__.py b/npbackup/restic_wrapper/__init__.py index 28702a2..1468cdd 100644 --- a/npbackup/restic_wrapper/__init__.py +++ b/npbackup/restic_wrapper/__init__.py @@ -253,6 +253,8 @@ class ResticRunner: errors_allowed: bool = False, no_output_queues: bool = False, timeout: int = None, + stdin: sys.stdin = None + ) -> Tuple[bool, str]: """ Executes restic with given command @@ -276,6 +278,7 @@ class ResticRunner: timeout=timeout, split_streams=False, encoding="utf-8", + stdin=stdin, stdout=self.stdout if not no_output_queues else None, stderr=self.stderr if not no_output_queues else None, no_close_queues=True, @@ -665,8 +668,8 @@ class ResticRunner: @check_if_init def backup( self, - paths: List[str], - source_type: str, + paths: List[str] = None, + source_type: str = None, exclude_patterns: List[str] = [], exclude_files: List[str] = [], excludes_case_ignore: bool = False, @@ -675,6 +678,8 @@ class ResticRunner: use_fs_snapshot: bool = False, tags: List[str] = [], one_file_system: bool = False, + read_from_stdin: bool = False, + stdin_filename: str = "stdin.data", additional_backup_only_parameters: str = None, ) -> Union[bool, str, dict]: """ @@ -683,78 +688,88 @@ class ResticRunner: kwargs = locals() kwargs.pop("self") - # Handle various source types - if source_type in [ - "files_from", - "files_from_verbatim", - "files_from_raw", - ]: - cmd = "backup" - if source_type == "files_from": - source_parameter = "--files-from" - elif source_type == "files_from_verbatim": - source_parameter = "--files-from-verbatim" - elif source_type == "files_from_raw": - source_parameter = "--files-from-raw" - else: - self.write_logs("Bogus source type given", level="error") - return False, "" - - for path in paths: - cmd += ' {} "{}"'.format(source_parameter, path) + if read_from_stdin: + cmd = "backup --stdin" + if stdin_filename: + cmd += f' --stdin-filename "{stdin_filename}"' else: - # make sure path is a list and does not have trailing slashes, unless we're backing up root - # We don't need to scan files for ETA, so let's add --no-scan - cmd = "backup --no-scan {}".format( - " ".join( - [ - '"{}"'.format(path.rstrip("/\\")) if path != "/" else path - for path in paths - ] + # Handle various source types + if source_type in [ + "files_from", + "files_from_verbatim", + "files_from_raw", + ]: + cmd = "backup" + if source_type == "files_from": + source_parameter = "--files-from" + elif source_type == "files_from_verbatim": + source_parameter = "--files-from-verbatim" + elif source_type == "files_from_raw": + source_parameter = "--files-from-raw" + else: + self.write_logs("Bogus source type given", level="error") + return False, "" + + for path in paths: + cmd += ' {} "{}"'.format(source_parameter, path) + else: + # make sure path is a list and does not have trailing slashes, unless we're backing up root + # We don't need to scan files for ETA, so let's add --no-scan + cmd = "backup --no-scan {}".format( + " ".join( + [ + '"{}"'.format(path.rstrip("/\\")) if path != "/" else path + for path in paths + ] + ) ) - ) - case_ignore_param = "" - # Always use case ignore excludes under windows - if os.name == "nt" or excludes_case_ignore: - case_ignore_param = "i" + case_ignore_param = "" + # Always use case ignore excludes under windows + if os.name == "nt" or excludes_case_ignore: + case_ignore_param = "i" - for exclude_pattern in exclude_patterns: - if exclude_pattern: - cmd += f' --{case_ignore_param}exclude "{exclude_pattern}"' - for exclude_file in exclude_files: - if exclude_file: - if os.path.isfile(exclude_file): - cmd += f' --{case_ignore_param}exclude-file "{exclude_file}"' + for exclude_pattern in exclude_patterns: + if exclude_pattern: + cmd += f' --{case_ignore_param}exclude "{exclude_pattern}"' + for exclude_file in exclude_files: + if exclude_file: + if os.path.isfile(exclude_file): + cmd += f' --{case_ignore_param}exclude-file "{exclude_file}"' + else: + self.write_logs( + f"Exclude file '{exclude_file}' not found", level="error" + ) + if exclude_caches: + cmd += " --exclude-caches" + if exclude_files_larger_than: + cmd += f" --exclude-files-larger-than {exclude_files_larger_than}" + if one_file_system: + cmd += " --one-file-system" + if use_fs_snapshot: + if os.name == "nt": + cmd += " --use-fs-snapshot" + self.write_logs("Using VSS snapshot to backup", level="info") else: self.write_logs( - f"Exclude file '{exclude_file}' not found", level="error" + "Parameter --use-fs-snapshot was given, which is only compatible with Windows", + level="warning", ) - if exclude_caches: - cmd += " --exclude-caches" - if exclude_files_larger_than: - cmd += f" --exclude-files-larger-than {exclude_files_larger_than}" - if one_file_system: - cmd += " --one-file-system" - if use_fs_snapshot: - if os.name == "nt": - cmd += " --use-fs-snapshot" - self.write_logs("Using VSS snapshot to backup", level="info") - else: - self.write_logs( - "Parameter --use-fs-snapshot was given, which is only compatible with Windows", - level="warning", - ) for tag in tags: if tag: tag = tag.strip() cmd += " --tag {}".format(tag) if additional_backup_only_parameters: cmd += " {}".format(additional_backup_only_parameters) - result, output = self.executor(cmd) + + # Run backup + if read_from_stdin: + result, output = self.executor(cmd, stdin=sys.stdin.buffer) + else: + result, output = self.executor(cmd) if ( - use_fs_snapshot + not read_from_stdin and use_fs_snapshot and not result and re.search("VSS Error", output, re.IGNORECASE) ): @@ -762,6 +777,7 @@ class ResticRunner: "VSS cannot be used. Backup will be done without VSS.", level="error" ) result, output = self.executor(cmd.replace(" --use-fs-snapshot", "")) + if self.json_output: return self.convert_to_json_output(result, output, **kwargs) if result: