mirror of
https://github.com/netinvent/npbackup.git
synced 2024-09-21 07:16:16 +08:00
Implemented --stdin backup source
This commit is contained in:
Orsiris de Jong
parent
9fed9e31a9
commit
9e9bda9807
|
|
@ -191,6 +191,17 @@ This is free software, and you are welcome to redistribute it under certain cond
|
|||
action="store_true",
|
||||
help="Run in JSON API mode. Nothing else than JSON will be printed to stdout",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stdin",
|
||||
action="store_true",
|
||||
help="Backup using data from stdin input"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stdin-filename",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Alternate filename for stdin, defaults to 'stdin.data'"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v", "--verbose", action="store_true", help="Show verbose output"
|
||||
)
|
||||
|
|
@ -289,7 +300,14 @@ This is free software, and you are welcome to redistribute it under certain cond
|
|||
"op_args": {},
|
||||
}
|
||||
|
||||
if args.backup:
|
||||
if args.stdin:
|
||||
cli_args["operation"] = "backup"
|
||||
cli_args["op_args"] = {
|
||||
"force": True,
|
||||
"read_from_stdin": True,
|
||||
"stdin_filename": args.stdin_filename if args.stdin_filename else None
|
||||
}
|
||||
elif args.backup:
|
||||
cli_args["operation"] = "backup"
|
||||
cli_args["op_args"] = {"force": args.force}
|
||||
elif args.restore:
|
||||
|
|
|
|||
|
|
@ -823,87 +823,88 @@ class NPBackupRunner:
|
|||
@is_ready
|
||||
@apply_config_to_restic_runner
|
||||
@catch_exceptions
|
||||
def backup(self, force: bool = False) -> bool:
|
||||
def backup(self, force: bool = False, read_from_stdin: bool = False, stdin_filename: str = "stdin.data") -> bool:
|
||||
"""
|
||||
Run backup after checking if no recent backup exists, unless force == True
|
||||
"""
|
||||
# Preflight checks
|
||||
paths = self.repo_config.g("backup_opts.paths")
|
||||
if not paths:
|
||||
self.write_logs(
|
||||
f"No paths to backup defined for repo {self.repo_config.g('name')}.",
|
||||
level="error",
|
||||
)
|
||||
return False
|
||||
|
||||
# Make sure we convert paths to list if only one path is give
|
||||
# Also make sure we remove trailing and ending spaces
|
||||
try:
|
||||
if not isinstance(paths, list):
|
||||
paths = [paths]
|
||||
paths = [path.strip() for path in paths]
|
||||
for path in paths:
|
||||
if path == self.repo_config.g("repo_uri"):
|
||||
self.write_logs(
|
||||
f"You cannot backup source into it's own path in repo {self.repo_config.g('name')}. No inception allowed !",
|
||||
level="critical",
|
||||
)
|
||||
return False
|
||||
except KeyError:
|
||||
self.write_logs(
|
||||
f"No backup source given for repo {self.repo_config.g('name')}.",
|
||||
level="error",
|
||||
)
|
||||
return False
|
||||
|
||||
source_type = self.repo_config.g("backup_opts.source_type")
|
||||
|
||||
# MSWindows does not support one-file-system option
|
||||
exclude_patterns = self.repo_config.g("backup_opts.exclude_patterns")
|
||||
if not isinstance(exclude_patterns, list):
|
||||
exclude_patterns = [exclude_patterns]
|
||||
|
||||
exclude_files = self.repo_config.g("backup_opts.exclude_files")
|
||||
if not isinstance(exclude_files, list):
|
||||
exclude_files = [exclude_files]
|
||||
|
||||
excludes_case_ignore = self.repo_config.g("backup_opts.excludes_case_ignore")
|
||||
exclude_caches = self.repo_config.g("backup_opts.exclude_caches")
|
||||
|
||||
exclude_files_larger_than = self.repo_config.g(
|
||||
"backup_opts.exclude_files_larger_than"
|
||||
)
|
||||
if exclude_files_larger_than:
|
||||
if not exclude_files_larger_than[-1] in (
|
||||
"k",
|
||||
"K",
|
||||
"m",
|
||||
"M",
|
||||
"g",
|
||||
"G",
|
||||
"t",
|
||||
"T",
|
||||
):
|
||||
if not read_from_stdin:
|
||||
paths = self.repo_config.g("backup_opts.paths")
|
||||
if not paths:
|
||||
self.write_logs(
|
||||
f"Bogus suffix for exclude_files_larger_than value given: {exclude_files_larger_than}",
|
||||
level="warning",
|
||||
f"No paths to backup defined for repo {self.repo_config.g('name')}.",
|
||||
level="error",
|
||||
)
|
||||
exclude_files_larger_than = None
|
||||
return False
|
||||
|
||||
# Make sure we convert paths to list if only one path is give
|
||||
# Also make sure we remove trailing and ending spaces
|
||||
try:
|
||||
float(exclude_files_larger_than[:-1])
|
||||
except (ValueError, TypeError):
|
||||
if not isinstance(paths, list):
|
||||
paths = [paths]
|
||||
paths = [path.strip() for path in paths]
|
||||
for path in paths:
|
||||
if path == self.repo_config.g("repo_uri"):
|
||||
self.write_logs(
|
||||
f"You cannot backup source into it's own path in repo {self.repo_config.g('name')}. No inception allowed !",
|
||||
level="critical",
|
||||
)
|
||||
return False
|
||||
except KeyError:
|
||||
self.write_logs(
|
||||
f"Cannot check whether excludes_files_larger_than is a float: {exclude_files_larger_than}",
|
||||
level="warning",
|
||||
f"No backup source given for repo {self.repo_config.g('name')}.",
|
||||
level="error",
|
||||
)
|
||||
exclude_files_larger_than = None
|
||||
return False
|
||||
|
||||
one_file_system = (
|
||||
self.repo_config.g("backup_opts.one_file_system")
|
||||
if os.name != "nt"
|
||||
else False
|
||||
)
|
||||
use_fs_snapshot = self.repo_config.g("backup_opts.use_fs_snapshot")
|
||||
source_type = self.repo_config.g("backup_opts.source_type")
|
||||
|
||||
# MSWindows does not support one-file-system option
|
||||
exclude_patterns = self.repo_config.g("backup_opts.exclude_patterns")
|
||||
if not isinstance(exclude_patterns, list):
|
||||
exclude_patterns = [exclude_patterns]
|
||||
|
||||
exclude_files = self.repo_config.g("backup_opts.exclude_files")
|
||||
if not isinstance(exclude_files, list):
|
||||
exclude_files = [exclude_files]
|
||||
|
||||
excludes_case_ignore = self.repo_config.g("backup_opts.excludes_case_ignore")
|
||||
exclude_caches = self.repo_config.g("backup_opts.exclude_caches")
|
||||
|
||||
exclude_files_larger_than = self.repo_config.g(
|
||||
"backup_opts.exclude_files_larger_than"
|
||||
)
|
||||
if exclude_files_larger_than:
|
||||
if not exclude_files_larger_than[-1] in (
|
||||
"k",
|
||||
"K",
|
||||
"m",
|
||||
"M",
|
||||
"g",
|
||||
"G",
|
||||
"t",
|
||||
"T",
|
||||
):
|
||||
self.write_logs(
|
||||
f"Bogus suffix for exclude_files_larger_than value given: {exclude_files_larger_than}",
|
||||
level="warning",
|
||||
)
|
||||
exclude_files_larger_than = None
|
||||
try:
|
||||
float(exclude_files_larger_than[:-1])
|
||||
except (ValueError, TypeError):
|
||||
self.write_logs(
|
||||
f"Cannot check whether excludes_files_larger_than is a float: {exclude_files_larger_than}",
|
||||
level="warning",
|
||||
)
|
||||
exclude_files_larger_than = None
|
||||
|
||||
one_file_system = (
|
||||
self.repo_config.g("backup_opts.one_file_system")
|
||||
if os.name != "nt"
|
||||
else False
|
||||
)
|
||||
use_fs_snapshot = self.repo_config.g("backup_opts.use_fs_snapshot")
|
||||
|
||||
minimum_backup_size_error = self.repo_config.g(
|
||||
"backup_opts.minimum_backup_size_error"
|
||||
|
|
@ -957,16 +958,19 @@ class NPBackupRunner:
|
|||
self.restic_runner.verbose = self.verbose
|
||||
|
||||
# Run backup here
|
||||
if source_type not in ["folder_list", None]:
|
||||
self.write_logs(
|
||||
f"Running backup of files in {paths} list to repo {self.repo_config.g('name')}",
|
||||
level="info",
|
||||
)
|
||||
if not read_from_stdin:
|
||||
if source_type not in ["folder_list", None]:
|
||||
self.write_logs(
|
||||
f"Running backup of files in {paths} list to repo {self.repo_config.g('name')}",
|
||||
level="info",
|
||||
)
|
||||
else:
|
||||
self.write_logs(
|
||||
f"Running backup of {paths} to repo {self.repo_config.g('name')}",
|
||||
level="info",
|
||||
)
|
||||
else:
|
||||
self.write_logs(
|
||||
f"Running backup of {paths} to repo {self.repo_config.g('name')}",
|
||||
level="info",
|
||||
)
|
||||
self.write_logs(f"Running backup of piped stdin data as name {stdin_filename} to repo {self.repo_config.g('name')}", level="info")
|
||||
|
||||
pre_exec_commands_success = True
|
||||
if pre_exec_commands:
|
||||
|
|
@ -989,19 +993,27 @@ class NPBackupRunner:
|
|||
)
|
||||
|
||||
self.restic_runner.dry_run = self.dry_run
|
||||
result, result_string = self.restic_runner.backup(
|
||||
paths=paths,
|
||||
source_type=source_type,
|
||||
exclude_patterns=exclude_patterns,
|
||||
exclude_files=exclude_files,
|
||||
excludes_case_ignore=excludes_case_ignore,
|
||||
exclude_caches=exclude_caches,
|
||||
exclude_files_larger_than=exclude_files_larger_than,
|
||||
one_file_system=one_file_system,
|
||||
use_fs_snapshot=use_fs_snapshot,
|
||||
tags=tags,
|
||||
additional_backup_only_parameters=additional_backup_only_parameters,
|
||||
)
|
||||
if not read_from_stdin:
|
||||
result, result_string = self.restic_runner.backup(
|
||||
paths=paths,
|
||||
source_type=source_type,
|
||||
exclude_patterns=exclude_patterns,
|
||||
exclude_files=exclude_files,
|
||||
excludes_case_ignore=excludes_case_ignore,
|
||||
exclude_caches=exclude_caches,
|
||||
exclude_files_larger_than=exclude_files_larger_than,
|
||||
one_file_system=one_file_system,
|
||||
use_fs_snapshot=use_fs_snapshot,
|
||||
tags=tags,
|
||||
additional_backup_only_parameters=additional_backup_only_parameters,
|
||||
)
|
||||
else:
|
||||
result, result_string = self.restic_runner.backup(
|
||||
read_from_stdin=read_from_stdin,
|
||||
stdin_filename=stdin_filename,
|
||||
tags=tags,
|
||||
additional_backup_only_parameters=additional_backup_only_parameters
|
||||
)
|
||||
self.write_logs(f"Restic output:\n{result_string}", level="debug")
|
||||
# Extract backup size from result_string
|
||||
|
||||
|
|
|
|||
|
|
@ -253,6 +253,8 @@ class ResticRunner:
|
|||
errors_allowed: bool = False,
|
||||
no_output_queues: bool = False,
|
||||
timeout: int = None,
|
||||
stdin: sys.stdin = None
|
||||
|
||||
) -> Tuple[bool, str]:
|
||||
"""
|
||||
Executes restic with given command
|
||||
|
|
@ -276,6 +278,7 @@ class ResticRunner:
|
|||
timeout=timeout,
|
||||
split_streams=False,
|
||||
encoding="utf-8",
|
||||
stdin=stdin,
|
||||
stdout=self.stdout if not no_output_queues else None,
|
||||
stderr=self.stderr if not no_output_queues else None,
|
||||
no_close_queues=True,
|
||||
|
|
@ -665,8 +668,8 @@ class ResticRunner:
|
|||
@check_if_init
|
||||
def backup(
|
||||
self,
|
||||
paths: List[str],
|
||||
source_type: str,
|
||||
paths: List[str] = None,
|
||||
source_type: str = None,
|
||||
exclude_patterns: List[str] = [],
|
||||
exclude_files: List[str] = [],
|
||||
excludes_case_ignore: bool = False,
|
||||
|
|
@ -675,6 +678,8 @@ class ResticRunner:
|
|||
use_fs_snapshot: bool = False,
|
||||
tags: List[str] = [],
|
||||
one_file_system: bool = False,
|
||||
read_from_stdin: bool = False,
|
||||
stdin_filename: str = "stdin.data",
|
||||
additional_backup_only_parameters: str = None,
|
||||
) -> Union[bool, str, dict]:
|
||||
"""
|
||||
|
|
@ -683,78 +688,88 @@ class ResticRunner:
|
|||
kwargs = locals()
|
||||
kwargs.pop("self")
|
||||
|
||||
# Handle various source types
|
||||
if source_type in [
|
||||
"files_from",
|
||||
"files_from_verbatim",
|
||||
"files_from_raw",
|
||||
]:
|
||||
cmd = "backup"
|
||||
if source_type == "files_from":
|
||||
source_parameter = "--files-from"
|
||||
elif source_type == "files_from_verbatim":
|
||||
source_parameter = "--files-from-verbatim"
|
||||
elif source_type == "files_from_raw":
|
||||
source_parameter = "--files-from-raw"
|
||||
else:
|
||||
self.write_logs("Bogus source type given", level="error")
|
||||
return False, ""
|
||||
|
||||
for path in paths:
|
||||
cmd += ' {} "{}"'.format(source_parameter, path)
|
||||
if read_from_stdin:
|
||||
cmd = "backup --stdin"
|
||||
if stdin_filename:
|
||||
cmd += f' --stdin-filename "{stdin_filename}"'
|
||||
else:
|
||||
# make sure path is a list and does not have trailing slashes, unless we're backing up root
|
||||
# We don't need to scan files for ETA, so let's add --no-scan
|
||||
cmd = "backup --no-scan {}".format(
|
||||
" ".join(
|
||||
[
|
||||
'"{}"'.format(path.rstrip("/\\")) if path != "/" else path
|
||||
for path in paths
|
||||
]
|
||||
# Handle various source types
|
||||
if source_type in [
|
||||
"files_from",
|
||||
"files_from_verbatim",
|
||||
"files_from_raw",
|
||||
]:
|
||||
cmd = "backup"
|
||||
if source_type == "files_from":
|
||||
source_parameter = "--files-from"
|
||||
elif source_type == "files_from_verbatim":
|
||||
source_parameter = "--files-from-verbatim"
|
||||
elif source_type == "files_from_raw":
|
||||
source_parameter = "--files-from-raw"
|
||||
else:
|
||||
self.write_logs("Bogus source type given", level="error")
|
||||
return False, ""
|
||||
|
||||
for path in paths:
|
||||
cmd += ' {} "{}"'.format(source_parameter, path)
|
||||
else:
|
||||
# make sure path is a list and does not have trailing slashes, unless we're backing up root
|
||||
# We don't need to scan files for ETA, so let's add --no-scan
|
||||
cmd = "backup --no-scan {}".format(
|
||||
" ".join(
|
||||
[
|
||||
'"{}"'.format(path.rstrip("/\\")) if path != "/" else path
|
||||
for path in paths
|
||||
]
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
case_ignore_param = ""
|
||||
# Always use case ignore excludes under windows
|
||||
if os.name == "nt" or excludes_case_ignore:
|
||||
case_ignore_param = "i"
|
||||
case_ignore_param = ""
|
||||
# Always use case ignore excludes under windows
|
||||
if os.name == "nt" or excludes_case_ignore:
|
||||
case_ignore_param = "i"
|
||||
|
||||
for exclude_pattern in exclude_patterns:
|
||||
if exclude_pattern:
|
||||
cmd += f' --{case_ignore_param}exclude "{exclude_pattern}"'
|
||||
for exclude_file in exclude_files:
|
||||
if exclude_file:
|
||||
if os.path.isfile(exclude_file):
|
||||
cmd += f' --{case_ignore_param}exclude-file "{exclude_file}"'
|
||||
for exclude_pattern in exclude_patterns:
|
||||
if exclude_pattern:
|
||||
cmd += f' --{case_ignore_param}exclude "{exclude_pattern}"'
|
||||
for exclude_file in exclude_files:
|
||||
if exclude_file:
|
||||
if os.path.isfile(exclude_file):
|
||||
cmd += f' --{case_ignore_param}exclude-file "{exclude_file}"'
|
||||
else:
|
||||
self.write_logs(
|
||||
f"Exclude file '{exclude_file}' not found", level="error"
|
||||
)
|
||||
if exclude_caches:
|
||||
cmd += " --exclude-caches"
|
||||
if exclude_files_larger_than:
|
||||
cmd += f" --exclude-files-larger-than {exclude_files_larger_than}"
|
||||
if one_file_system:
|
||||
cmd += " --one-file-system"
|
||||
if use_fs_snapshot:
|
||||
if os.name == "nt":
|
||||
cmd += " --use-fs-snapshot"
|
||||
self.write_logs("Using VSS snapshot to backup", level="info")
|
||||
else:
|
||||
self.write_logs(
|
||||
f"Exclude file '{exclude_file}' not found", level="error"
|
||||
"Parameter --use-fs-snapshot was given, which is only compatible with Windows",
|
||||
level="warning",
|
||||
)
|
||||
if exclude_caches:
|
||||
cmd += " --exclude-caches"
|
||||
if exclude_files_larger_than:
|
||||
cmd += f" --exclude-files-larger-than {exclude_files_larger_than}"
|
||||
if one_file_system:
|
||||
cmd += " --one-file-system"
|
||||
if use_fs_snapshot:
|
||||
if os.name == "nt":
|
||||
cmd += " --use-fs-snapshot"
|
||||
self.write_logs("Using VSS snapshot to backup", level="info")
|
||||
else:
|
||||
self.write_logs(
|
||||
"Parameter --use-fs-snapshot was given, which is only compatible with Windows",
|
||||
level="warning",
|
||||
)
|
||||
for tag in tags:
|
||||
if tag:
|
||||
tag = tag.strip()
|
||||
cmd += " --tag {}".format(tag)
|
||||
if additional_backup_only_parameters:
|
||||
cmd += " {}".format(additional_backup_only_parameters)
|
||||
result, output = self.executor(cmd)
|
||||
|
||||
# Run backup
|
||||
if read_from_stdin:
|
||||
result, output = self.executor(cmd, stdin=sys.stdin.buffer)
|
||||
else:
|
||||
result, output = self.executor(cmd)
|
||||
|
||||
if (
|
||||
use_fs_snapshot
|
||||
not read_from_stdin and use_fs_snapshot
|
||||
and not result
|
||||
and re.search("VSS Error", output, re.IGNORECASE)
|
||||
):
|
||||
|
|
@ -762,6 +777,7 @@ class ResticRunner:
|
|||
"VSS cannot be used. Backup will be done without VSS.", level="error"
|
||||
)
|
||||
result, output = self.executor(cmd.replace(" --use-fs-snapshot", ""))
|
||||
|
||||
if self.json_output:
|
||||
return self.convert_to_json_output(result, output, **kwargs)
|
||||
if result:
|
||||
|
|
|
|||
Loading…
Reference in a new issue