mirror of
https://github.com/netinvent/npbackup.git
synced 2025-10-14 07:26:09 +08:00
WIP: Refactor runner into decorator oblivion
This commit is contained in:
Orsiris de Jong
parent
b3a03a8fbd
commit
9733e4e26b
6 changed files with 213 additions and 140 deletions
|
|
@ -428,7 +428,7 @@ def get_repo_config(
|
|||
_config_inheritance.s(key, False)
|
||||
|
||||
return _repo_config, _config_inheritance
|
||||
|
||||
|
||||
return _inherit_group_settings(_repo_config, _group_config, _config_inheritance)
|
||||
|
||||
try:
|
||||
|
|
@ -443,6 +443,7 @@ def get_repo_config(
|
|||
except KeyError:
|
||||
logger.warning(f"Repo {repo_name} has no group")
|
||||
else:
|
||||
repo_config.s("name", repo_name)
|
||||
repo_config, config_inheritance = inherit_group_settings(
|
||||
repo_config, group_config
|
||||
)
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import queue
|
|||
from datetime import datetime, timedelta
|
||||
from functools import wraps
|
||||
import queue
|
||||
from copy import deepcopy
|
||||
from command_runner import command_runner
|
||||
from ofunctions.threading import threaded
|
||||
from ofunctions.platform import os_arch
|
||||
|
|
@ -29,7 +30,6 @@ from time import sleep
|
|||
|
||||
logger = logging.getLogger()
|
||||
|
||||
|
||||
def metric_writer(
|
||||
repo_config: dict, restic_result: bool, result_string: str, dry_run: bool
|
||||
):
|
||||
|
|
@ -114,25 +114,35 @@ class NPBackupRunner:
|
|||
# NPF-SEC-00002: password commands, pre_exec and post_exec commands will be executed with npbackup privileges
|
||||
# This can lead to a problem when the config file can be written by users other than npbackup
|
||||
|
||||
def __init__(self, repo_config: Optional[dict] = None):
|
||||
def __init__(self):
|
||||
self._stdout = None
|
||||
self._stderr = None
|
||||
|
||||
self._is_ready = False
|
||||
if repo_config:
|
||||
self.repo_config = repo_config
|
||||
|
||||
self._dry_run = False
|
||||
self._verbose = False
|
||||
self._stdout = None
|
||||
self.restic_runner = None
|
||||
self.minimum_backup_age = None
|
||||
self._exec_time = None
|
||||
self._repo_config = None
|
||||
|
||||
# Create an instance of restic wrapper
|
||||
self.create_restic_runner()
|
||||
# Configure that instance
|
||||
self.apply_config_to_restic_runner()
|
||||
self._dry_run = False
|
||||
self._verbose = False
|
||||
self._stdout = None
|
||||
self.restic_runner = None
|
||||
self.minimum_backup_age = None
|
||||
self._exec_time = None
|
||||
|
||||
self._using_dev_binary = False
|
||||
|
||||
|
||||
@property
|
||||
def repo_config(self) -> dict:
|
||||
return self._repo_config
|
||||
|
||||
@repo_config.setter
|
||||
def repo_config(self, value: dict):
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"Bogus repo config given: {value}")
|
||||
self._repo_config = deepcopy(value)
|
||||
# Create an instance of restic wrapper
|
||||
self.create_restic_runner()
|
||||
|
||||
@property
|
||||
def backend_version(self) -> bool:
|
||||
|
|
@ -149,7 +159,6 @@ class NPBackupRunner:
|
|||
if not isinstance(value, bool):
|
||||
raise ValueError("Bogus dry_run parameter given: {}".format(value))
|
||||
self._dry_run = value
|
||||
self.apply_config_to_restic_runner()
|
||||
|
||||
@property
|
||||
def verbose(self):
|
||||
|
|
@ -160,7 +169,6 @@ class NPBackupRunner:
|
|||
if not isinstance(value, bool):
|
||||
raise ValueError("Bogus verbose parameter given: {}".format(value))
|
||||
self._verbose = value
|
||||
self.apply_config_to_restic_runner()
|
||||
|
||||
@property
|
||||
def stdout(self):
|
||||
|
|
@ -176,7 +184,6 @@ class NPBackupRunner:
|
|||
):
|
||||
raise ValueError("Bogus stdout parameter given: {}".format(value))
|
||||
self._stdout = value
|
||||
self.apply_config_to_restic_runner()
|
||||
|
||||
@property
|
||||
def stderr(self):
|
||||
|
|
@ -192,7 +199,6 @@ class NPBackupRunner:
|
|||
):
|
||||
raise ValueError("Bogus stdout parameter given: {}".format(value))
|
||||
self._stderr = value
|
||||
self.apply_config_to_restic_runner()
|
||||
|
||||
@property
|
||||
def has_binary(self) -> bool:
|
||||
|
|
@ -208,19 +214,28 @@ class NPBackupRunner:
|
|||
def exec_time(self, value: int):
|
||||
self._exec_time = value
|
||||
|
||||
def write_logs(self, msg: str, error: bool = False):
|
||||
def write_logs(self, msg: str, level: str = None):
|
||||
"""
|
||||
Write logs to log file and stdout / stderr queues if exist for GUI usage
|
||||
"""
|
||||
if error:
|
||||
if level == 'warning':
|
||||
logger.warning(msg)
|
||||
if self.stderr:
|
||||
self.stderr.put(msg)
|
||||
elif level == 'error':
|
||||
logger.error(msg)
|
||||
if self.stderr:
|
||||
self.stderr.put(msg)
|
||||
elif level == 'critical':
|
||||
logger.critical(msg)
|
||||
if self.stderr:
|
||||
self.stderr.put(msg)
|
||||
else:
|
||||
logger.info(msg)
|
||||
if self.stdout:
|
||||
self.stdout.put(msg)
|
||||
|
||||
|
||||
# pylint does not understand why this function does not take a self parameter
|
||||
# It's a decorator, and the inner function will have the self argument instead
|
||||
# pylint: disable=no-self-argument
|
||||
|
|
@ -235,7 +250,7 @@ class NPBackupRunner:
|
|||
# pylint: disable=E1102 (not-callable)
|
||||
result = fn(self, *args, **kwargs)
|
||||
self.exec_time = (datetime.utcnow() - start_time).total_seconds()
|
||||
logger.info(f"Runner took {self.exec_time} seconds for {fn.__name__}")
|
||||
self.write_logs(f"Runner took {self.exec_time} seconds for {fn.__name__}")
|
||||
return result
|
||||
|
||||
return wrapper
|
||||
|
|
@ -268,7 +283,7 @@ class NPBackupRunner:
|
|||
if not self._is_ready:
|
||||
self.write_logs(
|
||||
f"Runner cannot execute {fn.__name__}. Backend not ready",
|
||||
error=True,
|
||||
level="error",
|
||||
)
|
||||
return False
|
||||
return fn(self, *args, **kwargs)
|
||||
|
|
@ -290,6 +305,7 @@ class NPBackupRunner:
|
|||
"find": ["backup", "restore", "full"],
|
||||
"restore": ["restore", "full"],
|
||||
"check": ["restore", "full"],
|
||||
"repair": ["full"],
|
||||
"forget": ["full"],
|
||||
"prune": ["full"],
|
||||
"raw": ["full"],
|
||||
|
|
@ -301,11 +317,23 @@ class NPBackupRunner:
|
|||
f"Permissions required are {required_permissions[operation]}"
|
||||
)
|
||||
except (IndexError, KeyError):
|
||||
self.write_logs("You don't have sufficient permissions")
|
||||
self.write_logs("You don't have sufficient permissions", level="error")
|
||||
return False
|
||||
return fn(self, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
def apply_config_to_restic_runner(fn: Callable):
|
||||
"""
|
||||
Decorator to update backend before every run
|
||||
"""
|
||||
|
||||
@wraps(fn)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
if not self._apply_config_to_restic_runner():
|
||||
return False
|
||||
return fn(self, *args, **kwargs)
|
||||
return wrapper
|
||||
|
||||
def create_restic_runner(self) -> None:
|
||||
can_run = True
|
||||
|
|
@ -314,12 +342,12 @@ class NPBackupRunner:
|
|||
if not repository:
|
||||
raise KeyError
|
||||
except (KeyError, AttributeError):
|
||||
logger.error("Repo cannot be empty")
|
||||
self.write_logs("Repo cannot be empty", level="error")
|
||||
can_run = False
|
||||
try:
|
||||
password = self.repo_config.g("repo_opts.repo_password")
|
||||
except (KeyError, AttributeError):
|
||||
logger.error("Repo password cannot be empty")
|
||||
self.write_logs("Repo password cannot be empty", level="error")
|
||||
can_run = False
|
||||
if not password or password == "":
|
||||
try:
|
||||
|
|
@ -334,27 +362,25 @@ class NPBackupRunner:
|
|||
)
|
||||
cr_logger.setLevel(cr_loglevel)
|
||||
if exit_code != 0 or output == "":
|
||||
logger.error(
|
||||
"Password command failed to produce output:\n{}".format(
|
||||
output
|
||||
)
|
||||
self.write_logs(
|
||||
f"Password command failed to produce output:\n{output}", level="error"
|
||||
)
|
||||
can_run = False
|
||||
elif "\n" in output.strip():
|
||||
logger.error(
|
||||
"Password command returned multiline content instead of a string"
|
||||
self.write_logs(
|
||||
"Password command returned multiline content instead of a string", level="error"
|
||||
)
|
||||
can_run = False
|
||||
else:
|
||||
password = output
|
||||
else:
|
||||
logger.error(
|
||||
"No password nor password command given. Repo password cannot be empty"
|
||||
self.write_logs(
|
||||
"No password nor password command given. Repo password cannot be empty", level="error"
|
||||
)
|
||||
can_run = False
|
||||
except KeyError:
|
||||
logger.error(
|
||||
"No password nor password command given. Repo password cannot be empty"
|
||||
self.write_logs(
|
||||
"No password nor password command given. Repo password cannot be empty", level="error"
|
||||
)
|
||||
can_run = False
|
||||
self._is_ready = can_run
|
||||
|
|
@ -366,21 +392,20 @@ class NPBackupRunner:
|
|||
binary_search_paths=[BASEDIR, CURRENT_DIR],
|
||||
)
|
||||
|
||||
self.restic_runner.stdout = self.stdout
|
||||
self.restic_runner.stderr = self.stderr
|
||||
|
||||
if self.restic_runner.binary is None:
|
||||
# Let's try to load our internal binary for dev purposes
|
||||
arch = os_arch()
|
||||
binary = get_restic_internal_binary(arch)
|
||||
if binary:
|
||||
logger.info("Using dev binary !")
|
||||
if not self._using_dev_binary:
|
||||
self._using_dev_binary = True
|
||||
self.write_logs("Using dev binary !", level='warning')
|
||||
self.restic_runner.binary = binary
|
||||
|
||||
def apply_config_to_restic_runner(self) -> None:
|
||||
if not self._is_ready:
|
||||
self.write_logs("Runner settings cannot be applied to backend", error=True)
|
||||
return None
|
||||
def _apply_config_to_restic_runner(self) -> bool:
|
||||
if not isinstance(self.restic_runner, ResticRunner):
|
||||
self.write_logs("Backend not ready", level="error")
|
||||
return False
|
||||
try:
|
||||
if self.repo_config.g("repo_opts.upload_speed"):
|
||||
self.restic_runner.limit_upload = self.repo_config.g(
|
||||
|
|
@ -389,7 +414,7 @@ class NPBackupRunner:
|
|||
except KeyError:
|
||||
pass
|
||||
except ValueError:
|
||||
logger.error("Bogus upload limit given.")
|
||||
self.write_logs("Bogus upload limit given.", level="error")
|
||||
try:
|
||||
if self.repo_config.g("repo_opts.download_speed"):
|
||||
self.restic_runner.limit_download = self.repo_config.g(
|
||||
|
|
@ -398,7 +423,7 @@ class NPBackupRunner:
|
|||
except KeyError:
|
||||
pass
|
||||
except ValueError:
|
||||
logger.error("Bogus download limit given.")
|
||||
self.write_logs("Bogus download limit given.", level="error")
|
||||
try:
|
||||
if self.repo_config.g("repo_opts.backend_connections"):
|
||||
self.restic_runner.backend_connections = self.repo_config.g(
|
||||
|
|
@ -407,14 +432,14 @@ class NPBackupRunner:
|
|||
except KeyError:
|
||||
pass
|
||||
except ValueError:
|
||||
logger.error("Bogus backend connections value given.")
|
||||
self.write_logs("Bogus backend connections value given.", level="erorr")
|
||||
try:
|
||||
if self.repo_config.g("backup_opts.priority"):
|
||||
self.restic_runner.priority = self.repo_config.g("backup_opts.priority")
|
||||
except KeyError:
|
||||
pass
|
||||
except ValueError:
|
||||
logger.warning("Bogus backup priority in config file.")
|
||||
self.write_logs("Bogus backup priority in config file.", level="warning")
|
||||
try:
|
||||
if self.repo_config.g("backup_opts.ignore_cloud_files"):
|
||||
self.restic_runner.ignore_cloud_files = self.repo_config.g(
|
||||
|
|
@ -423,7 +448,7 @@ class NPBackupRunner:
|
|||
except KeyError:
|
||||
pass
|
||||
except ValueError:
|
||||
logger.warning("Bogus ignore_cloud_files value given")
|
||||
self.write_logs("Bogus ignore_cloud_files value given", level="warning")
|
||||
|
||||
try:
|
||||
if self.repo_config.g("backup_opts.additional_parameters"):
|
||||
|
|
@ -433,7 +458,7 @@ class NPBackupRunner:
|
|||
except KeyError:
|
||||
pass
|
||||
except ValueError:
|
||||
logger.warning("Bogus additional parameters given")
|
||||
self.write_logs("Bogus additional parameters given", level="warning")
|
||||
self.restic_runner.stdout = self.stdout
|
||||
|
||||
try:
|
||||
|
|
@ -462,19 +487,17 @@ class NPBackupRunner:
|
|||
value = os.path.expandvars(value)
|
||||
expanded_env_vars[key.strip()] = value.strip()
|
||||
except ValueError:
|
||||
logger.error(
|
||||
'Bogus environment variable "{}" defined in configuration.'.format(
|
||||
env_variable
|
||||
)
|
||||
self.write_logs(
|
||||
f'Bogus environment variable "{env_variable}" defined in configuration.', level="error"
|
||||
)
|
||||
except (KeyError, AttributeError, TypeError):
|
||||
logger.error("Bogus environment variables defined in configuration.")
|
||||
logger.debug("Trace:", exc_info=True)
|
||||
self.write_logs("Bogus environment variables defined in configuration.", level="error")
|
||||
logger.error("Trace:", exc_info=True)
|
||||
|
||||
try:
|
||||
self.restic_runner.environment_variables = expanded_env_vars
|
||||
except ValueError:
|
||||
logger.error("Cannot initialize additional environment variables")
|
||||
self.write_logs("Cannot initialize additional environment variables", level="error")
|
||||
|
||||
try:
|
||||
self.minimum_backup_age = int(
|
||||
|
|
@ -484,9 +507,10 @@ class NPBackupRunner:
|
|||
self.minimum_backup_age = 1440
|
||||
|
||||
self.restic_runner.verbose = self.verbose
|
||||
# TODO
|
||||
# self.restic_runner.stdout = self.stdout
|
||||
# self.restic_runner.stderr = self.stderr
|
||||
self.restic_runner.stdout = self.stdout
|
||||
self.restic_runner.stderr = self.stderr
|
||||
|
||||
return True
|
||||
|
||||
###########################
|
||||
# ACTUAL RUNNER FUNCTIONS #
|
||||
|
|
@ -495,38 +519,42 @@ class NPBackupRunner:
|
|||
@exec_timer
|
||||
@has_permission
|
||||
@is_ready
|
||||
@apply_config_to_restic_runner
|
||||
@close_queues
|
||||
def list(self) -> Optional[dict]:
|
||||
logger.info("Listing snapshots")
|
||||
self.write_logs(f"Listing snapshots of repo {self.repo_config.g('name')}", level="error")
|
||||
snapshots = self.restic_runner.snapshots()
|
||||
return snapshots
|
||||
|
||||
@exec_timer
|
||||
@has_permission
|
||||
@is_ready
|
||||
@apply_config_to_restic_runner
|
||||
@close_queues
|
||||
def find(self, path: str) -> bool:
|
||||
logger.info("Searching for path {}".format(path))
|
||||
self.write_logs(f"Searching for path {path} in repo {self.repo_config.g('name')}", level="error")
|
||||
result = self.restic_runner.find(path=path)
|
||||
if result:
|
||||
logger.info("Found path in:\n")
|
||||
self.write_logs("Found path in:\n")
|
||||
for line in result:
|
||||
logger.info(line)
|
||||
self.write_logs(line)
|
||||
return True
|
||||
return False
|
||||
|
||||
@exec_timer
|
||||
@has_permission
|
||||
@is_ready
|
||||
@apply_config_to_restic_runner
|
||||
@close_queues
|
||||
def ls(self, snapshot: str) -> Optional[dict]:
|
||||
logger.info("Showing content of snapshot {}".format(snapshot))
|
||||
self.write_logs(f"Showing content of snapshot {snapshot} in repo {self.repo_config.g('name')}")
|
||||
result = self.restic_runner.ls(snapshot)
|
||||
return result
|
||||
|
||||
@exec_timer
|
||||
@has_permission
|
||||
@is_ready
|
||||
@apply_config_to_restic_runner
|
||||
@close_queues
|
||||
def check_recent_backups(self) -> bool:
|
||||
"""
|
||||
|
|
@ -535,12 +563,10 @@ class NPBackupRunner:
|
|||
Returns None if no information is available
|
||||
"""
|
||||
if self.minimum_backup_age == 0:
|
||||
logger.info("No minimal backup age set. Set for backup")
|
||||
self.write_logs("No minimal backup age set. Set for backup")
|
||||
|
||||
logger.info(
|
||||
"Searching for a backup newer than {} ago".format(
|
||||
str(timedelta(minutes=self.minimum_backup_age))
|
||||
)
|
||||
self.write_logs(
|
||||
f"Searching for a backup newer than {str(timedelta(minutes=self.minimum_backup_age))} ago"
|
||||
)
|
||||
self.restic_runner.verbose = False
|
||||
result, backup_tz = self.restic_runner.has_snapshot_timedelta(
|
||||
|
|
@ -548,18 +574,19 @@ class NPBackupRunner:
|
|||
)
|
||||
self.restic_runner.verbose = self.verbose
|
||||
if result:
|
||||
logger.info("Most recent backup is from {}".format(backup_tz))
|
||||
self.write_logs(f"Most recent backup in repo {self.repo_config.g('name')} is from {backup_tz}")
|
||||
elif result is False and backup_tz == datetime(1, 1, 1, 0, 0):
|
||||
logger.info("No snapshots found in repo.")
|
||||
self.write_logs(f"No snapshots found in repo {self.repo_config.g('name')}.")
|
||||
elif result is False:
|
||||
logger.info("No recent backup found. Newest is from {}".format(backup_tz))
|
||||
self.write_logs(f"No recent backup found in repo {self.repo_config.g('name')}. Newest is from {backup_tz}")
|
||||
elif result is None:
|
||||
logger.error("Cannot connect to repository or repository empty.")
|
||||
self.write_logs("Cannot connect to repository or repository empty.", level="error")
|
||||
return result, backup_tz
|
||||
|
||||
@exec_timer
|
||||
@has_permission
|
||||
@is_ready
|
||||
@apply_config_to_restic_runner
|
||||
@close_queues
|
||||
def backup(self, force: bool = False) -> bool:
|
||||
"""
|
||||
|
|
@ -568,7 +595,7 @@ class NPBackupRunner:
|
|||
# Preflight checks
|
||||
paths = self.repo_config.g("backup_opts.paths")
|
||||
if not paths:
|
||||
logger.error("No backup paths defined.")
|
||||
self.write_logs(f"No paths to backup defined for repo {self.repo_config.g('name')}.", level="error")
|
||||
return False
|
||||
|
||||
# Make sure we convert paths to list if only one path is give
|
||||
|
|
@ -579,12 +606,12 @@ class NPBackupRunner:
|
|||
paths = [path.strip() for path in paths]
|
||||
for path in paths:
|
||||
if path == self.repo_config.g("repo_uri"):
|
||||
logger.critical(
|
||||
"You cannot backup source into it's own path. No inception allowed !"
|
||||
self.write_logs(
|
||||
f"You cannot backup source into it's own path in repo {self.repo_config.g('name')}. No inception allowed !", level='critical'
|
||||
)
|
||||
return False
|
||||
except KeyError:
|
||||
logger.error("No backup source given.")
|
||||
self.write_logs(f"No backup source given for repo {self.repo_config.g('name')}.", level='error')
|
||||
return False
|
||||
|
||||
exclude_patterns_source_type = self.repo_config.g(
|
||||
|
|
@ -643,18 +670,18 @@ class NPBackupRunner:
|
|||
self.restic_runner.verbose = False
|
||||
if not self.restic_runner.is_init:
|
||||
if not self.restic_runner.init():
|
||||
logger.error("Cannot continue.")
|
||||
self.write_logs(f"Cannot continue, repo {self.repo_config.g('name')} is not defined.", level="critical")
|
||||
return False
|
||||
if self.check_recent_backups() and not force:
|
||||
logger.info("No backup necessary.")
|
||||
self.write_logs("No backup necessary.")
|
||||
return True
|
||||
self.restic_runner.verbose = self.verbose
|
||||
|
||||
# Run backup here
|
||||
if exclude_patterns_source_type not in ["folder_list", None]:
|
||||
logger.info("Running backup of files in {} list".format(paths))
|
||||
self.write_logs(f"Running backup of files in {paths} list to repo {self.repo_config.g('name')}")
|
||||
else:
|
||||
logger.info("Running backup of {}".format(paths))
|
||||
self.write_logs(f"Running backup of {paths} to repo {self.repo_config.g('name')}")
|
||||
|
||||
if pre_exec_commands:
|
||||
for pre_exec_command in pre_exec_commands:
|
||||
|
|
@ -662,18 +689,14 @@ class NPBackupRunner:
|
|||
pre_exec_command, shell=True, timeout=pre_exec_per_command_timeout
|
||||
)
|
||||
if exit_code != 0:
|
||||
logger.error(
|
||||
"Pre-execution of command {} failed with:\n{}".format(
|
||||
pre_exec_command, output
|
||||
)
|
||||
self.write_logs(
|
||||
f"Pre-execution of command {pre_exec_command} failed with:\n{output}", level="error"
|
||||
)
|
||||
if pre_exec_failure_is_fatal:
|
||||
return False
|
||||
else:
|
||||
logger.info(
|
||||
"Pre-execution of command {} success with:\n{}.".format(
|
||||
pre_exec_command, output
|
||||
)
|
||||
self.write_logs(
|
||||
"Pre-execution of command {pre_exec_command} success with:\n{output}."
|
||||
)
|
||||
|
||||
self.restic_runner.dry_run = self.dry_run
|
||||
|
|
@ -689,7 +712,7 @@ class NPBackupRunner:
|
|||
tags=tags,
|
||||
additional_backup_only_parameters=additional_backup_only_parameters,
|
||||
)
|
||||
logger.debug("Restic output:\n{}".format(result_string))
|
||||
logger.debug(f"Restic output:\n{result_string}")
|
||||
metric_writer(
|
||||
self.repo_config, result, result_string, self.restic_runner.dry_run
|
||||
)
|
||||
|
|
@ -700,32 +723,27 @@ class NPBackupRunner:
|
|||
post_exec_command, shell=True, timeout=post_exec_per_command_timeout
|
||||
)
|
||||
if exit_code != 0:
|
||||
logger.error(
|
||||
"Post-execution of command {} failed with:\n{}".format(
|
||||
post_exec_command, output
|
||||
)
|
||||
self.write_logs(
|
||||
f"Post-execution of command {post_exec_command} failed with:\n{output}", level="error"
|
||||
)
|
||||
if post_exec_failure_is_fatal:
|
||||
return False
|
||||
else:
|
||||
logger.info(
|
||||
"Post-execution of command {} success with:\n{}.".format(
|
||||
post_exec_command, output
|
||||
)
|
||||
self.write_logs(
|
||||
"Post-execution of command {post_exec_command} success with:\n{output}.", level="error"
|
||||
)
|
||||
return result
|
||||
|
||||
@exec_timer
|
||||
@has_permission
|
||||
@is_ready
|
||||
@apply_config_to_restic_runner
|
||||
@close_queues
|
||||
def restore(self, snapshot: str, target: str, restore_includes: List[str]) -> bool:
|
||||
if not self.repo_config.g("permissions") in ["restore", "full"]:
|
||||
msg = "You don't have permissions to restore this repo"
|
||||
self.output_queue.put(msg)
|
||||
logger.critical(msg)
|
||||
self.write_logs(f"You don't have permissions to restore repo {self.repo_config.g('name')}", level="error")
|
||||
return False
|
||||
logger.info("Launching restore to {}".format(target))
|
||||
self.write_logs(f"Launching restore to {target}")
|
||||
result = self.restic_runner.restore(
|
||||
snapshot=snapshot,
|
||||
target=target,
|
||||
|
|
@ -736,18 +754,24 @@ class NPBackupRunner:
|
|||
@exec_timer
|
||||
@has_permission
|
||||
@is_ready
|
||||
@apply_config_to_restic_runner
|
||||
@close_queues
|
||||
def forget(self, snapshot: str) -> bool:
|
||||
logger.info("Forgetting snapshot {}".format(snapshot))
|
||||
self.write_logs(f"Forgetting snapshot {snapshot}")
|
||||
result = self.restic_runner.forget(snapshot)
|
||||
return result
|
||||
|
||||
@exec_timer
|
||||
@has_permission
|
||||
@is_ready
|
||||
@apply_config_to_restic_runner
|
||||
@threaded
|
||||
@close_queues
|
||||
def check(self, read_data: bool = True) -> bool:
|
||||
self.write_logs("Checking repository")
|
||||
if read_data:
|
||||
self.write_logs(f"Running full data check of repository {self.repo_config.g('name')}")
|
||||
else:
|
||||
self.write_logs(f"Running metadata consistency check of repository {self.repo_config.g('name')}")
|
||||
sleep(1)
|
||||
result = self.restic_runner.check(read_data)
|
||||
return result
|
||||
|
|
@ -755,48 +779,63 @@ class NPBackupRunner:
|
|||
@exec_timer
|
||||
@has_permission
|
||||
@is_ready
|
||||
@apply_config_to_restic_runner
|
||||
@close_queues
|
||||
def prune(self) -> bool:
|
||||
logger.info("Pruning snapshots")
|
||||
self.write_logs(f"Pruning snapshots for repo {self.repo_config.g('name')}")
|
||||
result = self.restic_runner.prune()
|
||||
return result
|
||||
|
||||
@exec_timer
|
||||
@has_permission
|
||||
@is_ready
|
||||
@apply_config_to_restic_runner
|
||||
@close_queues
|
||||
def repair(self, order: str) -> bool:
|
||||
logger.info("Repairing {} in repo".format(order))
|
||||
result = self.restic_runner.repair(order)
|
||||
def repair(self, subject: str) -> bool:
|
||||
self.write_logs(f"Repairing {subject} in repo {self.repo_config.g('name')}")
|
||||
result = self.restic_runner.repair(subject)
|
||||
return result
|
||||
|
||||
@exec_timer
|
||||
@has_permission
|
||||
@is_ready
|
||||
@apply_config_to_restic_runner
|
||||
@close_queues
|
||||
def raw(self, command: str) -> bool:
|
||||
logger.info("Running raw command: {}".format(command))
|
||||
self.write_logs(f"Running raw command: {command}")
|
||||
result = self.restic_runner.raw(command=command)
|
||||
return result
|
||||
|
||||
@exec_timer
|
||||
def group_runner(self, repo_list: list, operation: str, **kwargs) -> bool:
|
||||
def group_runner(self, repo_config_list: list, operation: str, **kwargs) -> bool:
|
||||
group_result = True
|
||||
|
||||
# Make sure we don't close the stdout/stderr queues when running multiple operations
|
||||
kwargs = {**kwargs, **{"close_queues": False}}
|
||||
# Also make sure we don't thread functions
|
||||
kwargs = {
|
||||
**kwargs,
|
||||
**{
|
||||
"close_queues": False,
|
||||
#"__no_threads": True,
|
||||
}
|
||||
}
|
||||
|
||||
for repo in repo_list:
|
||||
self.write_logs(f"Running {operation} for repo {repo}")
|
||||
for repo_name, repo_config in repo_config_list:
|
||||
self.write_logs(f"Running {operation} for repo {repo_name}")
|
||||
self.repo_config = repo_config
|
||||
result = self.__getattribute__(operation)(**kwargs)
|
||||
if result:
|
||||
self.write_logs(f"Finished {operation} for repo {repo}")
|
||||
self.write_logs(f"Finished {operation} for repo {repo_name}")
|
||||
else:
|
||||
self.write_logs(
|
||||
f"Operation {operation} failed for repo {repo}", error=True
|
||||
f"Operation {operation} failed for repo {repo_name}", level="error"
|
||||
)
|
||||
group_result = False
|
||||
self.write_logs("Finished execution group operations")
|
||||
sleep(2)
|
||||
self.close_queues(lambda *args, **kwargs: None, **kwargs)
|
||||
# Manually close the queues at the end
|
||||
if self.stdout:
|
||||
self.stdout.put(None)
|
||||
if self.stderr:
|
||||
self.stderr.put(None)
|
||||
#sleep(1) # TODO this is arbitrary to allow queues to be read entirely
|
||||
return group_result
|
||||
|
|
|
|||
|
|
@ -131,7 +131,8 @@ def _about_gui(version_string: str, full_config: dict) -> None:
|
|||
|
||||
@threaded
|
||||
def _get_gui_data(repo_config: dict) -> Future:
|
||||
runner = NPBackupRunner(repo_config=repo_config)
|
||||
runner = NPBackupRunner()
|
||||
runner.repo_config = repo_config
|
||||
snapshots = runner.list()
|
||||
current_state, backup_tz = runner.check_recent_backups()
|
||||
snapshot_list = []
|
||||
|
|
@ -173,9 +174,10 @@ def get_gui_data(repo_config: dict) -> Tuple[bool, List[str]]:
|
|||
sg.Popup(_t("main_gui.repository_not_configured"))
|
||||
return None, None
|
||||
try:
|
||||
runner = NPBackupRunner(repo_config=repo_config)
|
||||
except ValueError:
|
||||
sg.Popup(_t("config_gui.no_runner"))
|
||||
runner = NPBackupRunner()
|
||||
runner.repo_config = repo_config
|
||||
except ValueError as exc:
|
||||
sg.Popup(f'{_t("config_gui.no_runner")}: {exc}')
|
||||
return None, None
|
||||
if not runner._is_ready:
|
||||
sg.Popup(_t("config_gui.runner_not_configured"))
|
||||
|
|
@ -283,14 +285,16 @@ def _make_treedata_from_json(ls_result: List[dict]) -> sg.TreeData:
|
|||
|
||||
@threaded
|
||||
def _forget_snapshot(repo_config: dict, snapshot_id: str) -> Future:
|
||||
runner = NPBackupRunner(repo_config=repo_config)
|
||||
runner = NPBackupRunner()
|
||||
runner.repo_config = repo_config
|
||||
result = runner.forget(snapshot=snapshot_id)
|
||||
return result
|
||||
|
||||
|
||||
@threaded
|
||||
def _ls_window(repo_config: dict, snapshot_id: str) -> Future:
|
||||
runner = NPBackupRunner(repo_config=repo_config)
|
||||
runner = NPBackupRunner()
|
||||
runner.repo_config = repo_config
|
||||
result = runner.ls(snapshot=snapshot_id)
|
||||
if not result:
|
||||
return result, None
|
||||
|
|
@ -464,7 +468,8 @@ def ls_window(config: dict, snapshot_id: str) -> bool:
|
|||
def _restore_window(
|
||||
repo_config: dict, snapshot: str, target: str, restore_includes: Optional[List]
|
||||
) -> Future:
|
||||
runner = NPBackupRunner(repo_config=repo_config)
|
||||
runner = NPBackupRunner()
|
||||
runner.repo_config = repo_config
|
||||
runner.verbose = True
|
||||
result = runner.restore(snapshot, target, restore_includes)
|
||||
THREAD_SHARED_DICT["exec_time"] = runner.exec_time
|
||||
|
|
@ -534,7 +539,8 @@ def restore_window(
|
|||
|
||||
@threaded
|
||||
def _gui_backup(repo_config, stdout, stderr) -> Future:
|
||||
runner = NPBackupRunner(repo_config=repo_config)
|
||||
runner = NPBackupRunner()
|
||||
runner.rep_config = repo_config
|
||||
runner.verbose = (
|
||||
True # We must use verbose so we get progress output from ResticRunner
|
||||
)
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ def gui_update_state(window, full_config: dict) -> list:
|
|||
or repo_config.g(f"repo_opts.repo_password_command")
|
||||
):
|
||||
backend_type, repo_uri = get_anon_repo_uri(repo_config.g(f"repo_uri"))
|
||||
repo_list.append([backend_type, repo_uri])
|
||||
repo_list.append([repo_name, backend_type, repo_uri])
|
||||
else:
|
||||
logger.warning("Incomplete operations repo {}".format(repo_name))
|
||||
except KeyError:
|
||||
|
|
@ -63,7 +63,7 @@ def operations_gui(full_config: dict) -> dict:
|
|||
"""
|
||||
|
||||
# This is a stupid hack to make sure uri column is large enough
|
||||
headings = ["Backend", "URI "]
|
||||
headings = ["Name ", "Backend", "URI "]
|
||||
|
||||
layout = [
|
||||
[
|
||||
|
|
@ -101,6 +101,19 @@ def operations_gui(full_config: dict) -> dict:
|
|||
_t("operations_gui.full_check"), key="--FULL-CHECK--"
|
||||
),
|
||||
],
|
||||
[
|
||||
sg.Button(
|
||||
_t("operations_gui.repair_index"), key="--REPAIR-INDEX--"
|
||||
),
|
||||
sg.Button(
|
||||
_t("operations_gui.repair_snapshots"), key="--REPAIR-SNAPSHOTS--"
|
||||
),
|
||||
],
|
||||
[
|
||||
sg.Button(
|
||||
_t("operations.gui.unlock"), key="--UNLOCK--"
|
||||
)
|
||||
],
|
||||
[
|
||||
sg.Button(
|
||||
_t("operations_gui.forget_using_retention_policy"),
|
||||
|
|
@ -150,13 +163,15 @@ def operations_gui(full_config: dict) -> dict:
|
|||
|
||||
if event in (sg.WIN_CLOSED, sg.WIN_X_EVENT, "--EXIT--"):
|
||||
break
|
||||
if event in [
|
||||
if event in (
|
||||
"--FORGET--",
|
||||
"--QUICK-CHECK--",
|
||||
"--FULL-CHECK--",
|
||||
"--REPAIR-INDEX--",
|
||||
"--REPAIR-SNAPSHOTS--",
|
||||
"--STANDARD-PRUNE--",
|
||||
"--MAX-PRUNE--",
|
||||
]:
|
||||
):
|
||||
if not values["repo-list"]:
|
||||
result = sg.popup(
|
||||
_t("operations_gui.apply_to_all"),
|
||||
|
|
@ -171,7 +186,10 @@ def operations_gui(full_config: dict) -> dict:
|
|||
runner = NPBackupRunner()
|
||||
runner.stdout = stdout_queue
|
||||
runner.stderr = stderr_queue
|
||||
group_runner_repo_list = [repo_name for backend_type, repo_name in repos]
|
||||
repo_config_list = []
|
||||
for repo_name, backend_type, repo_uri in repos:
|
||||
repo_config, config_inheritance = configuration.get_repo_config(full_config, repo_name)
|
||||
repo_config_list.append((repo_name, repo_config))
|
||||
|
||||
if event == "--FORGET--":
|
||||
operation = "forget"
|
||||
|
|
@ -182,6 +200,12 @@ def operations_gui(full_config: dict) -> dict:
|
|||
if event == "--FULL-CHECK--":
|
||||
operation = "check"
|
||||
op_args = {"read_data": True}
|
||||
if event == "--REPAIR-INDEX--":
|
||||
operation = "repair"
|
||||
op_args = {"subject": "index"}
|
||||
if event == "--REPAIR-SNAPSHOTS--":
|
||||
operation = "repair"
|
||||
op_args = {"subject": "snapshots"}
|
||||
if event == "--STANDARD-PRUNE--":
|
||||
operation = "prune"
|
||||
op_args = {}
|
||||
|
|
@ -190,13 +214,10 @@ def operations_gui(full_config: dict) -> dict:
|
|||
op_args = {}
|
||||
|
||||
@threaded
|
||||
def _group_runner(group_runner_repo_list, operation, **op_args) -> Future:
|
||||
return runner.group_runner(group_runner_repo_list, operation, **op_args)
|
||||
def _group_runner(repo_config_list, operation, **op_args) -> Future:
|
||||
return runner.group_runner(repo_config_list, operation, **op_args)
|
||||
|
||||
thread = _group_runner(group_runner_repo_list, operation, **op_args)
|
||||
|
||||
read_stdout_queue = True
|
||||
read_sterr_queue = True
|
||||
thread = _group_runner(repo_config_list, operation, **op_args)
|
||||
|
||||
progress_layout = [
|
||||
[sg.Text(_t("operations_gui.last_message"))],
|
||||
|
|
@ -209,8 +230,10 @@ def operations_gui(full_config: dict) -> dict:
|
|||
progress_window = sg.Window("Operation status", progress_layout)
|
||||
event, values = progress_window.read(timeout=0.01)
|
||||
|
||||
# while read_stdout_queue or read_sterr_queue:
|
||||
while not thread.done() and not thread.cancelled():
|
||||
read_stdout_queue = True
|
||||
read_stderr_queue = True
|
||||
#while read_stdout_queue or read_stderr_queue: # TODO add queue read as while
|
||||
while (not thread.done() and not thread.cancelled()) or (read_stdout_queue or read_stderr_queue):
|
||||
# Read stdout queue
|
||||
try:
|
||||
stdout_data = stdout_queue.get(timeout=0.01)
|
||||
|
|
@ -231,7 +254,7 @@ def operations_gui(full_config: dict) -> dict:
|
|||
pass
|
||||
else:
|
||||
if stderr_data is None:
|
||||
read_sterr_queue = False
|
||||
read_stderr_queue = False
|
||||
else:
|
||||
progress_window["-OPERATIONS-PROGRESS-STDERR-"].Update(
|
||||
f"{progress_window['-OPERATIONS-PROGRESS-STDERR-'].get()}\n{stderr_data}"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ en:
|
|||
configured_repositories: Configured repositories
|
||||
quick_check: Quick check
|
||||
full_check: Full check
|
||||
repair_index: Repair index
|
||||
repair_snapshots: Repair snapshots
|
||||
forget_using_retention_policy: Forget using retention polic
|
||||
standard_prune: Normal prune operation
|
||||
max_prune: Prune with maximum efficiency
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ fr:
|
|||
configured_repositories: Dépots configurés
|
||||
quick_check: Vérification rapide
|
||||
full_check: Vérification complète
|
||||
repair_index: Réparer index
|
||||
repair_snapshots: Réparer les sauvegardes
|
||||
forget_using_retention_policy: Oublier en utilisant la stratégie de rétention
|
||||
standard_prune: Opération de purge normale
|
||||
max_prune: Opération de purge la plus efficace
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue