From d79f2094da3875f8bb01b13ecbeeab114804e20f Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 30 Sep 2024 22:35:15 +0200 Subject: [PATCH 1/7] Update CHANGELOG --- CHANGELOG | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 026a943..0041e0f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -50,6 +50,9 @@ - Added --no-cache option to disable cache for restic operations (neeeded on RO systems) - Added CRC32 logging for config files in order to know when a file was modified - Missing exclude files will now search in current binary directory for a excludes directory + - Splitted releases between legacy and non legacy + - Updated legacy tcl8.6.13 to tc8.6.15 + - Updated legacy Python 3.7 to Python 3.11 (with openssl 3.1.3) From bac4aaf4d5c4edf7a8a597de064637ece3b16420 Mon Sep 17 00:00:00 2001 From: deajan Date: Thu, 10 Oct 2024 11:02:43 +0200 Subject: [PATCH 2/7] CLI: Don't show bytes json object on output --- npbackup/runner_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/npbackup/runner_interface.py b/npbackup/runner_interface.py index a3f44c4..b4a232e 100644 --- a/npbackup/runner_interface.py +++ b/npbackup/runner_interface.py @@ -80,7 +80,7 @@ def entrypoint(*args, **kwargs): logger.error(f"Operation finished") else: if HAVE_MSGSPEC: - print(msgspec.json.encode(result)) + print(msgspec.json.encode(result).decode('utf-8', errors='ignore')) else: print(json.dumps(result, default=serialize_datetime)) sys.exit(0) From 40cd5773562c2045be8c4920be79e43b2494198c Mon Sep 17 00:00:00 2001 From: deajan Date: Thu, 10 Oct 2024 11:17:39 +0200 Subject: [PATCH 3/7] GUI: Flushing the queue requires us to check if queue exists --- npbackup/core/runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/npbackup/core/runner.py b/npbackup/core/runner.py index 164fe72..2e79522 100644 --- a/npbackup/core/runner.py +++ b/npbackup/core/runner.py @@ -649,8 +649,10 @@ class NPBackupRunner: f"Runner: Function {operation} failed with: {exc}", level="error" ) logger.error("Trace:", exc_info=True) - self.stdout.put(None) - self.stderr.put(None) + if self.stdout: + self.stdout.put(None) + if self.stderr: + self.stderr.put(None) # In case of error, we really need to write metrics # pylint: disable=E1101 (no-member) metric_writer(self.repo_config, False, None, fn.__name__, self.dry_run) From 81cd60969c7efb332601ca4c528e24f892c6ecea Mon Sep 17 00:00:00 2001 From: deajan Date: Sat, 12 Oct 2024 20:07:36 +0200 Subject: [PATCH 4/7] CLI: Allow errors msg propagation to --json output --- npbackup/__debug__.py | 10 +++++- npbackup/core/runner.py | 53 ++++++++++++++++----------- npbackup/restic_wrapper/__init__.py | 56 +++++++++++++++++++++-------- 3 files changed, 83 insertions(+), 36 deletions(-) diff --git a/npbackup/__debug__.py b/npbackup/__debug__.py index 652059a..ac7a2c5 100644 --- a/npbackup/__debug__.py +++ b/npbackup/__debug__.py @@ -8,11 +8,12 @@ __author__ = "Orsiris de Jong" __site__ = "https://www.netperfect.fr/npbackup" __description__ = "NetPerfect Backup Client" __copyright__ = "Copyright (C) 2023-2024 NetInvent" -__build__ = "2024081901" +__build__ = "2024101201" import sys import os +import traceback from typing import Callable from functools import wraps from logging import getLogger @@ -43,6 +44,13 @@ if not "_DEBUG" in globals(): _DEBUG = True +def exception_to_string(exc): + stack = traceback.extract_stack()[:-3] + traceback.extract_tb(exc.__traceback__) # add limit=?? + pretty = traceback.format_list(stack) + return ''.join(pretty) + '\n {} {}'.format(exc.__class__, exc) + + + def catch_exceptions(fn: Callable): """ Catch any exception and log it so we don't loose exceptions in thread diff --git a/npbackup/core/runner.py b/npbackup/core/runner.py index 2e79522..f54708f 100644 --- a/npbackup/core/runner.py +++ b/npbackup/core/runner.py @@ -7,7 +7,7 @@ __intname__ = "npbackup.gui.core.runner" __author__ = "Orsiris de Jong" __copyright__ = "Copyright (C) 2022-2024 NetInvent" __license__ = "GPL-3.0-only" -__build__ = "2024091501" +__build__ = "2024101201" from typing import Optional, Callable, Union, List @@ -34,7 +34,7 @@ from npbackup.restic_wrapper import ResticRunner from npbackup.core.restic_source_binary import get_restic_internal_binary from npbackup.path_helper import CURRENT_DIR, BASEDIR from npbackup.__version__ import __intname__ as NAME, __version__ as VERSION -from npbackup.__debug__ import _DEBUG +from npbackup.__debug__ import _DEBUG, exception_to_string logger = logging.getLogger() @@ -229,6 +229,10 @@ class NPBackupRunner: self.minimum_backup_age = None self._exec_time = None + # Error /warning messages to add for json output + self.errors_for_json = [] + self.warnings_for_json = [] + @property def repo_config(self) -> dict: return self._repo_config @@ -368,9 +372,10 @@ class NPBackupRunner: def exec_time(self, value: int): self._exec_time = value - def write_logs(self, msg: str, level: str, raise_error: str = None): + def write_logs(self, msg: str, level: str, raise_error: str = None, ignore_additional_json: bool = False): """ Write logs to log file and stdout / stderr queues if exist for GUI usage + Also collect errors and warnings for json output """ if level == "warning": logger.warning(msg) @@ -392,6 +397,12 @@ class NPBackupRunner: if self.stderr and level in ("critical", "error", "warning"): self.stderr.put(f"\n{msg}") + if not ignore_additional_json: + if level in ("critical", "error"): + self.errors_for_json.append(msg) + if level == "warning": + self.warnings_for_json.append(msg) + if raise_error == "ValueError": raise ValueError(msg) if raise_error: @@ -660,7 +671,7 @@ class NPBackupRunner: js = { "result": False, "operation": operation, - "reason": f"Runner catched exception: {exc}", + "reason": f"Runner catched exception: {exception_to_string(exc)}", } return js return False @@ -866,26 +877,30 @@ class NPBackupRunner: def convert_to_json_output( self, - result: bool, + result: Union[bool, dict], output: str = None, - backend_js: dict = None, - warnings: str = None, ): if self.json_output: - if backend_js: - js = backend_js if isinstance(result, dict): js = result else: js = { "result": result, + "additional_error_info": [], + "additional_warning_info": [], } - if warnings: - js["warnings"] = warnings - if result: - js["output"] = output - else: - js["reason"] = output + if result: + js["output"] = output + else: + js["reason"] = output + if self.errors_for_json: + js["additional_error_info"] += self.errors_for_json + if self.warnings_for_json: + js["additional_warning_info"] += self.warnings_for_json + if not js["additional_error_info"]: + js.pop("additional_error_info") + if not js["additional_warning_info"]: + js.pop("additional_warning_info") return js return result @@ -1054,9 +1069,6 @@ class NPBackupRunner: """ Run backup after checking if no recent backup exists, unless force == True """ - # Possible warnings to add to json output - warnings = [] - stdin_from_command = self.repo_config.g("backup_opts.stdin_from_command") if not stdin_filename: stdin_filename = self.repo_config.g("backup_opts.stdin_filename") @@ -1207,7 +1219,7 @@ class NPBackupRunner: if pre_exec_failure_is_fatal: return self.convert_to_json_output(False, msg) else: - warnings.append(msg) + self.write_logs(msg, level="warning") pre_exec_commands_success = False else: self.write_logs( @@ -1284,7 +1296,7 @@ class NPBackupRunner: if post_exec_failure_is_fatal: return self.convert_to_json_output(False, msg) else: - warnings.append(msg) + self.write_logs(msg, level="warning") else: self.write_logs( f"Post-execution of command {post_exec_command} succeeded with:\n{output}", @@ -1301,6 +1313,7 @@ class NPBackupRunner: self.write_logs( msg, level="info" if operation_result else "error", + ignore_additional_json=True, ) if not operation_result: # patch result if json diff --git a/npbackup/restic_wrapper/__init__.py b/npbackup/restic_wrapper/__init__.py index 40535d6..e31e191 100644 --- a/npbackup/restic_wrapper/__init__.py +++ b/npbackup/restic_wrapper/__init__.py @@ -7,7 +7,7 @@ __intname__ = "npbackup.restic_wrapper" __author__ = "Orsiris de Jong" __copyright__ = "Copyright (C) 2022-2024 NetInvent" __license__ = "GPL-3.0-only" -__build__ = "2024091501" +__build__ = "2024101201" __version__ = "2.3.0" @@ -98,6 +98,11 @@ class ResticRunner: # Internal value to check whether executor is running, accessed via self.executor_running property self._executor_running = False + # Error /warning messages to add for json output + self.errors_for_json = [] + self.warnings_for_json = [] + + def on_exit(self) -> bool: self._executor_running = False return self._executor_running @@ -258,7 +263,7 @@ class ResticRunner: def executor_running(self) -> bool: return self._executor_running - def write_logs(self, msg: str, level: str, raise_error: str = None): + def write_logs(self, msg: str, level: str, raise_error: str = None, ignore_additional_json: bool = False): """ Write logs to log file and stdout / stderr queues if exist for GUI usage """ @@ -284,6 +289,12 @@ class ResticRunner: # pylint: disable=E1101 (no-member) self.stderr.put(msg) + if not ignore_additional_json: + if level in ("critical", "error"): + self.errors_for_json.append(msg) + if level == "warning": + self.warnings_for_json.append(msg) + if raise_error == "ValueError": raise ValueError(msg) if raise_error: @@ -666,6 +677,7 @@ class ResticRunner: result, output = command_runner results msg will be logged and used as reason on failure + Converts restic --json output to parseable json as of restic 0.16.2: @@ -676,9 +688,10 @@ class ResticRunner: js = { "result": result, "operation": operation, - "extended_info": None, "args": kwargs, "output": [], + "additional_error_info": [], + "additional_warning_info": [], } if result: if output: @@ -703,9 +716,13 @@ class ResticRunner: js["output"].append(decoder.decode(line)) is_first_line = False except msgspec.DecodeError as exc: - msg = f"JSON decode error: {exc} on content '{line}'" - self.write_logs(msg, level="error") - js["extended_info"] = msg + # We may have a json decode error, but actually, we just want to get the output + # in any case, since restic might output non json data, but we need to + # convert it to json + + #msg = f"JSON decode error: {exc} on content '{line}'" + #self.write_logs(msg, level="error") + #js["extended_info"] = msg js["output"].append({"data": line}) js["result"] = False else: @@ -713,9 +730,11 @@ class ResticRunner: # pylint: disable=E0601 (used-before-assignment) js["output"].append(json.loads(line)) except json.JSONDecodeError as exc: - msg = f"JSON decode error: {exc} on content '{line}'" - self.write_logs(msg, level="error") - js["extended_info"] = msg + # Same as above + + #msg = f"JSON decode error: {exc} on content '{line}'" + #self.write_logs(msg, level="error") + #js["extended_info"] = msg js["output"].append({"data": line}) js["result"] = False # If we only have one output, we don't need a list @@ -733,19 +752,26 @@ class ResticRunner: try: js["output"] = msgspec.json.decode(output) except msgspec.DecodeError as exc: - msg = f"JSON decode error: {exc} on output '{output}'" - self.write_logs(msg, level="error") - js["extended_info"] = msg + # Save as above + + #msg = f"JSON decode error: {exc} on output '{output}'" + #self.write_logs(msg, level="error") + #js["extended_info"] = msg js["output"] = {"data": output} else: try: # pylint: disable=E0601 (used-before-assignment) js["output"] = json.loads(output) except json.JSONDecodeError as exc: - msg = f"JSON decode error: {exc} on output '{output}'" - self.write_logs(msg, level="error") - js["extended_info"] = msg + # same as above + #msg = f"JSON decode error: {exc} on output '{output}'" + #self.write_logs(msg, level="error") + #js["extended_info"] = msg js["output"] = {"data": output} + if self.errors_for_json: + js["additional_error_info"] += self.errors_for_json + if self.warnings_for_json: + js["additional_warning_info"] += self.warnings_for_json return js if result: From c2791e8d7fc2f31422c93cbdbb1905484983ddc6 Mon Sep 17 00:00:00 2001 From: deajan Date: Sat, 12 Oct 2024 20:08:56 +0200 Subject: [PATCH 5/7] Reformat files with black --- npbackup/__debug__.py | 7 +++--- npbackup/core/runner.py | 8 ++++++- npbackup/restic_wrapper/__init__.py | 35 ++++++++++++++++------------- npbackup/runner_interface.py | 2 +- 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/npbackup/__debug__.py b/npbackup/__debug__.py index ac7a2c5..d031c6f 100644 --- a/npbackup/__debug__.py +++ b/npbackup/__debug__.py @@ -45,10 +45,11 @@ if not "_DEBUG" in globals(): def exception_to_string(exc): - stack = traceback.extract_stack()[:-3] + traceback.extract_tb(exc.__traceback__) # add limit=?? + stack = traceback.extract_stack()[:-3] + traceback.extract_tb( + exc.__traceback__ + ) # add limit=?? pretty = traceback.format_list(stack) - return ''.join(pretty) + '\n {} {}'.format(exc.__class__, exc) - + return "".join(pretty) + "\n {} {}".format(exc.__class__, exc) def catch_exceptions(fn: Callable): diff --git a/npbackup/core/runner.py b/npbackup/core/runner.py index f54708f..8f76c11 100644 --- a/npbackup/core/runner.py +++ b/npbackup/core/runner.py @@ -372,7 +372,13 @@ class NPBackupRunner: def exec_time(self, value: int): self._exec_time = value - def write_logs(self, msg: str, level: str, raise_error: str = None, ignore_additional_json: bool = False): + def write_logs( + self, + msg: str, + level: str, + raise_error: str = None, + ignore_additional_json: bool = False, + ): """ Write logs to log file and stdout / stderr queues if exist for GUI usage Also collect errors and warnings for json output diff --git a/npbackup/restic_wrapper/__init__.py b/npbackup/restic_wrapper/__init__.py index e31e191..69517fb 100644 --- a/npbackup/restic_wrapper/__init__.py +++ b/npbackup/restic_wrapper/__init__.py @@ -102,7 +102,6 @@ class ResticRunner: self.errors_for_json = [] self.warnings_for_json = [] - def on_exit(self) -> bool: self._executor_running = False return self._executor_running @@ -263,7 +262,13 @@ class ResticRunner: def executor_running(self) -> bool: return self._executor_running - def write_logs(self, msg: str, level: str, raise_error: str = None, ignore_additional_json: bool = False): + def write_logs( + self, + msg: str, + level: str, + raise_error: str = None, + ignore_additional_json: bool = False, + ): """ Write logs to log file and stdout / stderr queues if exist for GUI usage """ @@ -720,9 +725,9 @@ class ResticRunner: # in any case, since restic might output non json data, but we need to # convert it to json - #msg = f"JSON decode error: {exc} on content '{line}'" - #self.write_logs(msg, level="error") - #js["extended_info"] = msg + # msg = f"JSON decode error: {exc} on content '{line}'" + # self.write_logs(msg, level="error") + # js["extended_info"] = msg js["output"].append({"data": line}) js["result"] = False else: @@ -732,9 +737,9 @@ class ResticRunner: except json.JSONDecodeError as exc: # Same as above - #msg = f"JSON decode error: {exc} on content '{line}'" - #self.write_logs(msg, level="error") - #js["extended_info"] = msg + # msg = f"JSON decode error: {exc} on content '{line}'" + # self.write_logs(msg, level="error") + # js["extended_info"] = msg js["output"].append({"data": line}) js["result"] = False # If we only have one output, we don't need a list @@ -754,19 +759,19 @@ class ResticRunner: except msgspec.DecodeError as exc: # Save as above - #msg = f"JSON decode error: {exc} on output '{output}'" - #self.write_logs(msg, level="error") - #js["extended_info"] = msg + # msg = f"JSON decode error: {exc} on output '{output}'" + # self.write_logs(msg, level="error") + # js["extended_info"] = msg js["output"] = {"data": output} else: try: # pylint: disable=E0601 (used-before-assignment) js["output"] = json.loads(output) except json.JSONDecodeError as exc: - # same as above - #msg = f"JSON decode error: {exc} on output '{output}'" - #self.write_logs(msg, level="error") - #js["extended_info"] = msg + # same as above + # msg = f"JSON decode error: {exc} on output '{output}'" + # self.write_logs(msg, level="error") + # js["extended_info"] = msg js["output"] = {"data": output} if self.errors_for_json: js["additional_error_info"] += self.errors_for_json diff --git a/npbackup/runner_interface.py b/npbackup/runner_interface.py index b4a232e..30551b6 100644 --- a/npbackup/runner_interface.py +++ b/npbackup/runner_interface.py @@ -80,7 +80,7 @@ def entrypoint(*args, **kwargs): logger.error(f"Operation finished") else: if HAVE_MSGSPEC: - print(msgspec.json.encode(result).decode('utf-8', errors='ignore')) + print(msgspec.json.encode(result).decode("utf-8", errors="ignore")) else: print(json.dumps(result, default=serialize_datetime)) sys.exit(0) From f9275d7caa6a0d11108a3c0090f56347161e6f41 Mon Sep 17 00:00:00 2001 From: deajan Date: Sat, 12 Oct 2024 20:09:46 +0200 Subject: [PATCH 6/7] Add note about compatibility --- COMPATIBILITY.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 COMPATIBILITY.md diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md new file mode 100644 index 0000000..7704825 --- /dev/null +++ b/COMPATIBILITY.md @@ -0,0 +1,15 @@ +## Compatibility for various platforms + +### Linux + +We need Python 3.7 to compile on RHEL 7, which uses glibc 2.17 +These builds will be "legacy" builds, 64 bit builds are sufficient. + +### Windows + +We need Python 3.7 to compile on Windows 7 / Server 2008 R2 +These builds will be "legacy" builds. We will need 32 and 64 bit legacy builds. + +Also, last restic version to run on Windows 7 is 0.16.2, see https://github.com/restic/restic/issues/4636 (basically go1.21 is not windows 7 compatible anymore) +So we actually need to compile restic ourselves with go1.20.12 + From 4a51513469a8d486b195b5c4eb783356342aef7c Mon Sep 17 00:00:00 2001 From: deajan Date: Sat, 12 Oct 2024 20:12:32 +0200 Subject: [PATCH 7/7] Cite sources ;) --- npbackup/__debug__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/npbackup/__debug__.py b/npbackup/__debug__.py index d031c6f..4d3878b 100644 --- a/npbackup/__debug__.py +++ b/npbackup/__debug__.py @@ -45,6 +45,10 @@ if not "_DEBUG" in globals(): def exception_to_string(exc): + """ + Transform a catched exception to a string + https://stackoverflow.com/a/37135014/2635443 + """ stack = traceback.extract_stack()[:-3] + traceback.extract_tb( exc.__traceback__ ) # add limit=??