netinvent/npbackup
1
1
Fork 0
mirror of https://github.com/netinvent/npbackup.git synced 2025-09-07 05:24:47 +08:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #101 from netinvent/main

Browse source 
Update tasks branch with current master fixes
This commit is contained in:
Orsiris de Jong 2024-10-12 20:14:09 +02:00 • committed by GitHub
commit 3615ec44ab
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 122 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG
View file

@ -50,6 +50,9 @@
- Added --no-cache option to disable cache for restic operations (neeeded on RO systems)
- Added CRC32 logging for config files in order to know when a file was modified
- Missing exclude files will now search in current binary directory for a excludes directory
- Splitted releases between legacy and non legacy
- Updated legacy tcl8.6.13 to tc8.6.15
- Updated legacy Python 3.7 to Python 3.11 (with openssl 3.1.3)

15
COMPATIBILITY.md Normal file
View file

@ -0,0 +1,15 @@
## Compatibility for various platforms
### Linux
We need Python 3.7 to compile on RHEL 7, which uses glibc 2.17
These builds will be "legacy" builds, 64 bit builds are sufficient.
### Windows
We need Python 3.7 to compile on Windows 7 / Server 2008 R2
These builds will be "legacy" builds. We will need 32 and 64 bit legacy builds.
Also, last restic version to run on Windows 7 is 0.16.2, see https://github.com/restic/restic/issues/4636 (basically go1.21 is not windows 7 compatible anymore)
So we actually need to compile restic ourselves with go1.20.12

15
npbackup/__debug__.py
View file

@ -8,11 +8,12 @@ __author__ = "Orsiris de Jong"
__site__ = "https://www.netperfect.fr/npbackup"
__description__ = "NetPerfect Backup Client"
__copyright__ = "Copyright (C) 2023-2024 NetInvent"
__build__ = "2024081901"
__build__ = "2024101201"
import sys
import os
import traceback
from typing import Callable
from functools import wraps
from logging import getLogger
@ -43,6 +44,18 @@ if not "_DEBUG" in globals():
_DEBUG = True
def exception_to_string(exc):
"""
Transform a catched exception to a string
https://stackoverflow.com/a/37135014/2635443
"""
stack = traceback.extract_stack()[:-3] + traceback.extract_tb(
exc.__traceback__
) # add limit=??
pretty = traceback.format_list(stack)
return "".join(pretty) + "\n {} {}".format(exc.__class__, exc)
def catch_exceptions(fn: Callable):
"""
Catch any exception and log it so we don't loose exceptions in thread

65
npbackup/core/runner.py
View file

@ -7,7 +7,7 @@ __intname__ = "npbackup.gui.core.runner"
__author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2022-2024 NetInvent"
__license__ = "GPL-3.0-only"
__build__ = "2024091501"
__build__ = "2024101201"
from typing import Optional, Callable, Union, List
@ -34,7 +34,7 @@ from npbackup.restic_wrapper import ResticRunner
from npbackup.core.restic_source_binary import get_restic_internal_binary
from npbackup.path_helper import CURRENT_DIR, BASEDIR
from npbackup.__version__ import __intname__ as NAME, __version__ as VERSION
from npbackup.__debug__ import _DEBUG
from npbackup.__debug__ import _DEBUG, exception_to_string
logger = logging.getLogger()
@ -229,6 +229,10 @@ class NPBackupRunner:
self.minimum_backup_age = None
self._exec_time = None
# Error /warning messages to add for json output
self.errors_for_json = []
self.warnings_for_json = []
@property
def repo_config(self) -> dict:
return self._repo_config
@ -368,9 +372,16 @@ class NPBackupRunner:
def exec_time(self, value: int):
self._exec_time = value
def write_logs(self, msg: str, level: str, raise_error: str = None):
def write_logs(
self,
msg: str,
level: str,
raise_error: str = None,
ignore_additional_json: bool = False,
):
"""
Write logs to log file and stdout / stderr queues if exist for GUI usage
Also collect errors and warnings for json output
"""
if level == "warning":
logger.warning(msg)
@ -392,6 +403,12 @@ class NPBackupRunner:
if self.stderr and level in ("critical", "error", "warning"):
self.stderr.put(f"\n{msg}")
if not ignore_additional_json:
if level in ("critical", "error"):
self.errors_for_json.append(msg)
if level == "warning":
self.warnings_for_json.append(msg)
if raise_error == "ValueError":
raise ValueError(msg)
if raise_error:
@ -649,8 +666,10 @@ class NPBackupRunner:
f"Runner: Function {operation} failed with: {exc}", level="error"
)
logger.error("Trace:", exc_info=True)
self.stdout.put(None)
self.stderr.put(None)
if self.stdout:
self.stdout.put(None)
if self.stderr:
self.stderr.put(None)
# In case of error, we really need to write metrics
# pylint: disable=E1101 (no-member)
metric_writer(self.repo_config, False, None, fn.__name__, self.dry_run)
@ -658,7 +677,7 @@ class NPBackupRunner:
js = {
"result": False,
"operation": operation,
"reason": f"Runner catched exception: {exc}",
"reason": f"Runner catched exception: {exception_to_string(exc)}",
}
return js
return False
@ -864,26 +883,30 @@ class NPBackupRunner:
def convert_to_json_output(
self,
result: bool,
result: Union[bool, dict],
output: str = None,
backend_js: dict = None,
warnings: str = None,
):
if self.json_output:
if backend_js:
js = backend_js
if isinstance(result, dict):
js = result
else:
js = {
"result": result,
"additional_error_info": [],
"additional_warning_info": [],
}
if warnings:
js["warnings"] = warnings
if result:
js["output"] = output
else:
js["reason"] = output
if result:
js["output"] = output
else:
js["reason"] = output
if self.errors_for_json:
js["additional_error_info"] += self.errors_for_json
if self.warnings_for_json:
js["additional_warning_info"] += self.warnings_for_json
if not js["additional_error_info"]:
js.pop("additional_error_info")
if not js["additional_warning_info"]:
js.pop("additional_warning_info")
return js
return result
@ -1052,9 +1075,6 @@ class NPBackupRunner:
"""
Run backup after checking if no recent backup exists, unless force == True
"""
# Possible warnings to add to json output
warnings = []
stdin_from_command = self.repo_config.g("backup_opts.stdin_from_command")
if not stdin_filename:
stdin_filename = self.repo_config.g("backup_opts.stdin_filename")
@ -1205,7 +1225,7 @@ class NPBackupRunner:
if pre_exec_failure_is_fatal:
return self.convert_to_json_output(False, msg)
else:
warnings.append(msg)
self.write_logs(msg, level="warning")
pre_exec_commands_success = False
else:
self.write_logs(
@ -1282,7 +1302,7 @@ class NPBackupRunner:
if post_exec_failure_is_fatal:
return self.convert_to_json_output(False, msg)
else:
warnings.append(msg)
self.write_logs(msg, level="warning")
else:
self.write_logs(
f"Post-execution of command {post_exec_command} succeeded with:\n{output}",
@ -1299,6 +1319,7 @@ class NPBackupRunner:
self.write_logs(
msg,
level="info" if operation_result else "error",
ignore_additional_json=True,
)
if not operation_result:
# patch result if json

61
npbackup/restic_wrapper/__init__.py
View file

@ -7,7 +7,7 @@ __intname__ = "npbackup.restic_wrapper"
__author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2022-2024 NetInvent"
__license__ = "GPL-3.0-only"
__build__ = "2024091501"
__build__ = "2024101201"
__version__ = "2.3.0"
@ -98,6 +98,10 @@ class ResticRunner:
# Internal value to check whether executor is running, accessed via self.executor_running property
self._executor_running = False
# Error /warning messages to add for json output
self.errors_for_json = []
self.warnings_for_json = []
def on_exit(self) -> bool:
self._executor_running = False
return self._executor_running
@ -258,7 +262,13 @@ class ResticRunner:
def executor_running(self) -> bool:
return self._executor_running
def write_logs(self, msg: str, level: str, raise_error: str = None):
def write_logs(
self,
msg: str,
level: str,
raise_error: str = None,
ignore_additional_json: bool = False,
):
"""
Write logs to log file and stdout / stderr queues if exist for GUI usage
"""
@ -284,6 +294,12 @@ class ResticRunner:
# pylint: disable=E1101 (no-member)
self.stderr.put(msg)
if not ignore_additional_json:
if level in ("critical", "error"):
self.errors_for_json.append(msg)
if level == "warning":
self.warnings_for_json.append(msg)
if raise_error == "ValueError":
raise ValueError(msg)
if raise_error:
@ -666,6 +682,7 @@ class ResticRunner:
result, output = command_runner results
msg will be logged and used as reason on failure
Converts restic --json output to parseable json
as of restic 0.16.2:
@ -676,9 +693,10 @@ class ResticRunner:
js = {
"result": result,
"operation": operation,
"extended_info": None,
"args": kwargs,
"output": [],
"additional_error_info": [],
"additional_warning_info": [],
}
if result:
if output:
@ -703,9 +721,13 @@ class ResticRunner:
js["output"].append(decoder.decode(line))
is_first_line = False
except msgspec.DecodeError as exc:
msg = f"JSON decode error: {exc} on content '{line}'"
self.write_logs(msg, level="error")
js["extended_info"] = msg
# We may have a json decode error, but actually, we just want to get the output
# in any case, since restic might output non json data, but we need to
# convert it to json
# msg = f"JSON decode error: {exc} on content '{line}'"
# self.write_logs(msg, level="error")
# js["extended_info"] = msg
js["output"].append({"data": line})
js["result"] = False
else:
@ -713,9 +735,11 @@ class ResticRunner:
# pylint: disable=E0601 (used-before-assignment)
js["output"].append(json.loads(line))
except json.JSONDecodeError as exc:
msg = f"JSON decode error: {exc} on content '{line}'"
self.write_logs(msg, level="error")
js["extended_info"] = msg
# Same as above
# msg = f"JSON decode error: {exc} on content '{line}'"
# self.write_logs(msg, level="error")
# js["extended_info"] = msg
js["output"].append({"data": line})
js["result"] = False
# If we only have one output, we don't need a list
@ -733,19 +757,26 @@ class ResticRunner:
try:
js["output"] = msgspec.json.decode(output)
except msgspec.DecodeError as exc:
msg = f"JSON decode error: {exc} on output '{output}'"
self.write_logs(msg, level="error")
js["extended_info"] = msg
# Save as above
# msg = f"JSON decode error: {exc} on output '{output}'"
# self.write_logs(msg, level="error")
# js["extended_info"] = msg
js["output"] = {"data": output}
else:
try:
# pylint: disable=E0601 (used-before-assignment)
js["output"] = json.loads(output)
except json.JSONDecodeError as exc:
msg = f"JSON decode error: {exc} on output '{output}'"
self.write_logs(msg, level="error")
js["extended_info"] = msg
# same as above
# msg = f"JSON decode error: {exc} on output '{output}'"
# self.write_logs(msg, level="error")
# js["extended_info"] = msg
js["output"] = {"data": output}
if self.errors_for_json:
js["additional_error_info"] += self.errors_for_json
if self.warnings_for_json:
js["additional_warning_info"] += self.warnings_for_json
return js
if result:

2
npbackup/runner_interface.py
View file

@ -80,7 +80,7 @@ def entrypoint(*args, **kwargs):
logger.error(f"Operation finished")
else:
if HAVE_MSGSPEC:
print(msgspec.json.encode(result))
print(msgspec.json.encode(result).decode("utf-8", errors="ignore"))
else:
print(json.dumps(result, default=serialize_datetime))
sys.exit(0)