netinvent/npbackup
1
1
Fork 0
mirror of https://github.com/netinvent/npbackup.git synced 2025-11-11 14:42:37 +08:00
Code Issues Projects Releases Packages Wiki Activity

Speed up ls operation by using command_runner monitor mode

Browse source
This commit is contained in:
deajan 2024-05-25 16:07:34 +02:00
parent 154011fca2
commit 5a442d71f3
2 changed files with 23 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
npbackup/restic_wrapper/__init__.py
View file

@ -7,8 +7,8 @@ __intname__ = "npbackup.restic_wrapper"
__author__ = "Orsiris de Jong" __author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2022-2024 NetInvent" __copyright__ = "Copyright (C) 2022-2024 NetInvent"
__license__ = "GPL-3.0-only" __license__ = "GPL-3.0-only"
__build__ = "2024042301" __build__ = "2024052501"
__version__ = "2.0.3" __version__ = "2.1.0"
from typing import Tuple, List, Optional, Callable, Union from typing import Tuple, List, Optional, Callable, Union
@ -264,6 +264,7 @@ class ResticRunner:
cmd: str, cmd: str,
errors_allowed: bool = False, errors_allowed: bool = False,
no_output_queues: bool = False, no_output_queues: bool = False,
method: str = "poller",
timeout: int = None, timeout: int = None,
stdin: sys.stdin = None, stdin: sys.stdin = None,
) -> Tuple[bool, str]: ) -> Tuple[bool, str]:
@ -290,14 +291,16 @@ class ResticRunner:
split_streams=False, split_streams=False,
encoding="utf-8", encoding="utf-8",
stdin=stdin, stdin=stdin,
stdout=self.stdout if not no_output_queues else None, stdout=self.stdout if not no_output_queues and method == "poller" else None,
stderr=self.stderr if not no_output_queues else None, stderr=self.stderr if not no_output_queues and method == "poller" else None,
no_close_queues=True, no_close_queues=True,
valid_exit_codes=errors_allowed, valid_exit_codes=errors_allowed,
stop_on=self.stop_on, stop_on=self.stop_on,
on_exit=self.on_exit, on_exit=self.on_exit,
method="poller", method=method,
live_output=self._live_output, # Only on CLI non json mode # Live output is only useful in CLI non json mode
# But must not be used with ls since restic may produce too much output
live_output=self._live_output if method != "monitor" else False,
check_interval=CHECK_INTERVAL, check_interval=CHECK_INTERVAL,
priority=self._priority, priority=self._priority,
io_priority=self._priority, io_priority=self._priority,
@ -704,12 +707,14 @@ class ResticRunner:
Using --json here does not return actual json content, but lines with each file being a json... Using --json here does not return actual json content, but lines with each file being a json...
Also, restic may return millions of lines, which command_runner will be slow to read in poller mode
so we need to setup monitor mode in this one
""" """
kwargs = locals() kwargs = locals()
kwargs.pop("self") kwargs.pop("self")
cmd = "ls {}".format(snapshot) cmd = "ls {}".format(snapshot)
result, output = self.executor(cmd) result, output = self.executor(cmd, method="monitor")
if result: if result:
msg = f"Successfuly listed snapshot {snapshot} content" msg = f"Successfuly listed snapshot {snapshot} content"
else: else:

18
npbackup/runner_interface.py
View file

@ -9,7 +9,7 @@ __site__ = "https://www.netperfect.fr/npbackup"
__description__ = "NetPerfect Backup Client" __description__ = "NetPerfect Backup Client"
__copyright__ = "Copyright (C) 2022-2024 NetInvent" __copyright__ = "Copyright (C) 2022-2024 NetInvent"
__license__ = "GPL-3.0-only" __license__ = "GPL-3.0-only"
__build__ = "2024010201" __build__ = "2024052501"
import sys import sys
@ -35,6 +35,7 @@ def serialize_datetime(obj):
def entrypoint(*args, **kwargs): def entrypoint(*args, **kwargs):
repo_config = kwargs.pop("repo_config") repo_config = kwargs.pop("repo_config")
json_output = kwargs.pop("json_output") json_output = kwargs.pop("json_output")
operation = kwargs.pop("operation")
backend_binary = kwargs.pop("backend_binary", None) backend_binary = kwargs.pop("backend_binary", None)
npbackup_runner = NPBackupRunner() npbackup_runner = NPBackupRunner()
@ -46,20 +47,23 @@ def entrypoint(*args, **kwargs):
npbackup_runner.json_output = json_output npbackup_runner.json_output = json_output
if backend_binary: if backend_binary:
npbackup_runner.binary = backend_binary npbackup_runner.binary = backend_binary
result = npbackup_runner.__getattribute__(kwargs.pop("operation"))( result = npbackup_runner.__getattribute__(operation)(
**kwargs.pop("op_args"), __no_threads=True **kwargs.pop("op_args"), __no_threads=True
) )
if not json_output: if not json_output:
if not isinstance(result, bool): if not isinstance(result, bool):
# We need to temprarily remove the stdout handler # We need to temprarily remove the stdout handler
# Since we already get live output from the runner # Since we already get live output from the runner
# Unless operation is "ls", because it's too slow for command_runner poller method that allows live_output
# But we still need to log the result to our logfile # But we still need to log the result to our logfile
for handler in logger.handlers: if not operation == "ls":
if handler.stream == sys.stdout: for handler in logger.handlers:
logger.removeHandler(handler) if handler.stream == sys.stdout:
break logger.removeHandler(handler)
break
logger.info(f"\n{result}") logger.info(f"\n{result}")
logger.addHandler(handler) if not operation == "ls":
logger.addHandler(handler)
if result: if result:
logger.info(f"Operation finished with success") logger.info(f"Operation finished with success")
else: else: