netinvent/npbackup
1
1
Fork 0
mirror of https://github.com/netinvent/npbackup.git synced 2025-11-08 05:04:45 +08:00
Code Issues Projects Releases Packages Wiki Activity

Speed up ls operation by using command_runner monitor mode

Browse source
This commit is contained in:
deajan 2024-05-25 16:07:34 +02:00
parent 154011fca2
commit 5a442d71f3
2 changed files with 23 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
npbackup/restic_wrapper/__init__.py
View file

@ -7,8 +7,8 @@ __intname__ = "npbackup.restic_wrapper"
__author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2022-2024 NetInvent"
__license__ = "GPL-3.0-only"
__build__ = "2024042301"
__version__ = "2.0.3"
__build__ = "2024052501"
__version__ = "2.1.0"
from typing import Tuple, List, Optional, Callable, Union
@ -264,6 +264,7 @@ class ResticRunner:
cmd: str,
errors_allowed: bool = False,
no_output_queues: bool = False,
method: str = "poller",
timeout: int = None,
stdin: sys.stdin = None,
) -> Tuple[bool, str]:
@ -290,14 +291,16 @@ class ResticRunner:
split_streams=False,
encoding="utf-8",
stdin=stdin,
stdout=self.stdout if not no_output_queues else None,
stderr=self.stderr if not no_output_queues else None,
stdout=self.stdout if not no_output_queues and method == "poller" else None,
stderr=self.stderr if not no_output_queues and method == "poller" else None,
no_close_queues=True,
valid_exit_codes=errors_allowed,
stop_on=self.stop_on,
on_exit=self.on_exit,
method="poller",
live_output=self._live_output, # Only on CLI non json mode
method=method,
# Live output is only useful in CLI non json mode
# But must not be used with ls since restic may produce too much output
live_output=self._live_output if method != "monitor" else False,
check_interval=CHECK_INTERVAL,
priority=self._priority,
io_priority=self._priority,
@ -704,12 +707,14 @@ class ResticRunner:
Using --json here does not return actual json content, but lines with each file being a json...
Also, restic may return millions of lines, which command_runner will be slow to read in poller mode
so we need to setup monitor mode in this one
"""
kwargs = locals()
kwargs.pop("self")
cmd = "ls {}".format(snapshot)
result, output = self.executor(cmd)
result, output = self.executor(cmd, method="monitor")
if result:
msg = f"Successfuly listed snapshot {snapshot} content"
else:

18
npbackup/runner_interface.py
View file

@ -9,7 +9,7 @@ __site__ = "https://www.netperfect.fr/npbackup"
__description__ = "NetPerfect Backup Client"
__copyright__ = "Copyright (C) 2022-2024 NetInvent"
__license__ = "GPL-3.0-only"
__build__ = "2024010201"
__build__ = "2024052501"
import sys
@ -35,6 +35,7 @@ def serialize_datetime(obj):
def entrypoint(*args, **kwargs):
repo_config = kwargs.pop("repo_config")
json_output = kwargs.pop("json_output")
operation = kwargs.pop("operation")
backend_binary = kwargs.pop("backend_binary", None)
npbackup_runner = NPBackupRunner()
@ -46,20 +47,23 @@ def entrypoint(*args, **kwargs):
npbackup_runner.json_output = json_output
if backend_binary:
npbackup_runner.binary = backend_binary
result = npbackup_runner.__getattribute__(kwargs.pop("operation"))(
result = npbackup_runner.__getattribute__(operation)(
**kwargs.pop("op_args"), __no_threads=True
)
if not json_output:
if not isinstance(result, bool):
# We need to temprarily remove the stdout handler
# Since we already get live output from the runner
# Unless operation is "ls", because it's too slow for command_runner poller method that allows live_output
# But we still need to log the result to our logfile
for handler in logger.handlers:
if handler.stream == sys.stdout:
logger.removeHandler(handler)
break
if not operation == "ls":
for handler in logger.handlers:
if handler.stream == sys.stdout:
logger.removeHandler(handler)
break
logger.info(f"\n{result}")
logger.addHandler(handler)
if not operation == "ls":
logger.addHandler(handler)
if result:
logger.info(f"Operation finished with success")
else: