From 5a442d71f3c43cf5b4e1a6a05f3c07a129513d2f Mon Sep 17 00:00:00 2001
From: deajan <ozy@netpower.fr>
Date: Sat, 25 May 2024 16:07:34 +0200
Subject: [PATCH] Speed up ls operation by using command_runner monitor mode

---
 npbackup/restic_wrapper/__init__.py | 19 ++++++++++++-------
 npbackup/runner_interface.py        | 18 +++++++++++-------
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/npbackup/restic_wrapper/__init__.py b/npbackup/restic_wrapper/__init__.py
index e9789e5..34d1c2c 100644
--- a/npbackup/restic_wrapper/__init__.py
+++ b/npbackup/restic_wrapper/__init__.py
@@ -7,8 +7,8 @@ __intname__ = "npbackup.restic_wrapper"
 __author__ = "Orsiris de Jong"
 __copyright__ = "Copyright (C) 2022-2024 NetInvent"
 __license__ = "GPL-3.0-only"
-__build__ = "2024042301"
-__version__ = "2.0.3"
+__build__ = "2024052501"
+__version__ = "2.1.0"
 
 
 from typing import Tuple, List, Optional, Callable, Union
@@ -264,6 +264,7 @@ class ResticRunner:
         cmd: str,
         errors_allowed: bool = False,
         no_output_queues: bool = False,
+        method: str = "poller",
         timeout: int = None,
         stdin: sys.stdin = None,
     ) -> Tuple[bool, str]:
@@ -290,14 +291,16 @@ class ResticRunner:
             split_streams=False,
             encoding="utf-8",
             stdin=stdin,
-            stdout=self.stdout if not no_output_queues else None,
-            stderr=self.stderr if not no_output_queues else None,
+            stdout=self.stdout if not no_output_queues and method == "poller" else None,
+            stderr=self.stderr if not no_output_queues and method == "poller" else None,
             no_close_queues=True,
             valid_exit_codes=errors_allowed,
             stop_on=self.stop_on,
             on_exit=self.on_exit,
-            method="poller",
-            live_output=self._live_output,  # Only on CLI non json mode
+            method=method,
+            # Live output is only useful in CLI non json mode
+            # But must not be used with ls since restic may produce too much output
+            live_output=self._live_output if method != "monitor" else False,
             check_interval=CHECK_INTERVAL,
             priority=self._priority,
             io_priority=self._priority,
@@ -704,12 +707,14 @@ class ResticRunner:
 
         Using --json here does not return actual json content, but lines with each file being a json...
 
+        Also, restic may return millions of lines, which command_runner will be slow to read in poller mode
+        so we need to setup monitor mode in this one
         """
         kwargs = locals()
         kwargs.pop("self")
 
         cmd = "ls {}".format(snapshot)
-        result, output = self.executor(cmd)
+        result, output = self.executor(cmd, method="monitor")
         if result:
             msg = f"Successfuly listed snapshot {snapshot} content"
         else:
diff --git a/npbackup/runner_interface.py b/npbackup/runner_interface.py
index 87941c0..d2fe843 100644
--- a/npbackup/runner_interface.py
+++ b/npbackup/runner_interface.py
@@ -9,7 +9,7 @@ __site__ = "https://www.netperfect.fr/npbackup"
 __description__ = "NetPerfect Backup Client"
 __copyright__ = "Copyright (C) 2022-2024 NetInvent"
 __license__ = "GPL-3.0-only"
-__build__ = "2024010201"
+__build__ = "2024052501"
 
 
 import sys
@@ -35,6 +35,7 @@ def serialize_datetime(obj):
 def entrypoint(*args, **kwargs):
     repo_config = kwargs.pop("repo_config")
     json_output = kwargs.pop("json_output")
+    operation = kwargs.pop("operation")
     backend_binary = kwargs.pop("backend_binary", None)
 
     npbackup_runner = NPBackupRunner()
@@ -46,20 +47,23 @@ def entrypoint(*args, **kwargs):
     npbackup_runner.json_output = json_output
     if backend_binary:
         npbackup_runner.binary = backend_binary
-    result = npbackup_runner.__getattribute__(kwargs.pop("operation"))(
+    result = npbackup_runner.__getattribute__(operation)(
         **kwargs.pop("op_args"), __no_threads=True
     )
     if not json_output:
         if not isinstance(result, bool):
             # We need to temprarily remove the stdout handler
             # Since we already get live output from the runner
+            # Unless operation is "ls", because it's too slow for command_runner poller method that allows live_output
             # But we still need to log the result to our logfile
-            for handler in logger.handlers:
-                if handler.stream == sys.stdout:
-                    logger.removeHandler(handler)
-                    break
+            if not operation == "ls":
+                for handler in logger.handlers:
+                    if handler.stream == sys.stdout:
+                        logger.removeHandler(handler)
+                        break
             logger.info(f"\n{result}")
-            logger.addHandler(handler)
+            if not operation == "ls":
+                logger.addHandler(handler)
         if result:
             logger.info(f"Operation finished with success")
         else: