Allow to use list files as backup sources

bin/npbackup.conf.o

@@ -0,0 +1,85 @@
+# NPBackup config file for npbackup v2.2+
+# (C) 2022-2023 NetInvent
+
+backup:
+  compression: auto
+  exclude_caches: true
+  exclude_files:
+  - excludes/generic_excluded_extensions
+  - excludes/generic_excludes
+  - excludes/windows_excludes
+  exclude_case_ignore: false   # Exclusions will always have case ignored on Windows systems regarless of this setting
+  one_file_system: true
+  ## Paths can contain multiple values, one per line, without quotation marks
+  paths: c:\liste
+  use_fs_snapshot: false # Use VSS snapshot on Windows (needs administrator rights), will fallback to non VSS on failure
+  pre_exec_command: ''
+  pre_exec_timeout: 3600
+  pre_exec_failure_is_fatal: false
+  post_exec_command: ''
+  post_exec_timeout: 3600
+  post_exec_failure_is_fatal: false
+  tags: ''
+  additional_parameters: ''
+  priority: high
+
+  ignore_cloud_files: false
+  exclude_patterns: ''
+  source_type: files_from_verbatim
+repo:
+  repository: __NPBACKUP__sDOwE0QLAcM7/Tqh5Xu1GBvk+f90S58fKHHrWSdC79YxNjg1MjkwMDEyLjY2Mzc2NA8PDw8PDw8PDw8PDw8PDwUf5vEGXQzkoV0Q14OQtz9Tjz2ViQUItqtjott1zg7AWxUAPJi6jHdo2fn4f+Q8S9jENBZ2rM8bBtgAUMJylPqYpD2lS2tqLTs48ONKNd/DLq3nSIaAZvRKMAqpXSeeghRUFQC+QeIgEXpTle6GNs/RwdU=__NPBACKUP__
+  password: __NPBACKUP__IXvTGdPjhjckQ8UT99vFJaUZS2+dXZOpRjgkB11a3GgxNjg1MjkwMDEyLjY2Mzc2NA8PDw8PDw8PDw8PDw8PD40S3Im+KAIfOVueVCT1ENoOJov2Bn6xIPLs1B4uIBI9YzYfYttJ9DRynvk=__NPBACKUP__
+  # Backup age, in minutes, which is the minimum time between two backups
+  minimum_backup_age: 1440
+  upload_speed: 0            # in KiB, use 0 for unlimited upload speed
+  download_speed: 0    # in KiB, use 0 for unlimited download speed
+  backend_connections: 0 # Fine tune simultaneous connections to backend, use 0 for standard configuration
+
+  password_command: ''
+identity:
+  # ${HOSTNAME} is a variable containing the hostname as exposed by platform.node()
+  # ${RANDOM}[n] is a variable containing 'n' random alphanumeric char
+  machine_id: ${HOSTNAME}-niN9
+  machine_group: NETPERFECT
+
+prometheus:
+  ## Supervision
+  metrics: true
+  # Available variables: ${HOSTNAME}, ${RANDOM}[n], ${MACHINE_ID}, ${MACHINE_GROUP}, ${BACKUP_JOB}
+  backup_job: ${MACHINE_ID}
+  # Prometheus metrics destination can be a http / https server with optional basic authentication (pushgateway), or a file path for node textfile collector to pickup
+  # example: https://push.monitoring.example.tld/metrics/job/${BACKUP_JOB} where ${BACKUP_JOB} is defined in prometheus_backup_job
+  # example: /var/lib/prometheus/collector/mytextfile
+  destination: https://push.supervision.netperfect.eu/metrics/job/npbackup
+  # prometheus instance, becomes exported_instance when using a push gateway
+  instance: ${MACHINE_ID}
+
+  # prometheus metrics upload password
+  http_username: __NPBACKUP__Pci9BBXLQMnqjour0sBafVOwaQ7GoHKsbinxvnr3GqAxNjg1MjkwMDEyLjY2Mzc2NA8PDw8PDw8PDw8PDw8PD+zj6vphDPxJz7IljOeItr1FpvVhjAU3__NPBACKUP__
+  http_password: __NPBACKUP__rfBD6XhG2F/OKR1bAycy3o/4WAPUaDNH7J2FiGCnAC0xNjg1MjkwMDEyLjY2NDc2NQ8PDw8PDw8PDw8PDw8PD2fM16ngjWuh1OoQbCRWXY9HLp8pvHiPlQflay+gJckTvY07__NPBACKUP__
+
+  # Arbitrary group to filter later backups on
+  group: ${MACHINE_GROUP}
+
+  # Additional prometheus labels
+  additional_labels: npf_tenant=beauvois
+
+  no_cert_verify: true
+env:
+  variables:
+  - x=y
+  - y=z
+  - z=a
+  encrypted_variables: __NPBACKUP__aH6C7eMkEvg+VJEencPJVRjUxGAfq08ukLdYPZgGlVcxNjg1MjkwMDEyLjY2NDc2NQ8PDw8PDw8PDw8PDw8PD69quPfLhMKGiOoHYY0bj6zf1XlfKA/uAkO1nNmPaLLt/LsEju4ctM7l8cKZQ77sYz9JzZOO4aY=__NPBACKUP__
+options:
+  auto_upgrade: true
+  auto_upgrade_server_url: https://upgrades.stash.netperfect.eu
+  auto_upgrade_server_username: __NPBACKUP__p+160CrTZbe5OOHVO+u7Jq2YR/BRD48seoT0CvaeDdYxNjg1MjkwMDEyLjY2NDc2NQ8PDw8PDw8PDw8PDw8PD2DV71RDNDTiAGwm0Fg4kexIpWEPHXETiLij2Z388nc6ng==__NPBACKUP__
+  auto_upgrade_server_password: __NPBACKUP__wE1Y0hEDL66grVrnRFwgbu8Rz1KzC9uTqEp0FubLOOQxNjg1MjkwMDEyLjY2NDc2NQ8PDw8PDw8PDw8PDw8PDx0mzO6ov/JHmjE7cHwpay0b/BZFRVL8+2/SnY6n93PtQhGG__NPBACKUP__
+ # every 10 NPBackup runs, we'll try an autoupgrade. Never set this lower than 2 since failed upgrades will prevent backups from succeeding
+  auto_upgrade_interval: 10
+  # Available variables: ${HOSTNAME}, ${RANDOM}[n], ${MACHINE_ID}, ${MACHINE_GROUP}, ${BACKUP_JOB}
+  auto_upgrade_host_identity: ${MACHINE_ID}
+  auto_upgrade_group: ${MACHINE_GROUP}
+
+  backup_admin_password:

npbackup/core/runner.py

@@ -7,7 +7,7 @@ __intname__ = "npbackup.gui.core.runner"
 __author__ = "Orsiris de Jong"
 __copyright__ = "Copyright (C) 2022-2023 NetInvent"
 __license__ = "GPL-3.0-only"
-__build__ = "2023052701"
+__build__ = "2023052801"
 
 
 from typing import Optional, Callable, Union, List
@@ -472,8 +472,13 @@ class NPBackupRunner:
                     )
                     return False
         except KeyError:
-            logger.error("No backup source path given.")
+            logger.error("No backup source given.")
             return False
+        
+        try:
+            source_type = self.config_dict["backup"]["source_type"]
+        except KeyError:
+            source_type = None
 
         # MSWindows does not support one-file-system option
         try:
@@ -565,7 +570,10 @@ class NPBackupRunner:
         self.restic_runner.verbose = self.verbose
 
         # Run backup here
-        logger.info("Running backup of {}".format(paths))
+        if source_type not in ['folder_list', None]:
+            logger.info("Running backup of files in {} list".format(paths))
+        else:
+            logger.info("Running backup of {}".format(paths))
 
         if pre_exec_command:
             exit_code, output = command_runner(
@@ -589,6 +597,7 @@ class NPBackupRunner:
         self.restic_runner.dry_run = self.dry_run
         result, result_string = self.restic_runner.backup(
             paths=paths,
+            source_type=source_type,
             exclude_patterns=exclude_patterns,
             exclude_files=exclude_files,
             exclude_case_ignore=exclude_case_ignore,

npbackup/gui/config.py

@@ -14,6 +14,7 @@ import os
 from logging import getLogger
 import PySimpleGUI as sg
 import npbackup.configuration as configuration
+from ofunctions.misc import get_key_from_value
 from npbackup.core.i18n_helper import _t
 from npbackup.path_helper import CURRENT_EXECUTABLE
 from npbackup.core.nuitka_helper import IS_COMPILED
@@ -53,6 +54,25 @@ def config_gui(config_dict: dict, config_file: str):
         suppress_key_guessing=True,
     )
 
+    combo_boxes = {
+        'compression': {
+            "auto": _t("config_gui.auto"),
+            "max": _t("config_gui.max"),
+            "off": _t("config_gui.off")
+        },
+        'source_type': {
+            "folder_list": _t("config_gui.folder_list"),
+            "files_from": _t("config_gui.files_from"),
+            "files_from_verbatim": _t("config_gui.files_from_verbatim"),
+            "files_from_raw": _t("config_gui.files_from_raw")
+        },
+        'priority': {
+            "low": _t("config_gui.low"),
+            "normal": _t("config_gui.normal"),
+            "high": _t("config_gui.high")
+        }
+    }
+
     ENCRYPTED_DATA_PLACEHOLDER = "<{}>".format(_t("config_gui.encrypted_data"))
 
     def update_gui(window, config_dict, unencrypted=False):
@@ -95,7 +115,10 @@ def config_gui(config_dict: dict, config_file: str):
                         value = "\n".join(value)
                     # window keys are section---entry
                     key = "{}---{}".format(section, entry)
-                    window[key].Update(value)
+                    if entry in combo_boxes:
+                        window[key].Update(combo_boxes[entry][value])
+                    else:
+                        window[key].Update(value)
                 except KeyError:
                     logger.error("No GUI equivalent for {}.".format(entry))
                 except TypeError as exc:
@@ -110,8 +133,11 @@ def config_gui(config_dict: dict, config_file: str):
             except ValueError:
                 # Don't bother with keys that don't begin with "---"
                 continue
+            # Handle combo boxes first to transform translation into key
+            if entry in combo_boxes:
+                value = get_key_from_value(combo_boxes[entry], value)
             # check whether we need to split into list
-            if not isinstance(value, bool):
+            elif not isinstance(value, bool):
                 result = value.split("\n")
                 if len(result) > 1:
                     value = result
@@ -139,7 +165,7 @@ def config_gui(config_dict: dict, config_file: str):
     backup_col = [
         [
             sg.Text(_t("config_gui.compression"), size=(40, 1)),
-            sg.Combo(["auto", "max", "off"], key="backup---compression", size=(48, 1)),
+            sg.Combo(list(combo_boxes['compression'].values()), key="backup---compression", size=(48, 1)),
         ],
         [
             sg.Text(
@@ -150,6 +176,10 @@ def config_gui(config_dict: dict, config_file: str):
             ),
             sg.Multiline(key="backup---paths", size=(48, 4)),
         ],
+        [
+            sg.Text(_t("config_gui.source_type"), size=(40, 1)),
+            sg.Combo(list(combo_boxes['source_type'].values()), key="backup---source_type", size=(48, 1)), #WIP
+        ],
         [
             sg.Text(
                 "{}\n({})".format(
@@ -237,7 +267,7 @@ def config_gui(config_dict: dict, config_file: str):
         ],
         [
             sg.Text(_t("config_gui.backup_priority"), size=(40, 1)),
-            sg.Combo(["low", "normal", "high"], key="backup---priority", size=(48, 1)),
+            sg.Combo(list(combo_boxes['priority'].values()), key="backup---priority", size=(48, 1)),
         ],
         [
             sg.Text(_t("config_gui.additional_parameters"), size=(40, 1)),
@@ -425,7 +455,7 @@ def config_gui(config_dict: dict, config_file: str):
         [
             sg.Tab(
                 _t("config_gui.backup"),
-                backup_col,
+                [[sg.Column(backup_col, scrollable=True, vertical_scroll_only=True)]],
                 font="helvetica 16",
                 key="--tab-backup--",
                 element_justification="C",

npbackup/restic_wrapper/__init__.py

@@ -7,8 +7,8 @@ __intname__ = "npbackup.restic_wrapper"
 __author__ = "Orsiris de Jong"
 __copyright__ = "Copyright (C) 2022-2023 NetInvent"
 __license__ = "GPL-3.0-only"
-__build__ = "2023041201"
-__version__ = "1.6.3"
+__build__ = "20230522801"
+__version__ = "1.7.0"
 
 
 from typing import Tuple, List, Optional, Callable, Union
@@ -512,6 +512,7 @@ class ResticRunner:
     def backup(
         self,
         paths: List[str],
+        source_type: str,
         exclude_patterns: List[str] = [],
         exclude_files: List[str] = [],
         exclude_case_ignore: bool = False,
@@ -526,10 +527,27 @@ class ResticRunner:
         """
         if not self.is_init:
             return None, None
-        # make sure path is a list and does not have trailing slashes
-        cmd = "backup {}".format(
-            " ".join(['"{}"'.format(path.rstrip("/\\")) for path in paths])
-        )
+        
+        # Handle various source types
+        if source_type in ["files_from", "files_from_verbatim", "files_from_raw"]:
+            cmd = "backup"
+            if source_type == "files_from":
+                source_parameter = "--files-from"
+            elif source_type == "files_from_verbatim":
+                source_parameter = "--files-from-verbatim"
+            elif source_type == "files_from_raw":
+                source_parameter = "--files-from-raw"
+            else:
+                logger.error("Bogus source type given")
+                return False, ""
+            
+            for path in paths:
+                cmd += " {} \"{}\"".format(source_parameter, path)
+        else:
+            # make sure path is a list and does not have trailing slashes
+            cmd = "backup {}".format(
+                " ".join(['"{}"'.format(path.rstrip("/\\")) for path in paths])
+            )
 
         case_ignore_param = ""
         # Always use case ignore excludes under windows

npbackup/translations/config_gui.en.yml

@@ -82,4 +82,21 @@ en:
   machine_group: Machine group
 
   show_decrypted: Show decrypted
-  not_allowed_on_not_compiled: Using this option is not allowed on non compiled or non private builds
+  not_allowed_on_not_compiled: Using this option is not allowed on non compiled or non private builds
+
+  # compression
+  auto: Automatic
+  max: Maximum
+  off: Disabled
+
+  # priorities
+  low: Low
+  normal: Normal
+  high: High
+
+  # source types
+  source_type: Sources type
+  folder_list: Folder list
+  files_from: Files from list
+  files_from_verbatim: Files from verbatim list
+  files_from_raw: Files from raw list

npbackup/translations/config_gui.fr.yml

@@ -82,4 +82,21 @@ fr:
   machine_group: Groupe machine
 
   show_decrypted: Voir déchiffré
-  not_allowed_on_not_compiled: Cette option n'est pas permise sur une version non compilée ou non privée
+  not_allowed_on_not_compiled: Cette option n'est pas permise sur une version non compilée ou non privée
+
+  # compression
+  auto: Automatique
+  max: Maximale
+  off: Aucune
+
+  # priorities
+  low: Basse
+  normal: Normale
+  high: Haute
+
+  # source types
+  source_type: Type de sources
+  folder_list: Liste de dossiers
+  files_from: Liste fichiers depuis un fichier
+  files_from_verbatim: Liste fichiers depuis un fichier "exact"
+  files_from_raw: Liste fichiers depuis un fichier "raw"