From 18b8829bffe47d529d7a46905d9d3c213b4376a1 Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 27 Jun 2025 19:57:52 +0200 Subject: [PATCH] Retention policies can now be applied only on certain tags --- npbackup/configuration.py | 75 ++++++++++++++++++++-- npbackup/core/runner.py | 17 ++++- npbackup/gui/config.py | 85 ++++++++++++++++++++----- npbackup/restic_wrapper/__init__.py | 15 +++-- npbackup/translations/config_gui.en.yml | 1 + npbackup/translations/config_gui.fr.yml | 3 +- 6 files changed, 167 insertions(+), 29 deletions(-) diff --git a/npbackup/configuration.py b/npbackup/configuration.py index fc2e2d2..8a55ea3 100644 --- a/npbackup/configuration.py +++ b/npbackup/configuration.py @@ -7,7 +7,7 @@ __intname__ = "npbackup.configuration" __author__ = "Orsiris de Jong" __copyright__ = "Copyright (C) 2022-2025 NetInvent" __license__ = "GPL-3.0-only" -__build__ = "2025061701" +__build__ = "2025072701" __version__ = "npbackup 3.0.3+" @@ -31,7 +31,8 @@ from resources.customization import ID_STRING from npbackup.key_management import AES_KEY, EARLIER_AES_KEY, IS_PRIV_BUILD, get_aes_key from npbackup.__version__ import __version__ as MAX_CONF_VERSION -MIN_CONF_VERSION = "3.0" +MIN_MIGRATABLE_CONF_VERSION = "3.0.0" +MIN_CONF_VERSION = "3.0.3" sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) @@ -190,7 +191,8 @@ empty_config_dict = { "weekly": 4, "monthly": 12, "yearly": 3, - "tags": [], + "keep_tags": [], + "apply_on_tags": [], "keep_within": True, "group_by_host": True, "group_by_tags": True, @@ -811,6 +813,63 @@ def _get_config_file_checksum(config_file: Path) -> str: return "%08X" % (cur_hash & 0xFFFFFFFF) +def _migrate_config_dict(full_config: dict, old_version: str, new_version: str) -> dict: + """ + Migrate config dict from old version to new version + This is used when config file version is not the same as current version + """ + logger.info(f"Migrating config file from version {old_version} to {new_version}") + + def _migrate_retetion_policy_3_0_0_to_3_0_3( + full_config: dict, + object_name: str, + object_type: str, + ) -> dict: + try: + if full_config.g( + f"{object_type}.{object_name}.repo_opts.retention_policy.tags" + ) is not None and not full_config.g( + f"{object_type}.{object_name}.repo_opts.retention_policy.keep_tags" + ): + full_config.s( + f"{object_type}.{object_name}.repo_opts.retention_policy.keep_tags", + full_config.g( + f"{object_type}.{object_name}.repo_opts.retention_policy.tags" + ), + ) + full_config.d( + f"{object_type}.{object_name}.repo_opts.retention_policy.tags" + ) + logger.info( + f"Migrated {object_name} retention policy tags to keep_tags" + ) + except KeyError: + logger.info( + f"{object_type} {object_name} has no retention policy, skipping migration" + ) + return full_config + + def _apply_migrations( + full_config: dict, + object_name: str, + object_type: str, + ) -> dict: + if version_parse(old_version) < version_parse("3.0.3"): + full_config = _migrate_retetion_policy_3_0_0_to_3_0_3( + full_config, object_name, object_type + ) + return full_config + + for repo in get_repo_list(full_config): + _apply_migrations(full_config, repo, "repos") + + for group in get_group_list(full_config): + _apply_migrations(full_config, group, "groups") + + full_config.s("conf_version", new_version) + return full_config + + def _load_config_file(config_file: Path) -> Union[bool, dict]: """ Checks whether config file is valid @@ -830,12 +889,18 @@ def _load_config_file(config_file: Path) -> Union[bool, dict]: ) return False if conf_version < version_parse( - MIN_CONF_VERSION + MIN_MIGRATABLE_CONF_VERSION ) or conf_version > version_parse(MAX_CONF_VERSION): logger.critical( - f"Config file version {str(conf_version)} is not in required version range min={MIN_CONF_VERSION}, max={MAX_CONF_VERSION}" + f"Config file version {str(conf_version)} is not in required version range min={MIN_MIGRATABLE_CONF_VERSION}, max={MAX_CONF_VERSION}" ) return False + if conf_version < version_parse(MIN_CONF_VERSION): + full_config = _migrate_config_dict( + full_config, str(conf_version), MIN_CONF_VERSION + ) + logger.info("Writing migrated config file") + save_config(config_file, full_config) except (AttributeError, TypeError, InvalidVersion) as exc: logger.critical( f"Cannot read conf version from config file {config_file}, which seems bogus: {exc}" diff --git a/npbackup/core/runner.py b/npbackup/core/runner.py index 578490b..c199712 100644 --- a/npbackup/core/runner.py +++ b/npbackup/core/runner.py @@ -1032,11 +1032,11 @@ class NPBackupRunner: @has_permission @is_ready @apply_config_to_restic_runner - def snapshots(self, id: str = None, errors_allowed: bool = False) -> Optional[dict]: + def snapshots(self, snapshot_id: str = None, errors_allowed: bool = False) -> Optional[dict]: self.write_logs( f"Listing snapshots of repo {self.repo_config.g('name')}", level="info" ) - snapshots = self.restic_runner.snapshots(id=id, errors_allowed=errors_allowed) + snapshots = self.restic_runner.snapshots(snapshot_id=snapshot_id, errors_allowed=errors_allowed) return snapshots @threaded @@ -1643,10 +1643,23 @@ class NPBackupRunner: unit = "d" value = value * 7 policy[f"keep-within-{entry}"] = f"{value}{unit}" + + # DEPRECATED: since we renamed tags to keep_tags, we still neeed to fetch + # old tag name. Will be removed in 3.1 keep_tags = self.repo_config.g("repo_opts.retention_policy.tags") if not isinstance(keep_tags, list) and keep_tags: keep_tags = [keep_tags] policy["keep-tags"] = keep_tags + keep_tags = self.repo_config.g("repo_opts.retention_policy.keep_tags") + if not isinstance(keep_tags, list) and keep_tags: + keep_tags = [keep_tags] + policy["keep-tags"] = keep_tags + apply_on_tags = self.repo_config.g( + "repo_opts.retention_policy.apply_on_tags" + ) + if not isinstance(apply_on_tags, list) and apply_on_tags: + apply_on_tags = [apply_on_tags] + policy["apply-on-tags"] = apply_on_tags # Fool proof, don't run without policy, or else we'll get if not policy: msg = "Empty retention policy. Won't run" diff --git a/npbackup/gui/config.py b/npbackup/gui/config.py index 27abd49..3e03bb0 100644 --- a/npbackup/gui/config.py +++ b/npbackup/gui/config.py @@ -447,7 +447,8 @@ def config_gui(full_config: dict, config_file: str): "backup_opts.post_exec_commands", "backup_opts.exclude_files", "backup_opts.exclude_patterns", - "repo_opts.retention_policy.tags", + "repo_opts.retention_policy.keep_tags", + "repo_opts.retention_policy.apply_tags", ): if key == "backup_opts.tags": tree = tags_tree @@ -459,8 +460,10 @@ def config_gui(full_config: dict, config_file: str): tree = exclude_files_tree elif key == "backup_opts.exclude_patterns": tree = exclude_patterns_tree - elif key == "repo_opts.retention_policy.tags": - tree = retention_policy_tags_tree + elif key == "repo_opts.retention_policy.keep_tags": + tree = retention_policy_keep_tags_tree + elif key == "repo_opts.retention_policy.apply_on_tags": + tree = retention_policy_apply_on_tags_tree else: tree = None @@ -618,7 +621,8 @@ def config_gui(full_config: dict, config_file: str): nonlocal tags_tree nonlocal exclude_files_tree nonlocal exclude_patterns_tree - nonlocal retention_policy_tags_tree + nonlocal retention_policy_keep_tags_tree + nonlocal retention_policy_apply_on_tags_tree nonlocal pre_exec_commands_tree nonlocal post_exec_commands_tree nonlocal env_variables_tree @@ -646,7 +650,8 @@ def config_gui(full_config: dict, config_file: str): tags_tree = sg.TreeData() exclude_patterns_tree = sg.TreeData() exclude_files_tree = sg.TreeData() - retention_policy_tags_tree = sg.TreeData() + retention_policy_keep_tags_tree = sg.TreeData() + retention_policy_apply_on_tags_tree = sg.TreeData() pre_exec_commands_tree = sg.TreeData() post_exec_commands_tree = sg.TreeData() env_variables_tree = sg.TreeData() @@ -755,7 +760,8 @@ def config_gui(full_config: dict, config_file: str): "backup_opts.post_exec_commands", "backup_opts.exclude_files", "backup_opts.exclude_patterns", - "repo_opts.retention_policy.tags", + "repo_opts.retention_policy.keep_tags", + "repo_opts.retention_policy.apply_on_tags", ] for tree_data_key in list_tree_data_keys: values[tree_data_key] = [] @@ -1813,8 +1819,12 @@ def config_gui(full_config: dict, config_file: str): [ sg.Column( [ - [sg.Button("+", key="--ADD-RETENTION-TAG--", size=(3, 1))], - [sg.Button("-", key="--REMOVE-RETENTION-TAG--", size=(3, 1))], + [sg.Button("+", key="--ADD-RETENTION-KEEP-TAG--", size=(3, 1))], + [ + sg.Button( + "-", key="--REMOVE-RETENTION-KEEP-TAG--", size=(3, 1) + ) + ], ], pad=0, ), @@ -1823,10 +1833,44 @@ def config_gui(full_config: dict, config_file: str): [ sg.Tree( sg.TreeData(), - key="repo_opts.retention_policy.tags", + key="repo_opts.retention_policy.keep_tags", headings=[], col0_heading=_t("config_gui.keep_tags"), - num_rows=4, + num_rows=3, + expand_x=True, + expand_y=True, + ) + ] + ], + pad=0, + expand_x=True, + ), + sg.Column( + [ + [ + sg.Button( + "+", key="--ADD-RETENTION-APPLY-ON-TAG--", size=(3, 1) + ) + ], + [ + sg.Button( + "-", + key="--REMOVE-RETENTION-APPLY-ON-TAG--", + size=(3, 1), + ) + ], + ], + pad=0, + ), + sg.Column( + [ + [ + sg.Tree( + sg.TreeData(), + key="repo_opts.retention_policy.apply_on_tags", + headings=[], + col0_heading=_t("config_gui.apply_on_tags"), + num_rows=3, expand_x=True, expand_y=True, ) @@ -1837,6 +1881,8 @@ def config_gui(full_config: dict, config_file: str): ), ], [sg.HorizontalSeparator()], + [], + [sg.HorizontalSeparator()], [ sg.Text( _t("config_gui.post_backup_housekeeping_percent_chance"), @@ -2577,7 +2623,8 @@ Google Cloud storage: GOOGLE_PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS\n\ tags_tree = sg.TreeData() exclude_patterns_tree = sg.TreeData() exclude_files_tree = sg.TreeData() - retention_policy_tags_tree = sg.TreeData() + retention_policy_keep_tags_tree = sg.TreeData() + retention_policy_apply_on_tags_tree = sg.TreeData() pre_exec_commands_tree = sg.TreeData() post_exec_commands_tree = sg.TreeData() global_prometheus_labels_tree = sg.TreeData() @@ -2716,7 +2763,8 @@ Google Cloud storage: GOOGLE_PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS\n\ if event in ( "--ADD-BACKUP-TAG--", "--ADD-EXCLUDE-PATTERN--", - "--ADD-RETENTION-TAG--", + "--ADD-RETENTION-KEEP-TAG--", + "--ADD-RETENTION-APPLY-ON-TAG--", "--ADD-PRE-EXEC-COMMAND--", "--ADD-POST-EXEC-COMMAND--", "--ADD-PROMETHEUS-LABEL--", @@ -2730,7 +2778,8 @@ Google Cloud storage: GOOGLE_PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS\n\ "--REMOVE-BACKUP-TAG--", "--REMOVE-EXCLUDE-PATTERN--", "--REMOVE-EXCLUDE-FILE--", - "--REMOVE-RETENTION-TAG--", + "--REMOVE-RETENTION-KEEP-TAG--", + "--REMOVE-RETENTION-APPLY-ON-TAG--", "--REMOVE-PRE-EXEC-COMMAND--", "--REMOVE-POST-EXEC-COMMAND--", "--REMOVE-PROMETHEUS-LABEL--", @@ -2754,10 +2803,14 @@ Google Cloud storage: GOOGLE_PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS\n\ popup_text = None tree = exclude_files_tree option_key = "backup_opts.exclude_files" - elif "RETENTION-TAG" in event: + elif "RETENTION-KEEP-TAG" in event: popup_text = _t("config_gui.enter_tag") - tree = retention_policy_tags_tree - option_key = "repo_opts.retention_policy.tags" + tree = retention_policy_keep_tags_tree + option_key = "repo_opts.retention_policy.keep_tags" + elif "RETENTION-APPLY-ON-TAG" in event: + popup_text = _t("config_gui.enter_tag") + tree = retention_policy_apply_on_tags_tree + option_key = "repo_opts.retention_policy.apply_on_tags" elif "PRE-EXEC-COMMAND" in event: popup_text = _t("config_gui.enter_command") tree = pre_exec_commands_tree diff --git a/npbackup/restic_wrapper/__init__.py b/npbackup/restic_wrapper/__init__.py index 36a99c2..5466dab 100644 --- a/npbackup/restic_wrapper/__init__.py +++ b/npbackup/restic_wrapper/__init__.py @@ -378,8 +378,8 @@ class ResticRunner: if not isinstance(output, str): logger.debug("Skipping output filter for non str output") return output - for filter in restic_output_filters: - output = filter.sub("", output) + for regex_filter in restic_output_filters: + output = regex_filter.sub("", output) return output def executor( @@ -923,7 +923,7 @@ class ResticRunner: return self.convert_to_json_output(result, output, msg=msg, **kwargs) def snapshots( - self, id: str = None, errors_allowed: bool = False + self, snapshot_id: str = None, errors_allowed: bool = False ) -> Union[bool, str, dict]: """ Returns a list of snapshots @@ -935,8 +935,8 @@ class ResticRunner: kwargs.pop("self") cmd = "snapshots" - if id: - cmd += f" {id}" + if snapshot_id: + cmd += f" {snapshot_id}" result, output = self.executor( cmd, timeout=FAST_COMMANDS_TIMEOUT, errors_allowed=errors_allowed ) @@ -1206,6 +1206,11 @@ class ResticRunner: for tag in value: if tag: cmd += f" --keep-tag {tag}" + elif key == "apply-on-tags": + if isinstance(value, list): + for tag in value: + if tag: + cmd += f" --tag {tag}" else: cmd += f" --{key.replace('_', '-')} {value}" if group_by: diff --git a/npbackup/translations/config_gui.en.yml b/npbackup/translations/config_gui.en.yml index 7235ad6..4841c3e 100644 --- a/npbackup/translations/config_gui.en.yml +++ b/npbackup/translations/config_gui.en.yml @@ -128,6 +128,7 @@ en: yearly: yearly snapshots keep_within: Keep snapshots within time period relative to current snapshot keep_tags: Keep snapshots with the following tags + apply_on_tagds: Apply only on snapshots with the following tags post_backup_housekeeping_percent_chance: Post backup housekeeping run chance (%%) post_backup_housekeeping_percent_chance_explanation: Randomize housekeeping runs after backup (0-100%%, 0 = never, 100 = always) post_backup_housekeeping_interval: Post backup housekeeping interval diff --git a/npbackup/translations/config_gui.fr.yml b/npbackup/translations/config_gui.fr.yml index 17d478a..e101820 100644 --- a/npbackup/translations/config_gui.fr.yml +++ b/npbackup/translations/config_gui.fr.yml @@ -128,7 +128,8 @@ fr: monthly: instantanés mensuels yearly: instantanés annuelles keep_within: Garder les instantanées dans une période relative au dernier instantané - keep_tags: Garder les instantanés avec les tags suivants + keep_tags: Garder les instantanés aux tags suivants + apply_on_tags: Appliquer uniquement sur les instantanés aux tags suivants post_backup_housekeeping_percent_chance: Chance (%%) de lancer maintenance post-sauvegarde post_backup_housekeeping_percent_chance_explanation: Rend aléatoire la maintenance après sauvegarde (0-100%%, 0 = jamais, 100 = toujours) post_backup_housekeeping_interval: Intervalle de maintenance post-sauvegarde