netinvent/npbackup
1
1
Fork 0
mirror of https://github.com/netinvent/npbackup.git synced 2025-10-09 21:17:47 +08:00
Code Issues Projects Releases Packages Wiki Activity

Retention policies can now be applied only on certain tags

Browse source
This commit is contained in:
deajan 2025-06-27 19:57:52 +02:00
parent 3916e4ec9f
commit 18b8829bff
6 changed files with 167 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
npbackup/configuration.py
View file

@ -7,7 +7,7 @@ __intname__ = "npbackup.configuration"
__author__ = "Orsiris de Jong" __author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2022-2025 NetInvent" __copyright__ = "Copyright (C) 2022-2025 NetInvent"
__license__ = "GPL-3.0-only" __license__ = "GPL-3.0-only"
__build__ = "2025061701" __build__ = "2025072701"
__version__ = "npbackup 3.0.3+" __version__ = "npbackup 3.0.3+"
@ -31,7 +31,8 @@ from resources.customization import ID_STRING
from npbackup.key_management import AES_KEY, EARLIER_AES_KEY, IS_PRIV_BUILD, get_aes_key from npbackup.key_management import AES_KEY, EARLIER_AES_KEY, IS_PRIV_BUILD, get_aes_key
from npbackup.__version__ import __version__ as MAX_CONF_VERSION from npbackup.__version__ import __version__ as MAX_CONF_VERSION
MIN_CONF_VERSION = "3.0" MIN_MIGRATABLE_CONF_VERSION = "3.0.0"
MIN_CONF_VERSION = "3.0.3"
sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), "..")))
@ -190,7 +191,8 @@ empty_config_dict = {
"weekly": 4, "weekly": 4,
"monthly": 12, "monthly": 12,
"yearly": 3, "yearly": 3,
"tags": [], "keep_tags": [],
"apply_on_tags": [],
"keep_within": True, "keep_within": True,
"group_by_host": True, "group_by_host": True,
"group_by_tags": True, "group_by_tags": True,
@ -811,6 +813,63 @@ def _get_config_file_checksum(config_file: Path) -> str:
return "%08X" % (cur_hash & 0xFFFFFFFF) return "%08X" % (cur_hash & 0xFFFFFFFF)
def _migrate_config_dict(full_config: dict, old_version: str, new_version: str) -> dict:
"""
Migrate config dict from old version to new version
This is used when config file version is not the same as current version
"""
logger.info(f"Migrating config file from version {old_version} to {new_version}")
def _migrate_retetion_policy_3_0_0_to_3_0_3(
full_config: dict,
object_name: str,
object_type: str,
) -> dict:
try:
if full_config.g(
f"{object_type}.{object_name}.repo_opts.retention_policy.tags"
) is not None and not full_config.g(
f"{object_type}.{object_name}.repo_opts.retention_policy.keep_tags"
):
full_config.s(
f"{object_type}.{object_name}.repo_opts.retention_policy.keep_tags",
full_config.g(
f"{object_type}.{object_name}.repo_opts.retention_policy.tags"
),
)
full_config.d(
f"{object_type}.{object_name}.repo_opts.retention_policy.tags"
)
logger.info(
f"Migrated {object_name} retention policy tags to keep_tags"
)
except KeyError:
logger.info(
f"{object_type} {object_name} has no retention policy, skipping migration"
)
return full_config
def _apply_migrations(
full_config: dict,
object_name: str,
object_type: str,
) -> dict:
if version_parse(old_version) < version_parse("3.0.3"):
full_config = _migrate_retetion_policy_3_0_0_to_3_0_3(
full_config, object_name, object_type
)
return full_config
for repo in get_repo_list(full_config):
_apply_migrations(full_config, repo, "repos")
for group in get_group_list(full_config):
_apply_migrations(full_config, group, "groups")
full_config.s("conf_version", new_version)
return full_config
def _load_config_file(config_file: Path) -> Union[bool, dict]: def _load_config_file(config_file: Path) -> Union[bool, dict]:
""" """
Checks whether config file is valid Checks whether config file is valid
@ -830,12 +889,18 @@ def _load_config_file(config_file: Path) -> Union[bool, dict]:
) )
return False return False
if conf_version < version_parse( if conf_version < version_parse(
MIN_CONF_VERSION MIN_MIGRATABLE_CONF_VERSION
) or conf_version > version_parse(MAX_CONF_VERSION): ) or conf_version > version_parse(MAX_CONF_VERSION):
logger.critical( logger.critical(
f"Config file version {str(conf_version)} is not in required version range min={MIN_CONF_VERSION}, max={MAX_CONF_VERSION}" f"Config file version {str(conf_version)} is not in required version range min={MIN_MIGRATABLE_CONF_VERSION}, max={MAX_CONF_VERSION}"
) )
return False return False
if conf_version < version_parse(MIN_CONF_VERSION):
full_config = _migrate_config_dict(
full_config, str(conf_version), MIN_CONF_VERSION
)
logger.info("Writing migrated config file")
save_config(config_file, full_config)
except (AttributeError, TypeError, InvalidVersion) as exc: except (AttributeError, TypeError, InvalidVersion) as exc:
logger.critical( logger.critical(
f"Cannot read conf version from config file {config_file}, which seems bogus: {exc}" f"Cannot read conf version from config file {config_file}, which seems bogus: {exc}"

17
npbackup/core/runner.py
View file

@ -1032,11 +1032,11 @@ class NPBackupRunner:
@has_permission @has_permission
@is_ready @is_ready
@apply_config_to_restic_runner @apply_config_to_restic_runner
def snapshots(self, id: str = None, errors_allowed: bool = False) -> Optional[dict]: def snapshots(self, snapshot_id: str = None, errors_allowed: bool = False) -> Optional[dict]:
self.write_logs( self.write_logs(
f"Listing snapshots of repo {self.repo_config.g('name')}", level="info" f"Listing snapshots of repo {self.repo_config.g('name')}", level="info"
) )
snapshots = self.restic_runner.snapshots(id=id, errors_allowed=errors_allowed) snapshots = self.restic_runner.snapshots(snapshot_id=snapshot_id, errors_allowed=errors_allowed)
return snapshots return snapshots
@threaded @threaded
@ -1643,10 +1643,23 @@ class NPBackupRunner:
unit = "d" unit = "d"
value = value * 7 value = value * 7
policy[f"keep-within-{entry}"] = f"{value}{unit}" policy[f"keep-within-{entry}"] = f"{value}{unit}"
# DEPRECATED: since we renamed tags to keep_tags, we still neeed to fetch
# old tag name. Will be removed in 3.1
keep_tags = self.repo_config.g("repo_opts.retention_policy.tags") keep_tags = self.repo_config.g("repo_opts.retention_policy.tags")
if not isinstance(keep_tags, list) and keep_tags: if not isinstance(keep_tags, list) and keep_tags:
keep_tags = [keep_tags] keep_tags = [keep_tags]
policy["keep-tags"] = keep_tags policy["keep-tags"] = keep_tags
keep_tags = self.repo_config.g("repo_opts.retention_policy.keep_tags")
if not isinstance(keep_tags, list) and keep_tags:
keep_tags = [keep_tags]
policy["keep-tags"] = keep_tags
apply_on_tags = self.repo_config.g(
"repo_opts.retention_policy.apply_on_tags"
)
if not isinstance(apply_on_tags, list) and apply_on_tags:
apply_on_tags = [apply_on_tags]
policy["apply-on-tags"] = apply_on_tags
# Fool proof, don't run without policy, or else we'll get # Fool proof, don't run without policy, or else we'll get
if not policy: if not policy:
msg = "Empty retention policy. Won't run" msg = "Empty retention policy. Won't run"

85
npbackup/gui/config.py
View file

@ -447,7 +447,8 @@ def config_gui(full_config: dict, config_file: str):
"backup_opts.post_exec_commands", "backup_opts.post_exec_commands",
"backup_opts.exclude_files", "backup_opts.exclude_files",
"backup_opts.exclude_patterns", "backup_opts.exclude_patterns",
"repo_opts.retention_policy.tags", "repo_opts.retention_policy.keep_tags",
"repo_opts.retention_policy.apply_tags",
): ):
if key == "backup_opts.tags": if key == "backup_opts.tags":
tree = tags_tree tree = tags_tree
@ -459,8 +460,10 @@ def config_gui(full_config: dict, config_file: str):
tree = exclude_files_tree tree = exclude_files_tree
elif key == "backup_opts.exclude_patterns": elif key == "backup_opts.exclude_patterns":
tree = exclude_patterns_tree tree = exclude_patterns_tree
elif key == "repo_opts.retention_policy.tags": elif key == "repo_opts.retention_policy.keep_tags":
tree = retention_policy_tags_tree tree = retention_policy_keep_tags_tree
elif key == "repo_opts.retention_policy.apply_on_tags":
tree = retention_policy_apply_on_tags_tree
else: else:
tree = None tree = None
@ -618,7 +621,8 @@ def config_gui(full_config: dict, config_file: str):
nonlocal tags_tree nonlocal tags_tree
nonlocal exclude_files_tree nonlocal exclude_files_tree
nonlocal exclude_patterns_tree nonlocal exclude_patterns_tree
nonlocal retention_policy_tags_tree nonlocal retention_policy_keep_tags_tree
nonlocal retention_policy_apply_on_tags_tree
nonlocal pre_exec_commands_tree nonlocal pre_exec_commands_tree
nonlocal post_exec_commands_tree nonlocal post_exec_commands_tree
nonlocal env_variables_tree nonlocal env_variables_tree
@ -646,7 +650,8 @@ def config_gui(full_config: dict, config_file: str):
tags_tree = sg.TreeData() tags_tree = sg.TreeData()
exclude_patterns_tree = sg.TreeData() exclude_patterns_tree = sg.TreeData()
exclude_files_tree = sg.TreeData() exclude_files_tree = sg.TreeData()
retention_policy_tags_tree = sg.TreeData() retention_policy_keep_tags_tree = sg.TreeData()
retention_policy_apply_on_tags_tree = sg.TreeData()
pre_exec_commands_tree = sg.TreeData() pre_exec_commands_tree = sg.TreeData()
post_exec_commands_tree = sg.TreeData() post_exec_commands_tree = sg.TreeData()
env_variables_tree = sg.TreeData() env_variables_tree = sg.TreeData()
@ -755,7 +760,8 @@ def config_gui(full_config: dict, config_file: str):
"backup_opts.post_exec_commands", "backup_opts.post_exec_commands",
"backup_opts.exclude_files", "backup_opts.exclude_files",
"backup_opts.exclude_patterns", "backup_opts.exclude_patterns",
"repo_opts.retention_policy.tags", "repo_opts.retention_policy.keep_tags",
"repo_opts.retention_policy.apply_on_tags",
] ]
for tree_data_key in list_tree_data_keys: for tree_data_key in list_tree_data_keys:
values[tree_data_key] = [] values[tree_data_key] = []
@ -1813,8 +1819,12 @@ def config_gui(full_config: dict, config_file: str):
[ [
sg.Column( sg.Column(
[ [
[sg.Button("+", key="--ADD-RETENTION-TAG--", size=(3, 1))], [sg.Button("+", key="--ADD-RETENTION-KEEP-TAG--", size=(3, 1))],
[sg.Button("-", key="--REMOVE-RETENTION-TAG--", size=(3, 1))], [
sg.Button(
"-", key="--REMOVE-RETENTION-KEEP-TAG--", size=(3, 1)
)
],
], ],
pad=0, pad=0,
), ),
@ -1823,10 +1833,44 @@ def config_gui(full_config: dict, config_file: str):
[ [
sg.Tree( sg.Tree(
sg.TreeData(), sg.TreeData(),
key="repo_opts.retention_policy.tags", key="repo_opts.retention_policy.keep_tags",
headings=[], headings=[],
col0_heading=_t("config_gui.keep_tags"), col0_heading=_t("config_gui.keep_tags"),
num_rows=4, num_rows=3,
expand_x=True,
expand_y=True,
)
]
],
pad=0,
expand_x=True,
),
sg.Column(
[
[
sg.Button(
"+", key="--ADD-RETENTION-APPLY-ON-TAG--", size=(3, 1)
)
],
[
sg.Button(
"-",
key="--REMOVE-RETENTION-APPLY-ON-TAG--",
size=(3, 1),
)
],
],
pad=0,
),
sg.Column(
[
[
sg.Tree(
sg.TreeData(),
key="repo_opts.retention_policy.apply_on_tags",
headings=[],
col0_heading=_t("config_gui.apply_on_tags"),
num_rows=3,
expand_x=True, expand_x=True,
expand_y=True, expand_y=True,
) )
@ -1837,6 +1881,8 @@ def config_gui(full_config: dict, config_file: str):
), ),
], ],
[sg.HorizontalSeparator()], [sg.HorizontalSeparator()],
[],
[sg.HorizontalSeparator()],
[ [
sg.Text( sg.Text(
_t("config_gui.post_backup_housekeeping_percent_chance"), _t("config_gui.post_backup_housekeeping_percent_chance"),
@ -2577,7 +2623,8 @@ Google Cloud storage: GOOGLE_PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS\n\
tags_tree = sg.TreeData() tags_tree = sg.TreeData()
exclude_patterns_tree = sg.TreeData() exclude_patterns_tree = sg.TreeData()
exclude_files_tree = sg.TreeData() exclude_files_tree = sg.TreeData()
retention_policy_tags_tree = sg.TreeData() retention_policy_keep_tags_tree = sg.TreeData()
retention_policy_apply_on_tags_tree = sg.TreeData()
pre_exec_commands_tree = sg.TreeData() pre_exec_commands_tree = sg.TreeData()
post_exec_commands_tree = sg.TreeData() post_exec_commands_tree = sg.TreeData()
global_prometheus_labels_tree = sg.TreeData() global_prometheus_labels_tree = sg.TreeData()
@ -2716,7 +2763,8 @@ Google Cloud storage: GOOGLE_PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS\n\
if event in ( if event in (
"--ADD-BACKUP-TAG--", "--ADD-BACKUP-TAG--",
"--ADD-EXCLUDE-PATTERN--", "--ADD-EXCLUDE-PATTERN--",
"--ADD-RETENTION-TAG--", "--ADD-RETENTION-KEEP-TAG--",
"--ADD-RETENTION-APPLY-ON-TAG--",
"--ADD-PRE-EXEC-COMMAND--", "--ADD-PRE-EXEC-COMMAND--",
"--ADD-POST-EXEC-COMMAND--", "--ADD-POST-EXEC-COMMAND--",
"--ADD-PROMETHEUS-LABEL--", "--ADD-PROMETHEUS-LABEL--",
@ -2730,7 +2778,8 @@ Google Cloud storage: GOOGLE_PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS\n\
"--REMOVE-BACKUP-TAG--", "--REMOVE-BACKUP-TAG--",
"--REMOVE-EXCLUDE-PATTERN--", "--REMOVE-EXCLUDE-PATTERN--",
"--REMOVE-EXCLUDE-FILE--", "--REMOVE-EXCLUDE-FILE--",
"--REMOVE-RETENTION-TAG--", "--REMOVE-RETENTION-KEEP-TAG--",
"--REMOVE-RETENTION-APPLY-ON-TAG--",
"--REMOVE-PRE-EXEC-COMMAND--", "--REMOVE-PRE-EXEC-COMMAND--",
"--REMOVE-POST-EXEC-COMMAND--", "--REMOVE-POST-EXEC-COMMAND--",
"--REMOVE-PROMETHEUS-LABEL--", "--REMOVE-PROMETHEUS-LABEL--",
@ -2754,10 +2803,14 @@ Google Cloud storage: GOOGLE_PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS\n\
popup_text = None popup_text = None
tree = exclude_files_tree tree = exclude_files_tree
option_key = "backup_opts.exclude_files" option_key = "backup_opts.exclude_files"
elif "RETENTION-TAG" in event: elif "RETENTION-KEEP-TAG" in event:
popup_text = _t("config_gui.enter_tag") popup_text = _t("config_gui.enter_tag")
tree = retention_policy_tags_tree tree = retention_policy_keep_tags_tree
option_key = "repo_opts.retention_policy.tags" option_key = "repo_opts.retention_policy.keep_tags"
elif "RETENTION-APPLY-ON-TAG" in event:
popup_text = _t("config_gui.enter_tag")
tree = retention_policy_apply_on_tags_tree
option_key = "repo_opts.retention_policy.apply_on_tags"
elif "PRE-EXEC-COMMAND" in event: elif "PRE-EXEC-COMMAND" in event:
popup_text = _t("config_gui.enter_command") popup_text = _t("config_gui.enter_command")
tree = pre_exec_commands_tree tree = pre_exec_commands_tree

15
npbackup/restic_wrapper/__init__.py
View file

@ -378,8 +378,8 @@ class ResticRunner:
if not isinstance(output, str): if not isinstance(output, str):
logger.debug("Skipping output filter for non str output") logger.debug("Skipping output filter for non str output")
return output return output
for filter in restic_output_filters: for regex_filter in restic_output_filters:
output = filter.sub("", output) output = regex_filter.sub("", output)
return output return output
def executor( def executor(
@ -923,7 +923,7 @@ class ResticRunner:
return self.convert_to_json_output(result, output, msg=msg, **kwargs) return self.convert_to_json_output(result, output, msg=msg, **kwargs)
def snapshots( def snapshots(
self, id: str = None, errors_allowed: bool = False self, snapshot_id: str = None, errors_allowed: bool = False
) -> Union[bool, str, dict]: ) -> Union[bool, str, dict]:
""" """
Returns a list of snapshots Returns a list of snapshots
@ -935,8 +935,8 @@ class ResticRunner:
kwargs.pop("self") kwargs.pop("self")
cmd = "snapshots" cmd = "snapshots"
if id: if snapshot_id:
cmd += f" {id}" cmd += f" {snapshot_id}"
result, output = self.executor( result, output = self.executor(
cmd, timeout=FAST_COMMANDS_TIMEOUT, errors_allowed=errors_allowed cmd, timeout=FAST_COMMANDS_TIMEOUT, errors_allowed=errors_allowed
) )
@ -1206,6 +1206,11 @@ class ResticRunner:
for tag in value: for tag in value:
if tag: if tag:
cmd += f" --keep-tag {tag}" cmd += f" --keep-tag {tag}"
elif key == "apply-on-tags":
if isinstance(value, list):
for tag in value:
if tag:
cmd += f" --tag {tag}"
else: else:
cmd += f" --{key.replace('_', '-')} {value}" cmd += f" --{key.replace('_', '-')} {value}"
if group_by: if group_by:

1
npbackup/translations/config_gui.en.yml
View file

@ -128,6 +128,7 @@ en:
yearly: yearly snapshots yearly: yearly snapshots
keep_within: Keep snapshots within time period relative to current snapshot keep_within: Keep snapshots within time period relative to current snapshot
keep_tags: Keep snapshots with the following tags keep_tags: Keep snapshots with the following tags
apply_on_tagds: Apply only on snapshots with the following tags
post_backup_housekeeping_percent_chance: Post backup housekeeping run chance (%%) post_backup_housekeeping_percent_chance: Post backup housekeeping run chance (%%)
post_backup_housekeeping_percent_chance_explanation: Randomize housekeeping runs after backup (0-100%%, 0 = never, 100 = always) post_backup_housekeeping_percent_chance_explanation: Randomize housekeeping runs after backup (0-100%%, 0 = never, 100 = always)
post_backup_housekeeping_interval: Post backup housekeeping interval post_backup_housekeeping_interval: Post backup housekeeping interval

3
npbackup/translations/config_gui.fr.yml
View file

@ -128,7 +128,8 @@ fr:
monthly: instantanés mensuels monthly: instantanés mensuels
yearly: instantanés annuelles yearly: instantanés annuelles
keep_within: Garder les instantanées dans une période relative au dernier instantané keep_within: Garder les instantanées dans une période relative au dernier instantané
keep_tags: Garder les instantanés avec les tags suivants keep_tags: Garder les instantanés aux tags suivants
apply_on_tags: Appliquer uniquement sur les instantanés aux tags suivants
post_backup_housekeeping_percent_chance: Chance (%%) de lancer maintenance post-sauvegarde post_backup_housekeeping_percent_chance: Chance (%%) de lancer maintenance post-sauvegarde
post_backup_housekeeping_percent_chance_explanation: Rend aléatoire la maintenance après sauvegarde (0-100%%, 0 = jamais, 100 = toujours) post_backup_housekeeping_percent_chance_explanation: Rend aléatoire la maintenance après sauvegarde (0-100%%, 0 = jamais, 100 = toujours)
post_backup_housekeeping_interval: Intervalle de maintenance post-sauvegarde post_backup_housekeeping_interval: Intervalle de maintenance post-sauvegarde