From b8fdcd5849e25b3b5295b79b9c5c046dc70fbda9 Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 7 Mar 2025 17:22:29 +0100 Subject: [PATCH] Implement job scheduler as per #152 --- npbackup/__main__.py | 7 +- npbackup/configuration.py | 2 + npbackup/core/jobs.py | 134 ++++++++++++++++++++++++++++++++ npbackup/core/runner.py | 15 +++- npbackup/core/upgrade_runner.py | 102 +----------------------- 5 files changed, 154 insertions(+), 106 deletions(-) create mode 100644 npbackup/core/jobs.py diff --git a/npbackup/__main__.py b/npbackup/__main__.py index 1ce41fc..245304a 100644 --- a/npbackup/__main__.py +++ b/npbackup/__main__.py @@ -25,6 +25,7 @@ from npbackup.__version__ import version_string, version_dict from npbackup.__debug__ import _DEBUG from npbackup.common import execution_logs from npbackup.core import upgrade_runner +from npbackup.core import jobs from npbackup import key_management from npbackup.task import create_scheduled_task @@ -553,7 +554,7 @@ This is free software, and you are welcome to redistribute it under certain cond "auto_upgrade_percent_chance" ] except KeyError: - auto_upgrade_percent_chance = 50 + auto_upgrade_percent_chance = None # TODO: Deprecated auto_upgrade_interval in favor of auto_upgrade_percent_chance try: @@ -563,8 +564,8 @@ This is free software, and you are welcome to redistribute it under certain cond if ( auto_upgrade - and upgrade_runner.need_upgrade( - auto_upgrade_percent_chance, auto_upgrade_interval + and jobs.schedule_on_chance_or_interval( + "auto_upgrade", auto_upgrade_percent_chance, auto_upgrade_interval ) ) or args.auto_upgrade: if args.auto_upgrade: diff --git a/npbackup/configuration.py b/npbackup/configuration.py index e87db83..30263cd 100644 --- a/npbackup/configuration.py +++ b/npbackup/configuration.py @@ -193,6 +193,7 @@ empty_config_dict = { "post_exec_failure_is_fatal": False, "post_exec_execute_even_on_backup_error": True, "post_backup_housekeeping_percent_chance": 0, # 0 means disabled, 100 means always + "post_backup_housekeeping_interval": 0, # how many runs between a housekeeping after backup operation }, "repo_opts": { "repo_password": None, @@ -243,6 +244,7 @@ empty_config_dict = { "global_options": { "auto_upgrade": False, "auto_upgrade_percent_chance": 5, # On all runs. On 15m interval runs, this could be 5% (ie once a day), on daily runs, this should be 95% (ie once a day) + "auto_upgrade_interval": 15, # How many NPBackup runs before an auto upgrade is attempted "auto_upgrade_server_url": None, "auto_upgrade_server_username": None, "auto_upgrade_server_password": None, diff --git a/npbackup/core/jobs.py b/npbackup/core/jobs.py new file mode 100644 index 0000000..8729d23 --- /dev/null +++ b/npbackup/core/jobs.py @@ -0,0 +1,134 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# +# This file is part of npbackup + +__intname__ = "npbackup.gui.core.jobs" +__author__ = "Orsiris de Jong" +__copyright__ = "Copyright (C) 2022-2025 NetInvent" +__license__ = "GPL-3.0-only" +__build__ = "2025030701" + +# This module helps scheduling jobs without using a daemon +# We can schedule on a random percentage or a fixed interval + + +import os +from typing import Optional +import tempfile +from logging import getLogger +from random import randint +from npbackup.path_helper import CURRENT_DIR + + +logger = getLogger() + + +def schedule_on_interval(job_name: str, interval: int) -> bool: + """ + Basic counter that returns true only every X times this is called + + We need to make to select a write counter file that is writable + So we actually test a local file and a temp file (less secure for obvious reasons, ie tmp file deletions) + We just have to make sure that once we can write to one file, we stick to it unless proven otherwise + + The for loop logic isn't straight simple, but allows file fallback + """ + if not interval: + logger.debug(f"No interval given for schedule: {interval}") + return False + + try: + interval = int(interval) + except ValueError: + logger.error(f"No valid interval given for schedule: {interval}") + return False + + # file counter, local, home, or temp if not available + counter_file = f"{__intname__}.{job_name}.log" + + def _write_count(file: str, count: int) -> bool: + try: + with open(file, "w", encoding="utf-8") as fpw: + fpw.write(str(count)) + return True + except OSError: + # We may not have write privileges, hence we need a backup plan + return False + + def _get_count(file: str) -> Optional[int]: + try: + with open(file, "r", encoding="utf-8") as fp: + count = int(fp.read()) + return count + except OSError as exc: + # We may not have read privileges + logger.error(f"Cannot read {job_name} counter file {file}: {exc}") + except ValueError as exc: + logger.error(f"Bogus {job_name} counter in {file}: {exc}") + return None + + path_list = [ + os.path.join(tempfile.gettempdir(), counter_file), + os.path.join(CURRENT_DIR, counter_file), + ] + if os.name != "nt": + path_list = [os.path.join("/var/log", counter_file)] + path_list + else: + path_list = [os.path.join(r"C:\Windows\Temp", counter_file)] + path_list + + for file in path_list: + if not os.path.isfile(file): + if _write_count(file, 1): + logger.debug(f"Initial job {job_name} counter written to {file}") + else: + logger.debug(f"Cannot write {job_name} counter file {file}") + continue + count = _get_count(file) + # Make sure we can write to the file before we make any assumptions + result = _write_count(file, count + 1) + if result: + if count >= interval: + # Reinitialize counter before we actually approve job run + if _write_count(file, 1): + logger.info( + f"schedule on inteval has decided {job_name} is required" + ) + return True + break + else: + logger.debug(f"Cannot write {job_name} counter to {file}") + continue + return False + + +def schedule_on_chance(job_name: str, chance_percent: int) -> bool: + """ + Randomly decide if we need to run a job according to chance_percent + """ + if not chance_percent: + return False + try: + chance_percent = int(chance_percent) + except ValueError: + logger.error( + f"No valid chance percent given for schedule: {chance_percent}, job {job_name}" + ) + return False + if randint(1, 100) <= chance_percent: + logger.debug(f"schedule on chance has decided {job_name} is required") + return True + return False + + +def schedule_on_chance_or_interval( + job_name: str, chance_percent: int, interval: int +) -> bool: + """ + Decide if we will run a job according to chance_percent or interval + """ + if schedule_on_chance(chance_percent, job_name) or schedule_on_interval( + interval, job_name + ): + return True + return False diff --git a/npbackup/core/runner.py b/npbackup/core/runner.py index 9604be0..2a07f80 100644 --- a/npbackup/core/runner.py +++ b/npbackup/core/runner.py @@ -33,6 +33,7 @@ from npbackup.restic_metrics import ( ) from npbackup.restic_wrapper import ResticRunner from npbackup.core.restic_source_binary import get_restic_internal_binary +from npbackup.core import jobs from npbackup.path_helper import CURRENT_DIR, BASEDIR from npbackup.__version__ import __intname__ as NAME, version_dict from npbackup.__debug__ import _DEBUG, exception_to_string @@ -1534,7 +1535,13 @@ class NPBackupRunner: post_backup_housekeeping_percent_chance = self.repo_config.g( "backup_opts.post_backup_housekeeping_percent_chance" ) - if post_backup_housekeeping_percent_chance: + post_backup_housekeeping_interval = self.repo_config.g( + "backup_opts.post_backup_houskeeping_interval" + ) + if ( + post_backup_housekeeping_percent_chance + or post_backup_housekeeping_interval + ): post_backup_op = "housekeeping" current_permissions = self.repo_config.g("permissions") @@ -1547,7 +1554,11 @@ class NPBackupRunner: level="critical", ) raise PermissionError - elif randint(1, 100) <= post_backup_housekeeping_percent_chance: + elif jobs.schedule_on_chance_or_interval( + "housekeeping-after-backup", + post_backup_housekeeping_percent_chance, + post_backup_housekeeping_interval, + ): self.write_logs("Running housekeeping after backup", level="info") # Housekeeping after backup needs to run without threads # We need to keep the queues open since we need to report back to GUI diff --git a/npbackup/core/upgrade_runner.py b/npbackup/core/upgrade_runner.py index 0f7f6f8..c2b5b96 100644 --- a/npbackup/core/upgrade_runner.py +++ b/npbackup/core/upgrade_runner.py @@ -7,118 +7,18 @@ __intname__ = "npbackup.gui.core.upgrade_runner" __author__ = "Orsiris de Jong" __copyright__ = "Copyright (C) 2022-2025 NetInvent" __license__ = "GPL-3.0-only" -__build__ = "2025030601" +__build__ = "2025030701" -import os -from typing import Optional -import tempfile from logging import getLogger from random import randint from npbackup.upgrade_client.upgrader import auto_upgrader, _check_new_version import npbackup.configuration -from npbackup.path_helper import CURRENT_DIR logger = getLogger() -def _need_upgrade_interval(upgrade_interval: int) -> bool: - """ - TODO: Counter is now deprecated - - Basic counter which allows an upgrade only every X times this is called so failed operations won't end in an endless upgrade loop - - We need to make to select a write counter file that is writable - So we actually test a local file and a temp file (less secure for obvious reasons) - We just have to make sure that once we can write to one file, we stick to it unless proven otherwise - - The for loop logic isn't straight simple, but allows file fallback - """ - - # file counter, local, home, or temp if not available - counter_file = "npbackup.autoupgrade.log" - - def _write_count(file: str, count: int) -> bool: - try: - with open(file, "w", encoding="utf-8") as fpw: - fpw.write(str(count)) - return True - except OSError: - # We may not have write privileges, hence we need a backup plan - return False - - def _get_count(file: str) -> Optional[int]: - try: - with open(file, "r", encoding="utf-8") as fp: - count = int(fp.read()) - return count - except OSError as exc: - # We may not have read privileges - logger.error(f"Cannot read upgrade counter file {file}: {exc}") - except ValueError as exc: - logger.error(f"Bogus upgrade counter in {file}: {exc}") - return None - - try: - upgrade_interval = int(upgrade_interval) - except ValueError: - logger.error("Bogus upgrade interval given. Will not upgrade") - return False - - path_list = [ - os.path.join(tempfile.gettempdir(), counter_file), - os.path.join(CURRENT_DIR, counter_file), - ] - if os.name != "nt": - path_list = [os.path.join("/var/log", counter_file)] + path_list - - for file in path_list: - if not os.path.isfile(file): - if _write_count(file, 1): - logger.debug("Initial upgrade counter written to %s", file) - else: - logger.debug("Cannot write to upgrade counter file %s", file) - continue - count = _get_count(file) - # Make sure we can write to the file before we make any assumptions - result = _write_count(file, count + 1) - if result: - if count >= upgrade_interval: - # Reinitialize upgrade counter before we actually approve upgrades - if _write_count(file, 1): - logger.info("Auto upgrade has decided upgrade check is required") - return True - break - else: - logger.debug("Cannot write upgrade counter to %s", file) - continue - return False - - -def _need_upgrade_percent(upgrade_percent: int) -> bool: - """ - Randomly decide if we need an upgrade according to upgrade_percent - """ - if not upgrade_percent: - return False - if randint(1, 100) <= upgrade_percent: - return True - return False - - -def need_upgrade(upgrade_percent: int, upgrade_interval: int) -> bool: - """ - Decide if we need an upgrade according to upgrade_interval and upgrade_percent - # TODO: Deprecate _need_upgrade_interval - """ - if _need_upgrade_percent(upgrade_percent) or _need_upgrade_interval( - upgrade_interval - ): - return True - return False - - def check_new_version(full_config: dict) -> bool: upgrade_url = full_config.g("global_options.auto_upgrade_server_url") username = full_config.g("global_options.auto_upgrade_server_username")