netinvent/npbackup
1
1
Fork 0
mirror of https://github.com/netinvent/npbackup.git synced 2025-11-17 14:19:48 +08:00
Code Issues Projects Releases Packages Wiki Activity

Implement job scheduler as per #152

Browse source
This commit is contained in:
deajan 2025-03-07 17:22:29 +01:00
parent cc31c11a4a
commit b8fdcd5849
5 changed files with 154 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
npbackup/__main__.py
View file

@ -25,6 +25,7 @@ from npbackup.__version__ import version_string, version_dict
from npbackup.__debug__ import _DEBUG
from npbackup.common import execution_logs
from npbackup.core import upgrade_runner
from npbackup.core import jobs
from npbackup import key_management
from npbackup.task import create_scheduled_task
@ -553,7 +554,7 @@ This is free software, and you are welcome to redistribute it under certain cond
"auto_upgrade_percent_chance"
]
except KeyError:
auto_upgrade_percent_chance = 50
auto_upgrade_percent_chance = None
# TODO: Deprecated auto_upgrade_interval in favor of auto_upgrade_percent_chance
try:
@ -563,8 +564,8 @@ This is free software, and you are welcome to redistribute it under certain cond
if (
auto_upgrade
and upgrade_runner.need_upgrade(
auto_upgrade_percent_chance, auto_upgrade_interval
and jobs.schedule_on_chance_or_interval(
"auto_upgrade", auto_upgrade_percent_chance, auto_upgrade_interval
)
) or args.auto_upgrade:
if args.auto_upgrade:

2
npbackup/configuration.py
View file

@ -193,6 +193,7 @@ empty_config_dict = {
"post_exec_failure_is_fatal": False,
"post_exec_execute_even_on_backup_error": True,
"post_backup_housekeeping_percent_chance": 0, # 0 means disabled, 100 means always
"post_backup_housekeeping_interval": 0, # how many runs between a housekeeping after backup operation
},
"repo_opts": {
"repo_password": None,
@ -243,6 +244,7 @@ empty_config_dict = {
"global_options": {
"auto_upgrade": False,
"auto_upgrade_percent_chance": 5, # On all runs. On 15m interval runs, this could be 5% (ie once a day), on daily runs, this should be 95% (ie once a day)
"auto_upgrade_interval": 15, # How many NPBackup runs before an auto upgrade is attempted
"auto_upgrade_server_url": None,
"auto_upgrade_server_username": None,
"auto_upgrade_server_password": None,

134
npbackup/core/jobs.py Normal file
View file

@ -0,0 +1,134 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# This file is part of npbackup
__intname__ = "npbackup.gui.core.jobs"
__author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2022-2025 NetInvent"
__license__ = "GPL-3.0-only"
__build__ = "2025030701"
# This module helps scheduling jobs without using a daemon
# We can schedule on a random percentage or a fixed interval
import os
from typing import Optional
import tempfile
from logging import getLogger
from random import randint
from npbackup.path_helper import CURRENT_DIR
logger = getLogger()
def schedule_on_interval(job_name: str, interval: int) -> bool:
"""
Basic counter that returns true only every X times this is called
We need to make to select a write counter file that is writable
So we actually test a local file and a temp file (less secure for obvious reasons, ie tmp file deletions)
We just have to make sure that once we can write to one file, we stick to it unless proven otherwise
The for loop logic isn't straight simple, but allows file fallback
"""
if not interval:
logger.debug(f"No interval given for schedule: {interval}")
return False
try:
interval = int(interval)
except ValueError:
logger.error(f"No valid interval given for schedule: {interval}")
return False
# file counter, local, home, or temp if not available
counter_file = f"{__intname__}.{job_name}.log"
def _write_count(file: str, count: int) -> bool:
try:
with open(file, "w", encoding="utf-8") as fpw:
fpw.write(str(count))
return True
except OSError:
# We may not have write privileges, hence we need a backup plan
return False
def _get_count(file: str) -> Optional[int]:
try:
with open(file, "r", encoding="utf-8") as fp:
count = int(fp.read())
return count
except OSError as exc:
# We may not have read privileges
logger.error(f"Cannot read {job_name} counter file {file}: {exc}")
except ValueError as exc:
logger.error(f"Bogus {job_name} counter in {file}: {exc}")
return None
path_list = [
os.path.join(tempfile.gettempdir(), counter_file),
os.path.join(CURRENT_DIR, counter_file),
]
if os.name != "nt":
path_list = [os.path.join("/var/log", counter_file)] + path_list
else:
path_list = [os.path.join(r"C:\Windows\Temp", counter_file)] + path_list
for file in path_list:
if not os.path.isfile(file):
if _write_count(file, 1):
logger.debug(f"Initial job {job_name} counter written to {file}")
else:
logger.debug(f"Cannot write {job_name} counter file {file}")
continue
count = _get_count(file)
# Make sure we can write to the file before we make any assumptions
result = _write_count(file, count + 1)
if result:
if count >= interval:
# Reinitialize counter before we actually approve job run
if _write_count(file, 1):
logger.info(
f"schedule on inteval has decided {job_name} is required"
)
return True
break
else:
logger.debug(f"Cannot write {job_name} counter to {file}")
continue
return False
def schedule_on_chance(job_name: str, chance_percent: int) -> bool:
"""
Randomly decide if we need to run a job according to chance_percent
"""
if not chance_percent:
return False
try:
chance_percent = int(chance_percent)
except ValueError:
logger.error(
f"No valid chance percent given for schedule: {chance_percent}, job {job_name}"
)
return False
if randint(1, 100) <= chance_percent:
logger.debug(f"schedule on chance has decided {job_name} is required")
return True
return False
def schedule_on_chance_or_interval(
job_name: str, chance_percent: int, interval: int
) -> bool:
"""
Decide if we will run a job according to chance_percent or interval
"""
if schedule_on_chance(chance_percent, job_name) or schedule_on_interval(
interval, job_name
):
return True
return False

15
npbackup/core/runner.py
View file

@ -33,6 +33,7 @@ from npbackup.restic_metrics import (
)
from npbackup.restic_wrapper import ResticRunner
from npbackup.core.restic_source_binary import get_restic_internal_binary
from npbackup.core import jobs
from npbackup.path_helper import CURRENT_DIR, BASEDIR
from npbackup.__version__ import __intname__ as NAME, version_dict
from npbackup.__debug__ import _DEBUG, exception_to_string
@ -1534,7 +1535,13 @@ class NPBackupRunner:
post_backup_housekeeping_percent_chance = self.repo_config.g(
"backup_opts.post_backup_housekeeping_percent_chance"
)
if post_backup_housekeeping_percent_chance:
post_backup_housekeeping_interval = self.repo_config.g(
"backup_opts.post_backup_houskeeping_interval"
)
if (
post_backup_housekeeping_percent_chance
or post_backup_housekeeping_interval
):
post_backup_op = "housekeeping"
current_permissions = self.repo_config.g("permissions")
@ -1547,7 +1554,11 @@ class NPBackupRunner:
level="critical",
)
raise PermissionError
elif randint(1, 100) <= post_backup_housekeeping_percent_chance:
elif jobs.schedule_on_chance_or_interval(
"housekeeping-after-backup",
post_backup_housekeeping_percent_chance,
post_backup_housekeeping_interval,
):
self.write_logs("Running housekeeping after backup", level="info")
# Housekeeping after backup needs to run without threads
# We need to keep the queues open since we need to report back to GUI

102
npbackup/core/upgrade_runner.py
View file

@ -7,118 +7,18 @@ __intname__ = "npbackup.gui.core.upgrade_runner"
__author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2022-2025 NetInvent"
__license__ = "GPL-3.0-only"
__build__ = "2025030601"
__build__ = "2025030701"
import os
from typing import Optional
import tempfile
from logging import getLogger
from random import randint
from npbackup.upgrade_client.upgrader import auto_upgrader, _check_new_version
import npbackup.configuration
from npbackup.path_helper import CURRENT_DIR
logger = getLogger()
def _need_upgrade_interval(upgrade_interval: int) -> bool:
"""
TODO: Counter is now deprecated
Basic counter which allows an upgrade only every X times this is called so failed operations won't end in an endless upgrade loop
We need to make to select a write counter file that is writable
So we actually test a local file and a temp file (less secure for obvious reasons)
We just have to make sure that once we can write to one file, we stick to it unless proven otherwise
The for loop logic isn't straight simple, but allows file fallback
"""
# file counter, local, home, or temp if not available
counter_file = "npbackup.autoupgrade.log"
def _write_count(file: str, count: int) -> bool:
try:
with open(file, "w", encoding="utf-8") as fpw:
fpw.write(str(count))
return True
except OSError:
# We may not have write privileges, hence we need a backup plan
return False
def _get_count(file: str) -> Optional[int]:
try:
with open(file, "r", encoding="utf-8") as fp:
count = int(fp.read())
return count
except OSError as exc:
# We may not have read privileges
logger.error(f"Cannot read upgrade counter file {file}: {exc}")
except ValueError as exc:
logger.error(f"Bogus upgrade counter in {file}: {exc}")
return None
try:
upgrade_interval = int(upgrade_interval)
except ValueError:
logger.error("Bogus upgrade interval given. Will not upgrade")
return False
path_list = [
os.path.join(tempfile.gettempdir(), counter_file),
os.path.join(CURRENT_DIR, counter_file),
]
if os.name != "nt":
path_list = [os.path.join("/var/log", counter_file)] + path_list
for file in path_list:
if not os.path.isfile(file):
if _write_count(file, 1):
logger.debug("Initial upgrade counter written to %s", file)
else:
logger.debug("Cannot write to upgrade counter file %s", file)
continue
count = _get_count(file)
# Make sure we can write to the file before we make any assumptions
result = _write_count(file, count + 1)
if result:
if count >= upgrade_interval:
# Reinitialize upgrade counter before we actually approve upgrades
if _write_count(file, 1):
logger.info("Auto upgrade has decided upgrade check is required")
return True
break
else:
logger.debug("Cannot write upgrade counter to %s", file)
continue
return False
def _need_upgrade_percent(upgrade_percent: int) -> bool:
"""
Randomly decide if we need an upgrade according to upgrade_percent
"""
if not upgrade_percent:
return False
if randint(1, 100) <= upgrade_percent:
return True
return False
def need_upgrade(upgrade_percent: int, upgrade_interval: int) -> bool:
"""
Decide if we need an upgrade according to upgrade_interval and upgrade_percent
# TODO: Deprecate _need_upgrade_interval
"""
if _need_upgrade_percent(upgrade_percent) or _need_upgrade_interval(
upgrade_interval
):
return True
return False
def check_new_version(full_config: dict) -> bool:
upgrade_url = full_config.g("global_options.auto_upgrade_server_url")
username = full_config.g("global_options.auto_upgrade_server_username")