Merge pull request #6 from netinvent/upgrade_server

Fixes during upgrade server dev
This commit is contained in:
Orsiris de Jong 2023-02-01 19:15:24 +01:00 committed by GitHub
commit 71ff27ff87
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 279 additions and 125 deletions

View file

@ -3,7 +3,9 @@
- Aupgrade client integrated into NPBackup, that can be called manually via --auto-upgrade or automatically run every n backups
- Upgrade server which servers files and their metadata
- Improved setup.py to provide launch scripts for both Linux and Windows platforms
! Made windows floud file filter optional (enabled by default)
- Made windows cloud file filter optional (enabled by default)
- Added default configuration settings
- Updated restic binary to restic 0.15.1 compiled with go1.19.5
## v2.1.0 - 29/01/2023
- Added execution time information

View file

@ -27,9 +27,7 @@ Is there a repository at the following location?
Is there a repository at the following location?
rest:https://user:***@good.example.tld/user/
- Implement remote upgrade procedure
- Linux installer script
- Windows installer GUI
- Create task from GUI
- Make cloud file filters optional
- Default config parameters

View file

@ -59,8 +59,8 @@ env:
options:
auto_upgrade: true
auto_upgrade_server_url:
auto_upgrade_server_username:
auto_upgrade_server_password:
server_url:
server_username:
server_password:
# every 10 NPBackup runs, we'll try an autoupgrade. Never set this lower than 2 since failed upgrades will prevent backups from succeeding
auto_upgrade_interval: 10
interval: 10

View file

@ -39,7 +39,8 @@ from npbackup.gui.main import main_gui
from npbackup.core.runner import NPBackupRunner
from npbackup.core.i18n_helper import _t
from npbackup.path_helper import CURRENT_DIR, CURRENT_EXECUTABLE
from npbackup.upgrade_client.upgrader import auto_upgrader, need_upgrade
from npbackup.upgrade_client.upgrader import need_upgrade
from npbackup.core.upgrade_runner import run_upgrade
del sys.path[0]
@ -216,9 +217,13 @@ This is free software, and you are welcome to redistribute it under certain cond
parser.add_argument("--license", action="store_true", help="Show license")
parser.add_argument(
"--auto-upgrade", action="store_true", help="Auto upgrade NPBackup")
"--auto-upgrade", action="store_true", help="Auto upgrade NPBackup"
)
parser.add_argument(
"--upgrade-conf", action="store_true", help="Add new configuration elements after upgrade")
"--upgrade-conf",
action="store_true",
help="Add new configuration elements after upgrade",
)
args = parser.parse_args()
if args.version:
@ -324,36 +329,20 @@ This is free software, and you are welcome to redistribute it under certain cond
except KeyError:
auto_upgrade = True
try:
auto_upgrade_interval = config_dict["options"]["auto_upgrade_interval"]
auto_upgrade_interval = config_dict["options"]["ainterval"]
except KeyError:
auto_upgrade_interval = 10
if (auto_upgrade and need_upgrade(auto_upgrade_interval)) or args.auto_upgrade:
try:
auto_upgrade_upgrade_url = config_dict["options"]["auto_upgrade_server_url"]
auto_upgrade_username = config_dict["options"][
"auto_upgrade_server_username"
]
auto_upgrade_password = config_dict["options"][
"auto_upgrade_server_password"
]
except KeyError as exc:
logger.error("Missing auto upgrade info: %s, cannot launch auto upgrade", exc)
if args.auto_upgrade:
logger.info("Running user initiated auto upgrade")
else:
if args.auto_upgrade:
logger.info("Running user initiated auto upgrade")
else:
logger.info("Running program initiated auto upgrade")
result = auto_upgrader(
upgrade_url=auto_upgrade_upgrade_url,
username=auto_upgrade_username,
password=auto_upgrade_password,
)
if args.auto_upgrade:
if result:
sys.exit(0)
else:
sys.exit(23)
logger.info("Running program initiated auto upgrade")
result = run_upgrade(config_dict)
if result:
sys.exit(0)
elif args.auto_upgrade:
sys.exit(23)
dry_run = False
if args.dry_run:

View file

@ -37,11 +37,23 @@ ENCRYPTED_OPTIONS = [
{"section": "repo", "name": "password", "type": str},
{"section": "prometheus", "name": "http_username", "type": str},
{"section": "prometheus", "name": "http_password", "type": str},
{"section": "options", "name": "auto_upgrade_server_username", "type": str},
{"section": "options", "name": "auto_upgrade_server_password", "type": str},
{"section": "options", "name": "server_username", "type": str},
{"section": "options", "name": "server_password", "type": str},
]
empty_config_dict = {"backup": {}, "repo": {}, "prometheus": {}, "env": {}}
empty_config_dict = {
"backup": {
"compression": "auto",
"use_fs_snapshot": True,
"ignore_cloud_files": True,
"exclude_caches": True,
"priority": "low",
},
"repo": {"minimum_backup_age": 86400},
"prometheus": {},
"env": {},
"options": {},
}
def decrypt_data(config_dict):

View file

@ -1,8 +1,20 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# This file is part of npbackup
__intname__ = "npbackup.gui.core.restic_source_binary"
__author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2022-2023 NetInvent"
__license__ = "GPL-3.0-only"
__build__ = "2023011701"
import os
import glob
from npbackup.path_helper import BASEDIR
RESTIC_SOURCE_FILES_DIR = os.path.join(BASEDIR, os.pardir, "RESTIC_SOURCE_FILES")

View file

@ -253,6 +253,15 @@ class NPBackupRunner:
pass
except ValueError:
logger.warning("Bogus backup priority in config file.")
try:
if self.config_dict["backup"]["ignore_cloud_files"]:
self.restic_runner.ignore_cloud_files = self.config_dict["backup"][
"ignore_cloud_files"
]
except KeyError:
pass
except ValueError:
logger.warning("Bogus ignore_cloud_files value given")
self.restic_runner.stdout = self.stdout

View file

@ -0,0 +1,34 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# This file is part of npbackup
__intname__ = "npbackup.gui.core.upgrade_runner"
__author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2022-2023 NetInvent"
__license__ = "GPL-3.0-only"
__build__ = "2023011701"
from logging import getLogger
from npbackup.upgrade_client.upgrader import auto_upgrader
logger = getLogger(__intname__)
def run_upgrade(config_dict):
try:
auto_upgrade_upgrade_url = config_dict["options"]["server_url"]
auto_upgrade_username = config_dict["options"]["server_username"]
auto_upgrade_password = config_dict["options"]["server_password"]
except KeyError as exc:
logger.error("Missing auto upgrade info: %s, cannot launch auto upgrade", exc)
return False
else:
result = auto_upgrader(
upgrade_url=auto_upgrade_upgrade_url,
username=auto_upgrade_username,
password=auto_upgrade_password,
)
return result

View file

@ -7,14 +7,16 @@ __intname__ = "npbackup.gui.config"
__author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2022-2023 NetInvent"
__license__ = "GPL-3.0-only"
__build__ = "2023012601"
__build__ = "2023020101"
import sys
from logging import getLogger
import PySimpleGUI as sg
import npbackup.configuration as configuration
from npbackup.core.i18n_helper import _t
logger = getLogger(__intname__)
@ -43,6 +45,8 @@ def config_gui(config_dict: dict, config_file: str):
"http_password",
"repository",
"password",
"server_username",
"server_password",
]:
try:
if config_dict[section][entry] is None:
@ -68,7 +72,11 @@ def config_gui(config_dict: dict, config_file: str):
for key, value in values.items():
if value == ENCRYPTED_DATA_PLACEHOLDER:
continue
section, entry = key.split("---")
try:
section, entry = key.split("---")
except ValueError:
# Don't bother with keys that don't begin with "---"
continue
# check whether we need to split into list
if not isinstance(value, bool):
result = value.split("\n")
@ -96,7 +104,6 @@ def config_gui(config_dict: dict, config_file: str):
right_click_menu = ["", [_t("generic.decrypt")]]
backup_col = [
[sg.Text(_t("config_gui.backup"), font="helvetica 16")],
[
sg.Text(_t("config_gui.compression"), size=(30, 1)),
sg.Combo(["auto", "max", "off"], key="backup---compression", size=(48, 1)),
@ -117,7 +124,16 @@ def config_gui(config_dict: dict, config_file: str):
),
size=(30, 2),
),
sg.Checkbox("", key="backup---use_fs_snapshot", size=(0, 1)),
sg.Checkbox("", key="backup---use_fs_snapshot", size=(41, 1)),
],
[
sg.Text(
"{}\n({})".format(
_t("config_gui.ignore_cloud_files"), _t("config_gui.windows_only")
),
size=(30, 2),
),
sg.Checkbox("", key="backup---ignore_cloud_files", size=(41, 1)),
],
[
sg.Text(
@ -136,15 +152,15 @@ def config_gui(config_dict: dict, config_file: str):
),
size=(30, 2),
),
sg.Checkbox("", key="backup---exclude_case_ignore", size=(0, 1)),
sg.Checkbox("", key="backup---exclude_case_ignore", size=(41, 1)),
],
[
sg.Text(_t("config_gui.exclude_cache_dirs"), size=(30, 1)),
sg.Checkbox("", key="backup---exclude_caches", size=(0, 1)),
sg.Checkbox("", key="backup---exclude_caches", size=(41, 1)),
],
[
sg.Text(_t("config_gui.one_file_system"), size=(30, 1)),
sg.Checkbox("", key="backup---one_file_system", size=(0, 1)),
sg.Checkbox("", key="backup---one_file_system", size=(41, 1)),
],
[
sg.Text(_t("config_gui.pre_exec_command"), size=(30, 1)),
@ -156,7 +172,7 @@ def config_gui(config_dict: dict, config_file: str):
],
[
sg.Text(_t("config_gui.exec_failure_is_fatal"), size=(30, 1)),
sg.Checkbox("", key="backup---pre_exec_failure_is_fatal", size=(0, 1)),
sg.Checkbox("", key="backup---pre_exec_failure_is_fatal", size=(41, 1)),
],
[
sg.Text(_t("config_gui.post_exec_command"), size=(30, 1)),
@ -168,7 +184,7 @@ def config_gui(config_dict: dict, config_file: str):
],
[
sg.Text(_t("config_gui.exec_failure_is_fatal"), size=(30, 1)),
sg.Checkbox("", key="backup---post_exec_failure_is_fatal", size=(0, 1)),
sg.Checkbox("", key="backup---post_exec_failure_is_fatal", size=(41, 1)),
],
[
sg.Text(
@ -188,7 +204,6 @@ def config_gui(config_dict: dict, config_file: str):
]
repo_col = [
[sg.Text(_t("config_gui.backup_destination"), font="helvetica 16")],
[
sg.Text(
"{}\n({})".format(
@ -220,12 +235,11 @@ def config_gui(config_dict: dict, config_file: str):
],
]
options_col = [
[sg.Text(_t("config_gui.prometheus_config"), font="helvetica 16")],
prometheus_col = [
[sg.Text(_t("config_gui.explanation"))],
[
sg.Text(_t("config_gui.enable_prometheus"), size=(30, 1)),
sg.Checkbox("", key="prometheus---metrics", size=(0, 1)),
sg.Checkbox("", key="prometheus---metrics", size=(41, 1)),
],
[
sg.Text(_t("config_gui.job_name"), size=(30, 1)),
@ -265,7 +279,6 @@ def config_gui(config_dict: dict, config_file: str):
]
env_col = [
[sg.Text(_t("config_gui.environment_variables"), font="helvetica 16")],
[
sg.Text(
"{}\n({}\n{})".format(
@ -279,6 +292,29 @@ def config_gui(config_dict: dict, config_file: str):
],
]
options_col = [
[
sg.Text(_t("config_gui.auto_upgrade"), size=(30, 1)),
sg.Checkbox("", key="options---auto_upgrade", size=(41, 1)),
],
[
sg.Text(_t("config_gui.auto_upgrade_server_url"), size=(30, 1)),
sg.Input(key="options---server_url", size=(50, 1)),
],
[
sg.Text(_t("config_gui.auto_upgrade_server_username"), size=(30, 1)),
sg.Input(key="options---server_username", size=(50, 1)),
],
[
sg.Text(_t("config_gui.auto_upgrade_server_password"), size=(30, 1)),
sg.Input(key="options---server_password", size=(50, 1)),
],
[
sg.Text(_t("config_gui.auto_upgrade_interval"), size=(30, 1)),
sg.Input(key="options---interval", size=(50, 1)),
],
]
buttons = [
[
sg.Text(" " * 135),
@ -287,66 +323,63 @@ def config_gui(config_dict: dict, config_file: str):
]
]
layout = [
tab_group_layout = [
[
sg.Column(
[
[
sg.Column(
backup_col,
element_justification="L",
scrollable=False,
vertical_scroll_only=False,
size=(620, 610),
vertical_alignment="top",
),
]
],
vertical_alignment="top",
),
sg.Column(
[
[
sg.Column(
repo_col,
element_justification="L",
scrollable=False,
vertical_scroll_only=False,
size=(620, 210),
vertical_alignment="top",
)
],
[
sg.Column(
options_col,
element_justification="L",
scrollable=False,
vertical_scroll_only=False,
size=(620, 300),
vertical_alignment="top",
)
],
[
sg.Column(
env_col,
element_justification="L",
scrollable=False,
vertical_scroll_only=False,
size=(620, 100),
vertical_alignment="top",
)
],
],
vertical_alignment="top",
),
sg.Tab(
_t("config_gui.backup"),
backup_col,
font="helvetica 16",
key="--tab-backup--",
element_justification="C",
)
],
[
sg.Tab(
_t("config_gui.backup_destination"),
repo_col,
font="helvetica 16",
key="--tab-repo--",
element_justification="C",
)
],
[
sg.Tab(
_t("config_gui.prometheus_config"),
prometheus_col,
font="helvetica 16",
key="--tab-prometheus--",
element_justification="C",
)
],
[
sg.Tab(
_t("config_gui.environment_variables"),
env_col,
font="helvetica 16",
key="--tab-env--",
element_justification="C",
)
],
[
sg.Tab(
_t("generic.options"),
options_col,
font="helvetica 16",
key="--tab-options--",
element_justification="C",
)
],
]
layout = [
[sg.TabGroup(tab_group_layout, enable_events=True, key="--tabgroup--")],
[sg.Column(buttons, element_justification="C")],
]
window = sg.Window(
"Configuration",
layout,
text_justification="L",
text_justification="C",
auto_size_text=True,
auto_size_buttons=False,
no_titlebar=False,

View file

@ -7,10 +7,11 @@ __intname__ = "npbackup.gui.main"
__author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2022-2023 NetInvent"
__license__ = "GPL-3.0-only"
__build__ = "2023012501"
__build__ = "2023020101"
from typing import List, Optional, Tuple
import sys
import os
from logging import getLogger
import re
@ -33,6 +34,7 @@ from npbackup.customization import (
from npbackup.gui.config import config_gui
from npbackup.core.runner import NPBackupRunner
from npbackup.core.i18n_helper import _t
from npbackup.core.upgrade_runner import run_upgrade
logger = getLogger(__intname__)
@ -42,7 +44,7 @@ logger = getLogger(__intname__)
THREAD_SHARED_DICT = {}
def _about_gui(version_string: str) -> None:
def _about_gui(version_string: str, config_dict: dict) -> None:
license_content = LICENSE_TEXT
try:
with open(LICENSE_FILE, "r") as file_handle:
@ -52,16 +54,34 @@ def _about_gui(version_string: str) -> None:
layout = [
[sg.Text(version_string)],
[
sg.Button(
_t("config_gui.auto_upgrade_launch"), key="autoupgrade", size=(12, 2)
)
],
[sg.Text("License: GNU GPLv3")],
[sg.Multiline(license_content, size=(65, 20))],
[sg.Button(_t("generic.accept"), key="exit")],
]
window = sg.Window(_t("generic.about"), layout, keep_on_top=True)
window = sg.Window(
_t("generic.about"), layout, keep_on_top=True, element_justification="C"
)
while True:
event, _ = window.read()
if event in [sg.WIN_CLOSED, "exit"]:
break
elif event == "autoupgrade":
result = sg.PopupOKCancel(
_t("config_gui.auto_ugprade_will_quit"), keep_on_top=True
)
if result == "OK":
logger.info("Running GUI initiated upgrade")
sub_result = run_upgrade(config_dict)
if sub_result:
sys.exit(0)
else:
sg.Popup(_t("config_gui.auto_upgrade_failed"))
window.close()
@ -574,7 +594,7 @@ def main_gui(config_dict: dict, config_file: str, version_string: str):
except (TypeError, KeyError):
sg.PopupNoFrame(_t("main_gui.unknown_repo"))
if event == "about":
_about_gui(version_string)
_about_gui(version_string, config_dict)
# Update GUI on every window.read timeout = every minute or everytime an event happens, including the "uptodate" button
current_state, snapshot_list = get_gui_data(config_dict)

View file

@ -54,6 +54,7 @@ class ResticRunner:
self._backend_type = backend.lower()
else:
self._backend_type = "local"
self._ignore_cloud_files = True
self._addition_parameters = None
self._environment_variables = {}
@ -127,6 +128,17 @@ class ResticRunner:
else:
raise ValueError("Bogus dry run value givne")
@property
def ignore_cloud_files(self) -> bool:
return self._ignore_cloud_files
@ignore_cloud_files.setter
def ignore_cloud_files(self, value):
if isinstance(value, bool):
self._ignore_cloud_files = value
else:
raise ValueError("Bogus ignore_cloud_files value given")
@property
def exec_time(self) -> Optional[int]:
return self._exec_time
@ -191,7 +203,7 @@ class ResticRunner:
if exit_code == 0:
self.last_command_status = True
return True, output
if exit_code == 3 and os.name == "nt":
if exit_code == 3 and os.name == "nt" and self.ignore_cloud_files:
# TEMP-FIX-4155, since we don't have reparse point support for Windows, see https://github.com/restic/restic/issues/4155, we have to filter manually for cloud errors which should not affect backup result
# exit_code = 3 when errors are present but snapshot could be created
# Since errors are always shown, we don't need restic --verbose option explicitly

View file

@ -4,6 +4,7 @@ en:
compression: Compression
backup_paths: Backup paths
use_fs_snapshot: Use VSS snapshots
ignore_cloud_files: Ignore in-cloud files
windows_only: Windows only
exclude_files: Exclude files
exclude_case_ignore: Ignore case for excludes
@ -48,4 +49,13 @@ en:
configuration_saved: Configuration saved
enter_backup_admin_password: Enter backup administrator password
wrong_password: Wrong password
wrong_password: Wrong password
auto_upgrade: Auto upgrade
auto_upgrade_server_url: Server URL
auto_upgrade_server_username: Server username
auto_upgrade_server_password: Server password
auto_upgrade_interval: Auto upgrade interval
auto_upgrade_launch: Launch auto upgrade
auto_ugprade_will_quit: Warning, launching an upgrade procedure will quit this program without notice. You will have to wait 5 minutes before launching it again for the upgrade to complete
auto_upgrade_failed: Auto upgrade procedure failed, see logs for further details

View file

@ -4,11 +4,12 @@ fr:
compression: Compression
backup_paths: Chemins à sauvegarder
use_fs_snapshot: Utiliser les instantanés VSS
ignore_cloud_files: Exclure le fichiers dans le cloud
windows_only: Windows seulement
exclude_files: Fichiers d'exclusion
exclude_case_ignore: Ignorer la casse aux exclusions
windows_always: toujours actif pour Windows
exclude_cache_dirs: Exclude dossiers cache
exclude_cache_dirs: Exclure dossiers cache
one_file_system: Ne pas suivre les points de montage
pre_exec_command: Commande pré-sauvegarde
maximum_exec_time: Temps maximal d'execution
@ -21,7 +22,7 @@ fr:
backup_destination: Destination de sauvegarde
maximum_backup_age: Age minimal d'une sauvegarde
backup_repo_uri: URL / chemin dépot de sauvegarde
backup_repo_uri: URL / chemin local dépot de sauvegarde
backup_repo_password: Mot de passe dépot de sauvegarde
upload_speed: Vitesse limite de téléversement (KB/s)
download_speed: Vitesse limite de téléchargement (KB/s)
@ -48,4 +49,13 @@ fr:
configuration_saved: Configuration sauvegardée
enter_backup_admin_password: Veuillez entrer le mot de passe administrateur de sauvegarde
wrong_password: Mot de passe érroné
wrong_password: Mot de passe érroné
auto_upgrade: Mise à niveau
auto_upgrade_server_url: Serveur de mise à niveau
auto_upgrade_server_username: Nom d'utilisateur serveur
auto_upgrade_server_password: Mot de passe serveur
auto_upgrade_interval: Intervalle de mise à niveau
auto_upgrade_launch: Lancer une mise à niveau
auto_ugprade_will_quit: Attnetion, la procédure de mise à niveau va quitter ce programme sans notification. Vous devrez attendre 5 minutes pour laisser la procédure se terminer avant de relancer le programme
auto_upgrade_failed: Procédure de mise à niveau échouée, veuillez consulter les journaux pour plus de détails

View file

@ -4,6 +4,7 @@ en:
quit: Exit
configure: Configure
about: About
options: Options
_yes: Yes
_no: No

View file

@ -4,6 +4,7 @@ fr:
quit: Quitter
configure: Configurer
about: A propos
options: Options
_yes: Oui
_no: Non

View file

@ -40,7 +40,7 @@ def sha256sum_data(data):
def need_upgrade(upgrade_interval: int) -> bool:
"""
Basic counter which allows an upgrade only every X times this is called so failed operations won't end in an endless upgrade loop
We need to make to select a write counter file that is writable
So we actually test a local file and a temp file (less secure for obvious reasons)
We just have to make sure that once we can write to one file, we stick to it unless proven otherwise
@ -48,11 +48,11 @@ def need_upgrade(upgrade_interval: int) -> bool:
The for loop logic isn't straight simple, but allows file fallback
"""
# file counter, local, home, or temp if not available
counter_file = 'npbackup.autoupgrade.log'
counter_file = "npbackup.autoupgrade.log"
def _write_count(file: str, counter: int) -> bool:
try:
with open(file, 'w') as fpw:
with open(file, "w") as fpw:
fpw.write(str(counter))
return True
except OSError:
@ -61,7 +61,7 @@ def need_upgrade(upgrade_interval: int) -> bool:
def _get_count(file: str) -> Optional[int]:
try:
with open(file, 'r') as fpr:
with open(file, "r") as fpr:
count = int(fpr.read())
return count
except OSError:
@ -71,12 +71,13 @@ def need_upgrade(upgrade_interval: int) -> bool:
logger.error("Bogus upgrade counter in %s", file)
return None
for file in [os.path.join(CURRENT_DIR, counter_file), os.path.join(tempfile.gettempdir(), counter_file)]:
for file in [
os.path.join(CURRENT_DIR, counter_file),
os.path.join(tempfile.gettempdir(), counter_file),
]:
if not os.path.isfile(file):
if _write_count(file, 1):
logger.debug('Initial upgrade counter written to %s', file)
logger.debug("Initial upgrade counter written to %s", file)
else:
logger.debug("Cannot write to upgrade counter file %s", file)
continue
@ -170,11 +171,21 @@ def auto_upgrader(upgrade_url: str, username: str, password: str) -> bool:
logger.info("Upgrade file written to %s", executable)
log_file = os.path.join(tempfile.gettempdir(), file_info["filename"] + ".log")
logger.info("Logging upgrade to %s", log_file)
# Actual upgrade process
new_executable = os.path.join(CURRENT_DIR, os.path.basename(CURRENT_EXECUTABLE))
cmd = 'del "{}"; move "{}" "{}"; del "{}" > {}'.format(
CURRENT_EXECUTABLE, executable, new_executable, executable, log_file
cmd = 'del "{}" > "{}" && move "{}" "{}" >> "{}" && del "{}" >> "{}" && "{}" --upgrade-conf >> "{}"'.format(
CURRENT_EXECUTABLE,
log_file,
executable,
log_file,
new_executable,
log_file,
executable,
log_file,
new_executable,
log_file,
)
logger.info(
"Launching upgrade. Current process will quit. Upgrade starts in %s seconds. Upgrade is done by OS logged in %s",