diff --git a/npbackup/__env__.py b/npbackup/__env__.py index daa446f..d38ee8c 100644 --- a/npbackup/__env__.py +++ b/npbackup/__env__.py @@ -30,3 +30,6 @@ HEARTBEAT_INTERVAL = 3600 # Arbitrary timeout for init / init checks. # If init takes more than a minute, we really have a problem in our backend FAST_COMMANDS_TIMEOUT = 180 + +# # Wait x seconds before we actually do the upgrade so current program could quit before being erased +UPGRADE_DEFER_TIME = 60 diff --git a/npbackup/__version__.py b/npbackup/__version__.py index 8bfeca5..717153c 100644 --- a/npbackup/__version__.py +++ b/npbackup/__version__.py @@ -28,15 +28,15 @@ try: CURRENT_USER = psutil.Process().username() except Exception: CURRENT_USER = "unknown" -version_string = f"{__intname__} v{__version__}-{'priv' if IS_PRIV_BUILD else 'pub'}-{sys.version_info[0]}.{sys.version_info[1]}-{python_arch()}{'-legacy' if IS_LEGACY else ''}{'-c' if IS_COMPILED else '-i'} {__build__} - {__copyright__} running as {CURRENT_USER}" version_dict = { "name": __intname__, "version": __version__, - "buildtype": "priv" if IS_PRIV_BUILD else "pub", + "build_type": "priv" if IS_PRIV_BUILD else "pub", "os": get_os_identifier(), - "arch": python_arch(), + "arch": python_arch() + ("-legacy" if IS_LEGACY else ""), "pv": sys.version_info, "comp": IS_COMPILED, "build": __build__, "copyright": __copyright__, } +version_string = f"{version_dict['name']} {version_dict['version']}-{version_dict['buildtype']}-{version_dict['pv'][0]}.{version_dict['pv'][1]}-{version_dict['arch']}{'-c' if IS_COMPILED else '-i'} {version_dict['build']} - {version_dict['copyright']} running as {CURRENT_USER}" diff --git a/npbackup/core/runner.py b/npbackup/core/runner.py index e84cba0..6d2a57a 100644 --- a/npbackup/core/runner.py +++ b/npbackup/core/runner.py @@ -33,7 +33,7 @@ from npbackup.restic_metrics import ( from npbackup.restic_wrapper import ResticRunner from npbackup.core.restic_source_binary import get_restic_internal_binary from npbackup.path_helper import CURRENT_DIR, BASEDIR -from npbackup.__version__ import __intname__ as NAME, version_dict as version_dict +from npbackup.__version__ import __intname__ as NAME, version_dict from npbackup.__debug__ import _DEBUG, exception_to_string @@ -53,7 +53,7 @@ def metric_writer( try: labels = { - "npversion": f"{NAME}{version_dict['version']}-{version_dict['buildtype']}" + "npversion": f"{NAME}{version_dict['version']}-{version_dict['build_type']}" } if repo_config.g("prometheus.metrics"): labels["instance"] = repo_config.g("prometheus.instance") diff --git a/npbackup/upgrade_client/upgrader.py b/npbackup/upgrade_client/upgrader.py index 412f505..2f4ac7e 100644 --- a/npbackup/upgrade_client/upgrader.py +++ b/npbackup/upgrade_client/upgrader.py @@ -26,12 +26,11 @@ from ofunctions.random import random_string from command_runner import deferred_command from npbackup.path_helper import CURRENT_DIR, CURRENT_EXECUTABLE from npbackup.core.nuitka_helper import IS_COMPILED -from npbackup.__version__ import __version__ as npbackup_version, IS_LEGACY +from npbackup.__version__ import version_dict +from npbackup.__env__ import UPGRADE_DEFER_TIME logger = getLogger() -UPGRADE_DEFER_TIME = 60 # Wait x seconds before we actually do the upgrade so current program could quit before being erased - # RAW ofunctions.checksum import def sha256sum_data(data): @@ -44,8 +43,40 @@ def sha256sum_data(data): return sha256.hexdigest() +def _get_target_id(auto_upgrade_host_identity: str, group: str) -> str: + """ + Get current target information string as + + {platform}/{arch}/{build_type}/{host_id}/{current_version}/{group} + """ + # We'll check python_arch instead of os_arch since we build 32 bit python executables for compat reasons + build_type = os.environ.get("NPBACKUP_BUILD_TYPE", None) + if not build_type: + logger.critical("Cannot determine build type for upgrade processs") + return False + target = "{}/{}/{}".format( + get_os(), + version_dict["arch"], + version_dict["build_type"], + version_dict["audience"], + ).lower() + try: + host_id = "{}/{}/{}".format( + auto_upgrade_host_identity, version_dict["version"], group + ) + target = "{}/{}".format(target, host_id) + except TypeError as exc: + logger.debug(f"No other information to add to target: {exc}") + return target + + def _check_new_version( - upgrade_url: str, username: str, password: str, ignore_errors: bool = False + upgrade_url: str, + username: str, + password: str, + ignore_errors: bool = False, + auto_upgrade_host_identity: str = None, + group: str = None, ) -> bool: """ Check if we have a newer version of npbackup @@ -56,7 +87,7 @@ def _check_new_version( logger.debug("Upgrade server not set") return None requestor = Requestor(upgrade_url, username, password) - requestor.app_name = "npbackup" + npbackup_version + requestor.app_name = "npbackup" + version_dict["version"] requestor.user_agent = __intname__ requestor.ignore_errors = ignore_errors requestor.create_session(authenticated=True) @@ -83,7 +114,10 @@ def _check_new_version( logger.error(msg) return None - result = requestor.data_model("current_version") + target_id = _get_target_id( + auto_upgrade_host_identity=auto_upgrade_host_identity, group=group + ) + result = requestor.data_model("current_version", id_record=target_id) if result is False: msg = "Upgrade server didn't respond properly. Is it well configured ?" if ignore_errors: @@ -101,21 +135,29 @@ def _check_new_version( logger.error(msg) return None else: - if online_version: - if version.parse(online_version) > version.parse(npbackup_version): - logger.info( - "Current version %s is older than online version %s", - npbackup_version, - online_version, - ) - return True - else: - logger.info( - "Current version %s is up-to-date (online version %s)", - npbackup_version, - online_version, - ) - return False + try: + if online_version: + if version.parse(online_version) > version.parse( + version_dict["version"] + ): + logger.info( + "Current version %s is older than online version %s", + version_dict["version"], + online_version, + ) + return True + else: + logger.info( + "Current version %s is up-to-date (online version %s)", + version_dict["version"], + online_version, + ) + return False + except Exception as exc: + logger.error( + f"Cannot determine if online version '{online_version}' is newer than current version {version_dict['verison']}: {exc}" + ) + return False def auto_upgrader( @@ -139,7 +181,12 @@ def auto_upgrader( return False res = _check_new_version( - upgrade_url, username, password, ignore_errors=ignore_errors + upgrade_url, + username, + password, + ignore_errors=ignore_errors, + auto_upgrade_host_identity=auto_upgrade_host_identity, + group=group, ) # Let's set a global environment variable which we can check later in metrics os.environ["NPBACKUP_UPGRADE_STATE"] = "0" @@ -148,24 +195,16 @@ def auto_upgrader( os.environ["NPBACKUP_UPGRADE_STATE"] = "1" return False requestor = Requestor(upgrade_url, username, password) - requestor.app_name = "npbackup" + npbackup_version + requestor.app_name = "npbackup" + version_dict["version"] requestor.user_agent = __intname__ requestor.create_session(authenticated=True) - # We'll check python_arch instead of os_arch since we build 32 bit python executables for compat reasons - arch = python_arch() if not IS_LEGACY else f"{python_arch()}-legacy" - build_type = os.environ.get("NPBACKUP_BUILD_TYPE", None) - if not build_type: - logger.critical("Cannot determine build type for upgrade processs") - return False - target = "{}/{}/{}".format(get_os(), arch, build_type).lower() - try: - host_id = "{}/{}/{}".format(auto_upgrade_host_identity, npbackup_version, group) - id_record = "{}/{}".format(target, host_id) - except TypeError: - id_record = target + # This allows to get the current running target identification for upgrade server to return the right file + target_id = _get_target_id( + auto_upgrade_host_identity=auto_upgrade_host_identity, group=group + ) - file_info = requestor.data_model("upgrades", id_record=id_record) + file_info = requestor.data_model("upgrades", id_record=target_id) if not file_info: logger.error("Server didn't provide a file description") return False @@ -179,7 +218,7 @@ def auto_upgrader( logger.info("No upgrade file found has been found for me :/") return True - file_data = requestor.requestor("download/" + id_record, raw=True) + file_data = requestor.requestor(f"download/{target_id}", raw=True) if not file_data: logger.error("Cannot get update file") return False diff --git a/upgrade_server/upgrade_server/api.py b/upgrade_server/upgrade_server/api.py index d90f59d..7232773 100644 --- a/upgrade_server/upgrade_server/api.py +++ b/upgrade_server/upgrade_server/api.py @@ -18,7 +18,15 @@ from argparse import ArgumentParser from fastapi import FastAPI, HTTPException, Response, Depends, status, Request, Header from fastapi.security import HTTPBasic, HTTPBasicCredentials from fastapi_offline import FastAPIOffline -from upgrade_server.models.files import FileGet, FileSend, Platform, Arch, BuildType +from upgrade_server.models.files import ( + ClientTargetIdentification, + FileGet, + FileSend, + Platform, + Arch, + BuildType, + Audience, +) from upgrade_server.models.oper import CurrentVersion import upgrade_server.crud as crud import upgrade_server.configuration as configuration @@ -91,9 +99,35 @@ async def api_status(): return {"app": __appname__, "maintenance": "enabled"} -@app.get("/current_version", response_model=CurrentVersion, status_code=200) +@app.get( + "/current_version/{platform}/{arch}/{build_type}/{audience}", + response_model=CurrentVersion, + status_code=200, +) +@app.get( + "/current_version/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}", + response_model=CurrentVersion, + status_code=200, +) +@app.get( + "/current_version/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}", + response_model=CurrentVersion, + status_code=200, +) +@app.get( + "/current_version/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}/{group}", + response_model=CurrentVersion, + status_code=200, +) async def current_version( request: Request, + platform: Platform, + arch: Arch, + build_type: BuildType, + audience: Audience, + auto_upgrade_host_identity: str = None, + installed_version: str = None, + group: str = None, x_real_ip: Optional[str] = Header(default=None), x_forwarded_for: Optional[str] = Header(default=None), referer: Optional[str] = Header(default=None), @@ -108,11 +142,13 @@ async def current_version( data = { "action": "check_version", "ip": client_ip, - "auto_upgrade_host_identity": "", - "installed_version": referer, - "group": "", - "platform": "", - "arch": "", + "auto_upgrade_host_identity": auto_upgrade_host_identity, + "installed_version": installed_version, + "group": group, + "platform": platform.value, + "arch": arch.value, + "build": build_type.value, + "audience": audience.value, } try: @@ -124,7 +160,16 @@ async def current_version( return CurrentVersion(version="0.00") try: - result = crud.get_current_version() + target_id = ClientTargetIdentification( + platform=platform, + arch=arch, + build_type=build_type, + audience=audience, + auto_upgrade_host_identity=auto_upgrade_host_identity, + installed_version=installed_version, + group=group, + ) + result = crud.get_current_version(target_id) if not result: raise HTTPException(status_code=404, detail="Not found") return result @@ -139,22 +184,22 @@ async def current_version( @app.get( - "/upgrades/{platform}/{arch}/{build_type}", + "/upgrades/{platform}/{arch}/{build_type}/{audience}", response_model=Union[FileSend, dict], status_code=200, ) @app.get( - "/upgrades/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}", + "/upgrades/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}", response_model=Union[FileSend, dict], status_code=200, ) @app.get( - "/upgrades/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}/{installed_version}", + "/upgrades/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}", response_model=Union[FileSend, dict], status_code=200, ) @app.get( - "/upgrades/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}/{installed_version}/{group}", + "/upgrades/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}/{group}", response_model=Union[FileSend, dict], status_code=200, ) @@ -163,6 +208,7 @@ async def upgrades( platform: Platform, arch: Arch, build_type: BuildType, + audience: Audience, auto_upgrade_host_identity: str = None, installed_version: str = None, group: str = None, @@ -184,7 +230,8 @@ async def upgrades( "group": group, "platform": platform.value, "arch": arch.value, - "build_type": build_type.value, + "build": build_type.value, + "audience": audience.value, } try: @@ -201,6 +248,7 @@ async def upgrades( platform=platform, arch=arch, build_type=build_type, + audience=audience, auto_upgrade_host_identity=auto_upgrade_host_identity, installed_version=installed_version, group=group, @@ -221,20 +269,22 @@ async def upgrades( @app.get( - "/download/{platform}/{arch}/{build_type}", response_model=FileSend, status_code=200 -) -@app.get( - "/download/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}", + "/download/{platform}/{arch}/{build_type}/{audience}", response_model=FileSend, status_code=200, ) @app.get( - "/download/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}/{installed_version}", + "/download/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}", response_model=FileSend, status_code=200, ) @app.get( - "/download/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}/{installed_version}/{group}", + "/download/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}", + response_model=FileSend, + status_code=200, +) +@app.get( + "/download/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}/{group}", response_model=FileSend, status_code=200, ) @@ -243,6 +293,7 @@ async def download( platform: Platform, arch: Arch, build_type: BuildType, + audience: Audience, auto_upgrade_host_identity: str = None, installed_version: str = None, group: str = None, @@ -264,7 +315,8 @@ async def download( "group": group, "platform": platform.value, "arch": arch.value, - "build_type": build_type.value, + "build": build_type.value, + "audience": audience.value, } try: diff --git a/upgrade_server/upgrade_server/crud.py b/upgrade_server/upgrade_server/crud.py index e67bffc..aaee702 100644 --- a/upgrade_server/upgrade_server/crud.py +++ b/upgrade_server/upgrade_server/crud.py @@ -7,16 +7,16 @@ __intname__ = "npbackup.upgrade_server.crud" __author__ = "Orsiris de Jong" __copyright__ = "Copyright (C) 2023-2025 NetInvent" __license__ = "GPL-3.0-only" -__build__ = "2025011401" +__build__ = "2025011601" import os -from typing import Optional, Union +from typing import Optional, Union, Tuple from logging import getLogger import hashlib from argparse import ArgumentParser from datetime import datetime, timezone -from upgrade_server.models.files import FileGet, FileSend +from upgrade_server.models.files import ClientTargetIdentification, FileGet, FileSend from upgrade_server.models.oper import CurrentVersion import upgrade_server.configuration as configuration @@ -57,6 +57,43 @@ def is_enabled() -> bool: return not os.path.isfile(path) +def _get_path_from_target_id(target_id: ClientTargetIdentification) -> Tuple[str, str]: + """ + Determine specific or generic upgrade path depending on target_id sent by client + + NPBackup filenames are + npbackup-{platform}-{arch}-{build_type}-{audience}.{archive_extension}" + + """ + if target_id.platform.value == "windows": + extension = "zip" + else: + extension = "tar.gz" + + expected_filename = f"npbackup-{target_id.platform.value}-{target_id.build_type.value}-{target_id.audience.value}.{extension}" + + base_path = os.path.join( + config_dict["upgrades"]["data_root"], + ) + + for posssible_sub_path in [ + target_id.auto_upgrade_host_identity, + target_id.installed_version, + target_id.group, + ]: + if posssible_sub_path: + possibile_sub_path = os.path.join(base_path, posssible_sub_path) + if os.path.isdir(possibile_sub_path): + logger.info(f"Found specific upgrade path in {possibile_sub_path}") + base_path = possibile_sub_path + break + + archive_path = os.path.join(base_path, expected_filename) + version_file_path = os.path.join(base_path, "VERSION") + + return version_file_path, archive_path + + def store_host_info(destination: str, host_id: dict) -> None: try: data = ( @@ -72,52 +109,34 @@ def store_host_info(destination: str, host_id: dict) -> None: logger.error("Trace:", exc_info=True) -def get_current_version() -> Optional[CurrentVersion]: +def get_current_version( + target_id: ClientTargetIdentification, +) -> Optional[CurrentVersion]: try: - path = os.path.join(config_dict["upgrades"]["data_root"], "VERSION") - if os.path.isfile(path): - with open(path, "r", encoding="utf-8") as fh: + version_filename, _ = _get_path_from_target_id(target_id) + if os.path.isfile(version_filename): + with open(version_filename, "r", encoding="utf-8") as fh: ver = fh.readline().strip() return CurrentVersion(version=ver) except OSError as exc: - logger.error("Cannot get current version") + logger.error(f"Cannot get current version: {exc}") logger.error("Trace:", exc_info=True) except Exception as exc: - logger.error("Version seems to be bogus in VERSION file") + logger.error(f"Version seems to be bogus in VERSION file: {exc}") logger.error("Trace:", exc_info=True) def get_file( file: FileGet, content: bool = False ) -> Optional[Union[FileSend, bytes, dict]]: - if file.platform.value == "windows": - extension = "zip" - else: - extension = "tar.gz" - possible_filename = f"npbackup-{file.build_type.value}.{extension}" - base_path = os.path.join( - config_dict["upgrades"]["data_root"], - file.platform.value, - file.arch.value, + + _, archive_path = _get_path_from_target_id(file) + + logger.info( + f"Searching for file {'info' if not content else 'content'} in {archive_path}" ) - - for posssible_sub_path in [ - file.auto_upgrade_host_identity, - file.installed_version, - file.group, - ]: - if posssible_sub_path: - possibile_sub_path = os.path.join(base_path, posssible_sub_path) - if os.path.isdir(possibile_sub_path): - logger.info(f"Found specific upgrade path in {possibile_sub_path}") - base_path = possibile_sub_path - break - - path = os.path.join(base_path, possible_filename) - - logger.info(f"Searching for file {'info' if not content else 'content'} in {path}") - if not os.path.isfile(path): - logger.info(f"No upgrade file found in {path}") + if not os.path.isfile(archive_path): + logger.info(f"No upgrade file found in {archive_path}") return { "arch": file.arch.value, "platform": file.platform.value, @@ -127,7 +146,7 @@ def get_file( "file_length": 0, } - with open(path, "rb") as fh: + with open(archive_path, "rb") as fh: bytes = fh.read() if content: return bytes @@ -138,7 +157,7 @@ def get_file( platform=file.platform.value, build_type=file.build_type.value, sha256sum=sha256, - filename=possible_filename, + filename=archive_path, file_length=length, ) return file_send diff --git a/upgrade_server/upgrade_server/models/files.py b/upgrade_server/upgrade_server/models/files.py index 1e9d69a..8fbf48b 100644 --- a/upgrade_server/upgrade_server/models/files.py +++ b/upgrade_server/upgrade_server/models/files.py @@ -7,7 +7,7 @@ __intname__ = "npbackup.upgrade_server.models.files" __author__ = "Orsiris de Jong" __copyright__ = "Copyright (C) 2023-2025 NetInvent" __license__ = "GPL-3.0-only" -__build__ = "2025011401" +__build__ = "2025011601" from typing import Optional @@ -30,20 +30,26 @@ class BuildType(Enum): cli = "cli" -class FileBase(BaseModel): +class Audience(Enum): + public = "public" + private = "private" + + +class ClientTargetIdentification(BaseModel): arch: Arch platform: Platform build_type: BuildType + audience: Audience auto_upgrade_host_identity: Optional[str] = None installed_version: Optional[str] = None group: Optional[str] = None -class FileGet(FileBase): +class FileGet(ClientTargetIdentification): pass -class FileSend(FileBase): +class FileSend(ClientTargetIdentification): sha256sum: constr(min_length=64, max_length=64) filename: str file_length: int