upgrade_server: Allow simpler paths for updates, include audience in decision

npbackup/__env__.py

@@ -30,3 +30,6 @@ HEARTBEAT_INTERVAL = 3600
 # Arbitrary timeout for init / init checks.
 # If init takes more than a minute, we really have a problem in our backend
 FAST_COMMANDS_TIMEOUT = 180
+
+# # Wait x seconds before we actually do the upgrade so current program could quit before being erased
+UPGRADE_DEFER_TIME = 60

npbackup/__version__.py

@@ -28,15 +28,15 @@ try:
     CURRENT_USER = psutil.Process().username()
 except Exception:
     CURRENT_USER = "unknown"
-version_string = f"{__intname__} v{__version__}-{'priv' if IS_PRIV_BUILD else 'pub'}-{sys.version_info[0]}.{sys.version_info[1]}-{python_arch()}{'-legacy' if IS_LEGACY else ''}{'-c' if IS_COMPILED else '-i'} {__build__} - {__copyright__} running as {CURRENT_USER}"
 version_dict = {
     "name": __intname__,
     "version": __version__,
-    "buildtype": "priv" if IS_PRIV_BUILD else "pub",
+    "build_type": "priv" if IS_PRIV_BUILD else "pub",
     "os": get_os_identifier(),
-    "arch": python_arch(),
+    "arch": python_arch() + ("-legacy" if IS_LEGACY else ""),
     "pv": sys.version_info,
     "comp": IS_COMPILED,
     "build": __build__,
     "copyright": __copyright__,
 }
+version_string = f"{version_dict['name']} {version_dict['version']}-{version_dict['buildtype']}-{version_dict['pv'][0]}.{version_dict['pv'][1]}-{version_dict['arch']}{'-c' if IS_COMPILED else '-i'} {version_dict['build']} - {version_dict['copyright']} running as {CURRENT_USER}"

npbackup/core/runner.py

@@ -33,7 +33,7 @@ from npbackup.restic_metrics import (
 from npbackup.restic_wrapper import ResticRunner
 from npbackup.core.restic_source_binary import get_restic_internal_binary
 from npbackup.path_helper import CURRENT_DIR, BASEDIR
-from npbackup.__version__ import __intname__ as NAME, version_dict as version_dict
+from npbackup.__version__ import __intname__ as NAME, version_dict
 from npbackup.__debug__ import _DEBUG, exception_to_string
 
 
@@ -53,7 +53,7 @@ def metric_writer(
 
     try:
         labels = {
-            "npversion": f"{NAME}{version_dict['version']}-{version_dict['buildtype']}"
+            "npversion": f"{NAME}{version_dict['version']}-{version_dict['build_type']}"
         }
         if repo_config.g("prometheus.metrics"):
             labels["instance"] = repo_config.g("prometheus.instance")

npbackup/upgrade_client/upgrader.py

@@ -26,12 +26,11 @@ from ofunctions.random import random_string
 from command_runner import deferred_command
 from npbackup.path_helper import CURRENT_DIR, CURRENT_EXECUTABLE
 from npbackup.core.nuitka_helper import IS_COMPILED
-from npbackup.__version__ import __version__ as npbackup_version, IS_LEGACY
+from npbackup.__version__ import version_dict
+from npbackup.__env__ import UPGRADE_DEFER_TIME
 
 logger = getLogger()
 
-UPGRADE_DEFER_TIME = 60  # Wait x seconds before we actually do the upgrade so current program could quit before being erased
-
 
 # RAW ofunctions.checksum import
 def sha256sum_data(data):
@@ -44,8 +43,40 @@ def sha256sum_data(data):
     return sha256.hexdigest()
 
 
+def _get_target_id(auto_upgrade_host_identity: str, group: str) -> str:
+    """
+    Get current target information string as
+
+    {platform}/{arch}/{build_type}/{host_id}/{current_version}/{group}
+    """
+    # We'll check python_arch instead of os_arch since we build 32 bit python executables for compat reasons
+    build_type = os.environ.get("NPBACKUP_BUILD_TYPE", None)
+    if not build_type:
+        logger.critical("Cannot determine build type for upgrade processs")
+        return False
+    target = "{}/{}/{}".format(
+        get_os(),
+        version_dict["arch"],
+        version_dict["build_type"],
+        version_dict["audience"],
+    ).lower()
+    try:
+        host_id = "{}/{}/{}".format(
+            auto_upgrade_host_identity, version_dict["version"], group
+        )
+        target = "{}/{}".format(target, host_id)
+    except TypeError as exc:
+        logger.debug(f"No other information to add to target: {exc}")
+    return target
+
+
 def _check_new_version(
-    upgrade_url: str, username: str, password: str, ignore_errors: bool = False
+    upgrade_url: str,
+    username: str,
+    password: str,
+    ignore_errors: bool = False,
+    auto_upgrade_host_identity: str = None,
+    group: str = None,
 ) -> bool:
     """
     Check if we have a newer version of npbackup
@@ -56,7 +87,7 @@ def _check_new_version(
         logger.debug("Upgrade server not set")
         return None
     requestor = Requestor(upgrade_url, username, password)
-    requestor.app_name = "npbackup" + npbackup_version
+    requestor.app_name = "npbackup" + version_dict["version"]
     requestor.user_agent = __intname__
     requestor.ignore_errors = ignore_errors
     requestor.create_session(authenticated=True)
@@ -83,7 +114,10 @@ def _check_new_version(
             logger.error(msg)
         return None
 
-    result = requestor.data_model("current_version")
+    target_id = _get_target_id(
+        auto_upgrade_host_identity=auto_upgrade_host_identity, group=group
+    )
+    result = requestor.data_model("current_version", id_record=target_id)
     if result is False:
         msg = "Upgrade server didn't respond properly. Is it well configured ?"
         if ignore_errors:
@@ -101,21 +135,29 @@ def _check_new_version(
             logger.error(msg)
         return None
     else:
-        if online_version:
+        try:
-            if version.parse(online_version) > version.parse(npbackup_version):
+            if online_version:
-                logger.info(
+                if version.parse(online_version) > version.parse(
-                    "Current version %s is older than online version %s",
+                    version_dict["version"]
-                    npbackup_version,
+                ):
-                    online_version,
+                    logger.info(
-                )
+                        "Current version %s is older than online version %s",
-                return True
+                        version_dict["version"],
-            else:
+                        online_version,
-                logger.info(
+                    )
-                    "Current version %s is up-to-date (online version %s)",
+                    return True
-                    npbackup_version,
+                else:
-                    online_version,
+                    logger.info(
-                )
+                        "Current version %s is up-to-date (online version %s)",
-                return False
+                        version_dict["version"],
+                        online_version,
+                    )
+                    return False
+        except Exception as exc:
+            logger.error(
+                f"Cannot determine if online version '{online_version}' is newer than current version {version_dict['verison']}: {exc}"
+            )
+            return False
 
 
 def auto_upgrader(
@@ -139,7 +181,12 @@ def auto_upgrader(
         return False
 
     res = _check_new_version(
-        upgrade_url, username, password, ignore_errors=ignore_errors
+        upgrade_url,
+        username,
+        password,
+        ignore_errors=ignore_errors,
+        auto_upgrade_host_identity=auto_upgrade_host_identity,
+        group=group,
     )
     # Let's set a global environment variable which we can check later in metrics
     os.environ["NPBACKUP_UPGRADE_STATE"] = "0"
@@ -148,24 +195,16 @@ def auto_upgrader(
             os.environ["NPBACKUP_UPGRADE_STATE"] = "1"
         return False
     requestor = Requestor(upgrade_url, username, password)
-    requestor.app_name = "npbackup" + npbackup_version
+    requestor.app_name = "npbackup" + version_dict["version"]
     requestor.user_agent = __intname__
     requestor.create_session(authenticated=True)
 
-    # We'll check python_arch instead of os_arch since we build 32 bit python executables for compat reasons
+    # This allows to get the current running target identification for upgrade server to return the right file
-    arch = python_arch() if not IS_LEGACY else f"{python_arch()}-legacy"
+    target_id = _get_target_id(
-    build_type = os.environ.get("NPBACKUP_BUILD_TYPE", None)
+        auto_upgrade_host_identity=auto_upgrade_host_identity, group=group
-    if not build_type:
+    )
-        logger.critical("Cannot determine build type for upgrade processs")
-        return False
-    target = "{}/{}/{}".format(get_os(), arch, build_type).lower()
-    try:
-        host_id = "{}/{}/{}".format(auto_upgrade_host_identity, npbackup_version, group)
-        id_record = "{}/{}".format(target, host_id)
-    except TypeError:
-        id_record = target
 
-    file_info = requestor.data_model("upgrades", id_record=id_record)
+    file_info = requestor.data_model("upgrades", id_record=target_id)
     if not file_info:
         logger.error("Server didn't provide a file description")
         return False
@@ -179,7 +218,7 @@ def auto_upgrader(
         logger.info("No upgrade file found has been found for me :/")
         return True
 
-    file_data = requestor.requestor("download/" + id_record, raw=True)
+    file_data = requestor.requestor(f"download/{target_id}", raw=True)
     if not file_data:
         logger.error("Cannot get update file")
         return False

upgrade_server/upgrade_server/api.py

@@ -18,7 +18,15 @@ from argparse import ArgumentParser
 from fastapi import FastAPI, HTTPException, Response, Depends, status, Request, Header
 from fastapi.security import HTTPBasic, HTTPBasicCredentials
 from fastapi_offline import FastAPIOffline
-from upgrade_server.models.files import FileGet, FileSend, Platform, Arch, BuildType
+from upgrade_server.models.files import (
+    ClientTargetIdentification,
+    FileGet,
+    FileSend,
+    Platform,
+    Arch,
+    BuildType,
+    Audience,
+)
 from upgrade_server.models.oper import CurrentVersion
 import upgrade_server.crud as crud
 import upgrade_server.configuration as configuration
@@ -91,9 +99,35 @@ async def api_status():
         return {"app": __appname__, "maintenance": "enabled"}
 
 
-@app.get("/current_version", response_model=CurrentVersion, status_code=200)
+@app.get(
+    "/current_version/{platform}/{arch}/{build_type}/{audience}",
+    response_model=CurrentVersion,
+    status_code=200,
+)
+@app.get(
+    "/current_version/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}",
+    response_model=CurrentVersion,
+    status_code=200,
+)
+@app.get(
+    "/current_version/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}",
+    response_model=CurrentVersion,
+    status_code=200,
+)
+@app.get(
+    "/current_version/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}/{group}",
+    response_model=CurrentVersion,
+    status_code=200,
+)
 async def current_version(
     request: Request,
+    platform: Platform,
+    arch: Arch,
+    build_type: BuildType,
+    audience: Audience,
+    auto_upgrade_host_identity: str = None,
+    installed_version: str = None,
+    group: str = None,
     x_real_ip: Optional[str] = Header(default=None),
     x_forwarded_for: Optional[str] = Header(default=None),
     referer: Optional[str] = Header(default=None),
@@ -108,11 +142,13 @@ async def current_version(
     data = {
         "action": "check_version",
         "ip": client_ip,
-        "auto_upgrade_host_identity": "",
+        "auto_upgrade_host_identity": auto_upgrade_host_identity,
-        "installed_version": referer,
+        "installed_version": installed_version,
-        "group": "",
+        "group": group,
-        "platform": "",
+        "platform": platform.value,
-        "arch": "",
+        "arch": arch.value,
+        "build": build_type.value,
+        "audience": audience.value,
     }
 
     try:
@@ -124,7 +160,16 @@ async def current_version(
         return CurrentVersion(version="0.00")
 
     try:
-        result = crud.get_current_version()
+        target_id = ClientTargetIdentification(
+            platform=platform,
+            arch=arch,
+            build_type=build_type,
+            audience=audience,
+            auto_upgrade_host_identity=auto_upgrade_host_identity,
+            installed_version=installed_version,
+            group=group,
+        )
+        result = crud.get_current_version(target_id)
         if not result:
             raise HTTPException(status_code=404, detail="Not found")
         return result
@@ -139,22 +184,22 @@ async def current_version(
 
 
 @app.get(
-    "/upgrades/{platform}/{arch}/{build_type}",
+    "/upgrades/{platform}/{arch}/{build_type}/{audience}",
     response_model=Union[FileSend, dict],
     status_code=200,
 )
 @app.get(
-    "/upgrades/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}",
+    "/upgrades/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}",
     response_model=Union[FileSend, dict],
     status_code=200,
 )
 @app.get(
-    "/upgrades/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}/{installed_version}",
+    "/upgrades/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}",
     response_model=Union[FileSend, dict],
     status_code=200,
 )
 @app.get(
-    "/upgrades/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}/{installed_version}/{group}",
+    "/upgrades/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}/{group}",
     response_model=Union[FileSend, dict],
     status_code=200,
 )
@@ -163,6 +208,7 @@ async def upgrades(
     platform: Platform,
     arch: Arch,
     build_type: BuildType,
+    audience: Audience,
     auto_upgrade_host_identity: str = None,
     installed_version: str = None,
     group: str = None,
@@ -184,7 +230,8 @@ async def upgrades(
         "group": group,
         "platform": platform.value,
         "arch": arch.value,
-        "build_type": build_type.value,
+        "build": build_type.value,
+        "audience": audience.value,
     }
 
     try:
@@ -201,6 +248,7 @@ async def upgrades(
         platform=platform,
         arch=arch,
         build_type=build_type,
+        audience=audience,
         auto_upgrade_host_identity=auto_upgrade_host_identity,
         installed_version=installed_version,
         group=group,
@@ -221,20 +269,22 @@ async def upgrades(
 
 
 @app.get(
-    "/download/{platform}/{arch}/{build_type}", response_model=FileSend, status_code=200
+    "/download/{platform}/{arch}/{build_type}/{audience}",
-)
-@app.get(
-    "/download/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}",
     response_model=FileSend,
     status_code=200,
 )
 @app.get(
-    "/download/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}/{installed_version}",
+    "/download/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}",
     response_model=FileSend,
     status_code=200,
 )
 @app.get(
-    "/download/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}/{installed_version}/{group}",
+    "/download/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}",
+    response_model=FileSend,
+    status_code=200,
+)
+@app.get(
+    "/download/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}/{group}",
     response_model=FileSend,
     status_code=200,
 )
@@ -243,6 +293,7 @@ async def download(
     platform: Platform,
     arch: Arch,
     build_type: BuildType,
+    audience: Audience,
     auto_upgrade_host_identity: str = None,
     installed_version: str = None,
     group: str = None,
@@ -264,7 +315,8 @@ async def download(
         "group": group,
         "platform": platform.value,
         "arch": arch.value,
-        "build_type": build_type.value,
+        "build": build_type.value,
+        "audience": audience.value,
     }
 
     try:

upgrade_server/upgrade_server/crud.py

@@ -7,16 +7,16 @@ __intname__ = "npbackup.upgrade_server.crud"
 __author__ = "Orsiris de Jong"
 __copyright__ = "Copyright (C) 2023-2025 NetInvent"
 __license__ = "GPL-3.0-only"
-__build__ = "2025011401"
+__build__ = "2025011601"
 
 
 import os
-from typing import Optional, Union
+from typing import Optional, Union, Tuple
 from logging import getLogger
 import hashlib
 from argparse import ArgumentParser
 from datetime import datetime, timezone
-from upgrade_server.models.files import FileGet, FileSend
+from upgrade_server.models.files import ClientTargetIdentification, FileGet, FileSend
 from upgrade_server.models.oper import CurrentVersion
 import upgrade_server.configuration as configuration
 
@@ -57,6 +57,43 @@ def is_enabled() -> bool:
     return not os.path.isfile(path)
 
 
+def _get_path_from_target_id(target_id: ClientTargetIdentification) -> Tuple[str, str]:
+    """
+    Determine specific or generic upgrade path depending on target_id sent by client
+
+    NPBackup filenames are
+    npbackup-{platform}-{arch}-{build_type}-{audience}.{archive_extension}"
+
+    """
+    if target_id.platform.value == "windows":
+        extension = "zip"
+    else:
+        extension = "tar.gz"
+
+    expected_filename = f"npbackup-{target_id.platform.value}-{target_id.build_type.value}-{target_id.audience.value}.{extension}"
+
+    base_path = os.path.join(
+        config_dict["upgrades"]["data_root"],
+    )
+
+    for posssible_sub_path in [
+        target_id.auto_upgrade_host_identity,
+        target_id.installed_version,
+        target_id.group,
+    ]:
+        if posssible_sub_path:
+            possibile_sub_path = os.path.join(base_path, posssible_sub_path)
+            if os.path.isdir(possibile_sub_path):
+                logger.info(f"Found specific upgrade path in {possibile_sub_path}")
+                base_path = possibile_sub_path
+                break
+
+    archive_path = os.path.join(base_path, expected_filename)
+    version_file_path = os.path.join(base_path, "VERSION")
+
+    return version_file_path, archive_path
+
+
 def store_host_info(destination: str, host_id: dict) -> None:
     try:
         data = (
@@ -72,52 +109,34 @@ def store_host_info(destination: str, host_id: dict) -> None:
         logger.error("Trace:", exc_info=True)
 
 
-def get_current_version() -> Optional[CurrentVersion]:
+def get_current_version(
+    target_id: ClientTargetIdentification,
+) -> Optional[CurrentVersion]:
     try:
-        path = os.path.join(config_dict["upgrades"]["data_root"], "VERSION")
+        version_filename, _ = _get_path_from_target_id(target_id)
-        if os.path.isfile(path):
+        if os.path.isfile(version_filename):
-            with open(path, "r", encoding="utf-8") as fh:
+            with open(version_filename, "r", encoding="utf-8") as fh:
                 ver = fh.readline().strip()
                 return CurrentVersion(version=ver)
     except OSError as exc:
-        logger.error("Cannot get current version")
+        logger.error(f"Cannot get current version: {exc}")
         logger.error("Trace:", exc_info=True)
     except Exception as exc:
-        logger.error("Version seems to be bogus in VERSION file")
+        logger.error(f"Version seems to be bogus in VERSION file: {exc}")
         logger.error("Trace:", exc_info=True)
 
 
 def get_file(
     file: FileGet, content: bool = False
 ) -> Optional[Union[FileSend, bytes, dict]]:
-    if file.platform.value == "windows":
+
-        extension = "zip"
+    _, archive_path = _get_path_from_target_id(file)
-    else:
+
-        extension = "tar.gz"
+    logger.info(
-    possible_filename = f"npbackup-{file.build_type.value}.{extension}"
+        f"Searching for file {'info' if not content else 'content'} in {archive_path}"
-    base_path = os.path.join(
-        config_dict["upgrades"]["data_root"],
-        file.platform.value,
-        file.arch.value,
     )
-
+    if not os.path.isfile(archive_path):
-    for posssible_sub_path in [
+        logger.info(f"No upgrade file found in {archive_path}")
-        file.auto_upgrade_host_identity,
-        file.installed_version,
-        file.group,
-    ]:
-        if posssible_sub_path:
-            possibile_sub_path = os.path.join(base_path, posssible_sub_path)
-            if os.path.isdir(possibile_sub_path):
-                logger.info(f"Found specific upgrade path in {possibile_sub_path}")
-                base_path = possibile_sub_path
-                break
-
-    path = os.path.join(base_path, possible_filename)
-
-    logger.info(f"Searching for file {'info' if not content else 'content'} in {path}")
-    if not os.path.isfile(path):
-        logger.info(f"No upgrade file found in {path}")
         return {
             "arch": file.arch.value,
             "platform": file.platform.value,
@@ -127,7 +146,7 @@ def get_file(
             "file_length": 0,
         }
 
-    with open(path, "rb") as fh:
+    with open(archive_path, "rb") as fh:
         bytes = fh.read()
         if content:
             return bytes
@@ -138,7 +157,7 @@ def get_file(
             platform=file.platform.value,
             build_type=file.build_type.value,
             sha256sum=sha256,
-            filename=possible_filename,
+            filename=archive_path,
             file_length=length,
         )
         return file_send

upgrade_server/upgrade_server/models/files.py

@@ -7,7 +7,7 @@ __intname__ = "npbackup.upgrade_server.models.files"
 __author__ = "Orsiris de Jong"
 __copyright__ = "Copyright (C) 2023-2025 NetInvent"
 __license__ = "GPL-3.0-only"
-__build__ = "2025011401"
+__build__ = "2025011601"
 
 
 from typing import Optional
@@ -30,20 +30,26 @@ class BuildType(Enum):
     cli = "cli"
 
 
-class FileBase(BaseModel):
+class Audience(Enum):
+    public = "public"
+    private = "private"
+
+
+class ClientTargetIdentification(BaseModel):
     arch: Arch
     platform: Platform
     build_type: BuildType
+    audience: Audience
     auto_upgrade_host_identity: Optional[str] = None
     installed_version: Optional[str] = None
     group: Optional[str] = None
 
 
-class FileGet(FileBase):
+class FileGet(ClientTargetIdentification):
     pass
 
 
-class FileSend(FileBase):
+class FileSend(ClientTargetIdentification):
     sha256sum: constr(min_length=64, max_length=64)
     filename: str
     file_length: int