upgrade_server: Allow simpler paths for updates, include audience in decision

This commit is contained in:
deajan 2025-01-16 13:57:18 +01:00
parent 4026c69c36
commit 92cbdd6c92
7 changed files with 223 additions and 104 deletions

View file

@ -30,3 +30,6 @@ HEARTBEAT_INTERVAL = 3600
# Arbitrary timeout for init / init checks.
# If init takes more than a minute, we really have a problem in our backend
FAST_COMMANDS_TIMEOUT = 180
# # Wait x seconds before we actually do the upgrade so current program could quit before being erased
UPGRADE_DEFER_TIME = 60

View file

@ -28,15 +28,15 @@ try:
CURRENT_USER = psutil.Process().username()
except Exception:
CURRENT_USER = "unknown"
version_string = f"{__intname__} v{__version__}-{'priv' if IS_PRIV_BUILD else 'pub'}-{sys.version_info[0]}.{sys.version_info[1]}-{python_arch()}{'-legacy' if IS_LEGACY else ''}{'-c' if IS_COMPILED else '-i'} {__build__} - {__copyright__} running as {CURRENT_USER}"
version_dict = {
"name": __intname__,
"version": __version__,
"buildtype": "priv" if IS_PRIV_BUILD else "pub",
"build_type": "priv" if IS_PRIV_BUILD else "pub",
"os": get_os_identifier(),
"arch": python_arch(),
"arch": python_arch() + ("-legacy" if IS_LEGACY else ""),
"pv": sys.version_info,
"comp": IS_COMPILED,
"build": __build__,
"copyright": __copyright__,
}
version_string = f"{version_dict['name']} {version_dict['version']}-{version_dict['buildtype']}-{version_dict['pv'][0]}.{version_dict['pv'][1]}-{version_dict['arch']}{'-c' if IS_COMPILED else '-i'} {version_dict['build']} - {version_dict['copyright']} running as {CURRENT_USER}"

View file

@ -33,7 +33,7 @@ from npbackup.restic_metrics import (
from npbackup.restic_wrapper import ResticRunner
from npbackup.core.restic_source_binary import get_restic_internal_binary
from npbackup.path_helper import CURRENT_DIR, BASEDIR
from npbackup.__version__ import __intname__ as NAME, version_dict as version_dict
from npbackup.__version__ import __intname__ as NAME, version_dict
from npbackup.__debug__ import _DEBUG, exception_to_string
@ -53,7 +53,7 @@ def metric_writer(
try:
labels = {
"npversion": f"{NAME}{version_dict['version']}-{version_dict['buildtype']}"
"npversion": f"{NAME}{version_dict['version']}-{version_dict['build_type']}"
}
if repo_config.g("prometheus.metrics"):
labels["instance"] = repo_config.g("prometheus.instance")

View file

@ -26,12 +26,11 @@ from ofunctions.random import random_string
from command_runner import deferred_command
from npbackup.path_helper import CURRENT_DIR, CURRENT_EXECUTABLE
from npbackup.core.nuitka_helper import IS_COMPILED
from npbackup.__version__ import __version__ as npbackup_version, IS_LEGACY
from npbackup.__version__ import version_dict
from npbackup.__env__ import UPGRADE_DEFER_TIME
logger = getLogger()
UPGRADE_DEFER_TIME = 60 # Wait x seconds before we actually do the upgrade so current program could quit before being erased
# RAW ofunctions.checksum import
def sha256sum_data(data):
@ -44,8 +43,40 @@ def sha256sum_data(data):
return sha256.hexdigest()
def _get_target_id(auto_upgrade_host_identity: str, group: str) -> str:
"""
Get current target information string as
{platform}/{arch}/{build_type}/{host_id}/{current_version}/{group}
"""
# We'll check python_arch instead of os_arch since we build 32 bit python executables for compat reasons
build_type = os.environ.get("NPBACKUP_BUILD_TYPE", None)
if not build_type:
logger.critical("Cannot determine build type for upgrade processs")
return False
target = "{}/{}/{}".format(
get_os(),
version_dict["arch"],
version_dict["build_type"],
version_dict["audience"],
).lower()
try:
host_id = "{}/{}/{}".format(
auto_upgrade_host_identity, version_dict["version"], group
)
target = "{}/{}".format(target, host_id)
except TypeError as exc:
logger.debug(f"No other information to add to target: {exc}")
return target
def _check_new_version(
upgrade_url: str, username: str, password: str, ignore_errors: bool = False
upgrade_url: str,
username: str,
password: str,
ignore_errors: bool = False,
auto_upgrade_host_identity: str = None,
group: str = None,
) -> bool:
"""
Check if we have a newer version of npbackup
@ -56,7 +87,7 @@ def _check_new_version(
logger.debug("Upgrade server not set")
return None
requestor = Requestor(upgrade_url, username, password)
requestor.app_name = "npbackup" + npbackup_version
requestor.app_name = "npbackup" + version_dict["version"]
requestor.user_agent = __intname__
requestor.ignore_errors = ignore_errors
requestor.create_session(authenticated=True)
@ -83,7 +114,10 @@ def _check_new_version(
logger.error(msg)
return None
result = requestor.data_model("current_version")
target_id = _get_target_id(
auto_upgrade_host_identity=auto_upgrade_host_identity, group=group
)
result = requestor.data_model("current_version", id_record=target_id)
if result is False:
msg = "Upgrade server didn't respond properly. Is it well configured ?"
if ignore_errors:
@ -101,21 +135,29 @@ def _check_new_version(
logger.error(msg)
return None
else:
if online_version:
if version.parse(online_version) > version.parse(npbackup_version):
logger.info(
"Current version %s is older than online version %s",
npbackup_version,
online_version,
)
return True
else:
logger.info(
"Current version %s is up-to-date (online version %s)",
npbackup_version,
online_version,
)
return False
try:
if online_version:
if version.parse(online_version) > version.parse(
version_dict["version"]
):
logger.info(
"Current version %s is older than online version %s",
version_dict["version"],
online_version,
)
return True
else:
logger.info(
"Current version %s is up-to-date (online version %s)",
version_dict["version"],
online_version,
)
return False
except Exception as exc:
logger.error(
f"Cannot determine if online version '{online_version}' is newer than current version {version_dict['verison']}: {exc}"
)
return False
def auto_upgrader(
@ -139,7 +181,12 @@ def auto_upgrader(
return False
res = _check_new_version(
upgrade_url, username, password, ignore_errors=ignore_errors
upgrade_url,
username,
password,
ignore_errors=ignore_errors,
auto_upgrade_host_identity=auto_upgrade_host_identity,
group=group,
)
# Let's set a global environment variable which we can check later in metrics
os.environ["NPBACKUP_UPGRADE_STATE"] = "0"
@ -148,24 +195,16 @@ def auto_upgrader(
os.environ["NPBACKUP_UPGRADE_STATE"] = "1"
return False
requestor = Requestor(upgrade_url, username, password)
requestor.app_name = "npbackup" + npbackup_version
requestor.app_name = "npbackup" + version_dict["version"]
requestor.user_agent = __intname__
requestor.create_session(authenticated=True)
# We'll check python_arch instead of os_arch since we build 32 bit python executables for compat reasons
arch = python_arch() if not IS_LEGACY else f"{python_arch()}-legacy"
build_type = os.environ.get("NPBACKUP_BUILD_TYPE", None)
if not build_type:
logger.critical("Cannot determine build type for upgrade processs")
return False
target = "{}/{}/{}".format(get_os(), arch, build_type).lower()
try:
host_id = "{}/{}/{}".format(auto_upgrade_host_identity, npbackup_version, group)
id_record = "{}/{}".format(target, host_id)
except TypeError:
id_record = target
# This allows to get the current running target identification for upgrade server to return the right file
target_id = _get_target_id(
auto_upgrade_host_identity=auto_upgrade_host_identity, group=group
)
file_info = requestor.data_model("upgrades", id_record=id_record)
file_info = requestor.data_model("upgrades", id_record=target_id)
if not file_info:
logger.error("Server didn't provide a file description")
return False
@ -179,7 +218,7 @@ def auto_upgrader(
logger.info("No upgrade file found has been found for me :/")
return True
file_data = requestor.requestor("download/" + id_record, raw=True)
file_data = requestor.requestor(f"download/{target_id}", raw=True)
if not file_data:
logger.error("Cannot get update file")
return False

View file

@ -18,7 +18,15 @@ from argparse import ArgumentParser
from fastapi import FastAPI, HTTPException, Response, Depends, status, Request, Header
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from fastapi_offline import FastAPIOffline
from upgrade_server.models.files import FileGet, FileSend, Platform, Arch, BuildType
from upgrade_server.models.files import (
ClientTargetIdentification,
FileGet,
FileSend,
Platform,
Arch,
BuildType,
Audience,
)
from upgrade_server.models.oper import CurrentVersion
import upgrade_server.crud as crud
import upgrade_server.configuration as configuration
@ -91,9 +99,35 @@ async def api_status():
return {"app": __appname__, "maintenance": "enabled"}
@app.get("/current_version", response_model=CurrentVersion, status_code=200)
@app.get(
"/current_version/{platform}/{arch}/{build_type}/{audience}",
response_model=CurrentVersion,
status_code=200,
)
@app.get(
"/current_version/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}",
response_model=CurrentVersion,
status_code=200,
)
@app.get(
"/current_version/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}",
response_model=CurrentVersion,
status_code=200,
)
@app.get(
"/current_version/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}/{group}",
response_model=CurrentVersion,
status_code=200,
)
async def current_version(
request: Request,
platform: Platform,
arch: Arch,
build_type: BuildType,
audience: Audience,
auto_upgrade_host_identity: str = None,
installed_version: str = None,
group: str = None,
x_real_ip: Optional[str] = Header(default=None),
x_forwarded_for: Optional[str] = Header(default=None),
referer: Optional[str] = Header(default=None),
@ -108,11 +142,13 @@ async def current_version(
data = {
"action": "check_version",
"ip": client_ip,
"auto_upgrade_host_identity": "",
"installed_version": referer,
"group": "",
"platform": "",
"arch": "",
"auto_upgrade_host_identity": auto_upgrade_host_identity,
"installed_version": installed_version,
"group": group,
"platform": platform.value,
"arch": arch.value,
"build": build_type.value,
"audience": audience.value,
}
try:
@ -124,7 +160,16 @@ async def current_version(
return CurrentVersion(version="0.00")
try:
result = crud.get_current_version()
target_id = ClientTargetIdentification(
platform=platform,
arch=arch,
build_type=build_type,
audience=audience,
auto_upgrade_host_identity=auto_upgrade_host_identity,
installed_version=installed_version,
group=group,
)
result = crud.get_current_version(target_id)
if not result:
raise HTTPException(status_code=404, detail="Not found")
return result
@ -139,22 +184,22 @@ async def current_version(
@app.get(
"/upgrades/{platform}/{arch}/{build_type}",
"/upgrades/{platform}/{arch}/{build_type}/{audience}",
response_model=Union[FileSend, dict],
status_code=200,
)
@app.get(
"/upgrades/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}",
"/upgrades/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}",
response_model=Union[FileSend, dict],
status_code=200,
)
@app.get(
"/upgrades/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}/{installed_version}",
"/upgrades/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}",
response_model=Union[FileSend, dict],
status_code=200,
)
@app.get(
"/upgrades/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}/{installed_version}/{group}",
"/upgrades/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}/{group}",
response_model=Union[FileSend, dict],
status_code=200,
)
@ -163,6 +208,7 @@ async def upgrades(
platform: Platform,
arch: Arch,
build_type: BuildType,
audience: Audience,
auto_upgrade_host_identity: str = None,
installed_version: str = None,
group: str = None,
@ -184,7 +230,8 @@ async def upgrades(
"group": group,
"platform": platform.value,
"arch": arch.value,
"build_type": build_type.value,
"build": build_type.value,
"audience": audience.value,
}
try:
@ -201,6 +248,7 @@ async def upgrades(
platform=platform,
arch=arch,
build_type=build_type,
audience=audience,
auto_upgrade_host_identity=auto_upgrade_host_identity,
installed_version=installed_version,
group=group,
@ -221,20 +269,22 @@ async def upgrades(
@app.get(
"/download/{platform}/{arch}/{build_type}", response_model=FileSend, status_code=200
)
@app.get(
"/download/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}",
"/download/{platform}/{arch}/{build_type}/{audience}",
response_model=FileSend,
status_code=200,
)
@app.get(
"/download/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}/{installed_version}",
"/download/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}",
response_model=FileSend,
status_code=200,
)
@app.get(
"/download/{platform}/{arch}/{build_type}/{auto_upgrade_host_identity}/{installed_version}/{group}",
"/download/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}",
response_model=FileSend,
status_code=200,
)
@app.get(
"/download/{platform}/{arch}/{build_type}/{audience}/{auto_upgrade_host_identity}/{installed_version}/{group}",
response_model=FileSend,
status_code=200,
)
@ -243,6 +293,7 @@ async def download(
platform: Platform,
arch: Arch,
build_type: BuildType,
audience: Audience,
auto_upgrade_host_identity: str = None,
installed_version: str = None,
group: str = None,
@ -264,7 +315,8 @@ async def download(
"group": group,
"platform": platform.value,
"arch": arch.value,
"build_type": build_type.value,
"build": build_type.value,
"audience": audience.value,
}
try:

View file

@ -7,16 +7,16 @@ __intname__ = "npbackup.upgrade_server.crud"
__author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2023-2025 NetInvent"
__license__ = "GPL-3.0-only"
__build__ = "2025011401"
__build__ = "2025011601"
import os
from typing import Optional, Union
from typing import Optional, Union, Tuple
from logging import getLogger
import hashlib
from argparse import ArgumentParser
from datetime import datetime, timezone
from upgrade_server.models.files import FileGet, FileSend
from upgrade_server.models.files import ClientTargetIdentification, FileGet, FileSend
from upgrade_server.models.oper import CurrentVersion
import upgrade_server.configuration as configuration
@ -57,6 +57,43 @@ def is_enabled() -> bool:
return not os.path.isfile(path)
def _get_path_from_target_id(target_id: ClientTargetIdentification) -> Tuple[str, str]:
"""
Determine specific or generic upgrade path depending on target_id sent by client
NPBackup filenames are
npbackup-{platform}-{arch}-{build_type}-{audience}.{archive_extension}"
"""
if target_id.platform.value == "windows":
extension = "zip"
else:
extension = "tar.gz"
expected_filename = f"npbackup-{target_id.platform.value}-{target_id.build_type.value}-{target_id.audience.value}.{extension}"
base_path = os.path.join(
config_dict["upgrades"]["data_root"],
)
for posssible_sub_path in [
target_id.auto_upgrade_host_identity,
target_id.installed_version,
target_id.group,
]:
if posssible_sub_path:
possibile_sub_path = os.path.join(base_path, posssible_sub_path)
if os.path.isdir(possibile_sub_path):
logger.info(f"Found specific upgrade path in {possibile_sub_path}")
base_path = possibile_sub_path
break
archive_path = os.path.join(base_path, expected_filename)
version_file_path = os.path.join(base_path, "VERSION")
return version_file_path, archive_path
def store_host_info(destination: str, host_id: dict) -> None:
try:
data = (
@ -72,52 +109,34 @@ def store_host_info(destination: str, host_id: dict) -> None:
logger.error("Trace:", exc_info=True)
def get_current_version() -> Optional[CurrentVersion]:
def get_current_version(
target_id: ClientTargetIdentification,
) -> Optional[CurrentVersion]:
try:
path = os.path.join(config_dict["upgrades"]["data_root"], "VERSION")
if os.path.isfile(path):
with open(path, "r", encoding="utf-8") as fh:
version_filename, _ = _get_path_from_target_id(target_id)
if os.path.isfile(version_filename):
with open(version_filename, "r", encoding="utf-8") as fh:
ver = fh.readline().strip()
return CurrentVersion(version=ver)
except OSError as exc:
logger.error("Cannot get current version")
logger.error(f"Cannot get current version: {exc}")
logger.error("Trace:", exc_info=True)
except Exception as exc:
logger.error("Version seems to be bogus in VERSION file")
logger.error(f"Version seems to be bogus in VERSION file: {exc}")
logger.error("Trace:", exc_info=True)
def get_file(
file: FileGet, content: bool = False
) -> Optional[Union[FileSend, bytes, dict]]:
if file.platform.value == "windows":
extension = "zip"
else:
extension = "tar.gz"
possible_filename = f"npbackup-{file.build_type.value}.{extension}"
base_path = os.path.join(
config_dict["upgrades"]["data_root"],
file.platform.value,
file.arch.value,
_, archive_path = _get_path_from_target_id(file)
logger.info(
f"Searching for file {'info' if not content else 'content'} in {archive_path}"
)
for posssible_sub_path in [
file.auto_upgrade_host_identity,
file.installed_version,
file.group,
]:
if posssible_sub_path:
possibile_sub_path = os.path.join(base_path, posssible_sub_path)
if os.path.isdir(possibile_sub_path):
logger.info(f"Found specific upgrade path in {possibile_sub_path}")
base_path = possibile_sub_path
break
path = os.path.join(base_path, possible_filename)
logger.info(f"Searching for file {'info' if not content else 'content'} in {path}")
if not os.path.isfile(path):
logger.info(f"No upgrade file found in {path}")
if not os.path.isfile(archive_path):
logger.info(f"No upgrade file found in {archive_path}")
return {
"arch": file.arch.value,
"platform": file.platform.value,
@ -127,7 +146,7 @@ def get_file(
"file_length": 0,
}
with open(path, "rb") as fh:
with open(archive_path, "rb") as fh:
bytes = fh.read()
if content:
return bytes
@ -138,7 +157,7 @@ def get_file(
platform=file.platform.value,
build_type=file.build_type.value,
sha256sum=sha256,
filename=possible_filename,
filename=archive_path,
file_length=length,
)
return file_send

View file

@ -7,7 +7,7 @@ __intname__ = "npbackup.upgrade_server.models.files"
__author__ = "Orsiris de Jong"
__copyright__ = "Copyright (C) 2023-2025 NetInvent"
__license__ = "GPL-3.0-only"
__build__ = "2025011401"
__build__ = "2025011601"
from typing import Optional
@ -30,20 +30,26 @@ class BuildType(Enum):
cli = "cli"
class FileBase(BaseModel):
class Audience(Enum):
public = "public"
private = "private"
class ClientTargetIdentification(BaseModel):
arch: Arch
platform: Platform
build_type: BuildType
audience: Audience
auto_upgrade_host_identity: Optional[str] = None
installed_version: Optional[str] = None
group: Optional[str] = None
class FileGet(FileBase):
class FileGet(ClientTargetIdentification):
pass
class FileSend(FileBase):
class FileSend(ClientTargetIdentification):
sha256sum: constr(min_length=64, max_length=64)
filename: str
file_length: int