From 60de1d83532dd94dd36f6de986977221e73de6c6 Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 10 Dec 2024 21:04:39 +0100 Subject: [PATCH] Update restic binaries on tests/compilation --- RESTIC_SOURCE_FILES/update_restic.py | 112 +++++++++++++++++++++++++++ bin/COMPILE.cmd | 2 + bin/COMPILE.sh | 7 +- tests/test_npbackup-cli.py | 51 +----------- 4 files changed, 121 insertions(+), 51 deletions(-) create mode 100644 RESTIC_SOURCE_FILES/update_restic.py diff --git a/RESTIC_SOURCE_FILES/update_restic.py b/RESTIC_SOURCE_FILES/update_restic.py new file mode 100644 index 0000000..3cb41fa --- /dev/null +++ b/RESTIC_SOURCE_FILES/update_restic.py @@ -0,0 +1,112 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- + + +__intname__ = "npbackup.restic_update" +__author__ = "Orsiris de Jong" +__copyright__ = "Copyright (C) 2024 NetInvent" +__license__ = "BSD-3-Clause" +__build__ = "2024121001" + +import os +import sys +import bz2 +from pathlib import Path +import requests +import json +import shutil +from pprint import pprint + + +sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) + +from npbackup.path_helper import BASEDIR + + +def download_restic_binaries(arch: str = "amd64"): + """ + We must first download latest restic binaries to make sure we can run all tests and/or compile + """ + org = "restic" + repo = "restic" + + response = requests.get( + f"https://api.github.com/repos/{org}/{repo}/releases/latest" + ) + # print("RESPONSE: ", response) + json_response = json.loads(response.text) + current_version = json_response["tag_name"].lstrip("v") + # print("JSON RESPONSE") + # pprint(json_response, indent=5) + + dest_dir = Path(BASEDIR).absolute().parent.joinpath("RESTIC_SOURCE_FILES") + if os.name == "nt": + fname = f"_windows_{arch}.zip" + suffix = ".exe" + else: + fname = f"_linux_{arch}.bz2" + suffix = "" + + dest_file = dest_dir.joinpath("restic_" + current_version + fname).with_suffix( + suffix + ) + if dest_file.is_file(): + print(f"RESTIC SOURCE ALREADY PRESENT. NOT DOWNLOADING {dest_file}") + return True + else: + print(f"DOWNALOADING RESTIC {dest_file}") + + downloaded = False + for entry in json_response["assets"]: + if fname in entry["browser_download_url"]: + file_request = requests.get( + entry["browser_download_url"], allow_redirects=True + ) + print("FILE REQUEST RESPONSE", file_request) + filename = entry["browser_download_url"].rsplit("/", 1)[1] + full_path = dest_dir.joinpath(filename) + print("PATH TO DOWNLOADED ARCHIVE: ", full_path) + if fname.endswith("bz2"): + with open(full_path.with_suffix(""), "wb") as fp: + fp.write(bz2.decompress(file_request.content)) + # We also need to make that file executable + os.chmod(full_path.with_suffix(""), 0o775) + else: + with open(full_path, "wb") as fp: + fp.write(file_request.content) + # Assume we have a zip or tar.gz + shutil.unpack_archive(full_path, dest_dir) + try: + shutil.move(full_path, dest_dir.joinpath("ARCHIVES").joinpath(filename)) + except OSError: + print( + f'CANNOT MOVE TO ARCHIVE: {full_path} to {dest_dir.joinpath("ARCHIVES").joinpath(filename)}' + ) + return False + print(f"DOWNLOADED {dest_dir}") + downloaded = True + break + if not downloaded: + print(f"NO RESTIC BINARY FOUND for {arch}") + return False + return True + + +def download_restic_binaries_for_arch(): + """ + Shortcut to be used in compile script + """ + if os.name == "nt": + if not download_restic_binaries("amd64") or not download_restic_binaries("386"): + sys.exit(1) + else: + if ( + not download_restic_binaries("amd64") + or not download_restic_binaries("arm64") + or not download_restic_binaries("arm") + ): + sys.exit(1) + + +if __name__ == "__main__": + download_restic_binaries_for_arch() diff --git a/bin/COMPILE.cmd b/bin/COMPILE.cmd index 3be1417..96bf997 100644 --- a/bin/COMPILE.cmd +++ b/bin/COMPILE.cmd @@ -17,6 +17,8 @@ git pull || exit 1 SET OLD_PYTHONPATH=%PYTHONPATH% SET PYTHONPATH=c:\GIT\npbackup +%PYTHON64% RESTIC_SOURCE_FILES/update_restic.py || exit 1 + "%PYTHON64%" -m pip install --upgrade -r npbackup/requirements.txt || exit 1 "%PYTHON64%" bin\compile.py --audience all --sign "C:\ODJ\KEYS\NetInventEV.dat" diff --git a/bin/COMPILE.sh b/bin/COMPILE.sh index b0d5527..9045fb8 100755 --- a/bin/COMPILE.sh +++ b/bin/COMPILE.sh @@ -2,8 +2,6 @@ # This is an example compiler script -machine="$(uname -m)" - cd /opt/npbackup git pull || exit 1 @@ -13,7 +11,10 @@ export PYTHONPATH=/opt/npbackup # For RHEL 7 based builds, we need to define path to locally built tcl8.6 [ -d /usr/local/lib/tcl8.6 ] && export LD_LIBRARY_PATH=/usr/local/lib +/opt/npbackup/venv/bin/python RESTIC_SOURCE_FILES/update_restic.py || exit 1 + + /opt/npbackup/venv/bin/python -m pip install --upgrade -r npbackup/requirements.txt || exit 1 -/opt/npbackup/venv/bin/python bin/compile.py --audience all $opts +/opt/npbackup/venv/bin/python bin/compile.py --audience all $@ export PYTHONPATH="$OLD_PYTHONPATH" \ No newline at end of file diff --git a/tests/test_npbackup-cli.py b/tests/test_npbackup-cli.py index 0bf01a5..145925c 100644 --- a/tests/test_npbackup-cli.py +++ b/tests/test_npbackup-cli.py @@ -27,7 +27,6 @@ import json import requests import tempfile import bz2 -import fileinput from pprint import pprint sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) @@ -35,6 +34,7 @@ sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".." from npbackup import __main__ from npbackup.path_helper import BASEDIR from npbackup.configuration import load_config, get_repo_config +from RESTIC_SOURCE_FILES.update_restic import download_restic_binaries if os.name == "nt": @@ -89,54 +89,9 @@ class RedirectedStdout: def test_download_restic_binaries(): """ We must first download latest restic binaries to make sure we can run all tests + Currently we only run these on amd64 """ - org = "restic" - repo = "restic" - - dest_dir = Path(BASEDIR).absolute().parent.joinpath("RESTIC_SOURCE_FILES") - response = requests.get( - f"https://api.github.com/repos/{org}/{repo}/releases/latest" - ) - print("RESPONSE: ", response) - json_response = json.loads(response.text) - - if os.name == "nt": - fname = "_windows_amd64.zip" - suffix = ".exe" - else: - fname = "_linux_amd64.bz2" - suffix = "" - - if dest_dir.joinpath(fname).with_suffix(suffix).is_file(): - print("RESTIC SOURCE ALREADY PRESENT. NOT DOWNLOADING") - return - - print("JSON RESPONSE") - pprint(json_response, indent=5) - - for entry in json_response["assets"]: - if fname in entry["browser_download_url"]: - file_request = requests.get( - entry["browser_download_url"], allow_redirects=True - ) - print("FILE REQUEST RESPONSE", file_request) - filename = entry["browser_download_url"].rsplit("/", 1)[1] - full_path = dest_dir.joinpath(filename) - print("PATH TO DOWNLOADED ARCHIVE: ", full_path) - if fname.endswith("bz2"): - with open(full_path.with_suffix(""), "wb") as fp: - fp.write(bz2.decompress(file_request.content)) - # We also need to make that file executable - os.chmod(full_path.with_suffix(""), 0o775) - else: - with open(full_path, "wb") as fp: - fp.write(file_request.content) - # Assume we have a zip or tar.gz - shutil.unpack_archive(full_path, dest_dir) - try: - shutil.move(full_path, dest_dir.joinpath("ARCHIVES")) - except OSError: - pass + assert download_restic_binaries("amd64"), "Could not download restic binaries" def test_npbackup_cli_no_config():