From cb89b3de360c7ee8bc81b4abaaa261acbf2621eb Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 10 Dec 2024 23:37:14 +0100 Subject: [PATCH] tests: Improve restic download strategy --- RESTIC_SOURCE_FILES/update_restic.py | 60 +++++++++++++++++++--------- tests/test_npbackup-cli.py | 13 +----- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/RESTIC_SOURCE_FILES/update_restic.py b/RESTIC_SOURCE_FILES/update_restic.py index a41dedb..14d4221 100644 --- a/RESTIC_SOURCE_FILES/update_restic.py +++ b/RESTIC_SOURCE_FILES/update_restic.py @@ -23,7 +23,7 @@ sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".." from npbackup.path_helper import BASEDIR -def download_restic_binaries(arch: str = "amd64", move_is_fatal: bool = True) -> bool: +def download_restic_binaries(arch: str = "amd64") -> bool: """ We must first download latest restic binaries to make sure we can run all tests and/or compile """ @@ -36,29 +36,54 @@ def download_restic_binaries(arch: str = "amd64", move_is_fatal: bool = True) -> # print("RESPONSE: ", response) json_response = json.loads(response.text) current_version = json_response["tag_name"].lstrip("v") + # print("JSON RESPONSE") # pprint(json_response, indent=5) dest_dir = Path(BASEDIR).absolute().parent.joinpath("RESTIC_SOURCE_FILES") if os.name == "nt": - fname = f"_windows_{arch}.zip" + fname = f"_windows_{arch}" suffix = ".exe" + arch_suffix = ".zip" else: - fname = f"_linux_{arch}.bz2" + fname = f"_linux_{arch}" suffix = "" + arch_suffix = ".bz2" + + if not dest_dir.joinpath("ARCHIVES").is_dir(): + os.makedirs(dest_dir.joinpath("ARCHIVES")) + + dest_file = dest_dir.joinpath("restic_" + current_version + fname + arch_suffix) - dest_file = dest_dir.joinpath("restic_" + current_version + fname).with_suffix( - suffix - ) if dest_file.is_file(): print(f"RESTIC SOURCE ALREADY PRESENT. NOT DOWNLOADING {dest_file}") return True else: print(f"DOWNALOADING RESTIC {dest_file}") + # Also we need to move any earlier file that may not be current version to archives + for file in dest_dir.glob(f"restic_*{fname}{suffix}"): + # We need to keep legacy binary for Windows 7 32 bits + if ( + not file == "restic_0.16.2_windows_386.exe" + and not file == "restic_0.16.2_windows_amd64.exe" + ): + try: + archive_file = dest_dir.joinpath("ARCHIVES").joinpath(file.name) + if archive_file.is_file(): + archive_file.unlink() + shutil.move( + file, + archive_file, + ) + except OSError as exc: + print( + f"CANNOT MOVE OLD FILES ARCHIVE: {file} to {archive_file}: {exc}" + ) + return False downloaded = False for entry in json_response["assets"]: - if fname in entry["browser_download_url"]: + if f"{fname}{arch_suffix}" in entry["browser_download_url"]: file_request = requests.get( entry["browser_download_url"], allow_redirects=True ) @@ -66,26 +91,25 @@ def download_restic_binaries(arch: str = "amd64", move_is_fatal: bool = True) -> filename = entry["browser_download_url"].rsplit("/", 1)[1] full_path = dest_dir.joinpath(filename) print("PATH TO DOWNLOADED ARCHIVE: ", full_path) - if fname.endswith("bz2"): - with open(full_path.with_suffix(""), "wb") as fp: + if arch_suffix == ".bz2": + with open(str(full_path).rstrip("arch_suffix"), "wb") as fp: fp.write(bz2.decompress(file_request.content)) # We also need to make that file executable - os.chmod(full_path.with_suffix(""), 0o775) + os.chmod(str(full_path).rstrip(arch_suffix), 0o775) else: with open(full_path, "wb") as fp: fp.write(file_request.content) # Assume we have a zip or tar.gz shutil.unpack_archive(full_path, dest_dir) try: - if not dest_dir.joinpath("ARCHIVES").is_dir(): - os.makedirs(dest_dir.joinpath("ARCHIVES")) + if dest_dir.joinpath("ARCHIVES").joinpath(filename).is_file(): + dest_dir.joinpath("ARCHIVES").joinpath(filename).unlink() shutil.move(full_path, dest_dir.joinpath("ARCHIVES").joinpath(filename)) - except OSError: - if move_is_fatal: - print( - f'CANNOT MOVE TO ARCHIVE: {full_path} to {dest_dir.joinpath("ARCHIVES").joinpath(filename)}' - ) - return False + except OSError as exc: + print( + f'CANNOT MOVE TO ARCHIVE: {full_path} to {dest_dir.joinpath("ARCHIVES").joinpath(filename)}: {[exc]}' + ) + return False print(f"DOWNLOADED {dest_dir}") downloaded = True break diff --git a/tests/test_npbackup-cli.py b/tests/test_npbackup-cli.py index 6ea1728..145925c 100644 --- a/tests/test_npbackup-cli.py +++ b/tests/test_npbackup-cli.py @@ -65,15 +65,6 @@ repo_config, _ = get_repo_config(full_config) DUMP_FILE = "__version__.py" -def running_on_github_actions(): - """ - This is set in github actions workflow with - env: - RUNNING_ON_GITHUB_ACTIONS: true - """ - return os.environ.get("RUNNING_ON_GITHUB_ACTIONS", "False").lower() == "true" - - class RedirectedStdout: """ Balantly copied from https://stackoverflow.com/a/45899925/2635443 @@ -100,9 +91,7 @@ def test_download_restic_binaries(): We must first download latest restic binaries to make sure we can run all tests Currently we only run these on amd64 """ - assert download_restic_binaries( - "amd64", move_is_fatal=not running_on_github_actions() - ), "Could not download restic binaries" + assert download_restic_binaries("amd64"), "Could not download restic binaries" def test_npbackup_cli_no_config():