From 586c682cf67847088a44cbaafc193b09009ea272 Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 15 Sep 2024 19:48:48 +0200 Subject: [PATCH] GUI: Allow json fallback for Python 3.7 --- npbackup/gui/__main__.py | 144 +++++++++++++++++++--------- npbackup/requirements.txt | 1 + npbackup/restic_wrapper/__init__.py | 81 +++++++++++----- npbackup/restic_wrapper/schema.py | 17 +++- npbackup/runner_interface.py | 17 +++- 5 files changed, 183 insertions(+), 77 deletions(-) diff --git a/npbackup/gui/__main__.py b/npbackup/gui/__main__.py index 603dccf..b7add6a 100644 --- a/npbackup/gui/__main__.py +++ b/npbackup/gui/__main__.py @@ -58,7 +58,7 @@ from npbackup.path_helper import CURRENT_DIR from npbackup.__version__ import version_string from npbackup.__debug__ import _DEBUG from npbackup.restic_wrapper import ResticRunner -from npbackup.restic_wrapper import schema +from npbackup.restic_wrapper import schema, MSGSPEC logger = getLogger() @@ -188,58 +188,110 @@ def _make_treedata_from_json(ls_result: List[dict]) -> sg.TreeData: {'name': 'xmfbox.tcl', 'type': 'file', 'path': '/C/GIT/npbackup/npbackup.dist/tk/xmfbox.tcl', 'uid': 0, 'gid': 0, 'size': 27064, 'mode': 438, 'permissions': '-rw-rw-rw-', 'mtime': '2022-09-05T14:18:52+02:00', 'atime': '2022-09-05T14:18:52+02:00', 'ctime': '2022-09-05T14:18:52+02:00', 'struct_type': 'node'} ] - Since v3-rc6, we're actually using a msgspec.Struct represenation which uses dot notation + Since v3-rc6, we're actually using a msgspec.Struct represenation which uses dot notation, but only on Python 3.8+ + We still rely on json for Python 3.7 """ treedata = sg.TreeData() count = 0 + if not MSGSPEC: + logger.info("Using basic json representation for data which is slow and memory hungry. Consider using a newer OS that supports Python 3.8+") for entry in ls_result: # Make sure we drop the prefix '/' so sg.TreeData does not get an empty root + if MSGSPEC: + entry.path = entry.path.lstrip("/") + if os.name == "nt": + # On windows, we need to make sure tree keys don't get duplicate because of lower/uppercase + # Shown filenames aren't affected by this + entry.path = entry.path.lower() + parent = os.path.dirname(entry.path) - entry.path = entry.path.lstrip("/") - if os.name == "nt": - # On windows, we need to make sure tree keys don't get duplicate because of lower/uppercase - # Shown filenames aren't affected by this - entry.path = entry.path.lower() - parent = os.path.dirname(entry.path) + # Make sure we normalize mtime, and remove microseconds + # dateutil.parser.parse is *really* cpu hungry, let's replace it with a dumb alternative + # mtime = dateutil.parser.parse(entry["mtime"]).strftime("%Y-%m-%d %H:%M:%S") + mtime = entry.mtime.strftime("%Y-%m-%d %H:%M:%S") + name = os.path.basename(entry.path) + if ( + entry.type == schema.LsNodeType.DIR + and entry.path not in treedata.tree_dict + ): + treedata.Insert( + parent=parent, + key=entry.path, + text=name, + values=["", mtime], + icon=FOLDER_ICON, + ) + elif entry.type == schema.LsNodeType.FILE: + size = BytesConverter(entry.size).human + treedata.Insert( + parent=parent, + key=entry.path, + text=name, + values=[size, mtime], + icon=FILE_ICON, + ) + elif entry.type == schema.LsNodeType.SYMLINK: + treedata.Insert( + parent=parent, + key=entry.path, + text=name, + values=["", mtime], + icon=SYMLINK_ICON, + ) + elif entry.type == schema.LsNodeType.IRREGULAR: + treedata.Insert( + parent=parent, + key=entry.path, + text=name, + values=["", mtime], + icon=IRREGULAR_FILE_ICON, + ) + else: + entry["path"] = entry["path"].lstrip("/") + if os.name == "nt": + # On windows, we need to make sure tree keys don't get duplicate because of lower/uppercase + # Shown filenames aren't affected by this + entry["path"] = entry["path"].lower() + parent = os.path.dirname(entry["path"]) - # Make sure we normalize mtime, and remove microseconds - # dateutil.parser.parse is *really* cpu hungry, let's replace it with a dumb alternative - # mtime = dateutil.parser.parse(entry["mtime"]).strftime("%Y-%m-%d %H:%M:%S") - mtime = entry.mtime.strftime("%Y-%m-%d %H:%M:%S") - name = os.path.basename(entry.path) - if entry.type == schema.LsNodeType.DIR and entry.path not in treedata.tree_dict: - treedata.Insert( - parent=parent, - key=entry.path, - text=name, - values=["", mtime], - icon=FOLDER_ICON, - ) - elif entry.type == schema.LsNodeType.FILE: - size = BytesConverter(entry.size).human - treedata.Insert( - parent=parent, - key=entry.path, - text=name, - values=[size, mtime], - icon=FILE_ICON, - ) - elif entry.type == schema.LsNodeType.SYMLINK: - treedata.Insert( - parent=parent, - key=entry.path, - text=name, - values=["", mtime], - icon=SYMLINK_ICON, - ) - elif entry.type == schema.LsNodeType.IRREGULAR: - treedata.Insert( - parent=parent, - key=entry.path, - text=name, - values=["", mtime], - icon=IRREGULAR_FILE_ICON, - ) + # Make sure we normalize mtime, and remove microseconds + # dateutil.parser.parse is *really* cpu hungry, let's replace it with a dumb alternative + # mtime = dateutil.parser.parse(entry["mtime"]).strftime("%Y-%m-%d %H:%M:%S") + mtime = entry["mtime"][0:19] + name = os.path.basename(entry["name"]) + if entry["type"] == "dir" and entry["path"] not in treedata.tree_dict: + treedata.Insert( + parent=parent, + key=entry["path"], + text=name, + values=["", mtime], + icon=FOLDER_ICON, + ) + elif entry["type"] == "file": + size = BytesConverter(entry["size"]).human + treedata.Insert( + parent=parent, + key=entry["path"], + text=name, + values=[size, mtime], + icon=FILE_ICON, + ) + elif entry["type"] == "symlink": + treedata.Insert( + parent=parent, + key=entry["path"], + text=name, + values=["", mtime], + icon=SYMLINK_ICON, + ) + elif entry["type"] == "irregular": + treedata.Insert( + parent=parent, + key=entry["path"], + text=name, + values=["", mtime], + icon=IRREGULAR_FILE_ICON, + ) # Since the thread is heavily CPU bound, let's add a minimal # arbitrary sleep time to let GUI update diff --git a/npbackup/requirements.txt b/npbackup/requirements.txt index ecd7901..8c613c9 100644 --- a/npbackup/requirements.txt +++ b/npbackup/requirements.txt @@ -28,3 +28,4 @@ packaging pywin32; platform_system == "Windows" imageio; platform_system == "Darwin" ntplib>=0.4.0 +msgspec; python_version >= "3.8" diff --git a/npbackup/restic_wrapper/__init__.py b/npbackup/restic_wrapper/__init__.py index 74d6255..64e9c30 100644 --- a/npbackup/restic_wrapper/__init__.py +++ b/npbackup/restic_wrapper/__init__.py @@ -16,7 +16,6 @@ import os import sys from logging import getLogger import re -import msgspec from datetime import datetime, timezone import dateutil.parser import queue @@ -28,6 +27,16 @@ from npbackup.__env__ import FAST_COMMANDS_TIMEOUT, CHECK_INTERVAL from npbackup.path_helper import CURRENT_DIR from npbackup.restic_wrapper import schema +try: + import msgspec + + MSGSPEC = True +except ImportError: + # We may not have msgspec on Python 3.7 + import json + + MSGSPEC = False + logger = getLogger() @@ -672,30 +681,41 @@ class ResticRunner: } if result: if output: - decoder = msgspec.json.Decoder() - ls_decoder = msgspec.json.Decoder(schema.LsNode) + if MSGSPEC: + decoder = msgspec.json.Decoder() + ls_decoder = msgspec.json.Decoder(schema.LsNode) is_first_line = True for line in output.split("\n"): if not line: continue - try: - if ( - not is_first_line - and operation == "ls" - and self.struct_output - ): + if MSGSPEC: + try: + if ( + not is_first_line + and operation == "ls" + and self.struct_output + ): - js["output"].append(ls_decoder.decode(line)) - else: - js["output"].append(decoder.decode(line)) - is_first_line = False - except msgspec.DecodeError as exc: - msg = f"JSON decode error: {exc} on content '{line}'" - self.write_logs(msg, level="error") - js["extended_info"] = msg - js["output"].append({"data": line}) - js["result"] = False + js["output"].append(ls_decoder.decode(line)) + else: + js["output"].append(decoder.decode(line)) + is_first_line = False + except msgspec.DecodeError as exc: + msg = f"JSON decode error: {exc} on content '{line}'" + self.write_logs(msg, level="error") + js["extended_info"] = msg + js["output"].append({"data": line}) + js["result"] = False + else: + try: + js["output"].append(json.loads(line)) + except json.JSONDecodeError as exc: + msg = f"JSON decode error: {exc} on content '{line}'" + self.write_logs(msg, level="error") + js["extended_info"] = msg + js["output"].append({"data": line}) + js["result"] = False # If we only have one output, we don't need a list if len(js["output"]) == 1: js["output"] = js["output"][0] @@ -707,13 +727,22 @@ class ResticRunner: js["reason"] = msg self.write_logs(msg, level="error") if output: - try: - js["output"] = msgspec.json.decode(output) - except msgspec.DecodeError as exc: - msg = f"JSON decode error: {exc} on output '{output}'" - self.write_logs(msg, level="error") - js["extended_info"] = msg - js["output"] = {"data": output} + if MSGSPEC: + try: + js["output"] = msgspec.json.decode(output) + except msgspec.DecodeError as exc: + msg = f"JSON decode error: {exc} on output '{output}'" + self.write_logs(msg, level="error") + js["extended_info"] = msg + js["output"] = {"data": output} + else: + try: + js["output"] = json.loads(output) + except json.JSONDecodeError as exc: + msg = f"JSON decode error: {exc} on output '{output}'" + self.write_logs(msg, level="error") + js["extended_info"] = msg + js["output"] = {"data": output} return js if result: diff --git a/npbackup/restic_wrapper/schema.py b/npbackup/restic_wrapper/schema.py index a8e66ee..a73f48c 100644 --- a/npbackup/restic_wrapper/schema.py +++ b/npbackup/restic_wrapper/schema.py @@ -11,9 +11,22 @@ __description__ = "Restic json output schemas" from typing import Optional -from enum import StrEnum from datetime import datetime -from msgspec import Struct + +try: + from msgspec import Struct + from enum import StrEnum + MSGSPEC = True +except ImportError: + + class Struct: + def __init_subclass__(self, *args, **kwargs): + pass + pass + class StrEnum: + pass + + MSGSPEC = False class LsNodeType(StrEnum): diff --git a/npbackup/runner_interface.py b/npbackup/runner_interface.py index 8d18591..2596cba 100644 --- a/npbackup/runner_interface.py +++ b/npbackup/runner_interface.py @@ -14,7 +14,15 @@ __build__ = "2024091501" import sys from logging import getLogger -import msgspec.json + +try: + import msgspec.json + + MSGSPEC = True +except ImportError: + import json + + MSGSPEC = False import datetime from npbackup.core.runner import NPBackupRunner @@ -70,6 +78,9 @@ def entrypoint(*args, **kwargs): else: logger.error(f"Operation finished") else: - # print(json.dumps(result, default=serialize_datetime)) - print(msgspec.json.encode(result)) + if MSGSPEC: + print(msgspec.json.encode(result)) + else: + print(json.dumps(result, default=serialize_datetime)) + sys.exit(0)