mirror of
https://github.com/netinvent/npbackup.git
synced 2024-09-20 06:46:13 +08:00
GUI: Allow json fallback for Python 3.7
This commit is contained in:
parent
60a74067a4
commit
586c682cf6
|
@ -58,7 +58,7 @@ from npbackup.path_helper import CURRENT_DIR
|
|||
from npbackup.__version__ import version_string
|
||||
from npbackup.__debug__ import _DEBUG
|
||||
from npbackup.restic_wrapper import ResticRunner
|
||||
from npbackup.restic_wrapper import schema
|
||||
from npbackup.restic_wrapper import schema, MSGSPEC
|
||||
|
||||
|
||||
logger = getLogger()
|
||||
|
@ -188,58 +188,110 @@ def _make_treedata_from_json(ls_result: List[dict]) -> sg.TreeData:
|
|||
{'name': 'xmfbox.tcl', 'type': 'file', 'path': '/C/GIT/npbackup/npbackup.dist/tk/xmfbox.tcl', 'uid': 0, 'gid': 0, 'size': 27064, 'mode': 438, 'permissions': '-rw-rw-rw-', 'mtime': '2022-09-05T14:18:52+02:00', 'atime': '2022-09-05T14:18:52+02:00', 'ctime': '2022-09-05T14:18:52+02:00', 'struct_type': 'node'}
|
||||
]
|
||||
|
||||
Since v3-rc6, we're actually using a msgspec.Struct represenation which uses dot notation
|
||||
Since v3-rc6, we're actually using a msgspec.Struct represenation which uses dot notation, but only on Python 3.8+
|
||||
We still rely on json for Python 3.7
|
||||
"""
|
||||
treedata = sg.TreeData()
|
||||
count = 0
|
||||
if not MSGSPEC:
|
||||
logger.info("Using basic json representation for data which is slow and memory hungry. Consider using a newer OS that supports Python 3.8+")
|
||||
for entry in ls_result:
|
||||
# Make sure we drop the prefix '/' so sg.TreeData does not get an empty root
|
||||
if MSGSPEC:
|
||||
entry.path = entry.path.lstrip("/")
|
||||
if os.name == "nt":
|
||||
# On windows, we need to make sure tree keys don't get duplicate because of lower/uppercase
|
||||
# Shown filenames aren't affected by this
|
||||
entry.path = entry.path.lower()
|
||||
parent = os.path.dirname(entry.path)
|
||||
|
||||
entry.path = entry.path.lstrip("/")
|
||||
if os.name == "nt":
|
||||
# On windows, we need to make sure tree keys don't get duplicate because of lower/uppercase
|
||||
# Shown filenames aren't affected by this
|
||||
entry.path = entry.path.lower()
|
||||
parent = os.path.dirname(entry.path)
|
||||
# Make sure we normalize mtime, and remove microseconds
|
||||
# dateutil.parser.parse is *really* cpu hungry, let's replace it with a dumb alternative
|
||||
# mtime = dateutil.parser.parse(entry["mtime"]).strftime("%Y-%m-%d %H:%M:%S")
|
||||
mtime = entry.mtime.strftime("%Y-%m-%d %H:%M:%S")
|
||||
name = os.path.basename(entry.path)
|
||||
if (
|
||||
entry.type == schema.LsNodeType.DIR
|
||||
and entry.path not in treedata.tree_dict
|
||||
):
|
||||
treedata.Insert(
|
||||
parent=parent,
|
||||
key=entry.path,
|
||||
text=name,
|
||||
values=["", mtime],
|
||||
icon=FOLDER_ICON,
|
||||
)
|
||||
elif entry.type == schema.LsNodeType.FILE:
|
||||
size = BytesConverter(entry.size).human
|
||||
treedata.Insert(
|
||||
parent=parent,
|
||||
key=entry.path,
|
||||
text=name,
|
||||
values=[size, mtime],
|
||||
icon=FILE_ICON,
|
||||
)
|
||||
elif entry.type == schema.LsNodeType.SYMLINK:
|
||||
treedata.Insert(
|
||||
parent=parent,
|
||||
key=entry.path,
|
||||
text=name,
|
||||
values=["", mtime],
|
||||
icon=SYMLINK_ICON,
|
||||
)
|
||||
elif entry.type == schema.LsNodeType.IRREGULAR:
|
||||
treedata.Insert(
|
||||
parent=parent,
|
||||
key=entry.path,
|
||||
text=name,
|
||||
values=["", mtime],
|
||||
icon=IRREGULAR_FILE_ICON,
|
||||
)
|
||||
else:
|
||||
entry["path"] = entry["path"].lstrip("/")
|
||||
if os.name == "nt":
|
||||
# On windows, we need to make sure tree keys don't get duplicate because of lower/uppercase
|
||||
# Shown filenames aren't affected by this
|
||||
entry["path"] = entry["path"].lower()
|
||||
parent = os.path.dirname(entry["path"])
|
||||
|
||||
# Make sure we normalize mtime, and remove microseconds
|
||||
# dateutil.parser.parse is *really* cpu hungry, let's replace it with a dumb alternative
|
||||
# mtime = dateutil.parser.parse(entry["mtime"]).strftime("%Y-%m-%d %H:%M:%S")
|
||||
mtime = entry.mtime.strftime("%Y-%m-%d %H:%M:%S")
|
||||
name = os.path.basename(entry.path)
|
||||
if entry.type == schema.LsNodeType.DIR and entry.path not in treedata.tree_dict:
|
||||
treedata.Insert(
|
||||
parent=parent,
|
||||
key=entry.path,
|
||||
text=name,
|
||||
values=["", mtime],
|
||||
icon=FOLDER_ICON,
|
||||
)
|
||||
elif entry.type == schema.LsNodeType.FILE:
|
||||
size = BytesConverter(entry.size).human
|
||||
treedata.Insert(
|
||||
parent=parent,
|
||||
key=entry.path,
|
||||
text=name,
|
||||
values=[size, mtime],
|
||||
icon=FILE_ICON,
|
||||
)
|
||||
elif entry.type == schema.LsNodeType.SYMLINK:
|
||||
treedata.Insert(
|
||||
parent=parent,
|
||||
key=entry.path,
|
||||
text=name,
|
||||
values=["", mtime],
|
||||
icon=SYMLINK_ICON,
|
||||
)
|
||||
elif entry.type == schema.LsNodeType.IRREGULAR:
|
||||
treedata.Insert(
|
||||
parent=parent,
|
||||
key=entry.path,
|
||||
text=name,
|
||||
values=["", mtime],
|
||||
icon=IRREGULAR_FILE_ICON,
|
||||
)
|
||||
# Make sure we normalize mtime, and remove microseconds
|
||||
# dateutil.parser.parse is *really* cpu hungry, let's replace it with a dumb alternative
|
||||
# mtime = dateutil.parser.parse(entry["mtime"]).strftime("%Y-%m-%d %H:%M:%S")
|
||||
mtime = entry["mtime"][0:19]
|
||||
name = os.path.basename(entry["name"])
|
||||
if entry["type"] == "dir" and entry["path"] not in treedata.tree_dict:
|
||||
treedata.Insert(
|
||||
parent=parent,
|
||||
key=entry["path"],
|
||||
text=name,
|
||||
values=["", mtime],
|
||||
icon=FOLDER_ICON,
|
||||
)
|
||||
elif entry["type"] == "file":
|
||||
size = BytesConverter(entry["size"]).human
|
||||
treedata.Insert(
|
||||
parent=parent,
|
||||
key=entry["path"],
|
||||
text=name,
|
||||
values=[size, mtime],
|
||||
icon=FILE_ICON,
|
||||
)
|
||||
elif entry["type"] == "symlink":
|
||||
treedata.Insert(
|
||||
parent=parent,
|
||||
key=entry["path"],
|
||||
text=name,
|
||||
values=["", mtime],
|
||||
icon=SYMLINK_ICON,
|
||||
)
|
||||
elif entry["type"] == "irregular":
|
||||
treedata.Insert(
|
||||
parent=parent,
|
||||
key=entry["path"],
|
||||
text=name,
|
||||
values=["", mtime],
|
||||
icon=IRREGULAR_FILE_ICON,
|
||||
)
|
||||
|
||||
# Since the thread is heavily CPU bound, let's add a minimal
|
||||
# arbitrary sleep time to let GUI update
|
||||
|
|
|
@ -28,3 +28,4 @@ packaging
|
|||
pywin32; platform_system == "Windows"
|
||||
imageio; platform_system == "Darwin"
|
||||
ntplib>=0.4.0
|
||||
msgspec; python_version >= "3.8"
|
||||
|
|
|
@ -16,7 +16,6 @@ import os
|
|||
import sys
|
||||
from logging import getLogger
|
||||
import re
|
||||
import msgspec
|
||||
from datetime import datetime, timezone
|
||||
import dateutil.parser
|
||||
import queue
|
||||
|
@ -28,6 +27,16 @@ from npbackup.__env__ import FAST_COMMANDS_TIMEOUT, CHECK_INTERVAL
|
|||
from npbackup.path_helper import CURRENT_DIR
|
||||
from npbackup.restic_wrapper import schema
|
||||
|
||||
try:
|
||||
import msgspec
|
||||
|
||||
MSGSPEC = True
|
||||
except ImportError:
|
||||
# We may not have msgspec on Python 3.7
|
||||
import json
|
||||
|
||||
MSGSPEC = False
|
||||
|
||||
logger = getLogger()
|
||||
|
||||
|
||||
|
@ -672,30 +681,41 @@ class ResticRunner:
|
|||
}
|
||||
if result:
|
||||
if output:
|
||||
decoder = msgspec.json.Decoder()
|
||||
ls_decoder = msgspec.json.Decoder(schema.LsNode)
|
||||
if MSGSPEC:
|
||||
decoder = msgspec.json.Decoder()
|
||||
ls_decoder = msgspec.json.Decoder(schema.LsNode)
|
||||
is_first_line = True
|
||||
|
||||
for line in output.split("\n"):
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
if (
|
||||
not is_first_line
|
||||
and operation == "ls"
|
||||
and self.struct_output
|
||||
):
|
||||
if MSGSPEC:
|
||||
try:
|
||||
if (
|
||||
not is_first_line
|
||||
and operation == "ls"
|
||||
and self.struct_output
|
||||
):
|
||||
|
||||
js["output"].append(ls_decoder.decode(line))
|
||||
else:
|
||||
js["output"].append(decoder.decode(line))
|
||||
is_first_line = False
|
||||
except msgspec.DecodeError as exc:
|
||||
msg = f"JSON decode error: {exc} on content '{line}'"
|
||||
self.write_logs(msg, level="error")
|
||||
js["extended_info"] = msg
|
||||
js["output"].append({"data": line})
|
||||
js["result"] = False
|
||||
js["output"].append(ls_decoder.decode(line))
|
||||
else:
|
||||
js["output"].append(decoder.decode(line))
|
||||
is_first_line = False
|
||||
except msgspec.DecodeError as exc:
|
||||
msg = f"JSON decode error: {exc} on content '{line}'"
|
||||
self.write_logs(msg, level="error")
|
||||
js["extended_info"] = msg
|
||||
js["output"].append({"data": line})
|
||||
js["result"] = False
|
||||
else:
|
||||
try:
|
||||
js["output"].append(json.loads(line))
|
||||
except json.JSONDecodeError as exc:
|
||||
msg = f"JSON decode error: {exc} on content '{line}'"
|
||||
self.write_logs(msg, level="error")
|
||||
js["extended_info"] = msg
|
||||
js["output"].append({"data": line})
|
||||
js["result"] = False
|
||||
# If we only have one output, we don't need a list
|
||||
if len(js["output"]) == 1:
|
||||
js["output"] = js["output"][0]
|
||||
|
@ -707,13 +727,22 @@ class ResticRunner:
|
|||
js["reason"] = msg
|
||||
self.write_logs(msg, level="error")
|
||||
if output:
|
||||
try:
|
||||
js["output"] = msgspec.json.decode(output)
|
||||
except msgspec.DecodeError as exc:
|
||||
msg = f"JSON decode error: {exc} on output '{output}'"
|
||||
self.write_logs(msg, level="error")
|
||||
js["extended_info"] = msg
|
||||
js["output"] = {"data": output}
|
||||
if MSGSPEC:
|
||||
try:
|
||||
js["output"] = msgspec.json.decode(output)
|
||||
except msgspec.DecodeError as exc:
|
||||
msg = f"JSON decode error: {exc} on output '{output}'"
|
||||
self.write_logs(msg, level="error")
|
||||
js["extended_info"] = msg
|
||||
js["output"] = {"data": output}
|
||||
else:
|
||||
try:
|
||||
js["output"] = json.loads(output)
|
||||
except json.JSONDecodeError as exc:
|
||||
msg = f"JSON decode error: {exc} on output '{output}'"
|
||||
self.write_logs(msg, level="error")
|
||||
js["extended_info"] = msg
|
||||
js["output"] = {"data": output}
|
||||
return js
|
||||
|
||||
if result:
|
||||
|
|
|
@ -11,9 +11,22 @@ __description__ = "Restic json output schemas"
|
|||
|
||||
|
||||
from typing import Optional
|
||||
from enum import StrEnum
|
||||
from datetime import datetime
|
||||
from msgspec import Struct
|
||||
|
||||
try:
|
||||
from msgspec import Struct
|
||||
from enum import StrEnum
|
||||
MSGSPEC = True
|
||||
except ImportError:
|
||||
|
||||
class Struct:
|
||||
def __init_subclass__(self, *args, **kwargs):
|
||||
pass
|
||||
pass
|
||||
class StrEnum:
|
||||
pass
|
||||
|
||||
MSGSPEC = False
|
||||
|
||||
|
||||
class LsNodeType(StrEnum):
|
||||
|
|
|
@ -14,7 +14,15 @@ __build__ = "2024091501"
|
|||
|
||||
import sys
|
||||
from logging import getLogger
|
||||
import msgspec.json
|
||||
|
||||
try:
|
||||
import msgspec.json
|
||||
|
||||
MSGSPEC = True
|
||||
except ImportError:
|
||||
import json
|
||||
|
||||
MSGSPEC = False
|
||||
import datetime
|
||||
from npbackup.core.runner import NPBackupRunner
|
||||
|
||||
|
@ -70,6 +78,9 @@ def entrypoint(*args, **kwargs):
|
|||
else:
|
||||
logger.error(f"Operation finished")
|
||||
else:
|
||||
# print(json.dumps(result, default=serialize_datetime))
|
||||
print(msgspec.json.encode(result))
|
||||
if MSGSPEC:
|
||||
print(msgspec.json.encode(result))
|
||||
else:
|
||||
print(json.dumps(result, default=serialize_datetime))
|
||||
|
||||
sys.exit(0)
|
||||
|
|
Loading…
Reference in a new issue