diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4136100..33a5ba1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,11 +61,3 @@ repos: language: script pass_filenames: false stages: [commit] - - repo: local - hooks: - - id: update-readme-version - name: Update readme version - entry: ./scripts/pre-commit/update-readme-version.sh - language: script - pass_filenames: false - stages: [commit] diff --git a/CHANGELOG b/CHANGELOG index 88af256..92685d9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,11 @@ -# Requirements Updated -- qbittorrent-api==2024.9.67 +# New Updates +- Adds new config option `disable_qbt_default_share_limits` to allow qbit_manage to handle share limits and disable qbittorrent's default share limits +- Adds new config option `max_orphaned_files_to_delete` to set default safeguards against mass deletion when running remove orphaned. +- Adds new environment variables `QBT_LOG_SIZE` and `QBT_LOG_COUNT` to customize log retention (Closes #656) -**Full Changelog**: https://github.com/StuffAnThings/qbit_manage/compare/v4.1.9...v4.1.10 +# Bug Fixes +- Truncates Recyclebin JSON filename when its too long. (Closes #604) +- Uses Qbittorrent's torrent export to save .torrent files for qbittorrent version > 4.5.0 (Closes #650) +- Include orphaned files and recycle bin in the list of folders to ignore when looking for noHL (Closes #660) + +**Full Changelog**: https://github.com/StuffAnThings/qbit_manage/compare/v4.1.10...v4.1.11 diff --git a/VERSION b/VERSION index 5d30083..152e452 100755 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.1.10 +4.1.11 diff --git a/config/config.yml.sample b/config/config.yml.sample index 65de071..ce32caf 100755 --- a/config/config.yml.sample +++ b/config/config.yml.sample @@ -40,6 +40,7 @@ settings: share_limits_filter_completed: True # Filters for completed torrents only when running share_limits command tag_nohardlinks_filter_completed: True # Filters for completed torrents only when running tag_nohardlinks command cat_update_all: True # Checks and updates all torrent categories if set to True when running cat_update command, otherwise only update torrents that are uncategorized + disable_qbt_default_share_limits: True # Allows QBM to handle share limits by disabling qBittorrents default Share limits. Only active when the share_limits command is set to True directory: # Do not remove these @@ -276,6 +277,10 @@ orphaned: - "/data/torrents/temp/**" - "**/*.!qB" - "**/*_unpackerred" + # Set your desired threshold for the maximum number of orphaned files qbm will delete in a single run. (-1 to disable safeguards) + # This will help reduce the number of accidental large amount orphaned deletions in a single run + # WARNING: Setting this variable to -1 will not safeguard against any deletions + max_orphaned_files_to_delete: 50 apprise: # Apprise integration with webhooks diff --git a/docs/Commands.md b/docs/Commands.md index 91ded49..53a8a38 100644 --- a/docs/Commands.md +++ b/docs/Commands.md @@ -18,7 +18,9 @@ | `-sl` or `--share-limits` | QBT_SHARE_LIMITS | share_limits | Control how torrent share limits are set depending on the priority of your grouping. Each torrent will be matched with the share limit group with the highest priority that meets the group filter criteria. Each torrent can only be matched with one share limit group. | False | | `-sc` or `--skip-cleanup` | QBT_SKIP_CLEANUP | skip_cleanup | Use this to skip emptying the Recycle Bin folder (`/root_dir/.RecycleBin`) and Orphaned directory. (`/root_dir/orphaned_data`) | False | | `-dr` or `--dry-run` | QBT_DRY_RUN | dry_run | If you would like to see what is gonna happen but not actually move/delete or tag/categorize anything. | False | -| `-ll` or `--log-level LOGLEVEL` | QBT_LOG_LEVEL | N/A | Change the output log level. | INFO | +| `-ll` or `--log-level` | QBT_LOG_LEVEL | N/A | Change the output log level. | INFO | +| `-ls` or `--log-size` | QBT_LOG_SIZE | N/A | Maximum log size per file (in MB) | 10 | +| `-lc` or `--log-count` | QBT_LOG_COUNT | N/A | Maximum number of logs to keep | 5 | | `--debug` | QBT_DEBUG | N/A | Adds debug logs | False | | `--trace` | QBT_TRACE | N/A | Adds trace logs | False | | `-d` or `--divider` | QBT_DIVIDER | N/A | Character that divides the sections (Default: '=') | = | diff --git a/docs/Config-Setup.md b/docs/Config-Setup.md index e7f2703..ea35b67 100644 --- a/docs/Config-Setup.md +++ b/docs/Config-Setup.md @@ -58,6 +58,7 @@ This section defines any settings defined in the configuration. | `share_limits_filter_completed` | When running `--share-limits` function, it will filter for completed torrents only. | True |
❌
| | `tag_nohardlinks_filter_completed` | When running `--tag-nohardlinks` function, , it will filter for completed torrents only. | True |
❌
| | `cat_update_all` | When running `--cat-update` function, it will check and update all torrents categories, otherwise it will only update uncategorized torrents. | True |
❌
| +| `disable_qbt_default_share_limits` | When running `--share-limits` function, it allows QBM to handle share limits by disabling qBittorrents default Share limits. | True |
❌
| ## **directory:** --- @@ -198,6 +199,7 @@ This is handy when you have automatically generated files that certain OSs decid | :------------------- | :------------------------------------------------------------------------------------------------------------------------------------------ | :------------- | :----------------- | | `empty_after_x_days` | Will delete Orphaned data contents if the files have been in the Orphaned data for more than x days. (Uses date modified to track the time) | None |
❌
| | `exclude_patterns` | List of [patterns](https://commandbox.ortusbooks.com/usage/parameters/globbing-patterns) to exclude certain files from orphaned | None |
❌
| +| `max_orphaned_files_to_delete` | This will help reduce the number of accidental large amount orphaned deletions in a single run. Set your desired threshold for the maximum number of orphaned files qbm will delete in a single run. (-1 to disable safeguards) | 50 |
❌
| > Note: The more time you place for the `empty_after_x_days:` variable the better, allowing you more time to catch any mistakes by the script. If the variable is set to `0` it will delete contents immediately after every script run. If the variable is not set it will never delete the contents of the Orphaned Data. ## **apprise:** diff --git a/modules/config.py b/modules/config.py index 209b5a7..0859454 100755 --- a/modules/config.py +++ b/modules/config.py @@ -84,6 +84,8 @@ class Config: logger.debug(f" --config-file (QBT_CONFIG): {args['config_files']}") logger.debug(f" --log-file (QBT_LOGFILE): {args['log_file']}") logger.debug(f" --log-level (QBT_LOG_LEVEL): {args['log_level']}") + logger.debug(f" --log-size (QBT_LOG_SIZE): {args['log_size']}") + logger.debug(f" --log-count (QBT_LOG_COUNT): {args['log_count']}") logger.debug(f" --divider (QBT_DIVIDER): {args['divider']}") logger.debug(f" --width (QBT_WIDTH): {args['screen_width']}") logger.debug(f" --debug (QBT_DEBUG): {args['debug']}") @@ -232,6 +234,9 @@ class Config: "force_auto_tmm_ignore_tags": self.util.check_for_attribute( self.data, "force_auto_tmm_ignore_tags", parent="settings", var_type="list", default=[] ), + "disable_qbt_default_share_limits": self.util.check_for_attribute( + self.data, "disable_qbt_default_share_limits", parent="settings", var_type="bool", default=True + ), } self.tracker_error_tag = self.settings["tracker_error_tag"] @@ -723,6 +728,14 @@ class Config: self.orphaned["exclude_patterns"] = self.util.check_for_attribute( self.data, "exclude_patterns", parent="orphaned", var_type="list", default_is_none=True, do_print=False ) + self.orphaned["max_orphaned_files_to_delete"] = self.util.check_for_attribute( + self.data, + "max_orphaned_files_to_delete", + parent="orphaned", + var_type="int", + default=50, + min_int=-1, + ) if self.commands["rem_orphaned"]: exclude_orphaned = f"**{os.sep}{os.path.basename(self.orphaned_dir.rstrip(os.sep))}{os.sep}*" ( diff --git a/modules/core/category.py b/modules/core/category.py index b336784..d378362 100644 --- a/modules/core/category.py +++ b/modules/core/category.py @@ -32,8 +32,8 @@ class Category: new_cat.extend(self.get_tracker_cat(torrent) or self.qbt.get_category(torrent.save_path)) if not torrent.auto_tmm and torrent_category: logger.print_line( - f"{torrent.name} has Automatic Torrent Management disabled and already has a category" - f"{torrent_category}. Skipping..", + f"{torrent.name} has Automatic Torrent Management disabled and already has the category" + f" {torrent_category}. Skipping..", "DEBUG", ) continue diff --git a/modules/core/remove_orphaned.py b/modules/core/remove_orphaned.py index e4e7e5b..f25022d 100644 --- a/modules/core/remove_orphaned.py +++ b/modules/core/remove_orphaned.py @@ -44,8 +44,9 @@ class RemoveOrphaned: for fullpath in fullpathlist ] ) - - orphaned_files = set(root_files.result()) - set(torrent_files) + root_files_set = set(root_files.result()) + torrent_files_set = set(torrent_files) + orphaned_files = root_files_set - torrent_files_set if self.config.orphaned["exclude_patterns"]: logger.print_line("Processing orphan exclude patterns") @@ -59,7 +60,18 @@ class RemoveOrphaned: orphaned_files = set(orphaned_files) - set(excluded_orphan_files) - if orphaned_files: + # Check the threshold before deleting orphaned files + max_orphaned_files_to_delete = self.config.orphaned.get("max_orphaned_files_to_delete") + if len(orphaned_files) and len(orphaned_files) > max_orphaned_files_to_delete and max_orphaned_files_to_delete != -1: + e = ( + f"Too many orphaned files detected ({len(orphaned_files)}). " + f"Max Threshold for deletion is set to {max_orphaned_files_to_delete}. " + "Aborting deletion to avoid accidental data loss." + ) + self.config.notify(e, "Remove Orphaned", False) + logger.warning(e) + return + elif orphaned_files: orphaned_files = sorted(orphaned_files) os.makedirs(self.orphaned_dir, exist_ok=True) body = [] diff --git a/modules/core/tag_nohardlinks.py b/modules/core/tag_nohardlinks.py index 7d20a7f..8b05d6d 100644 --- a/modules/core/tag_nohardlinks.py +++ b/modules/core/tag_nohardlinks.py @@ -87,7 +87,7 @@ class TagNoHardLinks: """Tag torrents with no hardlinks""" logger.separator("Tagging Torrents with No Hardlinks", space=False, border=False) nohardlinks = self.nohardlinks - check_hardlinks = util.CheckHardLinks(self.root_dir, self.remote_dir) + check_hardlinks = util.CheckHardLinks(self.config) for category in nohardlinks: torrent_list = self.qbt.get_torrents({"category": category, "status_filter": self.status_filter}) if len(torrent_list) == 0: diff --git a/modules/logs.py b/modules/logs.py index c41a72e..70bc43d 100755 --- a/modules/logs.py +++ b/modules/logs.py @@ -34,7 +34,9 @@ _srcfile = os.path.normcase(fmt_filter.__code__.co_filename) class MyLogger: """Logger class""" - def __init__(self, logger_name, log_file, log_level, default_dir, screen_width, separating_character, ignore_ghost): + def __init__( + self, logger_name, log_file, log_level, default_dir, screen_width, separating_character, ignore_ghost, log_size, log_count + ): """Initialize logger""" self.logger_name = logger_name self.default_dir = default_dir @@ -49,6 +51,8 @@ class MyLogger: self.config_handlers = {} self.secrets = set() self.spacing = 0 + self.log_size = log_size + self.log_count = log_count os.makedirs(self.log_dir, exist_ok=True) self._logger = logging.getLogger(self.logger_name) logging.DRYRUN = DRYRUN @@ -69,13 +73,13 @@ class MyLogger: """Clear saved errors""" self.saved_errors = [] - def _get_handler(self, log_file, count=5): + def _get_handler(self, log_file): """Get handler for log file""" - max_bytes = 1024 * 1024 * 10 - _handler = RotatingFileHandler(log_file, delay=True, mode="w", maxBytes=max_bytes, backupCount=count, encoding="utf-8") + max_bytes = 1024 * 1024 * self.log_size + _handler = RotatingFileHandler( + log_file, delay=True, mode="w", maxBytes=max_bytes, backupCount=self.log_count, encoding="utf-8" + ) self._formatter(handler=_handler) - # if os.path.isfile(log_file): - # _handler.doRollover() return _handler def _formatter(self, handler=None, border=True, log_only=False, space=False): @@ -89,7 +93,7 @@ class MyLogger: def add_main_handler(self): """Add main handler to logger""" - self.main_handler = self._get_handler(self.main_log, count=19) + self.main_handler = self._get_handler(self.main_log) self.main_handler.addFilter(fmt_filter) self._logger.addHandler(self.main_handler) diff --git a/modules/qbittorrent.py b/modules/qbittorrent.py index 4211df4..62d6c43 100755 --- a/modules/qbittorrent.py +++ b/modules/qbittorrent.py @@ -83,6 +83,22 @@ class Qbt: self.torrent_list = self.get_torrents({"sort": "added_on"}) self.torrentfiles = {} # a map of torrent files to track cross-seeds + if ( + self.config.commands["share_limits"] + and self.config.settings["disable_qbt_default_share_limits"] + and self.client.app.preferences.max_ratio_act != 0 + ): + logger.info("Disabling qBittorrent default share limits to allow qbm to manage share limits.") + # max_ratio_act: 0 = Pause Torrent, 1 = Remove Torrent, 2 = superseeding, 3 = Remove Torrent and Files + self.client.app_set_preferences( + { + "max_ratio_act": 0, + "max_seeding_time_enabled": False, + "max_ratio_enabled": False, + "max_inactive_seeding_time_enabled": False, + } + ) + self.global_max_ratio_enabled = self.client.app.preferences.max_ratio_enabled self.global_max_ratio = self.client.app.preferences.max_ratio self.global_max_seeding_time_enabled = self.client.app.preferences.max_seeding_time_enabled @@ -427,15 +443,19 @@ class Qbt: else: recycle_path = self.config.recycle_dir # Create recycle bin if not exists - torrent_path = os.path.join(recycle_path, "torrents") + torrent_path = os.path.join(recycle_path, "torrents") # Export torrent/fastresume from BT_backup + torrent_export_path = os.path.join(recycle_path, "torrents_export") # Exported torrent file (qbittorrent v4.5.0+) torrents_json_path = os.path.join(recycle_path, "torrents_json") torrent_name = info["torrents"][0] + torrent_exportable = self.current_version >= "4.5.0" os.makedirs(recycle_path, exist_ok=True) if self.config.recyclebin["save_torrents"]: if os.path.isdir(torrent_path) is False: os.makedirs(torrent_path) if os.path.isdir(torrents_json_path) is False: os.makedirs(torrents_json_path) + if torrent_exportable and os.path.isdir(torrent_export_path) is False: + os.makedirs(torrent_export_path) torrent_json_file = os.path.join(torrents_json_path, f"{torrent_name}.json") torrent_json = util.load_json(torrent_json_file) if not torrent_json: @@ -445,6 +465,20 @@ class Qbt: else: logger.info(f"Adding {info['torrent_tracker']} to existing {os.path.basename(torrent_json_file)}") dot_torrent_files = [] + # Exporting torrent via Qbit API (v4.5.0+) + if torrent_exportable: + hash_suffix = f"{info_hash[-8:]}" # Get the last 8 hash characters of the torrent + torrent_export_file = os.path.join(torrent_export_path, f"{torrent_name} [{hash_suffix}].torrent") + truncated_torrent_export_file = util.truncate_filename(torrent_export_file, offset=11) + try: + with open(f"{truncated_torrent_export_file}", "wb") as file: + file.write(torrent.export()) + except Exception as ex: + logger.stacktrace() + self.config.notify(ex, "Deleting Torrent", False) + logger.warning(f"RecycleBin Warning: {ex}") + dot_torrent_files.append(os.path.basename(truncated_torrent_export_file)) + # Exporting torrent via torrent directory (backwards compatibility) for file in os.listdir(self.config.torrents_dir): if file.startswith(info_hash): dot_torrent_files.append(file) @@ -466,7 +500,7 @@ class Qbt: backup_str += val else: backup_str += f" and {val.replace(info_hash, '')}" - backup_str += f" to {torrent_path}" + backup_str += f" to {torrent_export_path if torrent_exportable else torrent_path}" logger.info(backup_str) torrent_json["tracker_torrent_files"] = tracker_torrent_files if "files" not in torrent_json: diff --git a/modules/util.py b/modules/util.py index 52aab3d..d028e6d 100755 --- a/modules/util.py +++ b/modules/util.py @@ -525,10 +525,16 @@ class CheckHardLinks: Class to check for hardlinks """ - def __init__(self, root_dir, remote_dir): - self.root_dir = root_dir - self.remote_dir = remote_dir - self.root_files = set(get_root_files(self.root_dir, self.remote_dir)) + def __init__(self, config): + self.root_dir = config.root_dir + self.remote_dir = config.remote_dir + self.orphaned_dir = config.orphaned_dir if config.orphaned_dir else "" + self.recycle_dir = config.recycle_dir if config.recycle_dir else "" + self.root_files = set( + get_root_files(self.root_dir, self.remote_dir) + + get_root_files(self.orphaned_dir, "") + + get_root_files(self.recycle_dir, "") + ) self.get_inode_count() def get_inode_count(self): @@ -641,6 +647,8 @@ class CheckHardLinks: def get_root_files(root_dir, remote_dir, exclude_dir=None): local_exclude_dir = exclude_dir.replace(remote_dir, root_dir) if exclude_dir and remote_dir != root_dir else exclude_dir + # if not root_dir: + # return [] root_files = [ os.path.join(path.replace(remote_dir, root_dir) if remote_dir != root_dir else path, name) for path, subdirs, files in os.walk(remote_dir if remote_dir != root_dir else root_dir) @@ -652,7 +660,7 @@ def get_root_files(root_dir, remote_dir, exclude_dir=None): def load_json(file): """Load json file if exists""" - if os.path.isfile(file): + if os.path.isfile(truncate_filename(file)): file = open(file) data = json.load(file) file.close() @@ -661,10 +669,48 @@ def load_json(file): return data +def truncate_filename(filename, max_length=255, offset=0): + """ + Truncate filename if necessary. + + Args: + filename (str): The original filename. + max_length (int, optional): The maximum length of the truncated filename. Defaults to 255. + offset (int, optional): The number of characters to keep from the end of the base name. Defaults to 0. + + Returns: + str: The truncated filename. + + """ + base, ext = os.path.splitext(filename) + if len(filename) > max_length: + max_base_length = max_length - len(ext) - offset + truncated_base = base[:max_base_length] + truncated_base_offset = base[-offset:] if offset > 0 else "" + truncated_filename = f"{truncated_base}{truncated_base_offset}{ext}" + else: + truncated_filename = filename + return truncated_filename + + def save_json(torrent_json, dest): - """Save json file to destination""" - with open(dest, "w", encoding="utf-8") as file: - json.dump(torrent_json, file, ensure_ascii=False, indent=4) + """Save json file to destination, truncating filename if necessary.""" + max_filename_length = 255 # Typical maximum filename length on many filesystems + directory, filename = os.path.split(dest) + filename, ext = os.path.splitext(filename) + + if len(filename) > (max_filename_length - len(ext)): + truncated_filename = truncate_filename(filename, max_filename_length) + dest = os.path.join(directory, truncated_filename) + logger.warning(f"Filename too long, truncated to: {dest}") + + try: + with open(dest, "w", encoding="utf-8") as file: + json.dump(torrent_json, file, ensure_ascii=False, indent=4) + except FileNotFoundError as e: + logger.error(f"Failed to save JSON file: {e.filename} - {e.strerror}.") + except OSError as e: + logger.error(f"OS error occurred: {e.filename} - {e.strerror}.") class GracefulKiller: diff --git a/qbit_manage.py b/qbit_manage.py index 43cf146..de0e201 100755 --- a/qbit_manage.py +++ b/qbit_manage.py @@ -200,6 +200,12 @@ parser.add_argument( "-d", "--divider", dest="divider", help="Character that divides the sections (Default: '=')", default="=", type=str ) parser.add_argument("-w", "--width", dest="width", help="Screen Width (Default: 100)", default=100, type=int) +parser.add_argument( + "-ls", "--log-size", dest="log_size", action="store", default=10, type=int, help="Maximum log size per file (in MB)" +) +parser.add_argument( + "-lc", "--log-count", dest="log_count", action="store", default=5, type=int, help="Maximum mumber of logs to keep" +) args = parser.parse_args() @@ -278,6 +284,8 @@ skip_cleanup = get_arg("QBT_SKIP_CLEANUP", args.skip_cleanup, arg_bool=True) skip_qb_version_check = get_arg("QBT_SKIP_QB_VERSION_CHECK", args.skip_qb_version_check, arg_bool=True) dry_run = get_arg("QBT_DRY_RUN", args.dry_run, arg_bool=True) log_level = get_arg("QBT_LOG_LEVEL", args.log_level) +log_size = get_arg("QBT_LOG_SIZE", args.log_size, arg_int=True) +log_count = get_arg("QBT_LOG_COUNT", args.log_count, arg_int=True) divider = get_arg("QBT_DIVIDER", args.divider) screen_width = get_arg("QBT_WIDTH", args.width, arg_int=True) debug = get_arg("QBT_DEBUG", args.debug, arg_bool=True) @@ -327,6 +335,8 @@ for v in [ "skip_qb_version_check", "dry_run", "log_level", + "log_size", + "log_count", "divider", "screen_width", "debug", @@ -354,7 +364,7 @@ except ValueError: sys.exit(1) -logger = MyLogger("qBit Manage", log_file, log_level, default_dir, screen_width, divider[0], False) +logger = MyLogger("qBit Manage", log_file, log_level, default_dir, screen_width, divider[0], False, log_size, log_count) from modules import util # noqa util.logger = logger