Merge pull request #269 from StuffAnThings/develop

3.6.1

.github/workflows/dependabot-approve-and-auto-merge.yml

@@ -19,7 +19,7 @@ jobs:
       # will not occur.
       - name: Dependabot metadata
         id: dependabot-metadata
-        uses: dependabot/fetch-metadata@v1.3.6
+        uses: dependabot/fetch-metadata@v1.4.0
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
       # Here the PR gets approved.

CHANGELOG

@@ -1,8 +1,8 @@
 # Requirements Updated
-- Updates qbitorrent api to 2023.4.45
+- Updates qbitorrent api to 2023.4.47
-- Updates Schedule to 1.2.0
 
-# Refactoring
+# Bug Fixes
-- Refactor qbit_manage to split up core functions into separate files
+- Fixes bug in not removing empty directories (Thanks to @buthed010203 #266)
+- Speed up remove_orphan by using multiprocessing (Thanks to @buthed010203 #266)
 
-**Full Changelog**: https://github.com/StuffAnThings/qbit_manage/compare/v3.5.1...v3.6.0
+**Full Changelog**: https://github.com/StuffAnThings/qbit_manage/compare/v3.6.0...v3.6.1

VERSION

@@ -1 +1 @@
-3.6.0
+3.6.1

modules/core/remove_orphaned.py

@@ -1,9 +1,13 @@
 import os
 from fnmatch import fnmatch
+from itertools import repeat
+from multiprocessing import cpu_count
+from multiprocessing import Pool
 
 from modules import util
 
 logger = util.logger
+_config = None
 
 
 class RemoveOrphaned:
@@ -17,7 +21,11 @@ class RemoveOrphaned:
         self.root_dir = qbit_manager.config.root_dir
         self.orphaned_dir = qbit_manager.config.orphaned_dir
 
+        global _config
+        _config = self.config
+        self.pool = Pool(processes=max(cpu_count() - 1, 1))
         self.rem_orphaned()
+        self.cleanup_pool()
 
     def rem_orphaned(self):
         """Remove orphaned files from remote directory"""
@@ -27,54 +35,64 @@ class RemoveOrphaned:
         root_files = []
         orphaned_files = []
         excluded_orphan_files = []
-        orphaned_parent_path = set()
 
         if self.remote_dir != self.root_dir:
+            local_orphaned_dir = self.orphaned_dir.replace(self.remote_dir, self.root_dir)
             root_files = [
                 os.path.join(path.replace(self.remote_dir, self.root_dir), name)
                 for path, subdirs, files in os.walk(self.remote_dir)
                 for name in files
-                if self.orphaned_dir.replace(self.remote_dir, self.root_dir) not in path
+                if local_orphaned_dir not in path
             ]
         else:
             root_files = [
                 os.path.join(path, name)
                 for path, subdirs, files in os.walk(self.root_dir)
                 for name in files
-                if self.orphaned_dir.replace(self.root_dir, self.remote_dir) not in path
+                if self.orphaned_dir not in path
             ]
 
         # Get an updated list of torrents
+        logger.print_line("Locating orphan files", self.config.loglevel)
         torrent_list = self.qbt.get_torrents({"sort": "added_on"})
+        torrent_files_and_save_path = []
         for torrent in torrent_list:
-            for file in torrent.files:
+            torrent_files = []
-                fullpath = os.path.join(torrent.save_path, file.name)
+            for torrent_files_dict in torrent.files:
-                # Replace fullpath with \\ if qbm is running in docker (linux) but qbt is on windows
+                torrent_files.append(torrent_files_dict.name)
-                fullpath = fullpath.replace(r"/", "\\") if ":\\" in fullpath else fullpath
+            torrent_files_and_save_path.append((torrent_files, torrent.save_path))
-                torrent_files.append(fullpath)
+        torrent_files.extend(
+            [
+                fullpath
+                for fullpathlist in self.pool.starmap(get_full_path_of_torrent_files, torrent_files_and_save_path)
+                for fullpath in fullpathlist
+                if fullpath not in torrent_files
+            ]
+        )
 
         orphaned_files = set(root_files) - set(torrent_files)
-        orphaned_files = sorted(orphaned_files)
 
         if self.config.orphaned["exclude_patterns"]:
-            exclude_patterns = self.config.orphaned["exclude_patterns"]
+            logger.print_line("Processing orphan exclude patterns")
+            exclude_patterns = [
+                exclude_pattern.replace(self.remote_dir, self.root_dir)
+                for exclude_pattern in self.config.orphaned["exclude_patterns"]
+            ]
             excluded_orphan_files = [
-                file
+                file for file in orphaned_files for exclude_pattern in exclude_patterns if fnmatch(file, exclude_pattern)
-                for file in orphaned_files
-                for exclude_pattern in exclude_patterns
-                if fnmatch(file, exclude_pattern.replace(self.remote_dir, self.root_dir))
             ]
 
         orphaned_files = set(orphaned_files) - set(excluded_orphan_files)
 
         if orphaned_files:
+            orphaned_files = sorted(orphaned_files)
             os.makedirs(self.orphaned_dir, exist_ok=True)
             body = []
             num_orphaned = len(orphaned_files)
             logger.print_line(f"{num_orphaned} Orphaned files found", self.config.loglevel)
             body += logger.print_line("\n".join(orphaned_files), self.config.loglevel)
             body += logger.print_line(
-                f"{'Did not move' if self.config.dry_run else 'Moved'} {num_orphaned} Orphaned files "
+                f"{'Not moving' if self.config.dry_run else 'Moving'} {num_orphaned} Orphaned files "
                 f"to {self.orphaned_dir.replace(self.remote_dir,self.root_dir)}",
                 self.config.loglevel,
             )
@@ -89,14 +107,31 @@ class RemoveOrphaned:
             }
             self.config.send_notifications(attr)
             # Delete empty directories after moving orphan files
-            logger.info("Cleaning up any empty directories...")
             if not self.config.dry_run:
-                for file in orphaned_files:
+                orphaned_parent_path = set(self.pool.map(move_orphan, orphaned_files))
-                    src = file.replace(self.root_dir, self.remote_dir)
+                logger.print_line("Removing newly empty directories", self.config.loglevel)
-                    dest = os.path.join(self.orphaned_dir, file.replace(self.root_dir, ""))
+                self.pool.starmap(util.remove_empty_directories, zip(orphaned_parent_path, repeat("**/*")))
-                    util.move_files(src, dest, True)
+
-                    orphaned_parent_path.add(os.path.dirname(file).replace(self.root_dir, self.remote_dir))
-                    for parent_path in orphaned_parent_path:
-                        util.remove_empty_directories(parent_path, "**/*")
         else:
             logger.print_line("No Orphaned Files found.", self.config.loglevel)
+
+    def cleanup_pool(self):
+        self.pool.close()
+        self.pool.join()
+
+
+def get_full_path_of_torrent_files(torrent_files, save_path):
+    fullpath_torrent_files = []
+    for file in torrent_files:
+        fullpath = os.path.join(save_path, file)
+        # Replace fullpath with \\ if qbm is running in docker (linux) but qbt is on windows
+        fullpath = fullpath.replace(r"/", "\\") if ":\\" in fullpath else fullpath
+        fullpath_torrent_files.append(fullpath)
+    return fullpath_torrent_files
+
+
+def move_orphan(file):
+    src = file.replace(_config.root_dir, _config.remote_dir)  # Could be optimized to only run when root != remote
+    dest = os.path.join(_config.orphaned_dir, file.replace(_config.root_dir, ""))
+    util.move_files(src, dest, True)
+    return os.path.dirname(file).replace(_config.root_dir, _config.remote_dir)  # Another candidate for micro optimizing

modules/util.py

@@ -265,7 +265,7 @@ def move_files(src, dest, mod=False):
     dest_path = os.path.dirname(dest)
     to_delete = False
     if os.path.isdir(dest_path) is False:
-        os.makedirs(dest_path)
+        os.makedirs(dest_path, exist_ok=True)
     try:
         if mod is True:
             mod_time = time.time()
@@ -305,6 +305,7 @@ def remove_empty_directories(pathlib_root_dir, pattern):
         key=lambda p: len(str(p)),
         reverse=True,
     )
+    longest.append(pathlib_root_dir)
     for pdir in longest:
         try:
             pdir.rmdir()  # remove directory if empty

requirements.txt

@@ -1,6 +1,6 @@
 flake8==6.0.0
 pre-commit==3.2.2
-qbittorrent-api==2023.4.45
+qbittorrent-api==2023.4.47
 requests==2.28.2
 retrying==1.3.4
 ruamel.yaml==0.17.21