Merge pull request #39 from StuffAnThings/develop

v2.0 Docker Release

.dockerignore

@@ -0,0 +1,21 @@
+**/dist
+**/build
+*.spec
+**/__pycache__
+/.vscode
+**/log
+README.md
+LICENSE
+.gitignore
+.dockerignore
+.git
+.github
+.vscode
+*.psd
+config/**/*
+config
+Dockerfile
+venv
+.idea
+test.py
+!config/config.yml.sample

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: bobokun

.github/workflows/develop.yml

@@ -0,0 +1,38 @@
+name: Docker Develop Release
+
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+
+  docker-develop:
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Check Out Repo
+        uses: actions/checkout@v2
+        with:
+          ref: develop
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKER_HUB_USERNAME }}/qbit_manage:develop

.github/workflows/latest.yml

@@ -0,0 +1,36 @@
+name: Docker Latest Release
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  docker-latest:
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Check Out Repo
+        uses: actions/checkout@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKER_HUB_USERNAME }}/qbit_manage:latest

.github/workflows/tag.yml

@@ -0,0 +1,18 @@
+name: Tag
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  tag-new-versions:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.PAT }}
+          fetch-depth: 2
+      - uses: salsify/action-detect-and-tag-new-version@v1.0.3
+        with:
+          version-command: |
+            cat VERSION

.github/workflows/version.yml

@@ -0,0 +1,39 @@
+name: Docker Version Release
+
+on:
+  create:
+    tags:
+      - v*
+
+jobs:
+
+  docker-develop:
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Check Out Repo
+        uses: actions/checkout@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Get the version
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKER_HUB_USERNAME }}/qbit_manage:${{ steps.get_version.outputs.VERSION }}

.gitignore

@@ -1,3 +1,9 @@
-*.log
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+*.log*
 *.yml
-.vscode/settings.json
+.vscode/*
+!.github/**

Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.9-slim
+COPY requirements.txt /
+RUN echo "**** install python packages ****" \
+ && pip3 install --no-cache-dir --upgrade --requirement /requirements.txt \
+ && apt-get autoremove -y \
+ && apt-get clean \
+ && rm -rf /requirements.txt /tmp/* /var/tmp/* /var/lib/apt/lists/*
+COPY . /
+VOLUME /config
+ENTRYPOINT ["python3","qbit_manage.py"]

README.md

@@ -9,7 +9,8 @@ This is a program used to manage your qBittorrent instance such as:
 * Recheck paused torrents sorted by lowest size and resume if completed
 * Remove orphaned files from your root directory that are not referenced by qBittorrent
 * Tag any torrents that have no hard links and allows optional cleanup to delete these torrents and contents based on maximum ratio and/or time seeded
-
+* RecycleBin function to move files into a RecycleBin folder instead of deleting the data directly when deleting a torrent
+* Built-in scheduler to run the script every x minutes. (Can use `--run` command to run without the scheduler)
 ## Installation
 
 Check out the [wiki](https://github.com/StuffAnThings/qbit_manage/wiki) for installation help
@@ -20,10 +21,12 @@ To run the script in an interactive terminal run:
 
 * copy the `config.yml.sample` file to `config.yml`
 * add your qBittorrent host, user and pass. If you are not using a username and password you can remove the `user` and `pass` lines.
-* add your `cross_seed` and `root_dir`. If you are using a docker container you must fill out `remote_dir` as well.
+* add your `cross_seed` and `root_dir`. If you're running cross-seed in a docker container you must fill out `remote_dir` as well.
 * Add your categories and save path to match with what is being used in your qBittorrent instance. I suggest using the full path when defining `save_path`
 * Add the `tag` definition based on tracker URL
 * Modify the  `nohardlinks` by specifying your completed movies/series category to match with qBittorrent. Please ensure the `root_dir` and/or  `remote_dir` is added in the `directory` section
+* `root_dir` needs to be defined in order to use the RecycleBin function. If optional `empty_after_x_days` is not defined then it will never empty the RecycleBin. Setting it to 0 will empty the RecycleBin immediately.
+* Modify the `orphaned` section to define file patterns not to consider as orphans. Use this to exclude your incomplete torrents directory, or to ignore auto-generated files such as Thumbs.db.
 * To run the script in an interactive terminal with a list of possible commands run:
 
 ```bash
@@ -32,21 +35,24 @@ python qbit_manage.py -h
 
 ## Commands
 
-| Shell Command | Description | Default Value |
-| :------------ | :------------  | :------------ |
-| `-c CONFIG` or `--config-file CONFIG`  | This is used if you want to use a different name for your config.yml. `Example: tv.yml`  | config.yml |
-| `-l LOGFILE,` or `--log-file LOGFILE,` | This is used if you want to use a different name for your log file. `Example: tv.log` | activity.log |
-| `-m` or `--manage` | Use this if you would like to update your tags, categories, remove unregistered torrents, AND recheck/resume paused torrents.  |  |
-| `-s` or `--cross-seed` | Use this after running [cross-seed script](https://github.com/mmgoodnow/cross-seed) to add torrents from the cross-seed output folder to qBittorrent  |  |
-| `-re` or `--recheck` | Recheck paused torrents sorted by lowest size. Resume if Completed.  |  |
-| `-g` or `--cat-update` |  Use this if you would like to update your categories.  |  |
-| `-t` or `--tag-update` |  Use this if you would like to update your tags. (Only adds tags to untagged torrents) |  |
-| `-r` or `--rem-unregistered` |  Use this if you would like to remove unregistered torrents. (It will the delete data & torrent if it is not being cross-seeded, otherwise it will just remove the torrent without deleting data) |  |
-| `-ro` or `--rem-orphaned` | Use this if you would like to remove orphaned files from your `root_dir` directory that are not referenced by any torrents. It will scan your `root_dir` directory and compare it with what is in qBittorrent. Any data not referenced in qBittorrent will be moved into `/data/torrents/orphaned_data` folder for you to review/delete. |  |
-| `-tnhl` or `--tag-nohardlinks` | Use this to tag any torrents that do not have any hard links associated with any of the files. This is useful for those that use Sonarr/Radarr that hard links your media files with the torrents for seeding. When files get upgraded they no longer become linked with your media therefore will be tagged with a new tag noHL. You can then safely delete/remove these torrents to free up any extra space that is not being used by your media folder. |  |
-| `--dry-run` |   If you would like to see what is gonna happen but not actually move/delete or tag/categorize anything. |  |
-| `--log LOGLEVEL` |   Change the ouput log level. | INFO |
-
+| Shell Command |Docker Environment Variable |Description | Default Value |
+| :------------ | :------------  | :------------ | :------------ |
+| `-r` or`--run` | QBT_RUN |Run without the scheduler. Script will exit after completion. | False |
+| `-sch` or `--schedule` | QBT_SCHEDULE  | Schedule to run every x minutes. (Default set to 30)  | 30 |
+| `-c CONFIG` or `--config-file CONFIG` | QBT_CONFIG  | This is used if you want to use a different name for your config.yml. `Example: tv.yml`  | config.yml |
+| `-lf LOGFILE,` or `--log-file LOGFILE,` | QBT_LOGFILE | This is used if you want to use a different name for your log file. `Example: tv.log` | activity.log |
+| `-cs` or `--cross-seed` | QBT_CROSS_SEED | Use this after running [cross-seed script](https://github.com/mmgoodnow/cross-seed) to add torrents from the cross-seed output folder to qBittorrent  | False |
+| `-re` or `--recheck` | QBT_RECHECK | Recheck paused torrents sorted by lowest size. Resume if Completed.  | False |
+| `-cu` or `--cat-update` | QBT_CAT_UPDATE |  Use this if you would like to update your categories.  | False |
+| `-tu` or `--tag-update` | QBT_TAG_UPDATE |  Use this if you would like to update your tags. (Only adds tags to untagged torrents) | False |
+| `-ru` or `--rem-unregistered` | QBT_REM_UNREGISTERED |  Use this if you would like to remove unregistered torrents. (It will the delete data & torrent if it is not being cross-seeded, otherwise it will just remove the torrent without deleting data) | False |
+| `-ro` or `--rem-orphaned` | QBT_REM_ORPHANED | Use this if you would like to remove orphaned files from your `root_dir` directory that are not referenced by any torrents. It will scan your `root_dir` directory and compare it with what is in qBittorrent. Any data not referenced in qBittorrent will be moved into `/data/torrents/orphaned_data` folder for you to review/delete. | False |
+| `-tnhl` or `--tag-nohardlinks` | QBT_TAG_NOHARDLINKS | Use this to tag any torrents that do not have any hard links associated with any of the files. This is useful for those that use Sonarr/Radarr that hard links your media files with the torrents for seeding. When files get upgraded they no longer become linked with your media therefore will be tagged with a new tag noHL. You can then safely delete/remove these torrents to free up any extra space that is not being used by your media folder. | False |
+| `-sr` or `--skip-recycle` | QBT_SKIP_RECYCLE | Use this to skip emptying the Reycle Bin folder (`/root_dir/.RecycleBin`). | False |
+| `-dr` or `--dry-run` | QBT_DRY_RUN |   If you would like to see what is gonna happen but not actually move/delete or tag/categorize anything. | False |
+| `-ll` or `--log-level LOGLEVEL` | QBT_LOG_LEVEL |   Change the ouput log level. | INFO |
+| `-d` or `--divider` | QBT_DIVIDER |   Character that divides the sections (Default: '=') | = |
+| `-w` or `--width` | QBT_WIDTH |   Screen Width (Default: 100) | 100 |
 ### Config
 
 To choose the location of the YAML config file

VERSION

@@ -0,0 +1 @@
+2.0

config.yml.sample

@@ -7,8 +7,8 @@ qbt:
 directory:
   # Do not remove these
   # Cross-seed var: </your/path/here/> #Output directory of cross-seed
-  # root_dir var: </your/path/here/> #Root downloads directory used to check for orphaned files
-  # <OPTIONAL> remote_dir var: </your/path/here/> # Path of docker host mapping of root_dir
+  # root_dir var: </your/path/here/> #Root downloads directory used to check for orphaned files and used in RecycleBin
+  # <OPTIONAL> remote_dir var: </your/path/here/> # Path of docker host mapping of root_dir. Must be set if you are using docker!
   cross_seed: "/your/path/here/"
   root_dir: "/data/torrents/"
   remote_dir: "/mnt/user/data/torrents/"
@@ -68,3 +68,20 @@ nohardlinks:
     max_ratio: 4.0
     #<OPTIONAL> seeding time var: Will set the torrent Maximum seeding time (min) until torrent is stopped from seeding
     max_seeding_time: 86400
+
+#Recycle Bin method of deletion will move files into the recycle bin instead of directly deleting them in qbit
+recyclebin:
+  enabled: true
+  #<OPTIONAL> empty_after_x_days var: Will automatically remove all files and folders in recycle bin after x days. 
+  #                                   If this variable is not defined it, the RecycleBin will never be emptied. 
+  #                                   Setting this variable to 0 will delete files immediately.
+  empty_after_x_days: 60
+
+# Orphaned files are those in the root_dir download directory that are not referenced by any active torrents.
+orphaned:
+    # File patterns that will not be considered orphaned files. Handy for generated files that aren't part of the torrent but belong with the torrent's files
+    exclude_patterns:
+        - "**/.DS_Store"
+        - "**/Thumbs.db"
+        - "**/@eaDir"
+        - "/data/torrents/temp/**"

config/config.yml.sample

@@ -0,0 +1,89 @@
+# qBittorrent parameters
+qbt:
+  host: "localhost:8080"
+  user: "username"
+  pass: "password"
+
+directory:
+  # Do not remove these
+  # Cross-seed var: </your/path/here/> #Output directory of cross-seed
+  # root_dir var: </your/path/here/> #Root downloads directory used to check for orphaned files, noHL, and RecycleBin.
+  # <OPTIONAL> remote_dir var: </your/path/here/> # Path of docker host mapping of root_dir.
+                              # Must be set if you're running qbit_manage locally and qBittorrent/cross_seed is in a docker
+  cross_seed: "/your/path/here/"
+  root_dir: "/data/torrents/"
+  remote_dir: "/mnt/user/data/torrents/"
+
+# Category/Pathing Parameters
+cat:
+  # <Category Name> : <save_path> #Path of your save directory. Can be a keyword or full path
+  movies: "/data/torrents/Movies"
+  tv: "TV"
+
+# Tag Parameters
+tags:
+  # <Tracker URL Keyword>: <Tag Name>
+  animebytes.tv: AnimeBytes
+  avistaz: Avistaz
+  beyond-hd: Beyond-HD
+  blutopia: Blutopia
+  cartoonchaos: CartoonChaos
+  digitalcore: DigitalCore
+  gazellegames: GGn
+  hdts: HDTorrents
+  landof.tv: BroadcasTheNet
+  myanonamouse: MaM
+  passthepopcorn: PassThePopcorn
+  privatehd: PrivateHD
+  tleechreload: TorrentLeech
+  torrentdb: TorrentDB
+  torrentleech: TorrentLeech
+  tv-vault: TV-Vault
+
+#Tag Movies/Series that are not hard linked
+nohardlinks:
+  # Mandatory to fill out directory parameter above to use this function (root_dir/remote_dir)
+  # This variable should be set to your category name of your completed movies/completed series in qbit. Acceptable variable can be any category you would like to tag if there are no hardlinks found
+  movies-completed:
+    #<OPTIONAL> exclude_tags var: Will exclude the following tags when searching through the category.
+    exclude_tags:
+      - Beyond-HD
+      - AnimeBytes
+      - MaM
+    #<OPTIONAL> cleanup var: WARNING!! Setting this as true Will remove and delete contents of any torrents that are in paused state and has the NoHL tag
+    cleanup: false
+    #<OPTIONAL> max_ratio var: Will set the torrent Maximum share ratio until torrent is stopped from seeding/uploading
+    max_ratio: 4.0
+    #<OPTIONAL> seeding time var: Will set the torrent Maximum seeding time (min) until torrent is stopped from seeding
+    max_seeding_time: 86400
+
+  #Can have additional categories set with separate ratio/seeding times defined.
+  series-completed:
+    #<OPTIONAL> exclude_tags var: Will exclude the following tags when searching through the category.
+    exclude_tags:
+      - Beyond-HD
+      - BroadcasTheNet
+    #<OPTIONAL> cleanup var: WARNING!! Setting this as true Will remove and delete contents of any torrents that are in paused state and has the NoHL tag
+    cleanup: false
+    #<OPTIONAL> max_ratio var: Will set the torrent Maximum share ratio until torrent is stopped from seeding/uploading
+    max_ratio: 4.0
+    #<OPTIONAL> seeding time var: Will set the torrent Maximum seeding time (min) until torrent is stopped from seeding
+    max_seeding_time: 86400
+
+#Recycle Bin method of deletion will move files into the recycle bin (Located in /root_dir/.RecycleBin) instead of directly deleting them in qbit
+#By default the Recycle Bin will be emptied on every run of the qbit_manage script if empty_after_x_days is defined.
+recyclebin:
+  enabled: true
+  #<OPTIONAL> empty_after_x_days var: Will automatically remove all files and folders in recycle bin after x days. (Checks every script run)
+  #                                   If this variable is not defined it, the RecycleBin will never be emptied. 
+  #                                   WARNING: Setting this variable to 0 will delete all files immediately upon script run!
+  empty_after_x_days: 60
+
+# Orphaned files are those in the root_dir download directory that are not referenced by any active torrents.
+orphaned:
+    # File patterns that will not be considered orphaned files. Handy for generated files that aren't part of the torrent but belong with the torrent's files
+    exclude_patterns:
+        - "**/.DS_Store"
+        - "**/Thumbs.db"
+        - "**/@eaDir"
+        - "/data/torrents/temp/**"

modules/docker.py

@@ -0,0 +1,10 @@
+import signal
+
+#Gracefully kill script when docker stops
+class GracefulKiller:
+  kill_now = False
+  def __init__(self):
+    #signal.signal(signal.SIGINT, self.exit_gracefully)
+    signal.signal(signal.SIGTERM, self.exit_gracefully)
+  def exit_gracefully(self, *args):
+    self.kill_now = True

modules/util.py

@@ -0,0 +1,95 @@
+import logging, traceback
+from logging.handlers import RotatingFileHandler
+
+logger = logging.getLogger("qBit Manage")
+
+class TimeoutExpired(Exception):
+    pass
+
+class Failed(Exception):
+    pass
+
+class NotScheduled(Exception):
+    pass
+
+separating_character = "="
+screen_width = 100
+spacing = 0
+
+
+def print_multiline(lines, loglevel='INFO'):
+    line_list = str(lines).split("\n") 
+    for i, line in enumerate(line_list):
+        if len(line) > 0 and i != len(line_list)-1:
+            logger.log(getattr(logging, loglevel),line)
+        if i == 0:
+            logger.handlers[1].setFormatter(logging.Formatter(" " * 37 + "| %(message)s"))
+    logger.handlers[1].setFormatter(logging.Formatter("[%(asctime)s] %(levelname)-10s | %(message)s"))
+
+def print_stacktrace():
+    print_multiline(traceback.format_exc())
+
+def my_except_hook(exctype, value, tb):
+    for line in traceback.format_exception(etype=exctype, value=value, tb=tb):
+        print_multiline(line, critical=True)
+
+
+def centered(text, sep=" "):
+    if len(text) > screen_width - 2:
+        return text
+    space = screen_width - len(text) - 2
+    text = f" {text} "
+    if space % 2 == 1:
+        text += sep
+        space -= 1
+    side = int(space / 2) - 1
+    final_text = f"{sep * side}{text}{sep * side}"
+    return final_text
+
+def separator(text=None, space=True, border=True, loglevel='INFO'):
+    sep = " " if space else separating_character
+    for handler in logger.handlers:
+        apply_formatter(handler, border=False)
+    border_text = f"|{separating_character * screen_width}|"
+    if border:
+        logger.log(getattr(logging, loglevel),border_text)
+    if text:
+        text_list = text.split("\n")
+        for t in text_list:
+            logger.log(getattr(logging, loglevel),f"|{sep}{centered(t, sep=sep)}{sep}|")
+        if border:
+            logger.log(getattr(logging, loglevel),border_text)
+    for handler in logger.handlers:
+        apply_formatter(handler)
+
+def apply_formatter(handler, border=True):
+    text = f"| %(message)-{screen_width - 2}s |" if border else f"%(message)-{screen_width - 2}s"
+    if isinstance(handler, RotatingFileHandler):
+        #text = f"[%(asctime)s] %(filename)-27s %(levelname)-10s {text}"
+        text = f"[%(asctime)s] %(levelname)-10s {text}"
+    handler.setFormatter(logging.Formatter(text))
+
+def adjust_space(display_title):
+    display_title = str(display_title)
+    space_length = spacing - len(display_title)
+    if space_length > 0:
+        display_title += " " * space_length
+    return display_title
+
+def insert_space(display_title,space_length=0):
+    display_title = str(display_title)
+    if space_length == 0: 
+        space_length = spacing - len(display_title)
+    if space_length > 0:
+        display_title = " " * space_length + display_title
+    return display_title
+
+def print_return(text):
+    print(adjust_space(f"| {text}"), end="\r")
+    global spacing
+    spacing = len(text) + 2
+
+def print_end():
+    print(adjust_space(" "), end="\r")
+    global spacing
+    spacing = 0

requirements.txt

@@ -1,2 +1,3 @@
 PyYAML
 qbittorrent-api
+schedule