Refinements and fix #252

README.md

@@ -1,6 +1,6 @@
-## yt-dlp-bot - YouTube Download Telegram Bot 🇺🇦
+## yt-dlp-bot - Video Download Telegram Bot 🇺🇦
 
-Simple and reliable self-hosted YouTube Download Telegram Bot.
+Simple and reliable self-hosted Video Download Telegram Bot.
 
 Version: 1.4.5. [Release details](RELEASES.md).
 
@@ -15,12 +15,16 @@ Version: 1.4.5. [Release details](RELEASES.md).
 
 ## 😂 Features
 
-* Download audio and videos from [yt-dlp](https://github.com/yt-dlp/yt-dlp) supported sites to your storage.
+* Download audio and free videos with Creative Commons (CC) License from [yt-dlp](https://github.com/yt-dlp/yt-dlp) sites to your storage.
 * Upload downloaded media to Telegram.
 * Interact with the bot in private or group chats.
 * Trigger video downloads via link to the API.
 * Track download tasks using the API.
 
+## Disclaimer
+
+- Intended to use only with videos that are under Creative Commons (CC) License
+
 ## ⚙ Quick Setup
 
 1. Create Telegram bot using [BotFather](https://t.me/BotFather) and get your `token`
@@ -145,7 +149,7 @@ documentations lives at `http://127.0.0.1:1984/docs`.
    [
        {
            "id": "7ab91ef7-461c-4ef6-a35b-d3704fe28e6c",
-           "url": "https://youtu.be/jMetnwUZBJQ",
+           "url": "https://www.youtube.com/watch?v=PavYAOpVpJI",
            "status": "DONE",
            "source": "BOT",
            "added_at": "2022-02-14T02:29:55.981622",
@@ -156,24 +160,7 @@ documentations lives at `http://127.0.0.1:1984/docs`.
                "id": "4b1c63ed-3e32-43e6-a0b7-c7fc8713b268",
                "created": "2022-02-14T02:29:59.597839",
                "updated": "2022-02-14T02:29:59.597845",
-               "name": "Ana Flora Vs. Dj Brizi - Conversa Fiada",
+               "name": "[Drone Freestyle] Mountain Landscape With Snow | Free Stock Footage | Creative Common Video",
-               "ext": "mp4"
-           }
-       },
-       {
-           "id": "952bfb7f-1ab3-4db9-8114-eb9995d0cf8d",
-           "url": "https://youtu.be/AWy1qiTF64M",
-           "status": "DONE",
-           "source": "API",
-           "added_at": "2022-02-14T00:36:21.398624",
-           "created": "2022-02-14T00:36:21.410999",
-           "updated": "2022-02-14T00:36:23.535844",
-           "message_id": null,
-           "file": {
-               "id": "ad1fef96-ce1c-4c5e-a426-58e2d5d3e907",
-               "created": "2022-02-14T00:36:23.537706",
-               "updated": "2022-02-14T00:36:23.537715",
-               "name": "Rufford Ford | part 47",
                "ext": "mp4"
            }
        }
@@ -184,7 +171,7 @@ documentations lives at `http://127.0.0.1:1984/docs`.
    Request
    ```json
    {
-       "url": "https://www.youtube.com/watch?v=zGDzdps75ns",
+       "url": "https://www.youtube.com/watch?v=PavYAOpVpJI",
        "download_media_type": "AUDIO_VIDEO",
        "save_to_storage": false
    }
@@ -193,7 +180,7 @@ documentations lives at `http://127.0.0.1:1984/docs`.
    ```json
    {
        "id": "5ac05808-b29c-40d6-b250-07e3e769d8a6",
-       "url": "https://youtu.be/AWy1qiTF64M",
+       "url": "https://www.youtube.com/watch?v=PavYAOpVpJI",
        "source": "API",
        "added_at": "2022-02-14T00:35:25.419962+00:00"
    }

app_api/api/core/services/task.py

@@ -74,6 +74,11 @@ class TaskService:
             source=source,
             download_media_type=task.download_media_type,
             save_to_storage=task.save_to_storage,
+            from_chat_id=None,
+            from_chat_type=None,
+            from_user_id=None,
+            message_id=None,
+            ack_message_id=None,
         )
         if not await publisher.send_for_download(payload):
             raise TaskServiceError('Failed to create task')

app_bot/bot/core/bot/bot.py

@@ -7,7 +7,7 @@ from pyrogram.enums import ParseMode
 from pyrogram.errors import RPCError
 
 from bot.core.config.config import get_main_config
-from bot.core.config.schema import UserSchema
+from bot.core.schema import UserSchema
 from bot.core.utils import bold
 
 

app_bot/bot/core/config/config.py

@@ -8,8 +8,8 @@ import yaml
 from pydantic import ValidationError
 from yt_shared.config import Settings
 
-from bot.core.config.schema import ConfigSchema
 from bot.core.exceptions import ConfigError
+from bot.core.schema import ConfigSchema
 
 
 class ConfigLoader:

app_bot/bot/core/handlers/abstract.py

@@ -6,13 +6,13 @@ from yt_shared.enums import TaskSource, TelegramChatType
 from yt_shared.schemas.error import ErrorDownloadGeneralPayload, ErrorDownloadPayload
 from yt_shared.schemas.success import SuccessDownloadPayload
 
-from bot.core.config.schema import AnonymousUserSchema, UserSchema
+from bot.core.schema import AnonymousUserSchema, UserSchema
 
 if TYPE_CHECKING:
     from bot.core.bot import VideoBot
 
 
-class AbstractDownloadHandler(metaclass=abc.ABCMeta):
+class AbstractDownloadHandler(abc.ABC):
     def __init__(
         self,
         body: SuccessDownloadPayload

app_bot/bot/core/handlers/success.py

@@ -15,7 +15,7 @@ from yt_shared.utils.tasks.tasks import create_task
 
 from bot.core.handlers.abstract import AbstractDownloadHandler
 from bot.core.tasks.upload import AudioUploadTask, VideoUploadTask
-from bot.core.utils import bold
+from bot.core.utils import bold, is_user_upload_silent
 
 
 class SuccessDownloadHandler(AbstractDownloadHandler):
@@ -47,12 +47,16 @@ class SuccessDownloadHandler(AbstractDownloadHandler):
             await self._delete_acknowledgment_message()
 
     async def _delete_acknowledgment_message(self) -> None:
-        await self._bot.delete_messages(
+        if self._body.from_chat_id and self._body.context.ack_message_id:
-            chat_id=self._body.from_chat_id,
+            await self._bot.delete_messages(
-            message_ids=[self._body.context.ack_message_id],
+                chat_id=self._body.from_chat_id,
-        )
+                message_ids=self._body.context.ack_message_id,
+            )
 
     async def _set_upload_message(self, media_object: BaseMedia) -> None:
+        if not (self._body.from_chat_id and self._body.context.ack_message_id):
+            return
+
         try:
             await self._bot.edit_message_text(
                 chat_id=self._body.from_chat_id,
@@ -142,16 +146,15 @@ class SuccessDownloadHandler(AbstractDownloadHandler):
     async def _send_success_text(self, media_object: BaseMedia) -> None:
         text = self._create_success_text(media_object)
         for user in self._receiving_users:
-            if user.upload.silent:
+            if not is_user_upload_silent(user=user, conf=self._bot.conf):
-                continue
+                kwargs = {
-            kwargs = {
+                    'chat_id': user.id,
-                'chat_id': user.id,
+                    'text': text,
-                'text': text,
+                    'parse_mode': ParseMode.HTML,
-                'parse_mode': ParseMode.HTML,
+                }
-            }
+                if self._body.message_id:
-            if self._body.message_id:
+                    kwargs['reply_to_message_id'] = self._body.message_id
-                kwargs['reply_to_message_id'] = self._body.message_id
+                await self._bot.send_message(**kwargs)
-            await self._bot.send_message(**kwargs)
 
     def _upload_is_enabled(self) -> bool:
         """Check whether upload is allowed for particular user configuration."""

app_bot/bot/core/schema.py

@@ -1,3 +1,5 @@
+import abc
+
 from pydantic import (
     StrictBool,
     StrictInt,
@@ -13,12 +15,12 @@ _LANG_CODE_LEN = 2
 _LANG_CODE_REGEX = rf'^[a-z]{{{_LANG_CODE_LEN}}}$'
 
 
-class AnonymousUserSchema(RealBaseModel):
+class _BaseUserSchema(RealBaseModel, abc.ABC):
     id: StrictInt
 
-    @property
+
-    def is_anonymous_user(self) -> bool:
+class AnonymousUserSchema(_BaseUserSchema):
-        return True
+    pass
 
 
 class VideoCaptionSchema(RealBaseModel):
@@ -37,7 +39,7 @@ class UploadSchema(RealBaseModel):
     video_caption: VideoCaptionSchema
 
 
-class UserSchema(AnonymousUserSchema):
+class UserSchema(_BaseUserSchema):
     is_admin: StrictBool
     send_startup_message: StrictBool
     download_media_type: DownMediaType
@@ -45,10 +47,6 @@ class UserSchema(AnonymousUserSchema):
     use_url_regex_match: StrictBool
     upload: UploadSchema
 
-    @property
-    def is_anonymous_user(self) -> bool:
-        return False
-
 
 class ApiSchema(RealBaseModel):
     upload_video_file: StrictBool

app_bot/bot/core/service.py

@@ -10,7 +10,7 @@ from yt_shared.rabbit.publisher import RmqPublisher
 from yt_shared.schemas.media import InbMediaPayload
 from yt_shared.schemas.url import URL
 
-from bot.core.config.schema import UserSchema
+from bot.core.schema import UserSchema
 from bot.core.utils import can_remove_url_params
 
 

app_bot/bot/core/tasks/upload.py

@@ -19,8 +19,8 @@ from yt_shared.utils.tasks.abstract import AbstractTask
 from yt_shared.utils.tasks.tasks import create_task
 
 from bot.core.config.config import get_main_config, settings
-from bot.core.config.schema import AnonymousUserSchema, UserSchema, VideoCaptionSchema
+from bot.core.schema import AnonymousUserSchema, UserSchema, VideoCaptionSchema
-from bot.core.utils import bold
+from bot.core.utils import bold, is_user_upload_silent
 
 if TYPE_CHECKING:
     from bot.core.bot import VideoBot
@@ -45,7 +45,7 @@ class AudioUploadContext(BaseUploadContext):
     pass
 
 
-class AbstractUploadTask(AbstractTask, metaclass=abc.ABCMeta):
+class AbstractUploadTask(AbstractTask, abc.ABC):
     _UPLOAD_ACTION: ChatAction
 
     def __init__(
@@ -103,16 +103,15 @@ class AbstractUploadTask(AbstractTask, metaclass=abc.ABCMeta):
         )
         coros = []
         for user in self._users:
-            if user.upload.silent:
+            if not is_user_upload_silent(user=user, conf=self._bot.conf):
-                continue
+                kwargs = {
-            kwargs = {
+                    'chat_id': user.id,
-                'chat_id': user.id,
+                    'text': text,
-                'text': text,
+                    'parse_mode': ParseMode.HTML,
-                'parse_mode': ParseMode.HTML,
+                }
-            }
+                if self._ctx.message_id:
-            if self._ctx.message_id:
+                    kwargs['reply_to_message_id'] = self._ctx.message_id
-                kwargs['reply_to_message_id'] = self._ctx.message_id
+                coros.append(self._bot.send_message(**kwargs))
-            coros.append(self._bot.send_message(**kwargs))
         await asyncio.gather(*coros)
 
     def _get_forward_chat_ids(self) -> list[int]:
@@ -257,7 +256,7 @@ class VideoUploadTask(AbstractUploadTask):
         )
 
     def _get_caption_conf(self) -> VideoCaptionSchema:
-        if self._users[0].is_anonymous_user:
+        if isinstance(self._users[0], AnonymousUserSchema):
             return self._bot.conf.telegram.api.video_caption
         return self._users[0].upload.video_caption
 

app_bot/bot/core/utils.py

@@ -1,4 +1,5 @@
 """Utils module."""
+
 import asyncio
 import random
 import string
@@ -11,6 +12,7 @@ from pyrogram.enums import ChatType
 from pyrogram.types import Message
 
 from bot.core.config import settings
+from bot.core.schema import AnonymousUserSchema, ConfigSchema, UserSchema
 
 
 async def shallow_sleep_async(sleep_time: float = 0.1) -> None:
@@ -92,3 +94,15 @@ def split_telegram_message(
 
 def can_remove_url_params(url: str, matching_hosts: Iterable[str]) -> bool:
     return urlparse(url).netloc in set(matching_hosts)
+
+
+def is_user_upload_silent(
+    user: UserSchema | AnonymousUserSchema, conf: ConfigSchema
+) -> bool:
+    if isinstance(user, AnonymousUserSchema):
+        if conf.telegram.api.silent:
+            return True
+    elif user.upload.silent:
+        return True
+    else:
+        return False

app_bot/bot/core/workers/abstract.py

@@ -1,4 +1,5 @@
 """RabbitMQ Queue abstract worker module."""
+
 import abc
 import enum
 from typing import TYPE_CHECKING, Type

app_bot/main.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 """Bot Launcher Module."""
+
 import asyncio
 
 import uvloop

app_worker/alembic/versions/0769fbebd121_.py

@@ -5,6 +5,7 @@ Revises: 63e7cae94c1d
 Create Date: 2022-02-18 23:34:39.587248
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

app_worker/alembic/versions/10ab08fc321b_.py

@@ -5,6 +5,7 @@ Revises: ba7716dca30a
 Create Date: 2023-02-25 15:47:37.542906
 
 """
+
 from alembic import op
 
 # revision identifiers, used by Alembic.

app_worker/alembic/versions/2689b03525f8_.py

@@ -5,6 +5,7 @@ Revises: 0769fbebd121
 Create Date: 2022-02-20 00:07:13.184353
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

app_worker/alembic/versions/50331b3c39bb_.py

@@ -5,6 +5,7 @@ Revises: 77c100300b5b
 Create Date: 2023-04-06 22:08:12.511699
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

app_worker/alembic/versions/6221c0018660_.py

@@ -5,6 +5,7 @@ Revises: 2689b03525f8
 Create Date: 2022-02-20 03:00:01.366172
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

app_worker/alembic/versions/63e7cae94c1d_.py

@@ -5,6 +5,7 @@ Revises:
 Create Date: 2022-02-06 21:18:09.738831
 
 """
+
 import sqlalchemy as sa
 import sqlalchemy_utils
 from sqlalchemy.dialects import postgresql

app_worker/alembic/versions/77c100300b5b_.py

@@ -5,6 +5,7 @@ Revises: 10ab08fc321b
 Create Date: 2023-04-06 10:45:47.289554
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

app_worker/alembic/versions/8021be777d1d_.py

@@ -5,6 +5,7 @@ Revises: ff03785a1f0d
 Create Date: 2022-06-13 20:16:10.845151
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

app_worker/alembic/versions/ba7716dca30a_.py

@@ -5,6 +5,7 @@ Revises: 8021be777d1d
 Create Date: 2022-06-13 22:51:29.775743
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

app_worker/alembic/versions/d3f89ea5e8b5_.py

@@ -5,6 +5,7 @@ Revises: da4a97a0fdb7
 Create Date: 2022-06-07 19:35:23.098590
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

app_worker/alembic/versions/da4a97a0fdb7_.py

@@ -5,6 +5,7 @@ Revises: 6221c0018660
 Create Date: 2022-03-19 10:46:57.624753
 
 """
+
 import sqlalchemy as sa
 import sqlalchemy_utils
 

app_worker/alembic/versions/ff03785a1f0d_.py

@@ -5,6 +5,7 @@ Revises: d3f89ea5e8b5
 Create Date: 2022-06-10 20:04:34.792613
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

app_worker/worker/core/downloader.py

@@ -68,13 +68,13 @@ class MediaDownloader:
 
                 meta: dict | None = ytdl.extract_info(url, download=True)
                 if not meta:
-                    err_msg = f'Error during media download. Check logs.'
+                    err_msg = 'Error during media download. Check logs.'
                     self._log.error('%s. Meta: %s', err_msg, meta)
                     raise MediaDownloaderError(err_msg)
 
                 current_files = os.listdir(curr_tmp_dir)
                 if not current_files:
-                    err_msg = f'Nothing downloaded. Is URL valid?'
+                    err_msg = 'Nothing downloaded. Is URL valid?'
                     self._log.error(err_msg)
                     raise MediaDownloaderError(err_msg)
 
@@ -126,7 +126,7 @@ class MediaDownloader:
             return create_dto(self._create_video_dto)
 
         def create_dto(
-            func: Callable[[dict, str, str], Audio | Video]
+            func: Callable[[dict, str, str], Audio | Video],
         ) -> Audio | Video:
             try:
                 return func(

app_worker/ytdl_opts/default.py

@@ -6,6 +6,7 @@ More here https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/options.py or 'yt-
 If you want to change any of these values or add new ones, copy all content to the `user.py` in the same
 directory as this file, and edit the values.
 """
+
 from worker.utils import get_cookies_opts_if_not_empty
 
 FINAL_AUDIO_FORMAT = 'mp3'

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.ruff]
 line-length = 88
-select = ["F", "E", "W", "I001"]
+lint.select = ["F", "E", "W", "I001"]
-ignore = ["E501"]  # Skip line length violations
+lint.ignore = ["E501"]  # Skip line length violations
 src = ["app_api", "app_bot", "app_worker"]
 
 [tool.ruff.format]

start.py

@@ -1,25 +1,25 @@
 """Over-engineered Python 3.10+ version of bash script with netcat (nc) just for fun.
 
-    #!/bin/bash
+#!/bin/bash
 
-    check_reachability() {
+check_reachability() {
-    while ! nc -z "$1" "${!2}"
+while ! nc -z "$1" "${!2}"
-    do
+do
-      echo "Waiting for $3 to be reachable on port ${!2}"
+  echo "Waiting for $3 to be reachable on port ${!2}"
-      sleep 1
+  sleep 1
-    done
+done
-    echo "Connection to $3 on port ${!2} verified"
+echo "Connection to $3 on port ${!2} verified"
-    return 0
+return 0
-    }
+}
 
 
-    wait_for_services_to_be_reachable() {
+wait_for_services_to_be_reachable() {
-      check_reachability rabbitmq RABBITMQ_PORT RabbitMQ
+  check_reachability rabbitmq RABBITMQ_PORT RabbitMQ
-      check_reachability postgres POSTGRES_PORT PostgreSQL
+  check_reachability postgres POSTGRES_PORT PostgreSQL
-    }
+}
 
-    wait_for_services_to_be_reachable
+wait_for_services_to_be_reachable
-    exit 0
+exit 0
 """
 
 import asyncio

yt_shared/yt_shared/schemas/cache.py

@@ -1,10 +1,12 @@
 import datetime
 
+from pydantic import StrictInt, StrictStr
+
 from yt_shared.schemas.base import RealBaseModel
 
 
 class CacheSchema(RealBaseModel):
-    cache_id: str
+    cache_id: StrictStr
-    cache_unique_id: str
+    cache_unique_id: StrictStr
-    file_size: int
+    file_size: StrictInt
     date_timestamp: datetime.datetime

yt_shared/yt_shared/utils/tasks/abstract.py

@@ -2,7 +2,7 @@ import abc
 import logging
 
 
-class AbstractTask(metaclass=abc.ABCMeta):
+class AbstractTask(abc.ABC):
     def __init__(self) -> None:
         self._log = logging.getLogger(self.__class__.__name__)