diff --git a/README.md b/README.md index 8b9fa77..33a6677 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -## yt-dlp-bot - YouTube Download Telegram Bot πŸ‡ΊπŸ‡¦ +## yt-dlp-bot - Video Download Telegram Bot πŸ‡ΊπŸ‡¦ -Simple and reliable self-hosted YouTube Download Telegram Bot. +Simple and reliable self-hosted Video Download Telegram Bot. Version: 1.4.5. [Release details](RELEASES.md). @@ -15,12 +15,16 @@ Version: 1.4.5. [Release details](RELEASES.md). ## πŸ˜‚ Features -* Download audio and videos from [yt-dlp](https://github.com/yt-dlp/yt-dlp) supported sites to your storage. +* Download audio and free videos with Creative Commons (CC) License from [yt-dlp](https://github.com/yt-dlp/yt-dlp) sites to your storage. * Upload downloaded media to Telegram. * Interact with the bot in private or group chats. * Trigger video downloads via link to the API. * Track download tasks using the API. +## Disclaimer + +- Intended to use only with videos that are under Creative Commons (CC) License + ## βš™ Quick Setup 1. Create Telegram bot using [BotFather](https://t.me/BotFather) and get your `token` @@ -145,7 +149,7 @@ documentations lives at `http://127.0.0.1:1984/docs`. [ { "id": "7ab91ef7-461c-4ef6-a35b-d3704fe28e6c", - "url": "https://youtu.be/jMetnwUZBJQ", + "url": "https://www.youtube.com/watch?v=PavYAOpVpJI", "status": "DONE", "source": "BOT", "added_at": "2022-02-14T02:29:55.981622", @@ -156,24 +160,7 @@ documentations lives at `http://127.0.0.1:1984/docs`. "id": "4b1c63ed-3e32-43e6-a0b7-c7fc8713b268", "created": "2022-02-14T02:29:59.597839", "updated": "2022-02-14T02:29:59.597845", - "name": "Ana Flora Vs. Dj Brizi - Conversa Fiada", - "ext": "mp4" - } - }, - { - "id": "952bfb7f-1ab3-4db9-8114-eb9995d0cf8d", - "url": "https://youtu.be/AWy1qiTF64M", - "status": "DONE", - "source": "API", - "added_at": "2022-02-14T00:36:21.398624", - "created": "2022-02-14T00:36:21.410999", - "updated": "2022-02-14T00:36:23.535844", - "message_id": null, - "file": { - "id": "ad1fef96-ce1c-4c5e-a426-58e2d5d3e907", - "created": "2022-02-14T00:36:23.537706", - "updated": "2022-02-14T00:36:23.537715", - "name": "Rufford Ford | part 47", + "name": "[Drone Freestyle] Mountain Landscape With Snow | Free Stock Footage | Creative Common Video", "ext": "mp4" } } @@ -184,7 +171,7 @@ documentations lives at `http://127.0.0.1:1984/docs`. Request ```json { - "url": "https://www.youtube.com/watch?v=zGDzdps75ns", + "url": "https://www.youtube.com/watch?v=PavYAOpVpJI", "download_media_type": "AUDIO_VIDEO", "save_to_storage": false } @@ -193,7 +180,7 @@ documentations lives at `http://127.0.0.1:1984/docs`. ```json { "id": "5ac05808-b29c-40d6-b250-07e3e769d8a6", - "url": "https://youtu.be/AWy1qiTF64M", + "url": "https://www.youtube.com/watch?v=PavYAOpVpJI", "source": "API", "added_at": "2022-02-14T00:35:25.419962+00:00" } diff --git a/app_api/api/core/services/task.py b/app_api/api/core/services/task.py index 544faac..aac5357 100644 --- a/app_api/api/core/services/task.py +++ b/app_api/api/core/services/task.py @@ -74,6 +74,11 @@ class TaskService: source=source, download_media_type=task.download_media_type, save_to_storage=task.save_to_storage, + from_chat_id=None, + from_chat_type=None, + from_user_id=None, + message_id=None, + ack_message_id=None, ) if not await publisher.send_for_download(payload): raise TaskServiceError('Failed to create task') diff --git a/app_bot/bot/core/bot/bot.py b/app_bot/bot/core/bot/bot.py index afbf8b4..1a77dfb 100644 --- a/app_bot/bot/core/bot/bot.py +++ b/app_bot/bot/core/bot/bot.py @@ -7,7 +7,7 @@ from pyrogram.enums import ParseMode from pyrogram.errors import RPCError from bot.core.config.config import get_main_config -from bot.core.config.schema import UserSchema +from bot.core.schema import UserSchema from bot.core.utils import bold diff --git a/app_bot/bot/core/config/config.py b/app_bot/bot/core/config/config.py index 2b80f11..4d26581 100644 --- a/app_bot/bot/core/config/config.py +++ b/app_bot/bot/core/config/config.py @@ -8,8 +8,8 @@ import yaml from pydantic import ValidationError from yt_shared.config import Settings -from bot.core.config.schema import ConfigSchema from bot.core.exceptions import ConfigError +from bot.core.schema import ConfigSchema class ConfigLoader: diff --git a/app_bot/bot/core/handlers/abstract.py b/app_bot/bot/core/handlers/abstract.py index 786b4f6..02ac92e 100644 --- a/app_bot/bot/core/handlers/abstract.py +++ b/app_bot/bot/core/handlers/abstract.py @@ -6,13 +6,13 @@ from yt_shared.enums import TaskSource, TelegramChatType from yt_shared.schemas.error import ErrorDownloadGeneralPayload, ErrorDownloadPayload from yt_shared.schemas.success import SuccessDownloadPayload -from bot.core.config.schema import AnonymousUserSchema, UserSchema +from bot.core.schema import AnonymousUserSchema, UserSchema if TYPE_CHECKING: from bot.core.bot import VideoBot -class AbstractDownloadHandler(metaclass=abc.ABCMeta): +class AbstractDownloadHandler(abc.ABC): def __init__( self, body: SuccessDownloadPayload diff --git a/app_bot/bot/core/handlers/success.py b/app_bot/bot/core/handlers/success.py index ed456ba..cffc27e 100644 --- a/app_bot/bot/core/handlers/success.py +++ b/app_bot/bot/core/handlers/success.py @@ -15,7 +15,7 @@ from yt_shared.utils.tasks.tasks import create_task from bot.core.handlers.abstract import AbstractDownloadHandler from bot.core.tasks.upload import AudioUploadTask, VideoUploadTask -from bot.core.utils import bold +from bot.core.utils import bold, is_user_upload_silent class SuccessDownloadHandler(AbstractDownloadHandler): @@ -47,12 +47,16 @@ class SuccessDownloadHandler(AbstractDownloadHandler): await self._delete_acknowledgment_message() async def _delete_acknowledgment_message(self) -> None: - await self._bot.delete_messages( - chat_id=self._body.from_chat_id, - message_ids=[self._body.context.ack_message_id], - ) + if self._body.from_chat_id and self._body.context.ack_message_id: + await self._bot.delete_messages( + chat_id=self._body.from_chat_id, + message_ids=self._body.context.ack_message_id, + ) async def _set_upload_message(self, media_object: BaseMedia) -> None: + if not (self._body.from_chat_id and self._body.context.ack_message_id): + return + try: await self._bot.edit_message_text( chat_id=self._body.from_chat_id, @@ -142,16 +146,15 @@ class SuccessDownloadHandler(AbstractDownloadHandler): async def _send_success_text(self, media_object: BaseMedia) -> None: text = self._create_success_text(media_object) for user in self._receiving_users: - if user.upload.silent: - continue - kwargs = { - 'chat_id': user.id, - 'text': text, - 'parse_mode': ParseMode.HTML, - } - if self._body.message_id: - kwargs['reply_to_message_id'] = self._body.message_id - await self._bot.send_message(**kwargs) + if not is_user_upload_silent(user=user, conf=self._bot.conf): + kwargs = { + 'chat_id': user.id, + 'text': text, + 'parse_mode': ParseMode.HTML, + } + if self._body.message_id: + kwargs['reply_to_message_id'] = self._body.message_id + await self._bot.send_message(**kwargs) def _upload_is_enabled(self) -> bool: """Check whether upload is allowed for particular user configuration.""" diff --git a/app_bot/bot/core/config/schema.py b/app_bot/bot/core/schema.py similarity index 89% rename from app_bot/bot/core/config/schema.py rename to app_bot/bot/core/schema.py index cfc3246..eb23e6f 100644 --- a/app_bot/bot/core/config/schema.py +++ b/app_bot/bot/core/schema.py @@ -1,3 +1,5 @@ +import abc + from pydantic import ( StrictBool, StrictInt, @@ -13,12 +15,12 @@ _LANG_CODE_LEN = 2 _LANG_CODE_REGEX = rf'^[a-z]{{{_LANG_CODE_LEN}}}$' -class AnonymousUserSchema(RealBaseModel): +class _BaseUserSchema(RealBaseModel, abc.ABC): id: StrictInt - @property - def is_anonymous_user(self) -> bool: - return True + +class AnonymousUserSchema(_BaseUserSchema): + pass class VideoCaptionSchema(RealBaseModel): @@ -37,7 +39,7 @@ class UploadSchema(RealBaseModel): video_caption: VideoCaptionSchema -class UserSchema(AnonymousUserSchema): +class UserSchema(_BaseUserSchema): is_admin: StrictBool send_startup_message: StrictBool download_media_type: DownMediaType @@ -45,10 +47,6 @@ class UserSchema(AnonymousUserSchema): use_url_regex_match: StrictBool upload: UploadSchema - @property - def is_anonymous_user(self) -> bool: - return False - class ApiSchema(RealBaseModel): upload_video_file: StrictBool diff --git a/app_bot/bot/core/service.py b/app_bot/bot/core/service.py index 412ec97..410f9b1 100644 --- a/app_bot/bot/core/service.py +++ b/app_bot/bot/core/service.py @@ -10,7 +10,7 @@ from yt_shared.rabbit.publisher import RmqPublisher from yt_shared.schemas.media import InbMediaPayload from yt_shared.schemas.url import URL -from bot.core.config.schema import UserSchema +from bot.core.schema import UserSchema from bot.core.utils import can_remove_url_params diff --git a/app_bot/bot/core/tasks/upload.py b/app_bot/bot/core/tasks/upload.py index 19f0a08..4313dbc 100644 --- a/app_bot/bot/core/tasks/upload.py +++ b/app_bot/bot/core/tasks/upload.py @@ -19,8 +19,8 @@ from yt_shared.utils.tasks.abstract import AbstractTask from yt_shared.utils.tasks.tasks import create_task from bot.core.config.config import get_main_config, settings -from bot.core.config.schema import AnonymousUserSchema, UserSchema, VideoCaptionSchema -from bot.core.utils import bold +from bot.core.schema import AnonymousUserSchema, UserSchema, VideoCaptionSchema +from bot.core.utils import bold, is_user_upload_silent if TYPE_CHECKING: from bot.core.bot import VideoBot @@ -45,7 +45,7 @@ class AudioUploadContext(BaseUploadContext): pass -class AbstractUploadTask(AbstractTask, metaclass=abc.ABCMeta): +class AbstractUploadTask(AbstractTask, abc.ABC): _UPLOAD_ACTION: ChatAction def __init__( @@ -103,16 +103,15 @@ class AbstractUploadTask(AbstractTask, metaclass=abc.ABCMeta): ) coros = [] for user in self._users: - if user.upload.silent: - continue - kwargs = { - 'chat_id': user.id, - 'text': text, - 'parse_mode': ParseMode.HTML, - } - if self._ctx.message_id: - kwargs['reply_to_message_id'] = self._ctx.message_id - coros.append(self._bot.send_message(**kwargs)) + if not is_user_upload_silent(user=user, conf=self._bot.conf): + kwargs = { + 'chat_id': user.id, + 'text': text, + 'parse_mode': ParseMode.HTML, + } + if self._ctx.message_id: + kwargs['reply_to_message_id'] = self._ctx.message_id + coros.append(self._bot.send_message(**kwargs)) await asyncio.gather(*coros) def _get_forward_chat_ids(self) -> list[int]: @@ -257,7 +256,7 @@ class VideoUploadTask(AbstractUploadTask): ) def _get_caption_conf(self) -> VideoCaptionSchema: - if self._users[0].is_anonymous_user: + if isinstance(self._users[0], AnonymousUserSchema): return self._bot.conf.telegram.api.video_caption return self._users[0].upload.video_caption diff --git a/app_bot/bot/core/utils.py b/app_bot/bot/core/utils.py index 5496141..748b3e4 100644 --- a/app_bot/bot/core/utils.py +++ b/app_bot/bot/core/utils.py @@ -1,4 +1,5 @@ """Utils module.""" + import asyncio import random import string @@ -11,6 +12,7 @@ from pyrogram.enums import ChatType from pyrogram.types import Message from bot.core.config import settings +from bot.core.schema import AnonymousUserSchema, ConfigSchema, UserSchema async def shallow_sleep_async(sleep_time: float = 0.1) -> None: @@ -92,3 +94,15 @@ def split_telegram_message( def can_remove_url_params(url: str, matching_hosts: Iterable[str]) -> bool: return urlparse(url).netloc in set(matching_hosts) + + +def is_user_upload_silent( + user: UserSchema | AnonymousUserSchema, conf: ConfigSchema +) -> bool: + if isinstance(user, AnonymousUserSchema): + if conf.telegram.api.silent: + return True + elif user.upload.silent: + return True + else: + return False diff --git a/app_bot/bot/core/workers/abstract.py b/app_bot/bot/core/workers/abstract.py index 06d02af..9cfd44b 100644 --- a/app_bot/bot/core/workers/abstract.py +++ b/app_bot/bot/core/workers/abstract.py @@ -1,4 +1,5 @@ """RabbitMQ Queue abstract worker module.""" + import abc import enum from typing import TYPE_CHECKING, Type diff --git a/app_bot/main.py b/app_bot/main.py index 5e8339b..af7de56 100644 --- a/app_bot/main.py +++ b/app_bot/main.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Bot Launcher Module.""" + import asyncio import uvloop diff --git a/app_worker/alembic/versions/0769fbebd121_.py b/app_worker/alembic/versions/0769fbebd121_.py index d415ed7..76a3955 100644 --- a/app_worker/alembic/versions/0769fbebd121_.py +++ b/app_worker/alembic/versions/0769fbebd121_.py @@ -5,6 +5,7 @@ Revises: 63e7cae94c1d Create Date: 2022-02-18 23:34:39.587248 """ + import sqlalchemy as sa from alembic import op diff --git a/app_worker/alembic/versions/10ab08fc321b_.py b/app_worker/alembic/versions/10ab08fc321b_.py index d4b2fa2..7f76f4c 100644 --- a/app_worker/alembic/versions/10ab08fc321b_.py +++ b/app_worker/alembic/versions/10ab08fc321b_.py @@ -5,6 +5,7 @@ Revises: ba7716dca30a Create Date: 2023-02-25 15:47:37.542906 """ + from alembic import op # revision identifiers, used by Alembic. diff --git a/app_worker/alembic/versions/2689b03525f8_.py b/app_worker/alembic/versions/2689b03525f8_.py index ac8fcf5..b38e9aa 100644 --- a/app_worker/alembic/versions/2689b03525f8_.py +++ b/app_worker/alembic/versions/2689b03525f8_.py @@ -5,6 +5,7 @@ Revises: 0769fbebd121 Create Date: 2022-02-20 00:07:13.184353 """ + import sqlalchemy as sa from alembic import op diff --git a/app_worker/alembic/versions/50331b3c39bb_.py b/app_worker/alembic/versions/50331b3c39bb_.py index 4e18a11..93b1e1b 100644 --- a/app_worker/alembic/versions/50331b3c39bb_.py +++ b/app_worker/alembic/versions/50331b3c39bb_.py @@ -5,6 +5,7 @@ Revises: 77c100300b5b Create Date: 2023-04-06 22:08:12.511699 """ + import sqlalchemy as sa from alembic import op diff --git a/app_worker/alembic/versions/6221c0018660_.py b/app_worker/alembic/versions/6221c0018660_.py index 40168bb..4ed980f 100644 --- a/app_worker/alembic/versions/6221c0018660_.py +++ b/app_worker/alembic/versions/6221c0018660_.py @@ -5,6 +5,7 @@ Revises: 2689b03525f8 Create Date: 2022-02-20 03:00:01.366172 """ + import sqlalchemy as sa from alembic import op diff --git a/app_worker/alembic/versions/63e7cae94c1d_.py b/app_worker/alembic/versions/63e7cae94c1d_.py index 8470518..ae3a5d6 100644 --- a/app_worker/alembic/versions/63e7cae94c1d_.py +++ b/app_worker/alembic/versions/63e7cae94c1d_.py @@ -5,6 +5,7 @@ Revises: Create Date: 2022-02-06 21:18:09.738831 """ + import sqlalchemy as sa import sqlalchemy_utils from sqlalchemy.dialects import postgresql diff --git a/app_worker/alembic/versions/77c100300b5b_.py b/app_worker/alembic/versions/77c100300b5b_.py index 644a199..be9fdfe 100644 --- a/app_worker/alembic/versions/77c100300b5b_.py +++ b/app_worker/alembic/versions/77c100300b5b_.py @@ -5,6 +5,7 @@ Revises: 10ab08fc321b Create Date: 2023-04-06 10:45:47.289554 """ + import sqlalchemy as sa from alembic import op diff --git a/app_worker/alembic/versions/8021be777d1d_.py b/app_worker/alembic/versions/8021be777d1d_.py index 444f547..c3340d9 100644 --- a/app_worker/alembic/versions/8021be777d1d_.py +++ b/app_worker/alembic/versions/8021be777d1d_.py @@ -5,6 +5,7 @@ Revises: ff03785a1f0d Create Date: 2022-06-13 20:16:10.845151 """ + import sqlalchemy as sa from alembic import op diff --git a/app_worker/alembic/versions/ba7716dca30a_.py b/app_worker/alembic/versions/ba7716dca30a_.py index 323e92e..41f9580 100644 --- a/app_worker/alembic/versions/ba7716dca30a_.py +++ b/app_worker/alembic/versions/ba7716dca30a_.py @@ -5,6 +5,7 @@ Revises: 8021be777d1d Create Date: 2022-06-13 22:51:29.775743 """ + import sqlalchemy as sa from alembic import op diff --git a/app_worker/alembic/versions/d3f89ea5e8b5_.py b/app_worker/alembic/versions/d3f89ea5e8b5_.py index 3c342f9..d1410b0 100644 --- a/app_worker/alembic/versions/d3f89ea5e8b5_.py +++ b/app_worker/alembic/versions/d3f89ea5e8b5_.py @@ -5,6 +5,7 @@ Revises: da4a97a0fdb7 Create Date: 2022-06-07 19:35:23.098590 """ + import sqlalchemy as sa from alembic import op diff --git a/app_worker/alembic/versions/da4a97a0fdb7_.py b/app_worker/alembic/versions/da4a97a0fdb7_.py index 0c8fc32..da6867b 100644 --- a/app_worker/alembic/versions/da4a97a0fdb7_.py +++ b/app_worker/alembic/versions/da4a97a0fdb7_.py @@ -5,6 +5,7 @@ Revises: 6221c0018660 Create Date: 2022-03-19 10:46:57.624753 """ + import sqlalchemy as sa import sqlalchemy_utils diff --git a/app_worker/alembic/versions/ff03785a1f0d_.py b/app_worker/alembic/versions/ff03785a1f0d_.py index a1f8e0f..f245d4b 100644 --- a/app_worker/alembic/versions/ff03785a1f0d_.py +++ b/app_worker/alembic/versions/ff03785a1f0d_.py @@ -5,6 +5,7 @@ Revises: d3f89ea5e8b5 Create Date: 2022-06-10 20:04:34.792613 """ + import sqlalchemy as sa from alembic import op diff --git a/app_worker/worker/core/downloader.py b/app_worker/worker/core/downloader.py index 7e1f63e..7a46722 100644 --- a/app_worker/worker/core/downloader.py +++ b/app_worker/worker/core/downloader.py @@ -68,13 +68,13 @@ class MediaDownloader: meta: dict | None = ytdl.extract_info(url, download=True) if not meta: - err_msg = f'Error during media download. Check logs.' + err_msg = 'Error during media download. Check logs.' self._log.error('%s. Meta: %s', err_msg, meta) raise MediaDownloaderError(err_msg) current_files = os.listdir(curr_tmp_dir) if not current_files: - err_msg = f'Nothing downloaded. Is URL valid?' + err_msg = 'Nothing downloaded. Is URL valid?' self._log.error(err_msg) raise MediaDownloaderError(err_msg) @@ -126,7 +126,7 @@ class MediaDownloader: return create_dto(self._create_video_dto) def create_dto( - func: Callable[[dict, str, str], Audio | Video] + func: Callable[[dict, str, str], Audio | Video], ) -> Audio | Video: try: return func( diff --git a/app_worker/ytdl_opts/default.py b/app_worker/ytdl_opts/default.py index 9691924..7fbde4b 100644 --- a/app_worker/ytdl_opts/default.py +++ b/app_worker/ytdl_opts/default.py @@ -6,6 +6,7 @@ More here https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/options.py or 'yt- If you want to change any of these values or add new ones, copy all content to the `user.py` in the same directory as this file, and edit the values. """ + from worker.utils import get_cookies_opts_if_not_empty FINAL_AUDIO_FORMAT = 'mp3' diff --git a/pyproject.toml b/pyproject.toml index 8b49132..5640ee6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.ruff] line-length = 88 -select = ["F", "E", "W", "I001"] -ignore = ["E501"] # Skip line length violations +lint.select = ["F", "E", "W", "I001"] +lint.ignore = ["E501"] # Skip line length violations src = ["app_api", "app_bot", "app_worker"] [tool.ruff.format] diff --git a/start.py b/start.py index f4d5230..5a4350c 100644 --- a/start.py +++ b/start.py @@ -1,25 +1,25 @@ """Over-engineered Python 3.10+ version of bash script with netcat (nc) just for fun. - #!/bin/bash +#!/bin/bash - check_reachability() { - while ! nc -z "$1" "${!2}" - do - echo "Waiting for $3 to be reachable on port ${!2}" - sleep 1 - done - echo "Connection to $3 on port ${!2} verified" - return 0 - } +check_reachability() { +while ! nc -z "$1" "${!2}" +do + echo "Waiting for $3 to be reachable on port ${!2}" + sleep 1 +done +echo "Connection to $3 on port ${!2} verified" +return 0 +} - wait_for_services_to_be_reachable() { - check_reachability rabbitmq RABBITMQ_PORT RabbitMQ - check_reachability postgres POSTGRES_PORT PostgreSQL - } +wait_for_services_to_be_reachable() { + check_reachability rabbitmq RABBITMQ_PORT RabbitMQ + check_reachability postgres POSTGRES_PORT PostgreSQL +} - wait_for_services_to_be_reachable - exit 0 +wait_for_services_to_be_reachable +exit 0 """ import asyncio diff --git a/yt_shared/yt_shared/schemas/cache.py b/yt_shared/yt_shared/schemas/cache.py index 99f6e73..7389ba8 100644 --- a/yt_shared/yt_shared/schemas/cache.py +++ b/yt_shared/yt_shared/schemas/cache.py @@ -1,10 +1,12 @@ import datetime +from pydantic import StrictInt, StrictStr + from yt_shared.schemas.base import RealBaseModel class CacheSchema(RealBaseModel): - cache_id: str - cache_unique_id: str - file_size: int + cache_id: StrictStr + cache_unique_id: StrictStr + file_size: StrictInt date_timestamp: datetime.datetime diff --git a/yt_shared/yt_shared/utils/tasks/abstract.py b/yt_shared/yt_shared/utils/tasks/abstract.py index 42655c5..3ce825c 100644 --- a/yt_shared/yt_shared/utils/tasks/abstract.py +++ b/yt_shared/yt_shared/utils/tasks/abstract.py @@ -2,7 +2,7 @@ import abc import logging -class AbstractTask(metaclass=abc.ABCMeta): +class AbstractTask(abc.ABC): def __init__(self) -> None: self._log = logging.getLogger(self.__class__.__name__)