diff --git a/README.md b/README.md index 22c7075..6c28bab 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Simple and reliable self-hosted Video Download Telegram Bot. -Version: 1.6. [Release details](RELEASES.md). +Version: 1.7. [Release details](RELEASES.md). ![frames](.assets/download_success.png) diff --git a/RELEASES.md b/RELEASES.md index c5cb5c6..91919cc 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,3 +1,22 @@ +## Release 1.7 + +Release date: May 30, 2024 + +## New Features + +- Choose `yt-dlp` release channel between `NIGHTLY`, `STABLE` or `MASTER`. Default is `NIGHTLY`. + +## Important + +- Update your `app_bot/config.yaml` with new `release_channel: "NIGHTLY"` line in `ytdlp` section +- `app_worker/requirements.txt` includes `NIGHTLY` `yt-dlp` package to install. + +## Misc + +N/A + +--- + ## Release 1.6 Release date: April 25, 2024 diff --git a/app_api/api/api/api_v1/endpoints/ytdlp.py b/app_api/api/api/api_v1/endpoints/ytdlp.py index b022d82..4b047fc 100644 --- a/app_api/api/api/api_v1/endpoints/ytdlp.py +++ b/app_api/api/api/api_v1/endpoints/ytdlp.py @@ -1,6 +1,9 @@ from fastapi import APIRouter, Depends from sqlalchemy.ext.asyncio import AsyncSession +from yt_shared.clients.github import YtdlpGithubClient from yt_shared.db.session import get_db +from yt_shared.enums import YtdlpReleaseChannelType +from yt_shared.repositories.ytdlp import YtdlpRepository from yt_shared.ytdlp.version_checker import YtdlpVersionChecker from api.api.api_v1.schemas.ytdlp import YTDLPLatestVersion @@ -9,8 +12,12 @@ router = APIRouter() @router.get('/', response_model_by_alias=False) -async def yt_dlp_version(db: AsyncSession = Depends(get_db)) -> YTDLPLatestVersion: - ctx = await YtdlpVersionChecker().get_version_context(db) +async def yt_dlp_version( + release_channel: YtdlpReleaseChannelType, db: AsyncSession = Depends(get_db) +) -> YTDLPLatestVersion: + ctx = await YtdlpVersionChecker( + client=YtdlpGithubClient(release_channel), repository=YtdlpRepository(db) + ).get_version_context() return YTDLPLatestVersion( current=ctx.current, latest=ctx.latest, need_upgrade=ctx.has_new_version ) diff --git a/app_bot/bot/bot/client.py b/app_bot/bot/bot/client.py index f946359..86f57bc 100644 --- a/app_bot/bot/bot/client.py +++ b/app_bot/bot/bot/client.py @@ -6,7 +6,7 @@ from pyrogram import Client from pyrogram.enums import ParseMode from pyrogram.errors import RPCError -from bot.core.schema import ConfigSchema, UserSchema +from bot.core.schemas import ConfigSchema, UserSchema from bot.core.utils import bold diff --git a/app_bot/bot/core/config/config.py b/app_bot/bot/core/config/config.py index 4d26581..4a05a97 100644 --- a/app_bot/bot/core/config/config.py +++ b/app_bot/bot/core/config/config.py @@ -9,7 +9,7 @@ from pydantic import ValidationError from yt_shared.config import Settings from bot.core.exceptions import ConfigError -from bot.core.schema import ConfigSchema +from bot.core.schemas import ConfigSchema class ConfigLoader: diff --git a/app_bot/bot/core/handlers/abstract.py b/app_bot/bot/core/handlers/abstract.py index 4d2da1d..4b39e45 100644 --- a/app_bot/bot/core/handlers/abstract.py +++ b/app_bot/bot/core/handlers/abstract.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING from yt_shared.enums import TaskSource, TelegramChatType from yt_shared.schemas.base_rabbit import BaseRabbitDownloadPayload -from bot.core.schema import AnonymousUserSchema, UserSchema +from bot.core.schemas import AnonymousUserSchema, UserSchema if TYPE_CHECKING: from bot.bot import VideoBotClient diff --git a/app_bot/bot/core/schema.py b/app_bot/bot/core/schemas.py similarity index 93% rename from app_bot/bot/core/schema.py rename to app_bot/bot/core/schemas.py index a82c039..f883453 100644 --- a/app_bot/bot/core/schema.py +++ b/app_bot/bot/core/schemas.py @@ -2,7 +2,7 @@ from abc import ABC from pydantic import Field, PositiveInt, StringConstraints, field_validator from typing_extensions import Annotated -from yt_shared.enums import DownMediaType +from yt_shared.enums import DownMediaType, YtdlpReleaseChannelType from yt_shared.schemas.base import StrictBaseConfigModel _LANG_CODE_LEN = 2 @@ -72,6 +72,7 @@ class YtdlpSchema(StrictBaseConfigModel): version_check_enabled: bool version_check_interval: PositiveInt notify_users_on_new_version: bool + release_channel: Annotated[YtdlpReleaseChannelType, Field(strict=False)] class ConfigSchema(StrictBaseConfigModel): diff --git a/app_bot/bot/core/service.py b/app_bot/bot/core/service.py index 6a7576b..5ff377e 100644 --- a/app_bot/bot/core/service.py +++ b/app_bot/bot/core/service.py @@ -10,7 +10,7 @@ from yt_shared.rabbit.publisher import RmqPublisher from yt_shared.schemas.media import InbMediaPayload from yt_shared.schemas.url import URL -from bot.core.schema import UserSchema +from bot.core.schemas import UserSchema from bot.core.utils import can_remove_url_params diff --git a/app_bot/bot/core/tasks/upload.py b/app_bot/bot/core/tasks/upload.py index cdda5ce..e0729cd 100644 --- a/app_bot/bot/core/tasks/upload.py +++ b/app_bot/bot/core/tasks/upload.py @@ -19,7 +19,7 @@ from yt_shared.utils.tasks.abstract import AbstractTask from yt_shared.utils.tasks.tasks import create_task from bot.core.config.config import get_main_config, settings -from bot.core.schema import AnonymousUserSchema, UserSchema, VideoCaptionSchema +from bot.core.schemas import AnonymousUserSchema, UserSchema, VideoCaptionSchema from bot.core.utils import bold, is_user_upload_silent if TYPE_CHECKING: diff --git a/app_bot/bot/core/tasks/ytdlp.py b/app_bot/bot/core/tasks/ytdlp.py index 89b857e..0cb57f5 100644 --- a/app_bot/bot/core/tasks/ytdlp.py +++ b/app_bot/bot/core/tasks/ytdlp.py @@ -2,8 +2,10 @@ import asyncio import datetime from typing import TYPE_CHECKING +from yt_shared.clients.github import YtdlpGithubClient from yt_shared.db.session import get_db from yt_shared.emoji import INFORMATION_EMOJI +from yt_shared.repositories.ytdlp import YtdlpRepository from yt_shared.schemas.ytdlp import VersionContext from yt_shared.utils.tasks.abstract import AbstractTask from yt_shared.ytdlp.version_checker import YtdlpVersionChecker @@ -12,53 +14,60 @@ from bot.core.config.config import get_main_config from bot.core.utils import bold, code if TYPE_CHECKING: - from bot.bot import VideoBotClient + from bot.bot.client import VideoBotClient class YtdlpNewVersionNotifyTask(AbstractTask): def __init__(self, bot: 'VideoBotClient') -> None: super().__init__() self._bot = bot - self._version_checker = YtdlpVersionChecker() self._startup_message_sent = False - - ytdlp_conf = get_main_config().ytdlp - self._version_check_enabled = ytdlp_conf.version_check_enabled - self._version_check_interval = ytdlp_conf.version_check_interval - self._notify_users_on_new_version = ytdlp_conf.notify_users_on_new_version + self._ytdlp_conf = get_main_config().ytdlp async def run(self) -> None: await self._run() async def _run(self) -> None: - if not self._version_check_enabled: - self._log.info('New "yt-dlp" version check disabled, exiting from task') + release_channel = self._ytdlp_conf.release_channel + if not self._ytdlp_conf.version_check_enabled: + self._log.info( + 'New %s "yt-dlp" version check disabled, exiting from task', + release_channel, + ) return while True: - self._log.info('Checking for new yt-dlp version') + self._log.info('Checking for new %s yt-dlp version', release_channel) try: await self._notify_if_new_version() except Exception: - self._log.exception('Failed check new yt-dlp version') + self._log.exception( + 'Failed check new %s yt-dlp version', release_channel + ) self._log.info( - 'Next yt-dlp version check planned at %s', + 'Next %s yt-dlp version check planned at %s', + release_channel, self._get_next_check_datetime().isoformat(' '), ) - await asyncio.sleep(self._version_check_interval) + await asyncio.sleep(self._ytdlp_conf.version_check_interval) def _get_next_check_datetime(self) -> datetime.datetime: return ( datetime.datetime.now(datetime.timezone.utc) - + datetime.timedelta(seconds=self._version_check_interval) + + datetime.timedelta(seconds=self._ytdlp_conf.version_check_interval) ).replace(microsecond=0) async def _notify_if_new_version(self) -> None: async for db in get_db(): - context = await self._version_checker.get_version_context(db) - if context.has_new_version and self._notify_users_on_new_version: - await self._notify_outdated(context) - return + context = await YtdlpVersionChecker( + client=YtdlpGithubClient(self._ytdlp_conf.release_channel), + repository=YtdlpRepository(db), + ).get_version_context() + if context.has_new_version: + self._log.info('yt-dlp has new version: %s', context.latest.version) + if self._ytdlp_conf.notify_users_on_new_version: + await self._notify_outdated(context) + return if not self._startup_message_sent: await self._notify_up_to_date( @@ -68,18 +77,19 @@ class YtdlpNewVersionNotifyTask(AbstractTask): async def _notify_outdated(self, ctx: VersionContext) -> None: text = ( - f'New {code("yt-dlp")} version available: {bold(ctx.latest.version)}\n' + f'New {bold(self._ytdlp_conf.release_channel)} {code("yt-dlp")} version available: ' + f'{bold(ctx.latest.version)}\n' f'Current version: {bold(ctx.current.version)}\n' - f'Rebuild worker with {code("docker compose build --no-cache worker")}' + f'Rebuild worker with {code("docker compose build --no-cache worker && docker compose up -d -t 0 worker")}' ) await self._bot.send_message_admins(text) async def _notify_up_to_date( self, ctx: VersionContext, user_ids: list[int] ) -> None: - """Send startup message that yt-dlp version is up to date.""" + """Send startup message that yt-dlp version is up-to-date.""" text = ( - f'{INFORMATION_EMOJI} Your {code("yt-dlp")} version ' - f'{bold(ctx.current.version)} is up to date, have fun' + f'{INFORMATION_EMOJI} Your {bold(self._ytdlp_conf.release_channel)} {code("yt-dlp")} ' + f'version {bold(ctx.current.version)} is up to date, have fun' ) await self._bot.send_message_to_users(text=text, user_ids=user_ids) diff --git a/app_bot/bot/core/utils.py b/app_bot/bot/core/utils.py index f87ef6d..477890f 100644 --- a/app_bot/bot/core/utils.py +++ b/app_bot/bot/core/utils.py @@ -9,7 +9,7 @@ from pyrogram.enums import ChatType from pyrogram.types import Message from bot.core.config import settings -from bot.core.schema import AnonymousUserSchema, ConfigSchema, UserSchema +from bot.core.schemas import AnonymousUserSchema, ConfigSchema, UserSchema async def shallow_sleep_async(sleep_time: float = 0.1) -> None: diff --git a/app_bot/config-example.yml b/app_bot/config-example.yml index 453a2ac..e6300d8 100644 --- a/app_bot/config-example.yml +++ b/app_bot/config-example.yml @@ -40,3 +40,4 @@ ytdlp: version_check_enabled: !!bool True version_check_interval: 86400 notify_users_on_new_version: !!bool True + release_channel: "NIGHTLY" diff --git a/app_worker/worker/core/launcher.py b/app_worker/worker/core/launcher.py index a3c9f88..4ac947a 100644 --- a/app_worker/worker/core/launcher.py +++ b/app_worker/worker/core/launcher.py @@ -55,7 +55,7 @@ class WorkerLauncher: 'Saving current yt-dlp version (%s) to the database', curr_version ) async for db in get_db(): - await YtdlpRepository().create_or_update_version(curr_version, db) + await YtdlpRepository(db).create_or_update_version(curr_version) async def _create_intermediate_directories(self) -> None: """Create temporary intermediate directories on start if they do not exist.""" diff --git a/yt_shared/yt_shared/clients/github.py b/yt_shared/yt_shared/clients/github.py index 123be2a..f674a2d 100644 --- a/yt_shared/yt_shared/clients/github.py +++ b/yt_shared/yt_shared/clients/github.py @@ -2,22 +2,31 @@ import datetime import logging import aiohttp +from yt_shared.enums import YtdlpReleaseChannelType from yt_shared.schemas.ytdlp import LatestVersion class YtdlpGithubClient: """yt-dlp Github version number checker.""" - LATEST_TAG_URL = 'https://github.com/yt-dlp/yt-dlp/releases/latest' + LATEST_TAG_URL_TPL = 'https://github.com/yt-dlp/{repository}/releases/latest' + LATEST_TAG_REPOSITORY_MAP = { + YtdlpReleaseChannelType.STABLE: 'yt-dlp', + YtdlpReleaseChannelType.NIGHTLY: 'yt-dlp-nightly-builds', + YtdlpReleaseChannelType.MASTER: 'yt-dlp-master-builds', + } - def __init__(self) -> None: + def __init__(self, release_channel: YtdlpReleaseChannelType) -> None: self._log = logging.getLogger(self.__class__.__name__) + self._release_channel = release_channel async def get_latest_version(self) -> LatestVersion: + tag_url = self.LATEST_TAG_URL_TPL.format( + repository=self.LATEST_TAG_REPOSITORY_MAP[self._release_channel] + ) async with aiohttp.ClientSession() as session: - async with session.get(self.LATEST_TAG_URL) as resp: + async with session.get(tag_url) as resp: version = resp.url.parts[-1] - self._log.info('Latest yt-dlp version: %s', version) return LatestVersion( version=version, retrieved_at=datetime.datetime.utcnow() ) diff --git a/yt_shared/yt_shared/enums.py b/yt_shared/yt_shared/enums.py index 18b434e..802c962 100644 --- a/yt_shared/yt_shared/enums.py +++ b/yt_shared/yt_shared/enums.py @@ -50,3 +50,9 @@ class DownMediaType(StrChoiceEnum): class MediaFileType(StrChoiceEnum): AUDIO = 'AUDIO' VIDEO = 'VIDEO' + + +class YtdlpReleaseChannelType(StrChoiceEnum): + STABLE = 'STABLE' + NIGHTLY = 'NIGHTLY' + MASTER = 'MASTER' diff --git a/yt_shared/yt_shared/repositories/ytdlp.py b/yt_shared/yt_shared/repositories/ytdlp.py index 8486a0d..dd98dbe 100644 --- a/yt_shared/yt_shared/repositories/ytdlp.py +++ b/yt_shared/yt_shared/repositories/ytdlp.py @@ -8,17 +8,18 @@ from yt_shared.models import YTDLP class YtdlpRepository: - def __init__(self) -> None: + def __init__(self, db: AsyncSession) -> None: self._log = logging.getLogger(self.__class__.__name__) + self._db = db - @staticmethod - async def get_current_version(db: AsyncSession) -> YTDLP: - result = await db.execute(select(YTDLP)) + async def get_current_version(self) -> YTDLP: + result = await self._db.execute(select(YTDLP)) return result.scalar_one() - @staticmethod - async def create_or_update_version(current_version: str, db: AsyncSession) -> None: - row_count: int = await db.scalar(select(func.count('*')).select_from(YTDLP)) + async def create_or_update_version(self, current_version: str) -> None: + row_count: int = await self._db.scalar( + select(func.count('*')).select_from(YTDLP) + ) if row_count > 1: raise MultipleResultsFound( 'Multiple yt-dlp version records found. Expected one.' @@ -30,5 +31,5 @@ class YtdlpRepository: .values({'current_version': current_version}) .execution_options(synchronize_session=False) ) - await db.execute(insert_or_update_stmt) - await db.commit() + await self._db.execute(insert_or_update_stmt) + await self._db.commit() diff --git a/yt_shared/yt_shared/ytdlp/version_checker.py b/yt_shared/yt_shared/ytdlp/version_checker.py index 5bd2f40..628371d 100644 --- a/yt_shared/yt_shared/ytdlp/version_checker.py +++ b/yt_shared/yt_shared/ytdlp/version_checker.py @@ -1,7 +1,6 @@ import asyncio import logging -from sqlalchemy.ext.asyncio import AsyncSession from yt_shared.clients.github import YtdlpGithubClient from yt_shared.repositories.ytdlp import YtdlpRepository from yt_shared.schemas.ytdlp import CurrentVersion, LatestVersion, VersionContext @@ -10,23 +9,23 @@ from yt_shared.schemas.ytdlp import CurrentVersion, LatestVersion, VersionContex class YtdlpVersionChecker: """yt-dlp version number checker.""" - LATEST_TAG_URL = 'https://github.com/yt-dlp/yt-dlp/releases/latest' - - def __init__(self) -> None: + def __init__(self, client: YtdlpGithubClient, repository: YtdlpRepository) -> None: self._log = logging.getLogger(self.__class__.__name__) - self._ytdlp_repository = YtdlpRepository() - self._ytdlp_client = YtdlpGithubClient() + self._ytdlp_repository = repository + self._ytdlp_client = client - async def get_version_context(self, db: AsyncSession) -> VersionContext: + async def get_version_context(self) -> VersionContext: latest, current = await asyncio.gather( - self.get_latest_version(), self.get_current_version(db) + self.get_latest_version(), self.get_current_version() ) return VersionContext(latest=latest, current=current) async def get_latest_version(self) -> LatestVersion: - return await self._ytdlp_client.get_latest_version() + latest_version = await self._ytdlp_client.get_latest_version() + self._log.info('Latest yt-dlp version: %s', latest_version.version) + return latest_version - async def get_current_version(self, db: AsyncSession) -> CurrentVersion: - ytdlp_ = await self._ytdlp_repository.get_current_version(db) + async def get_current_version(self) -> CurrentVersion: + ytdlp_ = await self._ytdlp_repository.get_current_version() self._log.info('Current yt-dlp version: %s', ytdlp_.current_version) return CurrentVersion.model_validate(ytdlp_)