Version 1.7

README.md

@@ -2,7 +2,7 @@
 
 Simple and reliable self-hosted Video Download Telegram Bot.
 
-Version: 1.6. [Release details](RELEASES.md).
+Version: 1.7. [Release details](RELEASES.md).
 
 ![frames](.assets/download_success.png)
 

RELEASES.md

@@ -1,3 +1,22 @@
+## Release 1.7
+
+Release date: May 30, 2024
+
+## New Features
+
+- Choose `yt-dlp` release channel between `NIGHTLY`, `STABLE` or `MASTER`. Default is `NIGHTLY`.
+
+## Important
+
+- Update your `app_bot/config.yaml` with new `release_channel: "NIGHTLY"` line in `ytdlp` section
+- `app_worker/requirements.txt` includes `NIGHTLY` `yt-dlp` package to install.
+
+## Misc
+
+N/A
+
+---
+
 ## Release 1.6
 
 Release date: April 25, 2024

app_api/api/api/api_v1/endpoints/ytdlp.py

@@ -1,6 +1,9 @@
 from fastapi import APIRouter, Depends
 from sqlalchemy.ext.asyncio import AsyncSession
+from yt_shared.clients.github import YtdlpGithubClient
 from yt_shared.db.session import get_db
+from yt_shared.enums import YtdlpReleaseChannelType
+from yt_shared.repositories.ytdlp import YtdlpRepository
 from yt_shared.ytdlp.version_checker import YtdlpVersionChecker
 
 from api.api.api_v1.schemas.ytdlp import YTDLPLatestVersion
@@ -9,8 +12,12 @@ router = APIRouter()
 
 
 @router.get('/', response_model_by_alias=False)
-async def yt_dlp_version(db: AsyncSession = Depends(get_db)) -> YTDLPLatestVersion:
-    ctx = await YtdlpVersionChecker().get_version_context(db)
+async def yt_dlp_version(
+    release_channel: YtdlpReleaseChannelType, db: AsyncSession = Depends(get_db)
+) -> YTDLPLatestVersion:
+    ctx = await YtdlpVersionChecker(
+        client=YtdlpGithubClient(release_channel), repository=YtdlpRepository(db)
+    ).get_version_context()
     return YTDLPLatestVersion(
         current=ctx.current, latest=ctx.latest, need_upgrade=ctx.has_new_version
     )

app_bot/bot/bot/client.py

@@ -6,7 +6,7 @@ from pyrogram import Client
 from pyrogram.enums import ParseMode
 from pyrogram.errors import RPCError
 
-from bot.core.schema import ConfigSchema, UserSchema
+from bot.core.schemas import ConfigSchema, UserSchema
 from bot.core.utils import bold
 
 

app_bot/bot/core/config/config.py

@@ -9,7 +9,7 @@ from pydantic import ValidationError
 from yt_shared.config import Settings
 
 from bot.core.exceptions import ConfigError
-from bot.core.schema import ConfigSchema
+from bot.core.schemas import ConfigSchema
 
 
 class ConfigLoader:

app_bot/bot/core/handlers/abstract.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING
 from yt_shared.enums import TaskSource, TelegramChatType
 from yt_shared.schemas.base_rabbit import BaseRabbitDownloadPayload
 
-from bot.core.schema import AnonymousUserSchema, UserSchema
+from bot.core.schemas import AnonymousUserSchema, UserSchema
 
 if TYPE_CHECKING:
     from bot.bot import VideoBotClient

app_bot/bot/core/schemas.py

@@ -2,7 +2,7 @@ from abc import ABC
 
 from pydantic import Field, PositiveInt, StringConstraints, field_validator
 from typing_extensions import Annotated
-from yt_shared.enums import DownMediaType
+from yt_shared.enums import DownMediaType, YtdlpReleaseChannelType
 from yt_shared.schemas.base import StrictBaseConfigModel
 
 _LANG_CODE_LEN = 2
@@ -72,6 +72,7 @@ class YtdlpSchema(StrictBaseConfigModel):
     version_check_enabled: bool
     version_check_interval: PositiveInt
     notify_users_on_new_version: bool
+    release_channel: Annotated[YtdlpReleaseChannelType, Field(strict=False)]
 
 
 class ConfigSchema(StrictBaseConfigModel):

app_bot/bot/core/service.py

@@ -10,7 +10,7 @@ from yt_shared.rabbit.publisher import RmqPublisher
 from yt_shared.schemas.media import InbMediaPayload
 from yt_shared.schemas.url import URL
 
-from bot.core.schema import UserSchema
+from bot.core.schemas import UserSchema
 from bot.core.utils import can_remove_url_params
 
 

app_bot/bot/core/tasks/upload.py

@@ -19,7 +19,7 @@ from yt_shared.utils.tasks.abstract import AbstractTask
 from yt_shared.utils.tasks.tasks import create_task
 
 from bot.core.config.config import get_main_config, settings
-from bot.core.schema import AnonymousUserSchema, UserSchema, VideoCaptionSchema
+from bot.core.schemas import AnonymousUserSchema, UserSchema, VideoCaptionSchema
 from bot.core.utils import bold, is_user_upload_silent
 
 if TYPE_CHECKING:

app_bot/bot/core/tasks/ytdlp.py

@@ -2,8 +2,10 @@ import asyncio
 import datetime
 from typing import TYPE_CHECKING
 
+from yt_shared.clients.github import YtdlpGithubClient
 from yt_shared.db.session import get_db
 from yt_shared.emoji import INFORMATION_EMOJI
+from yt_shared.repositories.ytdlp import YtdlpRepository
 from yt_shared.schemas.ytdlp import VersionContext
 from yt_shared.utils.tasks.abstract import AbstractTask
 from yt_shared.ytdlp.version_checker import YtdlpVersionChecker
@@ -12,53 +14,60 @@ from bot.core.config.config import get_main_config
 from bot.core.utils import bold, code
 
 if TYPE_CHECKING:
-    from bot.bot import VideoBotClient
+    from bot.bot.client import VideoBotClient
 
 
 class YtdlpNewVersionNotifyTask(AbstractTask):
     def __init__(self, bot: 'VideoBotClient') -> None:
         super().__init__()
         self._bot = bot
-        self._version_checker = YtdlpVersionChecker()
         self._startup_message_sent = False
-
-        ytdlp_conf = get_main_config().ytdlp
-        self._version_check_enabled = ytdlp_conf.version_check_enabled
-        self._version_check_interval = ytdlp_conf.version_check_interval
-        self._notify_users_on_new_version = ytdlp_conf.notify_users_on_new_version
+        self._ytdlp_conf = get_main_config().ytdlp
 
     async def run(self) -> None:
         await self._run()
 
     async def _run(self) -> None:
-        if not self._version_check_enabled:
-            self._log.info('New "yt-dlp" version check disabled, exiting from task')
+        release_channel = self._ytdlp_conf.release_channel
+        if not self._ytdlp_conf.version_check_enabled:
+            self._log.info(
+                'New %s "yt-dlp" version check disabled, exiting from task',
+                release_channel,
+            )
             return
 
         while True:
-            self._log.info('Checking for new yt-dlp version')
+            self._log.info('Checking for new %s yt-dlp version', release_channel)
             try:
                 await self._notify_if_new_version()
             except Exception:
-                self._log.exception('Failed check new yt-dlp version')
+                self._log.exception(
+                    'Failed check new %s yt-dlp version', release_channel
+                )
             self._log.info(
-                'Next yt-dlp version check planned at %s',
+                'Next %s yt-dlp version check planned at %s',
+                release_channel,
                 self._get_next_check_datetime().isoformat(' '),
             )
-            await asyncio.sleep(self._version_check_interval)
+            await asyncio.sleep(self._ytdlp_conf.version_check_interval)
 
     def _get_next_check_datetime(self) -> datetime.datetime:
         return (
             datetime.datetime.now(datetime.timezone.utc)
-            + datetime.timedelta(seconds=self._version_check_interval)
+            + datetime.timedelta(seconds=self._ytdlp_conf.version_check_interval)
         ).replace(microsecond=0)
 
     async def _notify_if_new_version(self) -> None:
         async for db in get_db():
-            context = await self._version_checker.get_version_context(db)
-            if context.has_new_version and self._notify_users_on_new_version:
-                await self._notify_outdated(context)
-                return
+            context = await YtdlpVersionChecker(
+                client=YtdlpGithubClient(self._ytdlp_conf.release_channel),
+                repository=YtdlpRepository(db),
+            ).get_version_context()
+            if context.has_new_version:
+                self._log.info('yt-dlp has new version: %s', context.latest.version)
+                if self._ytdlp_conf.notify_users_on_new_version:
+                    await self._notify_outdated(context)
+                    return
 
             if not self._startup_message_sent:
                 await self._notify_up_to_date(
@@ -68,18 +77,19 @@ class YtdlpNewVersionNotifyTask(AbstractTask):
 
     async def _notify_outdated(self, ctx: VersionContext) -> None:
         text = (
-            f'New {code("yt-dlp")} version available: {bold(ctx.latest.version)}\n'
+            f'New {bold(self._ytdlp_conf.release_channel)} {code("yt-dlp")} version available: '
+            f'{bold(ctx.latest.version)}\n'
             f'Current version: {bold(ctx.current.version)}\n'
-            f'Rebuild worker with {code("docker compose build --no-cache worker")}'
+            f'Rebuild worker with {code("docker compose build --no-cache worker && docker compose up -d -t 0 worker")}'
         )
         await self._bot.send_message_admins(text)
 
     async def _notify_up_to_date(
         self, ctx: VersionContext, user_ids: list[int]
     ) -> None:
-        """Send startup message that yt-dlp version is up to date."""
+        """Send startup message that yt-dlp version is up-to-date."""
         text = (
-            f'{INFORMATION_EMOJI} Your {code("yt-dlp")} version '
-            f'{bold(ctx.current.version)} is up to date, have fun'
+            f'{INFORMATION_EMOJI} Your {bold(self._ytdlp_conf.release_channel)} {code("yt-dlp")} '
+            f'version {bold(ctx.current.version)} is up to date, have fun'
         )
         await self._bot.send_message_to_users(text=text, user_ids=user_ids)

app_bot/bot/core/utils.py

@@ -9,7 +9,7 @@ from pyrogram.enums import ChatType
 from pyrogram.types import Message
 
 from bot.core.config import settings
-from bot.core.schema import AnonymousUserSchema, ConfigSchema, UserSchema
+from bot.core.schemas import AnonymousUserSchema, ConfigSchema, UserSchema
 
 
 async def shallow_sleep_async(sleep_time: float = 0.1) -> None:

app_bot/config-example.yml

@@ -40,3 +40,4 @@ ytdlp:
   version_check_enabled: !!bool True
   version_check_interval: 86400
   notify_users_on_new_version: !!bool True
+  release_channel: "NIGHTLY"

app_worker/worker/core/launcher.py

@@ -55,7 +55,7 @@ class WorkerLauncher:
             'Saving current yt-dlp version (%s) to the database', curr_version
         )
         async for db in get_db():
-            await YtdlpRepository().create_or_update_version(curr_version, db)
+            await YtdlpRepository(db).create_or_update_version(curr_version)
 
     async def _create_intermediate_directories(self) -> None:
         """Create temporary intermediate directories on start if they do not exist."""

yt_shared/yt_shared/clients/github.py

@@ -2,22 +2,31 @@ import datetime
 import logging
 
 import aiohttp
+from yt_shared.enums import YtdlpReleaseChannelType
 from yt_shared.schemas.ytdlp import LatestVersion
 
 
 class YtdlpGithubClient:
     """yt-dlp Github version number checker."""
 
-    LATEST_TAG_URL = 'https://github.com/yt-dlp/yt-dlp/releases/latest'
+    LATEST_TAG_URL_TPL = 'https://github.com/yt-dlp/{repository}/releases/latest'
+    LATEST_TAG_REPOSITORY_MAP = {
+        YtdlpReleaseChannelType.STABLE: 'yt-dlp',
+        YtdlpReleaseChannelType.NIGHTLY: 'yt-dlp-nightly-builds',
+        YtdlpReleaseChannelType.MASTER: 'yt-dlp-master-builds',
+    }
 
-    def __init__(self) -> None:
+    def __init__(self, release_channel: YtdlpReleaseChannelType) -> None:
         self._log = logging.getLogger(self.__class__.__name__)
+        self._release_channel = release_channel
 
     async def get_latest_version(self) -> LatestVersion:
+        tag_url = self.LATEST_TAG_URL_TPL.format(
+            repository=self.LATEST_TAG_REPOSITORY_MAP[self._release_channel]
+        )
         async with aiohttp.ClientSession() as session:
-            async with session.get(self.LATEST_TAG_URL) as resp:
+            async with session.get(tag_url) as resp:
                 version = resp.url.parts[-1]
-                self._log.info('Latest yt-dlp version: %s', version)
                 return LatestVersion(
                     version=version, retrieved_at=datetime.datetime.utcnow()
                 )

yt_shared/yt_shared/enums.py

@@ -50,3 +50,9 @@ class DownMediaType(StrChoiceEnum):
 class MediaFileType(StrChoiceEnum):
     AUDIO = 'AUDIO'
     VIDEO = 'VIDEO'
+
+
+class YtdlpReleaseChannelType(StrChoiceEnum):
+    STABLE = 'STABLE'
+    NIGHTLY = 'NIGHTLY'
+    MASTER = 'MASTER'

yt_shared/yt_shared/repositories/ytdlp.py

@@ -8,17 +8,18 @@ from yt_shared.models import YTDLP
 
 
 class YtdlpRepository:
-    def __init__(self) -> None:
+    def __init__(self, db: AsyncSession) -> None:
         self._log = logging.getLogger(self.__class__.__name__)
+        self._db = db
 
-    @staticmethod
-    async def get_current_version(db: AsyncSession) -> YTDLP:
-        result = await db.execute(select(YTDLP))
+    async def get_current_version(self) -> YTDLP:
+        result = await self._db.execute(select(YTDLP))
         return result.scalar_one()
 
-    @staticmethod
-    async def create_or_update_version(current_version: str, db: AsyncSession) -> None:
-        row_count: int = await db.scalar(select(func.count('*')).select_from(YTDLP))
+    async def create_or_update_version(self, current_version: str) -> None:
+        row_count: int = await self._db.scalar(
+            select(func.count('*')).select_from(YTDLP)
+        )
         if row_count > 1:
             raise MultipleResultsFound(
                 'Multiple yt-dlp version records found. Expected one.'
@@ -30,5 +31,5 @@ class YtdlpRepository:
             .values({'current_version': current_version})
             .execution_options(synchronize_session=False)
         )
-        await db.execute(insert_or_update_stmt)
-        await db.commit()
+        await self._db.execute(insert_or_update_stmt)
+        await self._db.commit()

yt_shared/yt_shared/ytdlp/version_checker.py

@@ -1,7 +1,6 @@
 import asyncio
 import logging
 
-from sqlalchemy.ext.asyncio import AsyncSession
 from yt_shared.clients.github import YtdlpGithubClient
 from yt_shared.repositories.ytdlp import YtdlpRepository
 from yt_shared.schemas.ytdlp import CurrentVersion, LatestVersion, VersionContext
@@ -10,23 +9,23 @@ from yt_shared.schemas.ytdlp import CurrentVersion, LatestVersion, VersionContex
 class YtdlpVersionChecker:
     """yt-dlp version number checker."""
 
-    LATEST_TAG_URL = 'https://github.com/yt-dlp/yt-dlp/releases/latest'
-
-    def __init__(self) -> None:
+    def __init__(self, client: YtdlpGithubClient, repository: YtdlpRepository) -> None:
         self._log = logging.getLogger(self.__class__.__name__)
-        self._ytdlp_repository = YtdlpRepository()
-        self._ytdlp_client = YtdlpGithubClient()
+        self._ytdlp_repository = repository
+        self._ytdlp_client = client
 
-    async def get_version_context(self, db: AsyncSession) -> VersionContext:
+    async def get_version_context(self) -> VersionContext:
         latest, current = await asyncio.gather(
-            self.get_latest_version(), self.get_current_version(db)
+            self.get_latest_version(), self.get_current_version()
         )
         return VersionContext(latest=latest, current=current)
 
     async def get_latest_version(self) -> LatestVersion:
-        return await self._ytdlp_client.get_latest_version()
+        latest_version = await self._ytdlp_client.get_latest_version()
+        self._log.info('Latest yt-dlp version: %s', latest_version.version)
+        return latest_version
 
-    async def get_current_version(self, db: AsyncSession) -> CurrentVersion:
-        ytdlp_ = await self._ytdlp_repository.get_current_version(db)
+    async def get_current_version(self) -> CurrentVersion:
+        ytdlp_ = await self._ytdlp_repository.get_current_version()
         self._log.info('Current yt-dlp version: %s', ytdlp_.current_version)
         return CurrentVersion.model_validate(ytdlp_)