Version 0.4. Details in /.releases/release_0.4.md

.releases/release_0.4.md

@@ -0,0 +1,19 @@
+# Release info
+
+Version: 0.4
+
+Release date: November 13, 2022
+
+# Important
+
+1. Changed default yt-dlp options in `worker/ytdl_opts/default.py`. Replaced `'max_downloads': 1` with `'playlist_items': '1:1'`
+   to properly handle the result. 
+2. It's important to know that the worker backend does not handle downloading more than one video from the playlist even if you change yt-dlp options. Only the first video will be downloaded and processed.
+
+# New features
+
+N/A
+
+# Misc
+
+N/A

README.md

@@ -2,7 +2,7 @@
 
 Simple and reliable YouTube Download Telegram Bot.
 
-Version: 0.3.1. [Release details](.releases/release_0.3.1.md).
+Version: 0.4. [Release details](.releases/release_0.4.md).
 
 ![frames](.assets/download_success.png)
 

api/api/api_v1/schemas/task.py

@@ -1,7 +1,7 @@
 import uuid
 from datetime import datetime
 
-from pydantic import StrictInt, StrictStr
+from pydantic import StrictFloat, StrictInt, StrictStr
 
 from api.api_v1.schemas.base import BaseOrmModel
 from yt_shared.enums import TaskSource, TaskStatus
@@ -25,7 +25,7 @@ class FileSimpleSchema(BaseOrmModel):
     title: StrictStr | None
     name: StrictStr | None
     thumb_name: StrictStr | None
-    duration: StrictInt | None
+    duration: StrictFloat | None
     width: StrictInt | None
     height: StrictInt | None
     cache: CacheSchema | None = ...

bot/core/tasks/upload.py

@@ -3,7 +3,7 @@ import os
 from itertools import chain
 from typing import Coroutine, TYPE_CHECKING
 
-from pydantic import StrictInt, StrictStr
+from pydantic import StrictFloat, StrictInt, StrictStr
 from pyrogram.enums import ChatAction, MessageMediaType, ParseMode
 from pyrogram.types import Animation, Message, Video
 from tenacity import retry, stop_after_attempt, wait_fixed
@@ -28,7 +28,7 @@ class VideoContext(RealBaseModel):
     caption: StrictStr
     file_name: StrictStr
     video_path: StrictStr
-    duration: StrictInt
+    duration: StrictFloat
     height: StrictInt
     width: StrictInt
     thumb: StrictStr
@@ -122,7 +122,7 @@ class UploadTask(AbstractTask):
             'chat_id': chat_id,
             'caption': self._video_ctx.caption,
             'file_name': self._video_ctx.file_name,
-            'duration': self._video_ctx.duration,
+            'duration': int(self._video_ctx.duration),
             'height': self._video_ctx.height,
             'width': self._video_ctx.width,
             'thumb': self._video_ctx.thumb,

bot/version.py

@@ -1 +1 @@
-__version__ = '0.3.1'
+__version__ = '0.4'

worker/core/downloader.py

@@ -11,6 +11,9 @@ except ImportError:
 
 
 class VideoDownloader:
+
+    _PLAYLIST = 'playlist'
+
     def __init__(self) -> None:
         self._log = logging.getLogger(self.__class__.__name__)
 
@@ -24,26 +27,47 @@ class VideoDownloader:
     def _download(self, url: str) -> DownVideo:
         self._log.info('Downloading %s', url)
         with yt_dlp.YoutubeDL(YTDL_OPTS) as ytdl:
-            meta = ytdl.extract_info(url, download=False)
-            meta = ytdl.sanitize_info(meta)
-            try:
-                ytdl.download(url)
-            except yt_dlp.utils.MaxDownloadsReached as err:
-                self._log.warning(
-                    'Check video URL %s. Looks like a page with videos. Stopped on %d: %s',
-                    url,
-                    YTDL_OPTS['max_downloads'],
-                    err,
-                )
+            meta = ytdl.extract_info(url, download=True)
+            meta_sanitized = ytdl.sanitize_info(meta)
 
         self._log.info('Finished downloading %s', url)
-        self._log.debug('Download meta: %s', meta)
-        filepath = ytdl.prepare_filename(meta)
+        self._log.debug('Download meta: %s', meta_sanitized)
+        duration, width, height = self._get_video_context(meta)
         return DownVideo(
             title=meta['title'],
-            name=filepath.rsplit('/', maxsplit=1)[-1],
-            duration=meta.get('duration'),
-            width=meta.get('width'),
-            height=meta.get('height'),
-            meta=meta,
+            name=self._get_filename(meta),
+            duration=duration,
+            width=width,
+            height=height,
+            meta=meta_sanitized,
         )
+
+    def _get_video_context(self, meta: dict) -> tuple[float | None, int | None, int | None]:
+        if meta['_type'] == self._PLAYLIST:
+            entry: dict = meta['entries'][0]
+            requested_video: dict = entry['requested_downloads'][0]
+            return (
+                self._to_float(entry.get('duration')),
+                requested_video.get('width'),
+                requested_video.get('height'),
+            )
+        return (
+            self._to_float(meta.get('duration')),
+            meta['requested_downloads'][0].get('width'),
+            meta['requested_downloads'][0].get('height'),
+        )
+
+    @staticmethod
+    def _to_float(duration: int | float | None) -> float | None:
+        try:
+            return float(duration)
+        except TypeError:
+            return duration
+
+    def _get_filename(self, meta: dict) -> str:
+        return self._get_filepath(meta).rsplit('/', maxsplit=1)[-1]
+
+    def _get_filepath(self, meta: dict) -> str:
+        if meta['_type'] == self._PLAYLIST:
+            return meta['entries'][0]['requested_downloads'][0]['filepath']
+        return meta['requested_downloads'][0]['filepath']

worker/core/tasks/ffprobe_context.py

@@ -16,18 +16,16 @@ class GetFfprobeContextTask(AbstractFfBinaryTask):
             return None
 
         stdout, stderr = await self._get_stdout_stderr(proc)
-        self._log.debug(
+        self._log.info(
             'Process %s returncode: %d, stderr: %s', cmd, proc.returncode, stderr
         )
         if proc.returncode:
-            self._log.error(
-                'Failed to make video context. Is file broken? %s?', self._file_path
-            )
-            return None
+            err_msg = f'Failed to make video context. Is file broken? {self._file_path}?'
+            self._log.error(err_msg)
+            raise RuntimeError(err_msg)
         try:
             return json.loads(stdout)
-        except Exception:
-            self._log.exception(
-                'Failed to load ffprobe output [type %s]: %s', type(stdout), stdout
-            )
-            return None
+        except Exception as err:
+            err_msg = f'Failed to load ffprobe output [type {type(stdout)}]: {stdout}'
+            self._log.exception(err_msg)
+            raise RuntimeError(err_msg) from err

worker/core/tasks/thumbnail.py

@@ -1,13 +1,11 @@
 from core.config import settings
 from core.tasks.abstract import AbstractFfBinaryTask
 
-from yt_shared.schemas.video import DownVideo
-
 
 class MakeThumbnailTask(AbstractFfBinaryTask):
     _CMD = 'ffmpeg -y -loglevel error -i "{filepath}" -ss {second} -vframes 1 -q:v 7 "{thumbpath}"'
 
-    def __init__(self, thumbnail_path: str, *args, duration: int, **kwargs) -> None:
+    def __init__(self, thumbnail_path: str, *args, duration: float, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self._thumbnail_path = thumbnail_path
         self._duration = duration

worker/core/video_service.py

@@ -73,7 +73,14 @@ class VideoService:
 
         # yt-dlp meta may not contain needed video metadata.
         if not all([video.duration, video.height, video.width]):
-            await self._set_probe_ctx(file_path, video)
+            # TODO: Move to higher level and re-raise as DownloadVideoServiceError with task,
+            # TODO: or create new exception type.
+            try:
+                await self._set_probe_ctx(file_path, video)
+            except RuntimeError as err:
+                exception = DownloadVideoServiceError(str(err))
+                exception.task = task
+                raise exception
 
         tasks = [self._create_thumbnail_task(file_path, thumb_path, video.duration)]
         if settings.SAVE_VIDEO_FILE:
@@ -91,8 +98,7 @@ class VideoService:
         video_streams = [
             stream for stream in probe_ctx['streams'] if stream['codec_type'] == 'video'
         ]
-
-        video.duration = int(float(probe_ctx['format']['duration']))
+        video.duration = float(probe_ctx['format']['duration'])
         video.width = video_streams[0]['width']
         video.height = video_streams[0]['height']
 
@@ -107,7 +113,7 @@ class VideoService:
         )
 
     def _create_thumbnail_task(
-        self, file_path: str, thumb_path: str, duration: int
+        self, file_path: str, thumb_path: str, duration: float
     ) -> asyncio.Task:
         return create_task(
             MakeThumbnailTask(thumb_path, file_path, duration=duration).run(),

worker/ytdl_opts/default.py

@@ -7,6 +7,6 @@ YTDL_OPTS = {
     'outtmpl': os.path.join(settings.TMP_DOWNLOAD_PATH, '%(title).200B.%(ext)s'),
     'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4',
     'noplaylist': True,
-    'max_downloads': 1,
+    'playlist_items': '1:1',
     'concurrent_fragment_downloads': 5,
 }

yt_shared/yt_shared/schemas/success.py

@@ -1,7 +1,8 @@
 import uuid
+from typing import ClassVar
 
 from pydantic import StrictStr, StrictInt
-from pydantic.types import ClassVar
+from pydantic.types import StrictFloat
 
 from yt_shared.enums import RabbitPayloadType, TelegramChatType
 from yt_shared.schemas.base import BaseRabbitPayloadModel
@@ -20,7 +21,7 @@ class SuccessPayload(BaseRabbitPayloadModel):
     title: StrictStr
     filename: StrictStr
     thumb_name: StrictStr
-    duration: StrictInt | None
+    duration: StrictFloat | None
     width: StrictInt | None
     height: StrictInt | None
     context: VideoPayload

yt_shared/yt_shared/schemas/video.py

@@ -1,7 +1,7 @@
 import uuid
 from datetime import datetime, timezone
 
-from pydantic import Field, StrictInt, StrictStr, root_validator
+from pydantic import Field, StrictFloat, StrictInt, StrictStr, root_validator
 
 from yt_shared.enums import TaskSource, TelegramChatType
 from yt_shared.schemas.base import RealBaseModel
@@ -24,7 +24,7 @@ class DownVideo(RealBaseModel):
     title: StrictStr
     name: StrictStr
     thumb_name: StrictStr | None = None
-    duration: int | None = None
+    duration: StrictFloat | None = None
     width: int | None = None
     height: int | None = None
     meta: dict