From 3c547e4bbd313f01ccbd03a0b5bd6bdd014d5d6a Mon Sep 17 00:00:00 2001 From: Taras Terletskyi <888784+tropicoo@users.noreply.github.com> Date: Sun, 22 Jan 2023 00:50:45 +0200 Subject: [PATCH] Improve error handling --- README.md | 28 ++++++++------ bot/core/handlers/success.py | 51 +++++++++++++++++++------- worker/core/payload_handler.py | 7 ++-- yt_shared/yt_shared/enums.py | 12 +++--- yt_shared/yt_shared/schemas/success.py | 1 + 5 files changed, 65 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index cac02db..5f74d89 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Version: 0.5. [Release details](.releases/release_0.5.md). ## 😂 Features * Download videos from any [yt-dlp](https://github.com/yt-dlp/yt-dlp) supported website + to your storage * Upload downloaded videos to the Telegram chat * Trigger video download by sending link to an API * Track download tasks via API @@ -75,6 +76,11 @@ or something went wrong. shorter, it will make it on `video length / 2` time point because the FFmpeg process will error out. Change the `THUMBNAIL_FRAME_SECOND` variable if needed in the `envs/.env_worker` file. +4. Max upload file size for non-premium Telegram user is 2GB (2147483648 bytes) which is + reflected in the example config `bot/config-example.yml`. If the configured user + is the premium user, you're allowed to upload files up to 4GB (4294967296 bytes) and + can change the default value stored in the `upload_video_max_file_size` config + variable. ## 🛑 Failed download @@ -96,17 +102,17 @@ details By default, API service will run on your `localhost` and `1984` port. API endpoint documentations lives at `http://127.0.0.1:1984/docs`. -| Endpoint | Method| Description| -|---|---|---| -| `/status` | `GET` | Get API healthcheck status, usually response is `{"status": "OK"}` | -| `/v1/yt-dlp` | `GET` | Get latest and currently installed `yt-dlp` version | -|`/v1/tasks/?include_meta=False&status=DONE`| `GET` | Get all tasks with filtering options like to include large file metadata and by task status: `PENDING`, `PROCESSING`, `FAILED` and `DONE`. | -| `/v1/tasks/f828714a-5c50-45de-87c0-3b51b7e04039?include_meta=True` | `GET` | Get info about task by ID | -| `/v1/tasks/latest?include_meta=True` | `GET` | Get info about latest task | -| `/v1/tasks/f828714a-5c50-45de-87c0-3b51b7e04039` | `DELETE` | Delete task by ID | -| `/v1/tasks/latest?include_meta=True` | `GET` | Get info about the latest task | -| `/v1/tasks` | `POST` | Create a download task by sending json payload `{"url": ""}` | -| `/v1/tasks/stats` | `GET` | Get overall tasks stats | +| Endpoint | Method | Description | +|--------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------| +| `/status` | `GET` | Get API healthcheck status, usually response is `{"status": "OK"}` | +| `/v1/yt-dlp` | `GET` | Get latest and currently installed `yt-dlp` version | +| `/v1/tasks/?include_meta=False&status=DONE` | `GET` | Get all tasks with filtering options like to include large file metadata and by task status: `PENDING`, `PROCESSING`, `FAILED` and `DONE`. | +| `/v1/tasks/f828714a-5c50-45de-87c0-3b51b7e04039?include_meta=True` | `GET` | Get info about task by ID | +| `/v1/tasks/latest?include_meta=True` | `GET` | Get info about latest task | +| `/v1/tasks/f828714a-5c50-45de-87c0-3b51b7e04039` | `DELETE` | Delete task by ID | +| `/v1/tasks/latest?include_meta=True` | `GET` | Get info about the latest task | +| `/v1/tasks` | `POST` | Create a download task by sending json payload `{"url": ""}` | +| `/v1/tasks/stats` | `GET` | Get overall tasks stats | ### API examples diff --git a/bot/core/handlers/success.py b/bot/core/handlers/success.py index 4ea36fe..96b5987 100644 --- a/bot/core/handlers/success.py +++ b/bot/core/handlers/success.py @@ -1,13 +1,17 @@ import asyncio import os +import traceback from pyrogram.enums import ParseMode from core.config import settings from core.handlers.abstract import AbstractHandler from core.tasks.upload import UploadTask + from yt_shared.emoji import SUCCESS_EMOJI from yt_shared.enums import TaskSource +from yt_shared.rabbit.publisher import Publisher +from yt_shared.schemas.error import ErrorGeneralPayload from yt_shared.schemas.success import SuccessPayload from yt_shared.utils.file import file_cleanup from yt_shared.utils.tasks.tasks import create_task @@ -16,8 +20,31 @@ from yt_shared.utils.tasks.tasks import create_task class SuccessHandler(AbstractHandler): _body: SuccessPayload + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._publisher = Publisher() + async def handle(self) -> None: - await self._handle() + try: + await self._handle() + except Exception as err: + await self._publish_error_message(err) + + async def _publish_error_message(self, err: Exception) -> None: + err_payload = ErrorGeneralPayload( + task_id=self._body.task_id, + message_id=self._body.message_id, + from_chat_id=self._body.from_chat_id, + from_chat_type=self._body.from_chat_type, + from_user_id=self._body.from_user_id, + message='Upload error', + url=self._body.context.url, + context=self._body.context, + yt_dlp_version=self._body.yt_dlp_version, + exception_msg=traceback.format_exc(), + exception_type=err.__class__.__name__, + ) + await self._publisher.send_download_error(err_payload) async def _handle(self) -> None: await self._send_success_text() @@ -26,14 +53,11 @@ class SuccessHandler(AbstractHandler): settings.TMP_DOWNLOAD_PATH, self._body.thumb_name ) try: - if not self._eligible_for_upload(video_path): - self._log.warning( - 'File %s will not be uploaded to Telegram', self._body.filename - ) - return + self._validate_file_size_for_upload(video_path) await self._create_upload_task() except Exception: self._log.error('Upload of "%s" failed, performing cleanup', video_path) + raise finally: file_cleanup(file_paths=(video_path, thumb_path), log=self._log) @@ -66,7 +90,7 @@ class SuccessHandler(AbstractHandler): kwargs['reply_to_message_id'] = self._body.message_id await self._bot.send_message(**kwargs) - def _eligible_for_upload(self, video_path: str) -> bool: + def _validate_file_size_for_upload(self, video_path: str) -> None: if self._body.context.source is TaskSource.API: upload_video_file = self._bot.conf.telegram.api.upload_video_file max_file_size = self._bot.conf.telegram.api.upload_video_max_file_size @@ -76,14 +100,13 @@ class SuccessHandler(AbstractHandler): max_file_size = user.upload.upload_video_max_file_size if not upload_video_file: - return False + raise ValueError(f'Video {video_path} not found') file_size = os.stat(video_path).st_size if file_size > max_file_size: - self._log.warning( - 'Video file size %d bigger then allowed %d. Will not upload', - file_size, - max_file_size, + err_msg = ( + f'Video file size {file_size} bytes bigger then allowed {max_file_size}' + f' bytes. Will not upload' ) - return False - return True + self._log.warning(err_msg) + raise ValueError(err_msg) diff --git a/worker/core/payload_handler.py b/worker/core/payload_handler.py index 73ea750..e021a27 100644 --- a/worker/core/payload_handler.py +++ b/worker/core/payload_handler.py @@ -57,7 +57,8 @@ class PayloadHandler: from_chat_id=video_payload.from_chat_id, from_chat_type=video_payload.from_chat_type, from_user_id=task.from_user_id, - context=video_payload.dict(), + context=video_payload, + yt_dlp_version=ytdlp_version.__version__, ) await self._publisher.send_download_finished(success_payload) @@ -75,7 +76,7 @@ class PayloadHandler: from_user_id=video_payload.from_user_id, message='Download error', url=video_payload.url, - context=video_payload.dict(), + context=video_payload, yt_dlp_version=ytdlp_version.__version__, exception_msg=str(err), exception_type=err.__class__.__name__, @@ -96,7 +97,7 @@ class PayloadHandler: from_user_id=video_payload.from_user_id, message='General worker error', url=video_payload.url, - context=video_payload.dict(), + context=video_payload, yt_dlp_version=ytdlp_version.__version__, exception_msg=traceback.format_exc(), exception_type=err.__class__.__name__, diff --git a/yt_shared/yt_shared/enums.py b/yt_shared/yt_shared/enums.py index 0cf8503..3ae5692 100644 --- a/yt_shared/yt_shared/enums.py +++ b/yt_shared/yt_shared/enums.py @@ -1,8 +1,8 @@ -import enum +from enum import Enum, auto, unique -@enum.unique -class ChoiceEnum(enum.Enum): +@unique +class ChoiceEnum(Enum): @classmethod def choices(cls) -> tuple[str, ...]: return tuple(x.value for x in cls) @@ -21,9 +21,9 @@ class TaskSource(str, ChoiceEnum): class RabbitPayloadType(ChoiceEnum): - DOWNLOAD_ERROR = 'ERROR_DOWNLOAD' - GENERAL_ERROR = 'GENERAL_ERROR' - SUCCESS = 'SUCCESS' + DOWNLOAD_ERROR = auto() + GENERAL_ERROR = auto() + SUCCESS = auto() class TelegramChatType(ChoiceEnum): diff --git a/yt_shared/yt_shared/schemas/success.py b/yt_shared/yt_shared/schemas/success.py index de894e0..aeee4ea 100644 --- a/yt_shared/yt_shared/schemas/success.py +++ b/yt_shared/yt_shared/schemas/success.py @@ -25,3 +25,4 @@ class SuccessPayload(BaseRabbitPayloadModel): width: StrictInt | None height: StrictInt | None context: VideoPayload + yt_dlp_version: StrictStr | None