From 78f1ffc4fefd397241db7829f03d5a87b3a0bd2b Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Mon, 28 Feb 2022 20:36:50 +0100 Subject: [PATCH 1/9] enh: download previously failed files which are added to ids_to_retry --- media_downloader.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/media_downloader.py b/media_downloader.py index a4e3054..3a8a729 100644 --- a/media_downloader.py +++ b/media_downloader.py @@ -27,6 +27,7 @@ logger = logging.getLogger("media_downloader") THIS_DIR = os.path.dirname(os.path.abspath(__file__)) FAILED_IDS: list = [] +DOWNLOADED_IDS: list = [] def update_config(config: dict): @@ -38,7 +39,9 @@ def update_config(config: dict): config: dict Configuraiton to be written into config file. """ - config["ids_to_retry"] = list(set(config["ids_to_retry"] + FAILED_IDS)) + config["ids_to_retry"] = ( + list(set(config["ids_to_retry"]) - set(DOWNLOADED_IDS)) + FAILED_IDS + ) with open("config.yaml", "w") as yaml_file: yaml.dump(config, yaml_file, default_flow_style=False) logger.info("Updated last read message_id to config file") @@ -193,6 +196,7 @@ async def download_media( ) if download_path: logger.info("Media downloaded - %s", download_path) + DOWNLOADED_IDS.append(message.message_id) break except pyrogram.errors.exceptions.bad_request_400.BadRequest: logger.warning( @@ -313,8 +317,16 @@ async def begin_import(config: dict, pagination_limit: int) -> dict: offset_id=last_read_message_id, reverse=True, ) - pagination_count: int = 0 messages_list: list = [] + pagination_count: int = 0 + if config["ids_to_retry"]: + logger.info("Downloading files failed during last run...") + skipped_messages: list = await client.get_messages( + chat_id=config["chat_id"], message_ids=config["ids_to_retry"] + ) + for message in skipped_messages: + pagination_count += 1 + messages_list.append(message) async for message in messages_iter: # type: ignore if pagination_count != pagination_limit: @@ -356,8 +368,7 @@ def main(): logger.info( "Downloading of %d files failed. " "Failed message ids are added to config file.\n" - "Functionality to re-download failed downloads will be added " - "in the next version of `Telegram-media-downloader`", + "These files will be downloaded on the next run.", len(set(FAILED_IDS)), ) update_config(updated_config) From d4585394ad1bcb8361fc05206cdd0307ccac1c45 Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Mon, 28 Feb 2022 20:39:44 +0100 Subject: [PATCH 2/9] docs: fix typos in docstrings --- media_downloader.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/media_downloader.py b/media_downloader.py index 3a8a729..caf34ea 100644 --- a/media_downloader.py +++ b/media_downloader.py @@ -32,12 +32,12 @@ DOWNLOADED_IDS: list = [] def update_config(config: dict): """ - Update exisitng configuration file. + Update existing configuration file. Parameters ---------- config: dict - Configuraiton to be written into config file. + Configuration to be written into config file. """ config["ids_to_retry"] = ( list(set(config["ids_to_retry"]) - set(DOWNLOADED_IDS)) + FAILED_IDS @@ -153,7 +153,7 @@ async def download_media( client: pyrogram.client.Client Client to interact with Telegram APIs. message: pyrogram.types.Message - Message object retrived from telegram. + Message object retrieved from telegram. media_types: list List of strings of media types to be downloaded. Ex : `["audio", "photo"]` @@ -217,7 +217,7 @@ async def download_media( except TypeError: # pylint: disable = C0301 logger.warning( - "Timeout Error occured when downloading Message[%d], retrying after 5 seconds", + "Timeout Error occurred when downloading Message[%d], retrying after 5 seconds", message.message_id, ) await asyncio.sleep(5) @@ -290,7 +290,7 @@ async def begin_import(config: dict, pagination_limit: int) -> dict: Create pyrogram client and initiate download. The pyrogram client is created using the ``api_id``, ``api_hash`` - from the config and iter throught message offset on the + from the config and iter through message offset on the ``last_message_id`` and the requested file_formats. Parameters @@ -303,7 +303,7 @@ async def begin_import(config: dict, pagination_limit: int) -> dict: Returns ------- dict - Updated configuraiton to be written into config file. + Updated configuration to be written into config file. """ client = pyrogram.Client( "media_downloader", From 5e34342cb9b5e7dc97d9a70391c7d06cd2e42aac Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Mon, 28 Feb 2022 20:45:34 +0100 Subject: [PATCH 3/9] types: update type hints --- media_downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media_downloader.py b/media_downloader.py index caf34ea..b99502d 100644 --- a/media_downloader.py +++ b/media_downloader.py @@ -321,7 +321,7 @@ async def begin_import(config: dict, pagination_limit: int) -> dict: pagination_count: int = 0 if config["ids_to_retry"]: logger.info("Downloading files failed during last run...") - skipped_messages: list = await client.get_messages( + skipped_messages: list = await client.get_messages( # type: ignore chat_id=config["chat_id"], message_ids=config["ids_to_retry"] ) for message in skipped_messages: From 6c23daaa89dc25f4ddc327d6babfd2083867b716 Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Mon, 28 Feb 2022 20:52:57 +0100 Subject: [PATCH 4/9] test: add test to download skipped files logic --- tests/test_media_downloader.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_media_downloader.py b/tests/test_media_downloader.py index 480d466..d203ffe 100644 --- a/tests/test_media_downloader.py +++ b/tests/test_media_downloader.py @@ -29,7 +29,7 @@ MOCK_CONF = { "api_hash": "hasw5Tgawsuj67", "last_read_message_id": 0, "chat_id": 8654123, - "ids_to_retry": [], + "ids_to_retry": [1], "media_types": ["audio", "voice"], "file_formats": {"audio": ["all"], "voice": ["all"]}, } @@ -203,6 +203,18 @@ class MockClient: mime_type="video/mov", ), ) + elif kwargs["message_ids"] == [1]: + message = [ + MockMessage( + id=1, + media=True, + chat_id=234568, + video=MockVideo( + file_name="sample_video.mov", + mime_type="video/mov", + ), + ) + ] return message async def download_media(self, *args, **kwargs): From 08a683492c79337b99f241c47c8394c962aaf6c5 Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Wed, 2 Mar 2022 16:50:55 +0100 Subject: [PATCH 5/9] test: add test for update checker utility module --- tests/utils/test_updates.py | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/utils/test_updates.py diff --git a/tests/utils/test_updates.py b/tests/utils/test_updates.py new file mode 100644 index 0000000..7f12c28 --- /dev/null +++ b/tests/utils/test_updates.py @@ -0,0 +1,50 @@ +"""Unittest module for update checker.""" +import os +import sys +import unittest + +import mock +from rich.markdown import Markdown + +sys.path.append("..") # Adds higher directory to python modules path. +from utils.updates import check_for_updates + + +class FakeHTTPSConnection: + def __init__(self, status): + self.status = status + + def request(self, *args, **kwargs): + pass + + def getresponse(self): + return FakeHTTPSResponse(self.status) + + +class FakeHTTPSResponse: + def __init__(self, status): + self.status = status + + def read(self): + return b'{"name":"v0.0.0 2022-03-02","tag_name":"v0.0.0", "html_url":"https://github.com/Dineshkarthik/telegram_media_downloader/releases/tag/v0.0.0"}' + + +class UpdatesTestCase(unittest.TestCase): + @mock.patch( + "utils.updates.http.client.HTTPSConnection", + new=mock.MagicMock(return_value=FakeHTTPSConnection(200)), + ) + @mock.patch("utils.updates.__version__", new="0.0.1") + @mock.patch("utils.updates.Console") + @mock.patch("utils.updates.Markdown") + def test_update(self, mock_markdown, mock_console): + check_for_updates() + name: str = "v0.0.0 2022-03-02" + html_url: str = "https://github.com/Dineshkarthik/telegram_media_downloader/releases/tag/v0.0.0" + expected_message: str = ( + f"## New versionof Telegram-Media-Downloader is available - {name}\n" + "You are using an outdated version v0.0.1 please pull in the changes using `git pull` or download the latest release.\n\n" + f"Find more details about the latest release here - {html_url}" + ) + mock_markdown.assert_called_with(expected_message) + mock_console.return_value.print.assert_called_once() From 2b84b8f7246e51b02a079ba6de8e6f6470df575d Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Wed, 2 Mar 2022 17:01:56 +0100 Subject: [PATCH 6/9] build(deps-dev): bump pyrogram from 1.4.7 to 1.4.8 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 85b6db6..385758c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Pyrogram==1.4.7 +Pyrogram==1.4.8 PyYAML==6.0 rich==11.2.0 TgCrypto==1.2.3 From 3c121da18a39ae8f3cf2c34f9b4330bbdfb2276f Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Wed, 2 Mar 2022 17:03:44 +0100 Subject: [PATCH 7/9] build: bump-up telegram-media-downloader version from 1.4.1 to 1.5.0 --- utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/__init__.py b/utils/__init__.py index 03b6abc..2f1a5fa 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -1,6 +1,6 @@ """Init namespace""" -__version__ = "1.4.1" +__version__ = "1.5.0" __license__ = "MIT License" __copyright__ = ( "Copyright (C) 2019 Dineshkarthik " From c4e3add0057806ad1295e55af154fc8d2585a643 Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Wed, 2 Mar 2022 17:42:46 +0100 Subject: [PATCH 8/9] enh: handle exceptions on update checker --- utils/updates.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/utils/updates.py b/utils/updates.py index ccd6ad1..40681cd 100644 --- a/utils/updates.py +++ b/utils/updates.py @@ -13,6 +13,7 @@ def check_for_updates() -> None: Using Github API checks for new release and prints information of new release if available. """ + console = Console() try: headers: dict = { "Content-Type": "application/json", @@ -28,11 +29,12 @@ def check_for_updates() -> None: latest_release: dict = json.loads(res.read().decode("utf-8")) if f"v{__version__}" != latest_release["tag_name"]: update_message: str = ( - f"## New versionof Telegram-Media-Downloader is available - {latest_release['name']}\n" + f"## New version of Telegram-Media-Downloader is available - {latest_release['name']}\n" f"You are using an outdated version v{__version__} please pull in the changes using `git pull` or download the latest release.\n\n" f"Find more details about the latest release here - {latest_release['html_url']}" ) - console = Console() console.print(Markdown(update_message)) - except Exception: - pass + except Exception as e: + console.log( + f"Following error occured when checking for updates\n{e.__class__}, {e}" + ) From a1871c528c89ccef695ee80ec108659b317c09cf Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Wed, 2 Mar 2022 17:43:16 +0100 Subject: [PATCH 9/9] test: add test for exception cases --- tests/utils/test_updates.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/utils/test_updates.py b/tests/utils/test_updates.py index 7f12c28..d04c2d3 100644 --- a/tests/utils/test_updates.py +++ b/tests/utils/test_updates.py @@ -26,7 +26,10 @@ class FakeHTTPSResponse: self.status = status def read(self): - return b'{"name":"v0.0.0 2022-03-02","tag_name":"v0.0.0", "html_url":"https://github.com/Dineshkarthik/telegram_media_downloader/releases/tag/v0.0.0"}' + if self.status == 200: + return b'{"name":"v0.0.0 2022-03-02","tag_name":"v0.0.0", "html_url":"https://github.com/Dineshkarthik/telegram_media_downloader/releases/tag/v0.0.0"}' + else: + return b"{error}" class UpdatesTestCase(unittest.TestCase): @@ -42,9 +45,22 @@ class UpdatesTestCase(unittest.TestCase): name: str = "v0.0.0 2022-03-02" html_url: str = "https://github.com/Dineshkarthik/telegram_media_downloader/releases/tag/v0.0.0" expected_message: str = ( - f"## New versionof Telegram-Media-Downloader is available - {name}\n" + f"## New version of Telegram-Media-Downloader is available - {name}\n" "You are using an outdated version v0.0.1 please pull in the changes using `git pull` or download the latest release.\n\n" f"Find more details about the latest release here - {html_url}" ) mock_markdown.assert_called_with(expected_message) mock_console.return_value.print.assert_called_once() + + @mock.patch( + "utils.updates.http.client.HTTPSConnection", + new=mock.MagicMock(return_value=FakeHTTPSConnection(500)), + ) + @mock.patch("utils.updates.Console") + def test_exception(self, mock_console): + check_for_updates() + exception_message: str = ( + "Following error occured when checking for updates\n" + ", Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" + ) + mock_console.return_value.log.assert_called_with(exception_message)