Merge pull request #183 from Dineshkarthik/65-download-skipped-files

Download skipped files

media_downloader.py

@@ -27,18 +27,21 @@ logger = logging.getLogger("media_downloader")
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 FAILED_IDS: list = []
+DOWNLOADED_IDS: list = []
 
 
 def update_config(config: dict):
     """
-    Update exisitng configuration file.
+    Update existing configuration file.
 
     Parameters
     ----------
     config: dict
-        Configuraiton to be written into config file.
+        Configuration to be written into config file.
     """
-    config["ids_to_retry"] = list(set(config["ids_to_retry"] + FAILED_IDS))
+    config["ids_to_retry"] = (
+        list(set(config["ids_to_retry"]) - set(DOWNLOADED_IDS)) + FAILED_IDS
+    )
     with open("config.yaml", "w") as yaml_file:
         yaml.dump(config, yaml_file, default_flow_style=False)
     logger.info("Updated last read message_id to config file")
@@ -150,7 +153,7 @@ async def download_media(
     client: pyrogram.client.Client
         Client to interact with Telegram APIs.
     message: pyrogram.types.Message
-        Message object retrived from telegram.
+        Message object retrieved from telegram.
     media_types: list
         List of strings of media types to be downloaded.
         Ex : `["audio", "photo"]`
@@ -193,6 +196,7 @@ async def download_media(
                         )
                     if download_path:
                         logger.info("Media downloaded - %s", download_path)
+                    DOWNLOADED_IDS.append(message.message_id)
             break
         except pyrogram.errors.exceptions.bad_request_400.BadRequest:
             logger.warning(
@@ -213,7 +217,7 @@ async def download_media(
         except TypeError:
             # pylint: disable = C0301
             logger.warning(
-                "Timeout Error occured when downloading Message[%d], retrying after 5 seconds",
+                "Timeout Error occurred when downloading Message[%d], retrying after 5 seconds",
                 message.message_id,
             )
             await asyncio.sleep(5)
@@ -286,7 +290,7 @@ async def begin_import(config: dict, pagination_limit: int) -> dict:
     Create pyrogram client and initiate download.
 
     The pyrogram client is created using the ``api_id``, ``api_hash``
-    from the config and iter throught message offset on the
+    from the config and iter through message offset on the
     ``last_message_id`` and the requested file_formats.
 
     Parameters
@@ -299,7 +303,7 @@ async def begin_import(config: dict, pagination_limit: int) -> dict:
     Returns
     -------
     dict
-        Updated configuraiton to be written into config file.
+        Updated configuration to be written into config file.
     """
     client = pyrogram.Client(
         "media_downloader",
@@ -313,8 +317,16 @@ async def begin_import(config: dict, pagination_limit: int) -> dict:
         offset_id=last_read_message_id,
         reverse=True,
     )
-    pagination_count: int = 0
     messages_list: list = []
+    pagination_count: int = 0
+    if config["ids_to_retry"]:
+        logger.info("Downloading files failed during last run...")
+        skipped_messages: list = await client.get_messages(  # type: ignore
+            chat_id=config["chat_id"], message_ids=config["ids_to_retry"]
+        )
+        for message in skipped_messages:
+            pagination_count += 1
+            messages_list.append(message)
 
     async for message in messages_iter:  # type: ignore
         if pagination_count != pagination_limit:
@@ -356,8 +368,7 @@ def main():
         logger.info(
             "Downloading of %d files failed. "
             "Failed message ids are added to config file.\n"
-            "Functionality to re-download failed downloads will be added "
-            "in the next version of `Telegram-media-downloader`",
+            "These files will be downloaded on the next run.",
             len(set(FAILED_IDS)),
         )
     update_config(updated_config)

requirements.txt

@@ -1,4 +1,4 @@
-Pyrogram==1.4.7
+Pyrogram==1.4.8
 PyYAML==6.0
 rich==11.2.0
 TgCrypto==1.2.3

tests/test_media_downloader.py

@@ -29,7 +29,7 @@ MOCK_CONF = {
     "api_hash": "hasw5Tgawsuj67",
     "last_read_message_id": 0,
     "chat_id": 8654123,
-    "ids_to_retry": [],
+    "ids_to_retry": [1],
     "media_types": ["audio", "voice"],
     "file_formats": {"audio": ["all"], "voice": ["all"]},
 }
@@ -203,6 +203,18 @@ class MockClient:
                     mime_type="video/mov",
                 ),
             )
+        elif kwargs["message_ids"] == [1]:
+            message = [
+                MockMessage(
+                    id=1,
+                    media=True,
+                    chat_id=234568,
+                    video=MockVideo(
+                        file_name="sample_video.mov",
+                        mime_type="video/mov",
+                    ),
+                )
+            ]
         return message
 
     async def download_media(self, *args, **kwargs):

tests/utils/test_updates.py

@@ -0,0 +1,66 @@
+"""Unittest module for update checker."""
+import os
+import sys
+import unittest
+
+import mock
+from rich.markdown import Markdown
+
+sys.path.append("..")  # Adds higher directory to python modules path.
+from utils.updates import check_for_updates
+
+
+class FakeHTTPSConnection:
+    def __init__(self, status):
+        self.status = status
+
+    def request(self, *args, **kwargs):
+        pass
+
+    def getresponse(self):
+        return FakeHTTPSResponse(self.status)
+
+
+class FakeHTTPSResponse:
+    def __init__(self, status):
+        self.status = status
+
+    def read(self):
+        if self.status == 200:
+            return b'{"name":"v0.0.0 2022-03-02","tag_name":"v0.0.0", "html_url":"https://github.com/Dineshkarthik/telegram_media_downloader/releases/tag/v0.0.0"}'
+        else:
+            return b"{error}"
+
+
+class UpdatesTestCase(unittest.TestCase):
+    @mock.patch(
+        "utils.updates.http.client.HTTPSConnection",
+        new=mock.MagicMock(return_value=FakeHTTPSConnection(200)),
+    )
+    @mock.patch("utils.updates.__version__", new="0.0.1")
+    @mock.patch("utils.updates.Console")
+    @mock.patch("utils.updates.Markdown")
+    def test_update(self, mock_markdown, mock_console):
+        check_for_updates()
+        name: str = "v0.0.0 2022-03-02"
+        html_url: str = "https://github.com/Dineshkarthik/telegram_media_downloader/releases/tag/v0.0.0"
+        expected_message: str = (
+            f"## New version of Telegram-Media-Downloader is available - {name}\n"
+            "You are using an outdated version v0.0.1 please pull in the changes using `git pull` or download the latest release.\n\n"
+            f"Find more details about the latest release here - {html_url}"
+        )
+        mock_markdown.assert_called_with(expected_message)
+        mock_console.return_value.print.assert_called_once()
+
+    @mock.patch(
+        "utils.updates.http.client.HTTPSConnection",
+        new=mock.MagicMock(return_value=FakeHTTPSConnection(500)),
+    )
+    @mock.patch("utils.updates.Console")
+    def test_exception(self, mock_console):
+        check_for_updates()
+        exception_message: str = (
+            "Following error occured when checking for updates\n"
+            "<class 'json.decoder.JSONDecodeError'>, Expecting property name enclosed in double quotes: line 1 column 2 (char 1)"
+        )
+        mock_console.return_value.log.assert_called_with(exception_message)

utils/__init__.py

@@ -1,6 +1,6 @@
 """Init namespace"""
 
-__version__ = "1.4.1"
+__version__ = "1.5.0"
 __license__ = "MIT License"
 __copyright__ = (
     "Copyright (C) 2019 Dineshkarthik <https://github.com/Dineshkarthik>"

utils/updates.py

@@ -13,6 +13,7 @@ def check_for_updates() -> None:
 
     Using Github API checks for new release and prints information of new release if available.
     """
+    console = Console()
     try:
         headers: dict = {
             "Content-Type": "application/json",
@@ -28,11 +29,12 @@ def check_for_updates() -> None:
         latest_release: dict = json.loads(res.read().decode("utf-8"))
         if f"v{__version__}" != latest_release["tag_name"]:
             update_message: str = (
-                f"## New versionof Telegram-Media-Downloader is available - {latest_release['name']}\n"
+                f"## New version of Telegram-Media-Downloader is available - {latest_release['name']}\n"
                 f"You are using an outdated version v{__version__} please pull in the changes using `git pull` or download the latest release.\n\n"
                 f"Find more details about the latest release here - {latest_release['html_url']}"
             )
-            console = Console()
             console.print(Markdown(update_message))
-    except Exception:
-        pass
+    except Exception as e:
+        console.log(
+            f"Following error occured when checking for updates\n{e.__class__}, {e}"
+        )