Merge pull request #277 from Dineshkarthik/v2.0.0-stable

Migrate to Pyrogram v2

.github/workflows/code-checks.yml

@@ -0,0 +1,24 @@
+name: Code Quality
+
+on:
+  pull_request:
+    branches: [ master ]
+    paths-ignore:
+      - 'README.md'
+  push:
+    branches: [ master ]
+    paths-ignore:
+      - 'README.md'
+
+jobs:
+  pre-commit:
+    name: Linting
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install dependencies
+      run: make dev_install
+    - uses: pre-commit/action@v3.0.0

.github/workflows/unittest.yml

@@ -3,8 +3,12 @@ name: Unittest
 on:
   push:
     branches: [ master ]
+    paths-ignore:
+      - 'README.md'
   pull_request:
     branches: [ master ]
+    paths-ignore:
+      - 'README.md'
 
 jobs:
   build:
@@ -13,13 +17,13 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [ '3.6', '3.7', '3.8', '3.9' ]
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11.0-beta.4' ]
     name: Test - Python ${{ matrix.python-version }} on ${{ matrix.os }}
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Get setuptools Unix
@@ -30,10 +34,6 @@ jobs:
       run: pip install --upgrade --user pip setuptools codecov
     - name: Install dependencies
       run: make dev_install
-    - name: Code Check - Pylint
-      run: make pylint
-    - name: Static Type Check - Mypy
-      run: make static_type_check
     - name: Test with pytest
       run: |
         make -e test

.pre-commit-config.yaml

@@ -0,0 +1,48 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.2.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+-   repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+    -   id: black
+        name: black
+        entry: black
+        types: [python]
+-   repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+    -   id: isort
+        name: isort
+        entry: isort
+        types: [python]
+        args: ["--profile", "black", "--filter-files"]
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.961
+    hooks:
+    -   id: mypy
+        name: mypy
+        entry: mypy
+        types: [python]
+        args: [--ignore-missing-imports]
+        files: utils/|media_downloader.py
+        exclude: tests/
+-   repo: https://github.com/pycqa/pylint
+    rev: v2.14.5
+    hooks:
+    -   id: pylint
+        name: pylint
+        entry: pylint
+        language: system
+        types: [python]
+        args: [
+          "-rn", # Only display messages
+          "-sn", # Don't display the score
+          "--rcfile=pylintrc" # Link to your config file
+        ]
+        files: utils/|media_downloader.py
+        exclude: tests/

CONTRIBUTING.md

@@ -179,10 +179,10 @@ Must be one of the following:
 -  **style**: Changes that do not affect the meaning of the code (white-space, formatting, etc)
 -  **test**: Additions/updates to tests
 -  **type**: Type annotations
-    
+
 #### Subject:
 
-Please reference the relevant GitHub issues in your commit message using  #1234. 
+Please reference the relevant GitHub issues in your commit message using  #1234.
 -  a subject line with  `<  80`  chars.
 -  summary in present tense.
 -  not capitalized.
@@ -196,4 +196,4 @@ Explain the motivation for the change in the commit message body. This commit me
 
 ### Code of Conduct
 
-As a contributor, you can help us keep the  community open and inclusive. Please read and follow our  [Code of Conduct](https://github.com/Dineshkarthik/telegram_media_downloader/blob/master/CODE_OF_CONDUCT.md).
+As a contributor, you can help us keep the  community open and inclusive. Please read and follow our  [Code of Conduct](https://github.com/Dineshkarthik/telegram_media_downloader/blob/master/CODE_OF_CONDUCT.md).

Makefile

@@ -4,7 +4,7 @@ TEST_ARTIFACTS ?= /tmp/coverage
 
 install:
 	python3 -m pip install --upgrade pip setuptools
-	python3 -m pip install -r requirements.txt 
+	python3 -m pip install -r requirements.txt
 
 dev_install: install
 	python3 -m pip install -r dev-requirements.txt
@@ -23,4 +23,4 @@ test:
 		--cov-report term-missing \
 		--cov-report html:${TEST_ARTIFACTS} \
 		--junit-xml=${TEST_ARTIFACTS}/media-downloader.xml \
-		tests/ 
+		tests/

README.md

@@ -25,7 +25,7 @@ A meta of last read/downloaded message is stored in the config file so that in s
 ### Support:
 | Category | Support |
 |--|--|
-|Language | `Python 3.6 ` and above|
+|Language | `Python 3.7 ` and above|
 |Download media types|  audio, document, photo, video, video_note, voice|
 
 ### ToDo:
@@ -39,21 +39,21 @@ $ git clone https://github.com/Dineshkarthik/telegram_media_downloader.git
 $ cd telegram_media_downloader
 $ make install
 ```
-For Windows which doesn't have `make` inbuilt 
+For Windows which doesn't have `make` inbuilt
 ```sh
 $ git clone https://github.com/Dineshkarthik/telegram_media_downloader.git
 $ cd telegram_media_downloader
 $ pip3 install -r requirements.txt
 ```
 
-## Configuration 
+## Configuration
 
 All the configurations are  passed to the Telegram Media Downloader via `config.yaml` file.
 
 **Getting your API Keys:**
 The very first step requires you to obtain a valid Telegram API key (API id/hash pair):
 1.  Visit  [https://my.telegram.org/apps](https://my.telegram.org/apps)  and log in with your Telegram Account.
-2.  Fill out the form to register a new Telegram application. 
+2.  Fill out the form to register a new Telegram application.
 3.  Done! The API key consists of two parts:  **api_id**  and  **api_hash**.
 
 
@@ -123,18 +123,17 @@ All the downloaded media will be stored inside  respective direcotry named  in t
 | voice_note | path/to/project/voice_note |
 
 ## Proxy
-`Socks5` proxy is supported in this project currently. To use it, simply create a `config.ini` file in the path of this project, and edit it with your proxy server info as follow:
+`socks4, socks5, http` proxies are supported in this project currently. To use it, add the following to the bottom of your `config.yaml` file
 
-```ini
-[proxy]
-enabled = True
-hostname = 127.0.0.1
-port = 1080
-username =
-password =
+```yaml
+proxy:
+  scheme: socks5
+  hostname: 11.22.33.44
+  port: 1234
+  username: your_username
+  password: your_password
 ```
-
-Then the proxy will automatically be enabled.
+If your proxy doesn’t require authorization you can omit username and password. Then the proxy will automatically be enabled.
 
 ## Contributing
 ### Contributing Guidelines

codecov.yml

@@ -10,4 +10,4 @@ coverage:
     patch: no
 
 comment:
-  require_changes: true
+  require_changes: true

dev-requirements.txt

@@ -1,6 +1,9 @@
+black==22.3.0
+isort==5.10.1
 mock==4.0.3
-mypy==0.942
-pylint==2.13.7
+mypy==0.961
+pre-commit==2.20.0
+pylint==2.14.5
 pytest==7.0.1
 pytest-cov==3.0.0
-types-PyYAML==6.0.7
+types-PyYAML==6.0.10

media_downloader.py

@@ -47,9 +47,7 @@ def update_config(config: dict):
     logger.info("Updated last read message_id to config file")
 
 
-def _can_download(
-    _type: str, file_formats: dict, file_format: Optional[str]
-) -> bool:
+def _can_download(_type: str, file_formats: dict, file_format: Optional[str]) -> bool:
     """
     Check if the given file format can be downloaded.
 
@@ -176,7 +174,7 @@ async def download_media(
     for retry in range(3):
         try:
             if message.media is None:
-                return message.message_id
+                return message.id
             for _type in media_types:
                 _media = getattr(message, _type, None)
                 if _media is None:
@@ -196,48 +194,48 @@ async def download_media(
                         )
                     if download_path:
                         logger.info("Media downloaded - %s", download_path)
-                    DOWNLOADED_IDS.append(message.message_id)
+                    DOWNLOADED_IDS.append(message.id)
             break
         except pyrogram.errors.exceptions.bad_request_400.BadRequest:
             logger.warning(
                 "Message[%d]: file reference expired, refetching...",
-                message.message_id,
+                message.id,
             )
             message = await client.get_messages(  # type: ignore
                 chat_id=message.chat.id,  # type: ignore
-                message_ids=message.message_id,
+                message_ids=message.id,
             )
             if retry == 2:
                 # pylint: disable = C0301
                 logger.error(
                     "Message[%d]: file reference expired for 3 retries, download skipped.",
-                    message.message_id,
+                    message.id,
                 )
-                FAILED_IDS.append(message.message_id)
+                FAILED_IDS.append(message.id)
         except TypeError:
             # pylint: disable = C0301
             logger.warning(
                 "Timeout Error occurred when downloading Message[%d], retrying after 5 seconds",
-                message.message_id,
+                message.id,
             )
             await asyncio.sleep(5)
             if retry == 2:
                 logger.error(
                     "Message[%d]: Timing out after 3 reties, download skipped.",
-                    message.message_id,
+                    message.id,
                 )
-                FAILED_IDS.append(message.message_id)
+                FAILED_IDS.append(message.id)
         except Exception as e:
             # pylint: disable = C0301
             logger.error(
                 "Message[%d]: could not be downloaded due to following exception:\n[%s].",
-                message.message_id,
+                message.id,
                 e,
                 exc_info=True,
             )
-            FAILED_IDS.append(message.message_id)
+            FAILED_IDS.append(message.id)
             break
-    return message.message_id
+    return message.id
 
 
 async def process_messages(
@@ -281,7 +279,7 @@ async def process_messages(
         ]
     )
 
-    last_message_id = max(message_ids)
+    last_message_id: int = max(message_ids)
     return last_message_id
 
 
@@ -312,10 +310,9 @@ async def begin_import(config: dict, pagination_limit: int) -> dict:
     )
     await client.start()
     last_read_message_id: int = config["last_read_message_id"]
-    messages_iter = client.iter_history(
+    messages_iter = client.get_chat_history(
         config["chat_id"],
         offset_id=last_read_message_id,
-        reverse=True,
     )
     messages_list: list = []
     pagination_count: int = 0

mypy.ini

@@ -0,0 +1,8 @@
+[mypy]
+warn_return_any = True
+
+[mypy-yaml.*]
+ignore_missing_imports = True
+
+[mypy-tests.*]
+ignore_errors = True

pylintrc

@@ -251,7 +251,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=80
+max-line-length=90
 
 # Maximum number of lines in a module.
 max-module-lines=

pyproject.toml

@@ -1,22 +0,0 @@
-[tool.black]
-line-length = 79
-target-version = ['py36', 'py37', 'py38']
-exclude = '''
-(
-  /(
-    \.eggs         # exclude a few common directories in the
-    | \.git          # root of the project
-    | \.hg
-    | \.mypy_cache
-    | \.tox
-    | \.venv
-    | _build
-    | buck-out
-    | build
-    | dist
-    | bin
-    | lib
-    | include
-  )/
-)
-'''

requirements.txt

@@ -1,4 +1,4 @@
-Pyrogram==1.4.12
+Pyrogram==2.0.33
 PyYAML==6.0
-rich==12.1.0
+rich==12.5.1
 TgCrypto==1.2.3

setup.py

@@ -12,7 +12,7 @@ setup(
     download_url="https://github.com/Dineshkarthik/telegram_media_downloader/releases/latest",
     py_modules=["media_downloader"],
     classifiers=[
-        "Development Status :: 4 - Beta",
+        "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
         "Operating System :: OS Independent",
         "Intended Audience :: Developers",
@@ -22,10 +22,11 @@ setup(
         "Natural Language :: English",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Topic :: Internet",
         "Topic :: Communications",
         "Topic :: Communications :: Chat",
@@ -37,5 +38,5 @@ setup(
         "Community": "https://t.me/tgmdnews",
         "Source": "https://github.com/Dineshkarthik/telegram_media_downloader",
     },
-    python_requires=">=3.6",
+    python_requires="~=3.7",
 )

tests/test_media_downloader.py

@@ -1,24 +1,24 @@
 """Unittest module for media downloader."""
-import os
+import asyncio
 import copy
 import logging
+import os
 import platform
 import unittest
 
-import asyncio
 import mock
 import pyrogram
 import pytest
 
 from media_downloader import (
-    _get_media_meta,
     _can_download,
+    _get_media_meta,
     _is_exist,
-    download_media,
-    update_config,
     begin_import,
-    process_messages,
+    download_media,
     main,
+    process_messages,
+    update_config,
 )
 
 MOCK_DIR: str = "/root/project"
@@ -53,7 +53,7 @@ class Chat:
 
 class MockMessage:
     def __init__(self, **kwargs):
-        self.message_id = kwargs.get("id")
+        self.id = kwargs.get("id")
         self.media = kwargs.get("media")
         self.audio = kwargs.get("audio", None)
         self.document = kwargs.get("document", None)
@@ -134,9 +134,7 @@ async def mock_process_message(*args, **kwargs):
 
 
 async def async_process_messages(client, messages, media_types, file_formats):
-    result = await process_messages(
-        client, messages, media_types, file_formats
-    )
+    result = await process_messages(client, messages, media_types, file_formats)
     return result
 
 
@@ -153,7 +151,7 @@ class MockClient:
     async def stop(self):
         pass
 
-    async def iter_history(self, *args, **kwargs):
+    async def get_chat_history(self, *args, **kwargs):
         items = [
             MockMessage(
                 id=1213,
@@ -219,11 +217,11 @@ class MockClient:
 
     async def download_media(self, *args, **kwargs):
         mock_message = args[0]
-        if mock_message.message_id in [7, 8]:
+        if mock_message.id in [7, 8]:
             raise pyrogram.errors.exceptions.bad_request_400.BadRequest
-        elif mock_message.message_id == 9:
+        elif mock_message.id == 9:
             raise pyrogram.errors.exceptions.unauthorized_401.Unauthorized
-        elif mock_message.message_id == 11:
+        elif mock_message.id == 11:
             raise TypeError
         return kwargs["file_name"]
 
@@ -289,9 +287,7 @@ class MediaDownloaderTestCase(unittest.TestCase):
         )
         self.assertEqual(
             (
-                platform_generic_path(
-                    "/root/project/document/sample_document.pdf"
-                ),
+                platform_generic_path("/root/project/document/sample_document.pdf"),
                 "pdf",
             ),
             result,
@@ -495,9 +491,7 @@ class MediaDownloaderTestCase(unittest.TestCase):
         }
         update_config(conf)
         mock_open.assert_called_with("config.yaml", "w")
-        mock_yaml.dump.assert_called_with(
-            conf, mock.ANY, default_flow_style=False
-        )
+        mock_yaml.dump.assert_called_with(conf, mock.ANY, default_flow_style=False)
 
     @mock.patch("media_downloader.update_config")
     @mock.patch("media_downloader.pyrogram.Client", new=MockClient)

tests/utils/test_file_management.py

@@ -27,7 +27,7 @@ class FileManagementTestCase(unittest.TestCase):
         result = get_next_name(self.test_file)
         excepted_result = os.path.join(self.this_dir, "file-test-copy3.txt")
         self.assertEqual(result, excepted_result)
-    
+
     def test_manage_duplicate_file(self):
         result = manage_duplicate_file(self.test_file_copy_2)
         self.assertEqual(result, self.test_file_copy_1)

tests/utils/test_log.py

@@ -27,4 +27,4 @@ class MetaTestCase(unittest.TestCase):
         self.assertEqual(result1, False)
 
         result2 = LogFilter().filter(MockLog(funcName="Synced"))
-        self.assertEqual(result2, True)
+        self.assertEqual(result2, True)

utils/__init__.py

@@ -1,7 +1,5 @@
 """Init namespace"""
 
-__version__ = "1.5.1"
+__version__ = "2.0.0"
 __license__ = "MIT License"
-__copyright__ = (
-    "Copyright (C) 2019 Dineshkarthik <https://github.com/Dineshkarthik>"
-)
+__copyright__ = "Copyright (C) 2019 Dineshkarthik <https://github.com/Dineshkarthik>"

utils/meta.py

@@ -6,9 +6,7 @@ from rich.console import Console
 from . import __copyright__, __license__, __version__
 
 APP_VERSION = f"Telegram Media Downloader {__version__}"
-DEVICE_MODEL = (
-    f"{platform.python_implementation()} {platform.python_version()}"
-)
+DEVICE_MODEL = f"{platform.python_implementation()} {platform.python_version()}"
 SYSTEM_VERSION = f"{platform.system()} {platform.release()}"
 LANG_CODE = "en"
 

utils/updates.py

@@ -7,6 +7,7 @@ from rich.markdown import Markdown
 
 from . import __version__
 
+
 # pylint: disable = C0301
 def check_for_updates() -> None:
     """Checks for new releases.