diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml new file mode 100644 index 0000000..0e89137 --- /dev/null +++ b/.github/workflows/code-checks.yml @@ -0,0 +1,24 @@ +name: Code Quality + +on: + pull_request: + branches: [ master ] + paths-ignore: + - 'README.md' + push: + branches: [ master ] + paths-ignore: + - 'README.md' + +jobs: + pre-commit: + name: Linting + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install dependencies + run: make dev_install + - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/main.yml b/.github/workflows/unittest.yml similarity index 78% rename from .github/workflows/main.yml rename to .github/workflows/unittest.yml index d5f4ed6..5eb0e9f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/unittest.yml @@ -3,8 +3,12 @@ name: Unittest on: push: branches: [ master ] + paths-ignore: + - 'README.md' pull_request: branches: [ master ] + paths-ignore: + - 'README.md' jobs: build: @@ -13,13 +17,13 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [ '3.6', '3.7', '3.8', '3.9' ] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11.0-beta.4' ] name: Test - Python ${{ matrix.python-version }} on ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Get setuptools Unix @@ -30,10 +34,6 @@ jobs: run: pip install --upgrade --user pip setuptools codecov - name: Install dependencies run: make dev_install - - name: Code Check - Pylint - run: make pylint - - name: Static Type Check - Mypy - run: make static_type_check - name: Test with pytest run: | make -e test diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..62a30db --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,48 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer +- repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + name: black + entry: black + types: [python] +- repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + name: isort + entry: isort + types: [python] + args: ["--profile", "black", "--filter-files"] +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.961 + hooks: + - id: mypy + name: mypy + entry: mypy + types: [python] + args: [--ignore-missing-imports] + files: utils/|media_downloader.py + exclude: tests/ +- repo: https://github.com/pycqa/pylint + rev: v2.14.5 + hooks: + - id: pylint + name: pylint + entry: pylint + language: system + types: [python] + args: [ + "-rn", # Only display messages + "-sn", # Don't display the score + "--rcfile=pylintrc" # Link to your config file + ] + files: utils/|media_downloader.py + exclude: tests/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f6a3678..5b03fa2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -179,10 +179,10 @@ Must be one of the following: - **style**: Changes that do not affect the meaning of the code (white-space, formatting, etc) - **test**: Additions/updates to tests - **type**: Type annotations - + #### Subject: -Please reference the relevant GitHub issues in your commit message using #1234. +Please reference the relevant GitHub issues in your commit message using #1234. - a subject line with `< 80` chars. - summary in present tense. - not capitalized. @@ -196,4 +196,4 @@ Explain the motivation for the change in the commit message body. This commit me ### Code of Conduct -As a contributor, you can help us keep the community open and inclusive. Please read and follow our [Code of Conduct](https://github.com/Dineshkarthik/telegram_media_downloader/blob/master/CODE_OF_CONDUCT.md). \ No newline at end of file +As a contributor, you can help us keep the community open and inclusive. Please read and follow our [Code of Conduct](https://github.com/Dineshkarthik/telegram_media_downloader/blob/master/CODE_OF_CONDUCT.md). diff --git a/Makefile b/Makefile index 57a56c1..5b2742f 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ TEST_ARTIFACTS ?= /tmp/coverage install: python3 -m pip install --upgrade pip setuptools - python3 -m pip install -r requirements.txt + python3 -m pip install -r requirements.txt dev_install: install python3 -m pip install -r dev-requirements.txt @@ -23,4 +23,4 @@ test: --cov-report term-missing \ --cov-report html:${TEST_ARTIFACTS} \ --junit-xml=${TEST_ARTIFACTS}/media-downloader.xml \ - tests/ + tests/ diff --git a/README.md b/README.md index 6a7b820..4a9f06e 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ A meta of last read/downloaded message is stored in the config file so that in s ### Support: | Category | Support | |--|--| -|Language | `Python 3.6 ` and above| +|Language | `Python 3.7 ` and above| |Download media types| audio, document, photo, video, video_note, voice| ### ToDo: @@ -39,21 +39,21 @@ $ git clone https://github.com/Dineshkarthik/telegram_media_downloader.git $ cd telegram_media_downloader $ make install ``` -For Windows which doesn't have `make` inbuilt +For Windows which doesn't have `make` inbuilt ```sh $ git clone https://github.com/Dineshkarthik/telegram_media_downloader.git $ cd telegram_media_downloader $ pip3 install -r requirements.txt ``` -## Configuration +## Configuration All the configurations are passed to the Telegram Media Downloader via `config.yaml` file. **Getting your API Keys:** The very first step requires you to obtain a valid Telegram API key (API id/hash pair): 1. Visit [https://my.telegram.org/apps](https://my.telegram.org/apps) and log in with your Telegram Account. -2. Fill out the form to register a new Telegram application. +2. Fill out the form to register a new Telegram application. 3. Done! The API key consists of two parts: **api_id** and **api_hash**. @@ -123,18 +123,17 @@ All the downloaded media will be stored inside respective direcotry named in t | voice_note | path/to/project/voice_note | ## Proxy -`Socks5` proxy is supported in this project currently. To use it, simply create a `config.ini` file in the path of this project, and edit it with your proxy server info as follow: +`socks4, socks5, http` proxies are supported in this project currently. To use it, add the following to the bottom of your `config.yaml` file -```ini -[proxy] -enabled = True -hostname = 127.0.0.1 -port = 1080 -username = -password = +```yaml +proxy: + scheme: socks5 + hostname: 11.22.33.44 + port: 1234 + username: your_username + password: your_password ``` - -Then the proxy will automatically be enabled. +If your proxy doesn’t require authorization you can omit username and password. Then the proxy will automatically be enabled. ## Contributing ### Contributing Guidelines diff --git a/codecov.yml b/codecov.yml index b18401c..1fbaabe 100644 --- a/codecov.yml +++ b/codecov.yml @@ -10,4 +10,4 @@ coverage: patch: no comment: - require_changes: true \ No newline at end of file + require_changes: true diff --git a/dev-requirements.txt b/dev-requirements.txt index bd2ba29..8b32322 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,9 @@ +black==22.3.0 +isort==5.10.1 mock==4.0.3 -mypy==0.942 -pylint==2.13.7 +mypy==0.961 +pre-commit==2.20.0 +pylint==2.14.5 pytest==7.0.1 pytest-cov==3.0.0 -types-PyYAML==6.0.7 +types-PyYAML==6.0.10 diff --git a/media_downloader.py b/media_downloader.py index b99502d..c68f8d6 100644 --- a/media_downloader.py +++ b/media_downloader.py @@ -47,9 +47,7 @@ def update_config(config: dict): logger.info("Updated last read message_id to config file") -def _can_download( - _type: str, file_formats: dict, file_format: Optional[str] -) -> bool: +def _can_download(_type: str, file_formats: dict, file_format: Optional[str]) -> bool: """ Check if the given file format can be downloaded. @@ -176,7 +174,7 @@ async def download_media( for retry in range(3): try: if message.media is None: - return message.message_id + return message.id for _type in media_types: _media = getattr(message, _type, None) if _media is None: @@ -196,48 +194,48 @@ async def download_media( ) if download_path: logger.info("Media downloaded - %s", download_path) - DOWNLOADED_IDS.append(message.message_id) + DOWNLOADED_IDS.append(message.id) break except pyrogram.errors.exceptions.bad_request_400.BadRequest: logger.warning( "Message[%d]: file reference expired, refetching...", - message.message_id, + message.id, ) message = await client.get_messages( # type: ignore chat_id=message.chat.id, # type: ignore - message_ids=message.message_id, + message_ids=message.id, ) if retry == 2: # pylint: disable = C0301 logger.error( "Message[%d]: file reference expired for 3 retries, download skipped.", - message.message_id, + message.id, ) - FAILED_IDS.append(message.message_id) + FAILED_IDS.append(message.id) except TypeError: # pylint: disable = C0301 logger.warning( "Timeout Error occurred when downloading Message[%d], retrying after 5 seconds", - message.message_id, + message.id, ) await asyncio.sleep(5) if retry == 2: logger.error( "Message[%d]: Timing out after 3 reties, download skipped.", - message.message_id, + message.id, ) - FAILED_IDS.append(message.message_id) + FAILED_IDS.append(message.id) except Exception as e: # pylint: disable = C0301 logger.error( "Message[%d]: could not be downloaded due to following exception:\n[%s].", - message.message_id, + message.id, e, exc_info=True, ) - FAILED_IDS.append(message.message_id) + FAILED_IDS.append(message.id) break - return message.message_id + return message.id async def process_messages( @@ -281,7 +279,7 @@ async def process_messages( ] ) - last_message_id = max(message_ids) + last_message_id: int = max(message_ids) return last_message_id @@ -312,10 +310,9 @@ async def begin_import(config: dict, pagination_limit: int) -> dict: ) await client.start() last_read_message_id: int = config["last_read_message_id"] - messages_iter = client.iter_history( + messages_iter = client.get_chat_history( config["chat_id"], offset_id=last_read_message_id, - reverse=True, ) messages_list: list = [] pagination_count: int = 0 diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..f96d433 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +warn_return_any = True + +[mypy-yaml.*] +ignore_missing_imports = True + +[mypy-tests.*] +ignore_errors = True diff --git a/pylintrc b/pylintrc index 8b57c74..b614094 100644 --- a/pylintrc +++ b/pylintrc @@ -251,7 +251,7 @@ indent-after-paren=4 indent-string=' ' # Maximum number of characters on a single line. -max-line-length=80 +max-line-length=90 # Maximum number of lines in a module. max-module-lines= diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 1c58150..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,22 +0,0 @@ -[tool.black] -line-length = 79 -target-version = ['py36', 'py37', 'py38'] -exclude = ''' -( - /( - \.eggs # exclude a few common directories in the - | \.git # root of the project - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist - | bin - | lib - | include - )/ -) -''' \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index def4add..e2c9d8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Pyrogram==1.4.12 +Pyrogram==2.0.33 PyYAML==6.0 -rich==12.1.0 +rich==12.5.1 TgCrypto==1.2.3 diff --git a/setup.py b/setup.py index c0d8a92..717a3bb 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( download_url="https://github.com/Dineshkarthik/telegram_media_downloader/releases/latest", py_modules=["media_downloader"], classifiers=[ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Environment :: Console", "Operating System :: OS Independent", "Intended Audience :: Developers", @@ -22,10 +22,11 @@ setup( "Natural Language :: English", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Internet", "Topic :: Communications", "Topic :: Communications :: Chat", @@ -37,5 +38,5 @@ setup( "Community": "https://t.me/tgmdnews", "Source": "https://github.com/Dineshkarthik/telegram_media_downloader", }, - python_requires=">=3.6", + python_requires="~=3.7", ) diff --git a/tests/test_media_downloader.py b/tests/test_media_downloader.py index d203ffe..758b44c 100644 --- a/tests/test_media_downloader.py +++ b/tests/test_media_downloader.py @@ -1,24 +1,24 @@ """Unittest module for media downloader.""" -import os +import asyncio import copy import logging +import os import platform import unittest -import asyncio import mock import pyrogram import pytest from media_downloader import ( - _get_media_meta, _can_download, + _get_media_meta, _is_exist, - download_media, - update_config, begin_import, - process_messages, + download_media, main, + process_messages, + update_config, ) MOCK_DIR: str = "/root/project" @@ -53,7 +53,7 @@ class Chat: class MockMessage: def __init__(self, **kwargs): - self.message_id = kwargs.get("id") + self.id = kwargs.get("id") self.media = kwargs.get("media") self.audio = kwargs.get("audio", None) self.document = kwargs.get("document", None) @@ -134,9 +134,7 @@ async def mock_process_message(*args, **kwargs): async def async_process_messages(client, messages, media_types, file_formats): - result = await process_messages( - client, messages, media_types, file_formats - ) + result = await process_messages(client, messages, media_types, file_formats) return result @@ -153,7 +151,7 @@ class MockClient: async def stop(self): pass - async def iter_history(self, *args, **kwargs): + async def get_chat_history(self, *args, **kwargs): items = [ MockMessage( id=1213, @@ -219,11 +217,11 @@ class MockClient: async def download_media(self, *args, **kwargs): mock_message = args[0] - if mock_message.message_id in [7, 8]: + if mock_message.id in [7, 8]: raise pyrogram.errors.exceptions.bad_request_400.BadRequest - elif mock_message.message_id == 9: + elif mock_message.id == 9: raise pyrogram.errors.exceptions.unauthorized_401.Unauthorized - elif mock_message.message_id == 11: + elif mock_message.id == 11: raise TypeError return kwargs["file_name"] @@ -289,9 +287,7 @@ class MediaDownloaderTestCase(unittest.TestCase): ) self.assertEqual( ( - platform_generic_path( - "/root/project/document/sample_document.pdf" - ), + platform_generic_path("/root/project/document/sample_document.pdf"), "pdf", ), result, @@ -495,9 +491,7 @@ class MediaDownloaderTestCase(unittest.TestCase): } update_config(conf) mock_open.assert_called_with("config.yaml", "w") - mock_yaml.dump.assert_called_with( - conf, mock.ANY, default_flow_style=False - ) + mock_yaml.dump.assert_called_with(conf, mock.ANY, default_flow_style=False) @mock.patch("media_downloader.update_config") @mock.patch("media_downloader.pyrogram.Client", new=MockClient) diff --git a/tests/utils/test_file_management.py b/tests/utils/test_file_management.py index 7ac188e..bf4fc4d 100644 --- a/tests/utils/test_file_management.py +++ b/tests/utils/test_file_management.py @@ -27,7 +27,7 @@ class FileManagementTestCase(unittest.TestCase): result = get_next_name(self.test_file) excepted_result = os.path.join(self.this_dir, "file-test-copy3.txt") self.assertEqual(result, excepted_result) - + def test_manage_duplicate_file(self): result = manage_duplicate_file(self.test_file_copy_2) self.assertEqual(result, self.test_file_copy_1) diff --git a/tests/utils/test_log.py b/tests/utils/test_log.py index 8cf8fd4..af7f935 100644 --- a/tests/utils/test_log.py +++ b/tests/utils/test_log.py @@ -27,4 +27,4 @@ class MetaTestCase(unittest.TestCase): self.assertEqual(result1, False) result2 = LogFilter().filter(MockLog(funcName="Synced")) - self.assertEqual(result2, True) \ No newline at end of file + self.assertEqual(result2, True) diff --git a/utils/__init__.py b/utils/__init__.py index 73171b5..6f90d1a 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -1,7 +1,5 @@ """Init namespace""" -__version__ = "1.5.1" +__version__ = "2.0.0" __license__ = "MIT License" -__copyright__ = ( - "Copyright (C) 2019 Dineshkarthik " -) +__copyright__ = "Copyright (C) 2019 Dineshkarthik " diff --git a/utils/meta.py b/utils/meta.py index 547c610..90354be 100644 --- a/utils/meta.py +++ b/utils/meta.py @@ -6,9 +6,7 @@ from rich.console import Console from . import __copyright__, __license__, __version__ APP_VERSION = f"Telegram Media Downloader {__version__}" -DEVICE_MODEL = ( - f"{platform.python_implementation()} {platform.python_version()}" -) +DEVICE_MODEL = f"{platform.python_implementation()} {platform.python_version()}" SYSTEM_VERSION = f"{platform.system()} {platform.release()}" LANG_CODE = "en" diff --git a/utils/updates.py b/utils/updates.py index 40681cd..e33a84b 100644 --- a/utils/updates.py +++ b/utils/updates.py @@ -7,6 +7,7 @@ from rich.markdown import Markdown from . import __version__ + # pylint: disable = C0301 def check_for_updates() -> None: """Checks for new releases.