From 69d44fef47cea331b63981b1f9f0d34fd37018c2 Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Fri, 10 Jul 2020 23:28:04 +0200 Subject: [PATCH 1/7] migrate to github actions --- .circleci/config.yml | 113 ------------------------------------- .github/workflows/main.yml | 28 +++++++++ 2 files changed, 28 insertions(+), 113 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/main.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index f0e74b5..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,113 +0,0 @@ -version: 2 - -jobs: - test-3.6: - docker: - - image: circleci/python:3.6.9 - environment: - CIRCLE_ARTIFACTS: /tmp/circleci-artifacts - CIRCLE_TEST_REPORTS: /tmp/circleci-test-results - LOG_DIR: ${CIRCLE_ARTIFACTS}/testresults/ - steps: - - checkout - - run: - name: Get setuptools - shell: /bin/bash - command: sudo pip install --upgrade pip setuptools codecov - - run: - name: Install dependencies - shell: /bin/bash - command: sudo make deps - - run: - name: Static Type Check - Mypy - shell: /bin/bash - command: sudo make static_type_check - - run: - name: Code Check - Pylint - shell: /bin/bash - command: sudo make pylint - - run: - name: Run test - shell: /bin/bash - command: | - sudo make -e test - codecov - - store_artifacts: - path: /tmp/circleci-artifacts - - test-3.7: - docker: - - image: circleci/python:3.7.8 - environment: - CIRCLE_ARTIFACTS: /tmp/circleci-artifacts - CIRCLE_TEST_REPORTS: /tmp/circleci-test-results - LOG_DIR: ${CIRCLE_ARTIFACTS}/testresults/ - steps: - - checkout - - run: - name: Get setuptools - shell: /bin/bash - command: sudo pip install --upgrade pip setuptools codecov - - run: - name: Install dependencies - shell: /bin/bash - command: sudo make deps - - run: - name: Static Type Check - Mypy - shell: /bin/bash - command: sudo make static_type_check - - run: - name: Code Check - Pylint - shell: /bin/bash - command: sudo make pylint - - run: - name: Run test - shell: /bin/bash - command: | - sudo make -e test - codecov - - store_artifacts: - path: /tmp/circleci-artifacts - - test-3.8: - docker: - - image: circleci/python:3.8.3 - environment: - CIRCLE_ARTIFACTS: /tmp/circleci-artifacts - CIRCLE_TEST_REPORTS: /tmp/circleci-test-results - LOG_DIR: ${CIRCLE_ARTIFACTS}/testresults/ - steps: - - checkout - - run: - name: Get setuptools - shell: /bin/bash - command: sudo pip install --upgrade pip setuptools codecov - - run: - name: Install dependencies - shell: /bin/bash - command: sudo make deps - - run: - name: Static Type Check - Mypy - shell: /bin/bash - command: sudo make static_type_check - - run: - name: Code Check - Pylint - shell: /bin/bash - command: sudo make pylint - - run: - name: Run test - shell: /bin/bash - command: | - sudo make -e test - codecov - - store_artifacts: - path: /tmp/circleci-artifacts - - -workflows: - version: 2 - tests: - jobs: - - test-3.6 - - test-3.7 - - test-3.8 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..4dd5931 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,28 @@ +name: Unittest + +on: [push] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: [ '3.6', '3.7', '3.8'] + name: Test - Python ${{ matrix.python-version }} on ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: make deps + - name: Code Check - Pylint + run: make pylint + - name: Static Type Check - Mypy + run: make static_type_check + - name: Test with pytest + run: make test \ No newline at end of file From 66aaedc94809a3f438c9ace2c4af7146b7ef0897 Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Fri, 10 Jul 2020 23:38:57 +0200 Subject: [PATCH 2/7] fix pylint C0411(wrong-import-order) --- media_downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media_downloader.py b/media_downloader.py index c68e4ed..f3b0bcf 100644 --- a/media_downloader.py +++ b/media_downloader.py @@ -5,8 +5,8 @@ from typing import List, Tuple, Optional from datetime import datetime as dt import asyncio -import yaml import pyrogram +import yaml logging.basicConfig(level=logging.INFO) From c91507d345bc87dab0f6807ab97ef595b69eedf6 Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Sat, 11 Jul 2020 08:40:17 +0200 Subject: [PATCH 3/7] platform generic paths --- tests/test_media_downloader.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tests/test_media_downloader.py b/tests/test_media_downloader.py index 02bd848..6a15341 100644 --- a/tests/test_media_downloader.py +++ b/tests/test_media_downloader.py @@ -27,6 +27,10 @@ MOCK_CONF = { } +def platform_generic_path(path: str) -> str: + return os.path.join("/", *path.split("/")) + + class MockMessage: def __init__(self, **kwargs): self.message_id = kwargs.get("id") @@ -130,9 +134,9 @@ class MockClient: async def download_media(self, *args, **kwargs): assert "AwADBQADbwAD2oTRVeHe5eXRFftfAg", args[0] - assert "/root/project/voice/voice_2019-07-25T14:53:50.ogg", kwargs[ - "file_name" - ] + assert platform_generic_path( + "/root/project/voice/voice_2019-07-25T14:53:50.ogg" + ), kwargs["file_name"] return kwargs["file_name"] @@ -160,7 +164,9 @@ class MediaDownloaderTestCase(unittest.TestCase): self.assertEqual( ( "AwADBQADbwAD2oTRVeHe5eXRFftfAg", - "/root/project/voice/voice_2019-07-25T14:53:50.ogg", + platform_generic_path( + "/root/project/voice/voice_2019-07-25T14:53:50.ogg" + ), "ogg", ), result, @@ -178,7 +184,11 @@ class MediaDownloaderTestCase(unittest.TestCase): async_get_media_meta(message.photo, "photo") ) self.assertEqual( - ("AgADBQAD5KkxG_FPQValJzQsJPyzhHcC", "/root/project/photo/", None), + ( + "AgADBQAD5KkxG_FPQValJzQsJPyzhHcC", + platform_generic_path("/root/project/photo/"), + None, + ), result, ) @@ -198,7 +208,9 @@ class MediaDownloaderTestCase(unittest.TestCase): self.assertEqual( ( "AQADAgADq7LfMgAEIdy5DwAE4w4AAwI", - "/root/project/document/sample_document.pdf", + platform_generic_path( + "/root/project/document/sample_document.pdf" + ), "pdf", ), result, @@ -220,7 +232,7 @@ class MediaDownloaderTestCase(unittest.TestCase): self.assertEqual( ( "AQADAgADq7LfMgAEIdy5DwAE5Q4AAgEC", - "/root/project/audio/sample_audio.mp3", + platform_generic_path("/root/project/audio/sample_audio.mp3"), "mp3", ), result, @@ -239,7 +251,11 @@ class MediaDownloaderTestCase(unittest.TestCase): async_get_media_meta(message.video, "video") ) self.assertEqual( - ("CQADBQADeQIAAlL60FUCNMBdK8OjlAI", "/root/project/video/", "mp4"), + ( + "CQADBQADeQIAAlL60FUCNMBdK8OjlAI", + platform_generic_path("/root/project/video/"), + "mp4", + ), result, ) From ce5201b2edfab54a3d5ea1a02f03688567aff14a Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Sat, 11 Jul 2020 09:23:38 +0200 Subject: [PATCH 4/7] enh: handle windows file path --- tests/test_media_downloader.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_media_downloader.py b/tests/test_media_downloader.py index 6a15341..0870e3b 100644 --- a/tests/test_media_downloader.py +++ b/tests/test_media_downloader.py @@ -2,6 +2,7 @@ import os import copy import logging +import platform import unittest import mock @@ -16,7 +17,9 @@ from media_downloader import ( process_messages, ) -MOCK_DIR = "/root/project" +MOCK_DIR: str = "/root/project" +if platform.system() == "Windows": + MOCK_DIR = "\\root\\project" MOCK_CONF = { "api_id": 123, "api_hash": "hasw5Tgawsuj67", @@ -27,8 +30,11 @@ MOCK_CONF = { } -def platform_generic_path(path: str) -> str: - return os.path.join("/", *path.split("/")) +def platform_generic_path(_path: str) -> str: + platform_specific_path: str = _path + if platform.system() == "Windows": + platform_specific_path = platform_specific_path.replace("/", "\\") + return platform_specific_path class MockMessage: From ca90f7f3f6b50e841ecbd00e51c98fe845679a56 Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Sat, 11 Jul 2020 10:09:24 +0200 Subject: [PATCH 5/7] add: codecov coverage --- .github/workflows/main.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4dd5931..cd0a500 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,4 +25,13 @@ jobs: - name: Static Type Check - Mypy run: make static_type_check - name: Test with pytest - run: make test \ No newline at end of file + run: make test + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: /tmp/coverage/media-downloader.xml + flags: unittests + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: true \ No newline at end of file From 9e89601092f13016b7307d8d8f54912efd54db4f Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Sat, 11 Jul 2020 10:37:01 +0200 Subject: [PATCH 6/7] update codecov step --- .github/workflows/main.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd0a500..4fe3db2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,6 +18,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Get setuptools + run: pip install --upgrade pip setuptools codecov - name: Install dependencies run: make deps - name: Code Check - Pylint @@ -25,13 +27,8 @@ jobs: - name: Static Type Check - Mypy run: make static_type_check - name: Test with pytest - run: make test - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: /tmp/coverage/media-downloader.xml - flags: unittests - env_vars: OS,PYTHON - name: codecov-umbrella - fail_ci_if_error: true \ No newline at end of file + run: | + make -e test + codecov + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file From eb3dec7f3858e765241ec95701eefbbaf80afa76 Mon Sep 17 00:00:00 2001 From: Dineshkarthik Date: Sat, 11 Jul 2020 12:39:17 +0200 Subject: [PATCH 7/7] upd: github action's status badge --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3c4886a..7af96e2 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,12 @@

Telegram Media Downloader

-Build Status +Unittest Coverage Status License: MIT Code style: black

- +![Unittest]() ### Overview: Download all media files from a conversation or a channel that you are a part of from telegram. A meta of last read/downloaded message is stored in the config file so that in such a way it won't download the same media file again.