add static typing and tests (qwant search engine discovery)

.github/workflows/theHarvester.yml

@@ -91,6 +91,10 @@ jobs:
       run: |
         python theHarvester.py -d yale.edu -b otx
 
+    - name: Run theHarvester module Qwant
+      run: |
+        python theHarvester.py -d yale.edu -b qwant
+
     - name: Run theHarvester module RapidDns
       run: |
         python theHarvester.py -d yale.edu -b rapiddns

.travis.yml

@@ -13,7 +13,7 @@ before_install:
 install:
 - python setup.py test
 script:
-- python theHarvester.py -d apple.com -b baidu,bing,bufferoverun,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,intelx,threatcrowd,trello,twitter,virustotal,yahoo,rapiddns
+- python theHarvester.py -d apple.com -b baidu,bing,bufferoverun,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,intelx,threatcrowd,trello,twitter,virustotal,yahoo,rapiddns,qwant
   -l 200
 - pytest
 - flake8 . --count --show-source --statistics

tests/discovery/test_qwantsearch.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# coding=utf-8
+from theHarvester.discovery import qwantsearch
+import pytest
+
+pytestmark = pytest.mark.asyncio
+
+
+class TestSearchQwant(object):
+
+    @staticmethod
+    def domain() -> str:
+        return 'example.com'
+
+    def test_get_start_offset_return_0(self):
+        search = qwantsearch.SearchQwant(TestSearchQwant.domain(), 0, 200)
+        assert search.get_start_offset() == 0
+
+    def test_get_start_offset_return_50(self):
+        search = qwantsearch.SearchQwant(TestSearchQwant.domain(), 55, 200)
+        assert search.get_start_offset() == 50
+
+    def test_get_start_offset_return_100(self):
+        search = qwantsearch.SearchQwant(TestSearchQwant.domain(), 100, 200)
+        assert search.get_start_offset() == 100
+
+    async def test_get_emails(self):
+        search = qwantsearch.SearchQwant(TestSearchQwant.domain(), 0, 200)
+        await search.process()
+        assert isinstance(await search.get_emails(), set)
+
+    async def test_get_hostnames(self):
+        search = qwantsearch.SearchQwant(TestSearchQwant.domain(), 0, 200)
+        await search.process()
+        assert isinstance(await search.get_hostnames(), list)

theHarvester/discovery/qwantsearch.py

@@ -12,7 +12,7 @@ def __init__(self, word, start, limit):
         self.start = int(start)
         self.proxy = False
 
-    def get_start_offset(self):
+    def get_start_offset(self) -> int:
         """
         print(get_start_offset(0))
         >>> 0
@@ -26,7 +26,7 @@ def get_start_offset(self):
         start = int(math.floor(self.start / 10.0)) * 10
         return max(start, 0)
 
-    async def do_search(self):
+    async def do_search(self) -> None:
         headers = {
             'Host': "api.qwant.com",
             'User-agent': Core.get_user_agent()
@@ -65,15 +65,15 @@ async def do_search(self):
                 self.total_results += " "
                 self.total_results += desc
 
-    async def get_emails(self):
+    async def get_emails(self) -> set:
         parser = myparser.Parser(self.total_results, self.word)
         return await parser.emails()
 
-    async def get_hostnames(self):
+    async def get_hostnames(self) -> list:
         parser = myparser.Parser(self.total_results, self.word)
         return await parser.hostnames()
 
-    async def process(self, api, proxy=False):
+    async def process(self, proxy=False) -> None:
         self.proxy = proxy
         await self.do_search()
         print(f'\tSearching {self.limit} results.')