Add new n45ht module, lots of results returned (#899)

* Add new fullhunt module

* Remove python checks in the run script as they are not needed anymore

* Remove netcraft from ci as it is not needed

* Update Dockerfile to fix security issues

* Add new n45ht module, lots of results returned

.github/workflows/theHarvester.yml

@@ -27,6 +27,7 @@ jobs:
     - name: Install dependencies
       run: |
         pip install --upgrade pip
+        pip install wheel
         pip install -r requirements/dev.txt
 
     - name: Lint with flake8
@@ -93,10 +94,14 @@ jobs:
       run: |
         python theHarvester.py -d yale.edu -b linkedin
 
-    - name: Run theHarvester module linkedin_links
+    - name: Run theHarvester module Linkedin links
       run: |
         python theHarvester.py -d yale.edu -b linkedin_links
 
+    - name: Run theHarvester module N45ht
+      run: |
+        python theHarvester.py -d yale.edu -b n45ht
+
     - name: Run theHarvester module Omnisint
       run: |
         python theHarvester.py -d yale.edu -b omnisint

README.md

@@ -50,6 +50,8 @@ Passive:
 
 * linkedin_links: specific search for LinkedIn users for target domain (Uses Google search.)
 
+* n45ht: - https://n45ht.or.id
+
 * omnisint: Project Crobat, A Centralised Searchable Open Source Project Sonar DNS Database - https://github.com/Cgboal/SonarSearch
 
 * otx: AlienVault Open Threat Exchange - https://otx.alienvault.com

tests/discovery/test_n45ht.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# coding=utf-8
+from theHarvester.lib.core import *
+from theHarvester.discovery import n45htsearch
+import os
+import requests
+import pytest
+
+pytestmark = pytest.mark.asyncio
+github_ci = os.getenv('GITHUB_ACTIONS')  # Github set this to be the following: true instead of True
+
+
+class TestN45ht(object):
+    @staticmethod
+    def domain() -> str:
+        return 'uber.com'
+
+    async def test_api(self):
+        base_url = f'https://api.n45ht.or.id/v1/subdomain-enumeration?domain={TestN45ht.domain()}'
+        headers = {'User-Agent': Core.get_user_agent()}
+        request = requests.get(base_url, headers=headers)
+        assert request.status_code == 200
+
+    async def test_do_search(self):
+        search = n45htsearch.SearchN45ht(TestN45ht.domain())
+        await search.process()
+        assert isinstance(await search.get_hostnames(), set)
+
+
+if __name__ == '__main__':
+    pytest.main()

theHarvester/__main__.py

@@ -35,7 +35,7 @@ async def start(rest_args=None):
     parser.add_argument('-f', '--filename', help='Save the results to an XML and JSON file.', default='', type=str)
     parser.add_argument('-b', '--source', help='''anubis, baidu, bing, binaryedge, bingapi, bufferoverun, censys, certspotter, crtsh,
                             dnsdumpster, duckduckgo, fullhunt, github-code, google,
-                            hackertarget, hunter, intelx, linkedin, linkedin_links,
+                            hackertarget, hunter, intelx, linkedin, linkedin_links, n45ht,
                             omnisint, otx, pentesttools, projectdiscovery,
                             qwant, rapiddns, rocketreach, securityTrails, spyse, sublist3r, threatcrowd, threatminer,
                             trello, twitter, urlscan, virustotal, yahoo, zoomeye''')
@@ -341,6 +341,14 @@ async def store(search_engine: Any, source: str, process_param: Any = None, stor
                     linkedin_links_search = linkedinsearch.SearchLinkedin(word, limit)
                     stor_lst.append(store(linkedin_links_search, 'linkedin', store_links=True))
 
+                elif engineitem == 'n45ht':
+                    from theHarvester.discovery import n45htsearch
+                    try:
+                        n45ht_search = n45htsearch.SearchN45ht(word)
+                        stor_lst.append(store(n45ht_search, engineitem, store_host=True))
+                    except Exception as e:
+                        print(e)
+
                 elif engineitem == 'omnisint':
                     from theHarvester.discovery import omnisint
                     try:

theHarvester/discovery/n45htsearch.py

@@ -0,0 +1,23 @@
+from theHarvester.lib.core import *
+
+
+class SearchN45ht:
+
+    def __init__(self, word):
+        self.word = word
+        self.totalhosts = set()
+        self.proxy = False
+
+    async def do_search(self):
+        url = f'https://api.n45ht.or.id/v1/subdomain-enumeration?domain={self.word}'
+        response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy)
+        responses = response[0]
+        dct = responses
+        self.totalhosts: set = {host for host in dct['subdomains']}
+
+    async def get_hostnames(self) -> set:
+        return self.totalhosts
+
+    async def process(self, proxy=False):
+        self.proxy = proxy
+        await self.do_search()

theHarvester/lib/core.py

@@ -136,6 +136,7 @@ def get_supportedengines() -> Set[Union[str, Any]]:
                             'intelx',
                             'linkedin',
                             'linkedin_links',
+                            'n45ht',
                             'omnisint',
                             'otx',
                             'pentesttools',