From 15fe11130c6165846f117890166e2b3ae26acfbc Mon Sep 17 00:00:00 2001 From: "J.Townsend" Date: Sun, 24 Oct 2021 04:10:37 +0100 Subject: [PATCH] Add new n45ht module, lots of results returned (#899) * Add new fullhunt module * Remove python checks in the run script as they are not needed anymore * Remove netcraft from ci as it is not needed * Update Dockerfile to fix security issues * Add new n45ht module, lots of results returned --- .github/workflows/theHarvester.yml | 7 +++++- README.md | 2 ++ tests/discovery/test_n45ht.py | 31 +++++++++++++++++++++++++++ theHarvester/__main__.py | 10 ++++++++- theHarvester/discovery/n45htsearch.py | 23 ++++++++++++++++++++ theHarvester/lib/core.py | 1 + 6 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 tests/discovery/test_n45ht.py create mode 100644 theHarvester/discovery/n45htsearch.py diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index c8a74ea2..c7c89aaf 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -27,6 +27,7 @@ jobs: - name: Install dependencies run: | pip install --upgrade pip + pip install wheel pip install -r requirements/dev.txt - name: Lint with flake8 @@ -93,10 +94,14 @@ jobs: run: | python theHarvester.py -d yale.edu -b linkedin - - name: Run theHarvester module linkedin_links + - name: Run theHarvester module Linkedin links run: | python theHarvester.py -d yale.edu -b linkedin_links + - name: Run theHarvester module N45ht + run: | + python theHarvester.py -d yale.edu -b n45ht + - name: Run theHarvester module Omnisint run: | python theHarvester.py -d yale.edu -b omnisint diff --git a/README.md b/README.md index a84ff0e3..a36c4f13 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,8 @@ Passive: * linkedin_links: specific search for LinkedIn users for target domain (Uses Google search.) +* n45ht: - https://n45ht.or.id + * omnisint: Project Crobat, A Centralised Searchable Open Source Project Sonar DNS Database - https://github.com/Cgboal/SonarSearch * otx: AlienVault Open Threat Exchange - https://otx.alienvault.com diff --git a/tests/discovery/test_n45ht.py b/tests/discovery/test_n45ht.py new file mode 100644 index 00000000..10a87000 --- /dev/null +++ b/tests/discovery/test_n45ht.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# coding=utf-8 +from theHarvester.lib.core import * +from theHarvester.discovery import n45htsearch +import os +import requests +import pytest + +pytestmark = pytest.mark.asyncio +github_ci = os.getenv('GITHUB_ACTIONS') # Github set this to be the following: true instead of True + + +class TestN45ht(object): + @staticmethod + def domain() -> str: + return 'uber.com' + + async def test_api(self): + base_url = f'https://api.n45ht.or.id/v1/subdomain-enumeration?domain={TestN45ht.domain()}' + headers = {'User-Agent': Core.get_user_agent()} + request = requests.get(base_url, headers=headers) + assert request.status_code == 200 + + async def test_do_search(self): + search = n45htsearch.SearchN45ht(TestN45ht.domain()) + await search.process() + assert isinstance(await search.get_hostnames(), set) + + +if __name__ == '__main__': + pytest.main() diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 1d5a2ee3..fc01a5ee 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -35,7 +35,7 @@ async def start(rest_args=None): parser.add_argument('-f', '--filename', help='Save the results to an XML and JSON file.', default='', type=str) parser.add_argument('-b', '--source', help='''anubis, baidu, bing, binaryedge, bingapi, bufferoverun, censys, certspotter, crtsh, dnsdumpster, duckduckgo, fullhunt, github-code, google, - hackertarget, hunter, intelx, linkedin, linkedin_links, + hackertarget, hunter, intelx, linkedin, linkedin_links, n45ht, omnisint, otx, pentesttools, projectdiscovery, qwant, rapiddns, rocketreach, securityTrails, spyse, sublist3r, threatcrowd, threatminer, trello, twitter, urlscan, virustotal, yahoo, zoomeye''') @@ -341,6 +341,14 @@ async def store(search_engine: Any, source: str, process_param: Any = None, stor linkedin_links_search = linkedinsearch.SearchLinkedin(word, limit) stor_lst.append(store(linkedin_links_search, 'linkedin', store_links=True)) + elif engineitem == 'n45ht': + from theHarvester.discovery import n45htsearch + try: + n45ht_search = n45htsearch.SearchN45ht(word) + stor_lst.append(store(n45ht_search, engineitem, store_host=True)) + except Exception as e: + print(e) + elif engineitem == 'omnisint': from theHarvester.discovery import omnisint try: diff --git a/theHarvester/discovery/n45htsearch.py b/theHarvester/discovery/n45htsearch.py new file mode 100644 index 00000000..5360115e --- /dev/null +++ b/theHarvester/discovery/n45htsearch.py @@ -0,0 +1,23 @@ +from theHarvester.lib.core import * + + +class SearchN45ht: + + def __init__(self, word): + self.word = word + self.totalhosts = set() + self.proxy = False + + async def do_search(self): + url = f'https://api.n45ht.or.id/v1/subdomain-enumeration?domain={self.word}' + response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy) + responses = response[0] + dct = responses + self.totalhosts: set = {host for host in dct['subdomains']} + + async def get_hostnames(self) -> set: + return self.totalhosts + + async def process(self, proxy=False): + self.proxy = proxy + await self.do_search() diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 091e99c3..2213bf0d 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -136,6 +136,7 @@ def get_supportedengines() -> Set[Union[str, Any]]: 'intelx', 'linkedin', 'linkedin_links', + 'n45ht', 'omnisint', 'otx', 'pentesttools',