diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index 72c03055..59dcf988 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -8,6 +8,7 @@ on: pull_request: branches: - '*' + jobs: Python: runs-on: ${{ matrix.os }} @@ -146,6 +147,7 @@ jobs: - name: Test with pytest run: | pytest + - name: Static type checking with mypy run: | - mypy --pretty theHarvester/*/*.py \ No newline at end of file + mypy --pretty theHarvester/*/*.py diff --git a/README.md b/README.md index 22147f00..19826a2d 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ Passive: * bufferoverun: Uses data from Rapid7's Project Sonar - www.rapid7.com/research/project-sonar/ +* censys: Censys search engine, will use certificates searches to enumerate subdomains (Requires an API key, see below.) - [censys.io](https://censys.io/) + * certspotter: Cert Spotter monitors Certificate Transparency logs - https://sslmate.com/certspotter/ * crtsh: Comodo Certificate search - https://crt.sh @@ -63,7 +65,7 @@ to enhance research and analyse changes around DNS for better insights - https:/ * securityTrails: Security Trails search engine, the world's largest repository of historical DNS data
(Requires an API key, see below.) - www.securitytrails.com -* shodan: Shodan search engine, will search for ports and banners from discovered hosts - www.shodanhq.com +* shodan: Shodan search engine, will search for ports and banners from discovered hosts (Requires an API key, see below.) - www.shodanhq.com * spyse: Web research tools for professionals (Requires an API key.) - https://spyse.com @@ -104,7 +106,7 @@ Documentation to setup API keys can be found at - https://github.com/laramies/th * securityTrails * shodan * spyse - need to have a paid account be able to use the api now - +* censys Install and dependencies: ------------------------- diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index a6dcfbde..e0f31c57 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -328,7 +328,7 @@ async def store(search_engine: Any, source: str, process_param: Any = None, stor if isinstance(e, MissingKey): print(e) else: - print(f'An exception has occurred in ProjectDiscovery') + print('An exception has occurred in ProjectDiscovery') elif engineitem == 'qwant': from theHarvester.discovery import qwantsearch diff --git a/theHarvester/discovery/bingsearch.py b/theHarvester/discovery/bingsearch.py index fd812ff5..b7ff5017 100644 --- a/theHarvester/discovery/bingsearch.py +++ b/theHarvester/discovery/bingsearch.py @@ -72,7 +72,7 @@ async def process(self, api, proxy=False): self.proxy = proxy if api == 'yes': if self.bingApi is None: - raise MissingKey(True, 'BingAPI') + raise MissingKey('BingAPI') else: if api == 'yes': await self.do_search_api() diff --git a/theHarvester/discovery/censys.py b/theHarvester/discovery/censys.py index 116ec8e3..347d9f93 100644 --- a/theHarvester/discovery/censys.py +++ b/theHarvester/discovery/censys.py @@ -1,29 +1,34 @@ -from theHarvester.discovery.constants import * -from theHarvester.lib.core import * -import censys.certificates -import censys.base +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import Core +from censys.certificates import CensysCertificates +from censys.exceptions import ( + CensysRateLimitExceededException, + CensysUnauthorizedException, +) class SearchCensys: - - def __init__(self, word): - self.word = word + def __init__(self, domain): + self.word = domain self.key = Core.censys_key() if self.key[0] is None or self.key[1] is None: - raise MissingKey(True, 'Censys ID or Secret') + raise MissingKey("Censys ID and/or Secret") self.totalhosts = set() self.proxy = False async def do_search(self): - cert = censys.certificates.CensysCertificates(api_id=self.key[0], api_secret=self.key[1]) - query = f'parsed.names: {self.word}' try: - response = cert.search(query=query, fields=['parsed.names'], page=1) - except censys.base.CensysRateLimitExceededException: - print('Censys rate limit exceeded') + c = CensysCertificates(api_id=self.key[0], api_secret=self.key[1]) + except CensysUnauthorizedException: + raise MissingKey("Censys ID and/or Secret") - for hosts in response: - self.totalhosts.update(hosts['parsed.names']) + query = f"parsed.names: {self.word}" + try: + response = c.search(query=query, fields=["parsed.names", "metadata"]) + for cert in response: + self.totalhosts.update(cert["parsed.names"]) + except CensysRateLimitExceededException: + print("Censys rate limit exceeded") async def get_hostnames(self) -> set: return self.totalhosts diff --git a/theHarvester/discovery/constants.py b/theHarvester/discovery/constants.py index c5b97ccb..e57b2548 100644 --- a/theHarvester/discovery/constants.py +++ b/theHarvester/discovery/constants.py @@ -1,5 +1,5 @@ from theHarvester.lib.core import * -from typing import Union +from typing import Union, Optional import random googleUA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 ' \ @@ -107,8 +107,8 @@ class MissingKey(Exception): """ :raise: When there is a module that has not been provided its API key """ - def __init__(self, identity_flag: bool, source: str): - if identity_flag: + def __init__(self, source: Optional[str]): + if source: self.message = f'\n\033[93m[!] Missing API key for {source}. \033[0m' else: self.message = '\n\033[93m[!] Missing CSE id. \033[0m' diff --git a/theHarvester/discovery/githubcode.py b/theHarvester/discovery/githubcode.py index 0410d740..14b53624 100644 --- a/theHarvester/discovery/githubcode.py +++ b/theHarvester/discovery/githubcode.py @@ -37,7 +37,7 @@ def __init__(self, word, limit): # rate limits you more severely # https://developer.github.com/v3/search/#rate-limit if self.key is None: - raise MissingKey(True, 'Github') + raise MissingKey('Github') self.proxy = False @staticmethod diff --git a/theHarvester/discovery/huntersearch.py b/theHarvester/discovery/huntersearch.py index 4c821959..82ba969a 100644 --- a/theHarvester/discovery/huntersearch.py +++ b/theHarvester/discovery/huntersearch.py @@ -11,7 +11,7 @@ def __init__(self, word, limit, start): self.start = start self.key = Core.hunter_key() if self.key is None: - raise MissingKey(True, 'Hunter') + raise MissingKey('Hunter') self.total_results = "" self.counter = start self.database = f'https://api.hunter.io/v2/domain-search?domain={self.word}&api_key={self.key}&limit={self.limit}' diff --git a/theHarvester/discovery/intelxsearch.py b/theHarvester/discovery/intelxsearch.py index 388359ae..a37e1f36 100644 --- a/theHarvester/discovery/intelxsearch.py +++ b/theHarvester/discovery/intelxsearch.py @@ -11,7 +11,7 @@ def __init__(self, word, limit): # default key is public key self.key = Core.intelx_key() if self.key is None: - raise MissingKey(True, 'Intelx') + raise MissingKey('Intelx') self.database = 'https://public.intelx.io/' self.results = None self.info = () diff --git a/theHarvester/discovery/pentesttools.py b/theHarvester/discovery/pentesttools.py index 351174ce..5ab8b8c8 100644 --- a/theHarvester/discovery/pentesttools.py +++ b/theHarvester/discovery/pentesttools.py @@ -11,7 +11,7 @@ def __init__(self, word): self.word = word self.key = Core.pentest_tools_key() if self.key is None: - raise MissingKey(True, 'PentestTools') + raise MissingKey('PentestTools') self.total_results = [] self.api = f'https://pentest-tools.com/api?key={self.key}' self.proxy = False diff --git a/theHarvester/discovery/projectdiscovery.py b/theHarvester/discovery/projectdiscovery.py index 0f70b125..5a730b6f 100644 --- a/theHarvester/discovery/projectdiscovery.py +++ b/theHarvester/discovery/projectdiscovery.py @@ -8,7 +8,7 @@ def __init__(self, word): self.word = word self.key = Core.projectdiscovery_key() if self.key is None: - raise MissingKey(True, 'ProjectDiscovery') + raise MissingKey('ProjectDiscovery') self.total_results = None self.proxy = False diff --git a/theHarvester/discovery/securitytrailssearch.py b/theHarvester/discovery/securitytrailssearch.py index 91c88727..f0b9b071 100644 --- a/theHarvester/discovery/securitytrailssearch.py +++ b/theHarvester/discovery/securitytrailssearch.py @@ -10,7 +10,7 @@ def __init__(self, word): self.word = word self.key = Core.security_trails_key() if self.key is None: - raise MissingKey(True, 'Securitytrail') + raise MissingKey('Securitytrail') self.results = "" self.totalresults = "" self.api = 'https://api.securitytrails.com/v1/' diff --git a/theHarvester/discovery/shodansearch.py b/theHarvester/discovery/shodansearch.py index 399dadc7..58683128 100644 --- a/theHarvester/discovery/shodansearch.py +++ b/theHarvester/discovery/shodansearch.py @@ -9,7 +9,7 @@ class SearchShodan: def __init__(self): self.key = Core.shodan_key() if self.key is None: - raise MissingKey(True, 'Shodan') + raise MissingKey('Shodan') self.api = Shodan(self.key) self.hostdatarow = [] diff --git a/theHarvester/discovery/spyse.py b/theHarvester/discovery/spyse.py index 158dbc4b..b5229e35 100644 --- a/theHarvester/discovery/spyse.py +++ b/theHarvester/discovery/spyse.py @@ -9,7 +9,7 @@ def __init__(self, word): self.word = word self.key = Core.spyse_key() if self.key is None: - raise MissingKey(True, 'Spyse') + raise MissingKey('Spyse') self.results = '' self.hosts = set() self.proxy = False