mirror of
https://github.com/laramies/theHarvester.git
synced 2024-09-20 23:37:00 +08:00
Merge pull request #323 from thehappydinoa/dev
Update code to pass CI and Censys Module
This commit is contained in:
commit
c27202975c
4
.github/workflows/theHarvester.yml
vendored
4
.github/workflows/theHarvester.yml
vendored
|
@ -8,6 +8,7 @@ on:
|
|||
pull_request:
|
||||
branches:
|
||||
- '*'
|
||||
|
||||
jobs:
|
||||
Python:
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
@ -146,6 +147,7 @@ jobs:
|
|||
- name: Test with pytest
|
||||
run: |
|
||||
pytest
|
||||
|
||||
- name: Static type checking with mypy
|
||||
run: |
|
||||
mypy --pretty theHarvester/*/*.py
|
||||
mypy --pretty theHarvester/*/*.py
|
||||
|
|
|
@ -20,6 +20,8 @@ Passive:
|
|||
|
||||
* bufferoverun: Uses data from Rapid7's Project Sonar - www.rapid7.com/research/project-sonar/
|
||||
|
||||
* censys: Censys search engine, will use certificates searches to enumerate subdomains (Requires an API key, see below.) - [censys.io](https://censys.io/)
|
||||
|
||||
* certspotter: Cert Spotter monitors Certificate Transparency logs - https://sslmate.com/certspotter/
|
||||
|
||||
* crtsh: Comodo Certificate search - https://crt.sh
|
||||
|
@ -63,7 +65,7 @@ to enhance research and analyse changes around DNS for better insights - https:/
|
|||
* securityTrails: Security Trails search engine, the world's largest repository of historical DNS data<br>
|
||||
(Requires an API key, see below.) - www.securitytrails.com
|
||||
|
||||
* shodan: Shodan search engine, will search for ports and banners from discovered hosts - www.shodanhq.com
|
||||
* shodan: Shodan search engine, will search for ports and banners from discovered hosts (Requires an API key, see below.) - www.shodanhq.com
|
||||
|
||||
* spyse: Web research tools for professionals (Requires an API key.) - https://spyse.com
|
||||
|
||||
|
@ -104,7 +106,7 @@ Documentation to setup API keys can be found at - https://github.com/laramies/th
|
|||
* securityTrails
|
||||
* shodan
|
||||
* spyse - need to have a paid account be able to use the api now
|
||||
|
||||
* censys
|
||||
|
||||
Install and dependencies:
|
||||
-------------------------
|
||||
|
|
|
@ -328,7 +328,7 @@ async def store(search_engine: Any, source: str, process_param: Any = None, stor
|
|||
if isinstance(e, MissingKey):
|
||||
print(e)
|
||||
else:
|
||||
print(f'An exception has occurred in ProjectDiscovery')
|
||||
print('An exception has occurred in ProjectDiscovery')
|
||||
|
||||
elif engineitem == 'qwant':
|
||||
from theHarvester.discovery import qwantsearch
|
||||
|
|
|
@ -72,7 +72,7 @@ async def process(self, api, proxy=False):
|
|||
self.proxy = proxy
|
||||
if api == 'yes':
|
||||
if self.bingApi is None:
|
||||
raise MissingKey(True, 'BingAPI')
|
||||
raise MissingKey('BingAPI')
|
||||
else:
|
||||
if api == 'yes':
|
||||
await self.do_search_api()
|
||||
|
|
|
@ -1,29 +1,34 @@
|
|||
from theHarvester.discovery.constants import *
|
||||
from theHarvester.lib.core import *
|
||||
import censys.certificates
|
||||
import censys.base
|
||||
from theHarvester.discovery.constants import MissingKey
|
||||
from theHarvester.lib.core import Core
|
||||
from censys.certificates import CensysCertificates
|
||||
from censys.exceptions import (
|
||||
CensysRateLimitExceededException,
|
||||
CensysUnauthorizedException,
|
||||
)
|
||||
|
||||
|
||||
class SearchCensys:
|
||||
|
||||
def __init__(self, word):
|
||||
self.word = word
|
||||
def __init__(self, domain):
|
||||
self.word = domain
|
||||
self.key = Core.censys_key()
|
||||
if self.key[0] is None or self.key[1] is None:
|
||||
raise MissingKey(True, 'Censys ID or Secret')
|
||||
raise MissingKey("Censys ID and/or Secret")
|
||||
self.totalhosts = set()
|
||||
self.proxy = False
|
||||
|
||||
async def do_search(self):
|
||||
cert = censys.certificates.CensysCertificates(api_id=self.key[0], api_secret=self.key[1])
|
||||
query = f'parsed.names: {self.word}'
|
||||
try:
|
||||
response = cert.search(query=query, fields=['parsed.names'], page=1)
|
||||
except censys.base.CensysRateLimitExceededException:
|
||||
print('Censys rate limit exceeded')
|
||||
c = CensysCertificates(api_id=self.key[0], api_secret=self.key[1])
|
||||
except CensysUnauthorizedException:
|
||||
raise MissingKey("Censys ID and/or Secret")
|
||||
|
||||
for hosts in response:
|
||||
self.totalhosts.update(hosts['parsed.names'])
|
||||
query = f"parsed.names: {self.word}"
|
||||
try:
|
||||
response = c.search(query=query, fields=["parsed.names", "metadata"])
|
||||
for cert in response:
|
||||
self.totalhosts.update(cert["parsed.names"])
|
||||
except CensysRateLimitExceededException:
|
||||
print("Censys rate limit exceeded")
|
||||
|
||||
async def get_hostnames(self) -> set:
|
||||
return self.totalhosts
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
from theHarvester.lib.core import *
|
||||
from typing import Union
|
||||
from typing import Union, Optional
|
||||
import random
|
||||
|
||||
googleUA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 ' \
|
||||
|
@ -107,8 +107,8 @@ class MissingKey(Exception):
|
|||
"""
|
||||
:raise: When there is a module that has not been provided its API key
|
||||
"""
|
||||
def __init__(self, identity_flag: bool, source: str):
|
||||
if identity_flag:
|
||||
def __init__(self, source: Optional[str]):
|
||||
if source:
|
||||
self.message = f'\n\033[93m[!] Missing API key for {source}. \033[0m'
|
||||
else:
|
||||
self.message = '\n\033[93m[!] Missing CSE id. \033[0m'
|
||||
|
|
|
@ -37,7 +37,7 @@ def __init__(self, word, limit):
|
|||
# rate limits you more severely
|
||||
# https://developer.github.com/v3/search/#rate-limit
|
||||
if self.key is None:
|
||||
raise MissingKey(True, 'Github')
|
||||
raise MissingKey('Github')
|
||||
self.proxy = False
|
||||
|
||||
@staticmethod
|
||||
|
|
|
@ -11,7 +11,7 @@ def __init__(self, word, limit, start):
|
|||
self.start = start
|
||||
self.key = Core.hunter_key()
|
||||
if self.key is None:
|
||||
raise MissingKey(True, 'Hunter')
|
||||
raise MissingKey('Hunter')
|
||||
self.total_results = ""
|
||||
self.counter = start
|
||||
self.database = f'https://api.hunter.io/v2/domain-search?domain={self.word}&api_key={self.key}&limit={self.limit}'
|
||||
|
|
|
@ -11,7 +11,7 @@ def __init__(self, word, limit):
|
|||
# default key is public key
|
||||
self.key = Core.intelx_key()
|
||||
if self.key is None:
|
||||
raise MissingKey(True, 'Intelx')
|
||||
raise MissingKey('Intelx')
|
||||
self.database = 'https://public.intelx.io/'
|
||||
self.results = None
|
||||
self.info = ()
|
||||
|
|
|
@ -11,7 +11,7 @@ def __init__(self, word):
|
|||
self.word = word
|
||||
self.key = Core.pentest_tools_key()
|
||||
if self.key is None:
|
||||
raise MissingKey(True, 'PentestTools')
|
||||
raise MissingKey('PentestTools')
|
||||
self.total_results = []
|
||||
self.api = f'https://pentest-tools.com/api?key={self.key}'
|
||||
self.proxy = False
|
||||
|
|
|
@ -8,7 +8,7 @@ def __init__(self, word):
|
|||
self.word = word
|
||||
self.key = Core.projectdiscovery_key()
|
||||
if self.key is None:
|
||||
raise MissingKey(True, 'ProjectDiscovery')
|
||||
raise MissingKey('ProjectDiscovery')
|
||||
self.total_results = None
|
||||
self.proxy = False
|
||||
|
||||
|
|
|
@ -10,7 +10,7 @@ def __init__(self, word):
|
|||
self.word = word
|
||||
self.key = Core.security_trails_key()
|
||||
if self.key is None:
|
||||
raise MissingKey(True, 'Securitytrail')
|
||||
raise MissingKey('Securitytrail')
|
||||
self.results = ""
|
||||
self.totalresults = ""
|
||||
self.api = 'https://api.securitytrails.com/v1/'
|
||||
|
|
|
@ -9,7 +9,7 @@ class SearchShodan:
|
|||
def __init__(self):
|
||||
self.key = Core.shodan_key()
|
||||
if self.key is None:
|
||||
raise MissingKey(True, 'Shodan')
|
||||
raise MissingKey('Shodan')
|
||||
self.api = Shodan(self.key)
|
||||
self.hostdatarow = []
|
||||
|
||||
|
|
|
@ -9,7 +9,7 @@ def __init__(self, word):
|
|||
self.word = word
|
||||
self.key = Core.spyse_key()
|
||||
if self.key is None:
|
||||
raise MissingKey(True, 'Spyse')
|
||||
raise MissingKey('Spyse')
|
||||
self.results = ''
|
||||
self.hosts = set()
|
||||
self.proxy = False
|
||||
|
|
Loading…
Reference in a new issue