mirror of
https://github.com/laramies/theHarvester.git
synced 2024-09-22 00:06:30 +08:00
Syncing.
This commit is contained in:
NotoriousRebel
parent
70caa8986e
commit
96c0fdde5e
13
lib/core.py
13
lib/core.py
|
|
@ -85,22 +85,9 @@ def get_supportedengines():
|
|||
}
|
||||
return supportedengines
|
||||
|
||||
@staticmethod
|
||||
def get_supportedengines():
|
||||
supportedengines = {'baidu', 'bing', 'bingapi', 'censys', 'crtsh', 'cymon', 'dogpile', 'duckduckgo', 'google',
|
||||
'googleCSE', 'google-certificates', 'google-profiles', 'hunter', 'linkedin', 'netcraft',
|
||||
'pgp', 'securityTrails', 'threatcrowd', 'trello', 'twitter', 'vhost', 'virustotal', 'yahoo',
|
||||
'all'}
|
||||
return supportedengines
|
||||
|
||||
@staticmethod
|
||||
def get_user_agent():
|
||||
<<<<<<< HEAD
|
||||
"""User-Agents from https://github.com/tamimibrahim17/List-of-user-agents"""
|
||||
||||||| merged common ancestors
|
||||
=======
|
||||
# User-Agents from https://github.com/tamimibrahim17/List-of-user-agents
|
||||
>>>>>>> 0d7b8a09ef12196954ed9f21fb8339335e940948
|
||||
user_agents = [
|
||||
'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36',
|
||||
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205',
|
||||
|
|
|
|||
397
theHarvester.py
397
theHarvester.py
|
|
@ -1,30 +1,18 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
from discovery import *
|
||||
from discovery.constants import *
|
||||
from lib import hostchecker
|
||||
from lib import htmlExport
|
||||
from lib import reportgraph
|
||||
from lib import statichtmlgenerator
|
||||
<<<<<<< HEAD
|
||||
from lib import stash
|
||||
||||||| merged common ancestors
|
||||
=======
|
||||
from lib import stash
|
||||
from lib.core import *
|
||||
from platform import python_version
|
||||
import argparse
|
||||
>>>>>>> 0d7b8a09ef12196954ed9f21fb8339335e940948
|
||||
import datetime
|
||||
import ipaddress
|
||||
import re
|
||||
<<<<<<< HEAD
|
||||
from platform import python_version
|
||||
||||||| merged common ancestors
|
||||
import stash
|
||||
=======
|
||||
>>>>>>> 0d7b8a09ef12196954ed9f21fb8339335e940948
|
||||
import time
|
||||
|
||||
try:
|
||||
|
|
@ -42,39 +30,6 @@
|
|||
Core.banner()
|
||||
|
||||
|
||||
<<<<<<< HEAD
|
||||
def start():
|
||||
parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon')
|
||||
parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain, default=False, params=True', default=False)
|
||||
parser.add_argument('-d', '--domain', help='Company name or domain to search', required=True)
|
||||
parser.add_argument('-t', '--dnstld', help='Perform a DNS TLD expansion discovery, default False', default=False)
|
||||
parser.add_argument('-l', '--limit', help='limit the number of search results, default 500', default=500, type=int)
|
||||
parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts, default=False, params=True', default=False)
|
||||
parser.add_argument('-S', '--start', help='start with result number X (default: 0)', default=0)
|
||||
parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str)
|
||||
parser.add_argument('-g', '--googleDork', help='use googledorks for google search, default False', default=False)
|
||||
parser.add_argument('-n', '--dns-lookup', help='Enable DNS server lookup, default=False, params=True', default=False)
|
||||
parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup')
|
||||
parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts params=basic, default=False', default=False)
|
||||
parser.add_argument('-p', '--portscan', help='port scan the detected hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True', default=False)
|
||||
parser.add_argument('-b', '--source', help='''source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile,
|
||||
google, googleCSE, google-certificates, google-profiles,
|
||||
hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd,
|
||||
trello, twitter, vhost, virustotal, yahoo, all''', required=True)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
||||||| merged common ancestors
|
||||
def start(argv):
|
||||
if len(sys.argv) < 4:
|
||||
Core.usage()
|
||||
sys.exit(1)
|
||||
try:
|
||||
opts, args = getopt.getopt(argv, 'l:d:b:s:u:vf:nhcgpte:')
|
||||
except getopt.GetoptError:
|
||||
Core.usage()
|
||||
sys.exit(1)
|
||||
=======
|
||||
def start():
|
||||
parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon')
|
||||
parser.add_argument('-d', '--domain', help='company name or domain to search', required=True)
|
||||
|
|
@ -96,7 +51,6 @@ def start():
|
|||
|
||||
args = parser.parse_args()
|
||||
|
||||
>>>>>>> 0d7b8a09ef12196954ed9f21fb8339335e940948
|
||||
try:
|
||||
db = stash.stash_manager()
|
||||
db.do_init()
|
||||
|
|
@ -107,181 +61,21 @@ def start():
|
|||
all_hosts = []
|
||||
all_ip = []
|
||||
bingapi = 'yes'
|
||||
<<<<<<< HEAD
|
||||
dnsbrute = args.dns_brute
|
||||
dnslookup = args.dns_lookup
|
||||
dnsserver = args.dns_server
|
||||
dnstld = args.dnstld
|
||||
filename = args.filename
|
||||
||||||| merged common ancestors
|
||||
dnsbrute = False
|
||||
dnslookup = False
|
||||
dnsserver = ""
|
||||
dnstld = False
|
||||
filename = ""
|
||||
=======
|
||||
dnsbrute = args.dns_brute
|
||||
dnslookup = args.dns_lookup
|
||||
dnsserver = args.dns_server
|
||||
dnstld = args.dns_tld
|
||||
filename = args.filename
|
||||
>>>>>>> 0d7b8a09ef12196954ed9f21fb8339335e940948
|
||||
full = []
|
||||
<<<<<<< HEAD
|
||||
google_dorking = args.googleDork
|
||||
||||||| merged common ancestors
|
||||
google_dorking = False
|
||||
=======
|
||||
google_dorking = args.google_dork
|
||||
>>>>>>> 0d7b8a09ef12196954ed9f21fb8339335e940948
|
||||
host_ip = []
|
||||
<<<<<<< HEAD
|
||||
limit = args.limit
|
||||
ports_scanning = args.portscan
|
||||
shodan = args.shodan
|
||||
start = args.start
|
||||
||||||| merged common ancestors
|
||||
limit = 500
|
||||
ports_scanning = False
|
||||
shodan = False
|
||||
start = 0
|
||||
=======
|
||||
limit = args.limit
|
||||
ports_scanning = args.port_scan
|
||||
shodan = args.shodan
|
||||
start = args.start
|
||||
>>>>>>> 0d7b8a09ef12196954ed9f21fb8339335e940948
|
||||
takeover_check = False
|
||||
trello_info = ([], False)
|
||||
vhost = []
|
||||
<<<<<<< HEAD
|
||||
virtual = args.virtual_host
|
||||
word = args.domain
|
||||
engines = set(args.source.split(','))
|
||||
if set(engines).issubset(Core.get_supportedengines()):
|
||||
print(f'\033[94m[*] Target domain: {word} \n \033[0m')
|
||||
for engineitem in engines:
|
||||
if engineitem == 'baidu':
|
||||
print('\033[94m[*] Searching Baidu. \033[0m')
|
||||
try:
|
||||
search = baidusearch.SearchBaidu(word, limit)
|
||||
search.process()
|
||||
all_emails = filter(search.get_emails())
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'baidu')
|
||||
db.store_all(word, all_emails, 'email', 'baidu')
|
||||
except Exception:
|
||||
pass
|
||||
||||||| merged common ancestors
|
||||
virtual = False
|
||||
|
||||
for value in enumerate(opts):
|
||||
opt = value[1][0]
|
||||
arg = value[1][1]
|
||||
opt = str(opt)
|
||||
arg = str(arg)
|
||||
if opt == '-l':
|
||||
limit = int(arg)
|
||||
elif opt == '-d':
|
||||
word = arg
|
||||
elif opt == '-g':
|
||||
google_dorking = True
|
||||
elif opt == '-s':
|
||||
start = int(arg)
|
||||
elif opt == '-v':
|
||||
virtual = 'basic'
|
||||
elif opt == '-f':
|
||||
filename = arg
|
||||
elif opt == '-n':
|
||||
dnslookup = True
|
||||
elif opt == '-c':
|
||||
dnsbrute = True
|
||||
elif opt == '-h':
|
||||
shodan = True
|
||||
elif opt == '-e':
|
||||
dnsserver = arg
|
||||
elif opt == '-p':
|
||||
ports_scanning = True
|
||||
elif opt == '-t':
|
||||
dnstld = True
|
||||
elif opt == '-b':
|
||||
engines = set(arg.split(','))
|
||||
supportedengines = set(['baidu', 'bing', 'bingapi', 'censys', 'crtsh', 'cymon', 'dogpile', 'duckduckgo',
|
||||
'google', 'googleCSE', 'google-certificates', 'google-profiles', 'hunter',
|
||||
'linkedin', 'netcraft', 'pgp', 'securityTrails', 'threatcrowd', 'trello',
|
||||
'twitter', 'vhost', 'virustotal', 'yahoo', 'all'])
|
||||
if set(engines).issubset(supportedengines):
|
||||
print(f'\033[94m[*] Target domain: {word} \n \033[0m')
|
||||
for engineitem in engines:
|
||||
if engineitem == 'baidu':
|
||||
print('\033[94m[*] Searching Baidu. \033[0m')
|
||||
try:
|
||||
search = baidusearch.SearchBaidu(word, limit)
|
||||
search.process()
|
||||
all_emails = filter(search.get_emails())
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'baidu')
|
||||
db.store_all(word, all_emails, 'email', 'baidu')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
elif engineitem == 'bing' or engineitem == 'bingapi':
|
||||
print('\033[94m[*] Searching Bing. \033[0m')
|
||||
try:
|
||||
search = bingsearch.SearchBing(word, limit, start)
|
||||
if engineitem == 'bingapi':
|
||||
bingapi = 'yes'
|
||||
else:
|
||||
bingapi = 'no'
|
||||
search.process(bingapi)
|
||||
all_emails = filter(search.get_emails())
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'email', 'bing')
|
||||
db.store_all(word, all_hosts, 'host', 'bing')
|
||||
except Exception as e:
|
||||
if isinstance(e, MissingKey):
|
||||
print(e)
|
||||
else:
|
||||
pass
|
||||
|
||||
elif engineitem == 'censys':
|
||||
print('\033[94m[*] Searching Censys. \033[0m')
|
||||
from discovery import censys
|
||||
# Import locally or won't work
|
||||
search = censys.SearchCensys(word, limit)
|
||||
search.process()
|
||||
all_ip = search.get_ipaddresses()
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'censys')
|
||||
db.store_all(word, all_ip, 'ip', 'censys')
|
||||
|
||||
elif engineitem == 'crtsh':
|
||||
print('\033[94m[*] Searching CRT.sh. \033[0m')
|
||||
search = crtsh.search_crtsh(word)
|
||||
search.process()
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'CRTsh')
|
||||
|
||||
elif engineitem == 'cymon':
|
||||
print('\033[94m[*] Searching Cymon. \033[0m')
|
||||
from discovery import cymon
|
||||
# Import locally or won't work.
|
||||
search = cymon.search_cymon(word)
|
||||
search.process()
|
||||
all_ip = search.get_ipaddresses()
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_ip, 'ip', 'cymon')
|
||||
=======
|
||||
virtual = args.virtual_host
|
||||
word = args.domain
|
||||
engines = set(args.source.split(','))
|
||||
|
|
@ -323,30 +117,6 @@ def start():
|
|||
print(e)
|
||||
else:
|
||||
pass
|
||||
>>>>>>> 0d7b8a09ef12196954ed9f21fb8339335e940948
|
||||
|
||||
<<<<<<< HEAD
|
||||
elif engineitem == 'bing' or engineitem == 'bingapi':
|
||||
print('\033[94m[*] Searching Bing. \033[0m')
|
||||
try:
|
||||
search = bingsearch.SearchBing(word, limit, start)
|
||||
bingapi = ''
|
||||
if engineitem == 'bingapi':
|
||||
bingapi += 'yes'
|
||||
else:
|
||||
bingapi += 'no'
|
||||
search.process(bingapi)
|
||||
all_emails = filter(search.get_emails())
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'email', 'bing')
|
||||
db.store_all(word, all_hosts, 'host', 'bing')
|
||||
except Exception as e:
|
||||
if isinstance(e, MissingKey):
|
||||
print(e)
|
||||
else:
|
||||
pass
|
||||
|
||||
elif engineitem == 'censys':
|
||||
print('\033[94m[*] Searching Censys. \033[0m')
|
||||
|
|
@ -424,112 +194,7 @@ def start():
|
|||
search.process()
|
||||
search.store_results()
|
||||
all_emails = filter(search.get_emails())
|
||||
||||||| merged common ancestors
|
||||
elif engineitem == 'dogpile':
|
||||
print('\033[94m[*] Searching Dogpile. \033[0m')
|
||||
search = dogpilesearch.SearchDogpile(word, limit)
|
||||
search.process()
|
||||
emails = filter(search.get_emails())
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
all_emails.extend(emails)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'email', 'dogpile')
|
||||
db.store_all(word, all_hosts, 'host', 'dogpile')
|
||||
|
||||
elif engineitem == 'duckduckgo':
|
||||
print('\033[94m[*] Searching DuckDuckGo. \033[0m')
|
||||
from discovery import duckduckgosearch
|
||||
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
|
||||
search.process()
|
||||
emails = filter(search.get_emails())
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
all_emails.extend(emails)
|
||||
=======
|
||||
elif engineitem == 'censys':
|
||||
print('\033[94m[*] Searching Censys. \033[0m')
|
||||
from discovery import censys
|
||||
# Import locally or won't work
|
||||
search = censys.SearchCensys(word, limit)
|
||||
search.process()
|
||||
all_ip = search.get_ipaddresses()
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'censys')
|
||||
db.store_all(word, all_ip, 'ip', 'censys')
|
||||
|
||||
elif engineitem == 'crtsh':
|
||||
print('\033[94m[*] Searching CRT.sh. \033[0m')
|
||||
search = crtsh.search_crtsh(word)
|
||||
search.process()
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'CRTsh')
|
||||
|
||||
elif engineitem == 'cymon':
|
||||
print('\033[94m[*] Searching Cymon. \033[0m')
|
||||
from discovery import cymon
|
||||
# Import locally or won't work.
|
||||
search = cymon.search_cymon(word)
|
||||
search.process()
|
||||
all_ip = search.get_ipaddresses()
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_ip, 'ip', 'cymon')
|
||||
|
||||
elif engineitem == 'dogpile':
|
||||
print('\033[94m[*] Searching Dogpile. \033[0m')
|
||||
search = dogpilesearch.SearchDogpile(word, limit)
|
||||
search.process()
|
||||
emails = filter(search.get_emails())
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
all_emails.extend(emails)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'email', 'dogpile')
|
||||
db.store_all(word, all_hosts, 'host', 'dogpile')
|
||||
|
||||
elif engineitem == 'duckduckgo':
|
||||
print('\033[94m[*] Searching DuckDuckGo. \033[0m')
|
||||
from discovery import duckduckgosearch
|
||||
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
|
||||
search.process()
|
||||
emails = filter(search.get_emails())
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
all_emails.extend(emails)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'email', 'duckduckgo')
|
||||
db.store_all(word, all_hosts, 'host', 'duckduckgo')
|
||||
|
||||
elif engineitem == 'google':
|
||||
print('\033[94m[*] Searching Google. \033[0m')
|
||||
search = googlesearch.search_google(word, limit, start)
|
||||
search.process(google_dorking)
|
||||
emails = filter(search.get_emails())
|
||||
all_emails.extend(emails)
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'google')
|
||||
db.store_all(word, all_emails, 'email', 'google')
|
||||
|
||||
elif engineitem == 'googleCSE':
|
||||
print('\033[94m[*] Searching Google Custom Search. \033[0m')
|
||||
try:
|
||||
search = googleCSE.SearchGoogleCSE(word, limit, start)
|
||||
search.process()
|
||||
search.store_results()
|
||||
all_emails = filter(search.get_emails())
|
||||
db = stash.stash_manager()
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db.store_all(word, all_hosts, 'email', 'googleCSE')
|
||||
>>>>>>> 0d7b8a09ef12196954ed9f21fb8339335e940948
|
||||
db = stash.stash_manager()
|
||||
<<<<<<< HEAD
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db.store_all(word, all_hosts, 'email', 'googleCSE')
|
||||
|
|
@ -538,66 +203,6 @@ def start():
|
|||
except Exception as e:
|
||||
if isinstance(e, MissingKey):
|
||||
print(e)
|
||||
||||||| merged common ancestors
|
||||
db.store_all(word, all_hosts, 'email', 'duckduckgo')
|
||||
db.store_all(word, all_hosts, 'host', 'duckduckgo')
|
||||
|
||||
elif engineitem == 'google':
|
||||
print('\033[94m[*] Searching Google. \033[0m')
|
||||
search = googlesearch.search_google(word, limit, start)
|
||||
search.process(google_dorking)
|
||||
emails = filter(search.get_emails())
|
||||
all_emails.extend(emails)
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'google')
|
||||
db.store_all(word, all_emails, 'email', 'google')
|
||||
|
||||
elif engineitem == 'googleCSE':
|
||||
print('\033[94m[*] Searching Google Custom Search. \033[0m')
|
||||
try:
|
||||
search = googleCSE.SearchGoogleCSE(word, limit, start)
|
||||
search.process()
|
||||
search.store_results()
|
||||
all_emails = filter(search.get_emails())
|
||||
db = stash.stash_manager()
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db.store_all(word, all_hosts, 'email', 'googleCSE')
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'googleCSE')
|
||||
except Exception as e:
|
||||
if isinstance(e, MissingKey):
|
||||
print(e)
|
||||
else:
|
||||
pass
|
||||
|
||||
elif engineitem == 'google-certificates':
|
||||
print('\033[94m[*] Searching Google Certificate transparency report. \033[0m')
|
||||
search = googlecertificates.SearchGoogleCertificates(word, limit, start)
|
||||
search.process()
|
||||
hosts = filter(search.get_domains())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'google-certificates')
|
||||
|
||||
elif engineitem == 'google-profiles':
|
||||
print('\033[94m[*] Searching Google profiles. \033[0m')
|
||||
search = googlesearch.search_google(word, limit, start)
|
||||
search.process_profiles()
|
||||
people = search.get_profiles()
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, people, 'name', 'google-profile')
|
||||
|
||||
if len(people) == 0:
|
||||
print('\n[*] No users found.\n\n')
|
||||
=======
|
||||
db.store_all(word, all_hosts, 'host', 'googleCSE')
|
||||
except Exception as e:
|
||||
if isinstance(e, MissingKey):
|
||||
print(e)
|
||||
>>>>>>> 0d7b8a09ef12196954ed9f21fb8339335e940948
|
||||
else:
|
||||
pass
|
||||
|
||||
|
|
@ -744,7 +349,7 @@ def start():
|
|||
db.store_all(word, people, 'name', 'twitter')
|
||||
|
||||
if len(people) == 0:
|
||||
print('\n[*] No users found on Twitter.\n\n')
|
||||
print('\n[*] No users found.\n\n')
|
||||
else:
|
||||
print('\n[*] Users found: ' + str(len(people)))
|
||||
print('---------------------')
|
||||
|
|
|
|||
Loading…
Reference in a new issue