remove -b all option as it does not work correctly

This commit is contained in:
Jay Townsend 2019-08-23 23:06:56 +01:00
parent d3ef0c4711
commit 3dde69f5b8
2 changed files with 1 additions and 542 deletions

View file

@ -56,7 +56,7 @@ def start():
dogpile, duckduckgo, exalead, github-code, google,
hunter, intelx,
linkedin, netcraft, securityTrails, threatcrowd,
trello, twitter, vhost, virustotal, yahoo, all''')
trello, twitter, vhost, virustotal, yahoo''')
parser.add_argument('-x', '--exclude', help='exclude options when using all sources', type=str)
args = parser.parse_args()
@ -403,299 +403,6 @@ def start():
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
elif engineitem == 'all':
print(('[*] Full harvest on ' + word))
all_emails = []
all_hosts = []
try:
print('\033[94m[*] Searching Baidu. \033[0m')
from theHarvester.discovery import baidusearch
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
try:
print('\033[94m[*] Searching Bing. \033[0m')
from theHarvester.discovery import bingsearch
bingapi = 'no'
search = bingsearch.SearchBing(word, limit, start)
search.process(bingapi)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'bing')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'bing')
except Exception:
pass
print('\033[94m[*] Searching Censys. \033[0m')
from theHarvester.discovery import censys
search = censys.SearchCensys(word, limit)
search.process()
ips = search.get_ipaddresses()
setips = set(ips)
uniqueips = list(setips) # Remove duplicates.
all_ip.extend(uniqueips)
hosts = filter(search.get_hostnames())
sethosts = set(hosts)
uniquehosts = list(sethosts) # Remove duplicates.
all_hosts.extend(uniquehosts)
db = stash.stash_manager()
db.store_all(word, uniquehosts, 'host', 'censys')
db.store_all(word, uniqueips, 'ip', 'censys')
try:
print('\033[94m[*] Searching CRT.sh. \033[0m')
from theHarvester.discovery import crtsh
search = crtsh.SearchCrtsh(word)
search.process()
hosts = filter(search.get_data())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
except Exception:
print(f'\033[93m[!] An timeout occurred with crtsh: cannot find {args.domain} \033[0m')
try:
print('\033[94m[*] Searching DNSdumpster. \033[0m')
from theHarvester.discovery import dnsdumpster
search = dnsdumpster.SearchDnsDumpster(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'dnsdumpster')
except Exception as e:
print(f'\033[93m[!] An error occurred with dnsdumpster: {e} \033[0m')
print('\033[94m[*] Searching Dogpile. \033[0m')
from theHarvester.discovery import dogpilesearch
try:
search = dogpilesearch.SearchDogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
except Exception as e:
print(f'An exception has occurred in Dogpile: {e}')
print('\033[94m[*] Searching DuckDuckGo. \033[0m')
from theHarvester.discovery import duckduckgosearch
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
print('\033[94m[*] Searching Exalead \033[0m')
try:
from theHarvester.discovery import exaleadsearch
search = exaleadsearch.search_exalead(word, limit, start)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'exalead')
db.store_all(word, all_emails, 'email', 'exalead')
except Exception:
pass
print('\033[94m[*] Searching Google. \033[0m')
from theHarvester.discovery import googlesearch
search = googlesearch.SearchGoogle(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'google')
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
print('\033[94m[*] Searching Hunter. \033[0m')
from theHarvester.discovery import huntersearch
# Import locally.
try:
search = huntersearch.SearchHunter(word, limit, start)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'hunter')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
print(f'Error occured in hunter: {e}')
print('\033[94m[*] Searching Intelx. \033[0m')
from theHarvester.discovery import intelxsearch
# Import locally or won't work.
try:
search = intelxsearch.SearchIntelx(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'intelx')
db.store_all(word, all_emails, 'email', 'intelx')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
print(e)
print('\033[94m[*] Searching Linkedin. \033[0m')
from theHarvester.discovery import linkedinsearch
search = linkedinsearch.SearchLinkedin(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
print('\033[94m[*] Searching Netcraft. \033[0m')
from theHarvester.discovery import netcraft
search = netcraft.SearchNetcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
print('\033[94m[*] Searching SecurityTrails. \033[0m')
from theHarvester.discovery import securitytrailssearch
try:
search = securitytrailssearch.SearchSecuritytrail(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'securityTrails')
ips = search.get_ips()
all_ip.extend(ips)
db = stash.stash_manager()
db.store_all(word, ips, 'ip', 'securityTrails')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
print('\033[94m[*] Searching Threatcrowd. \033[0m')
from theHarvester.discovery import threatcrowd
try:
search = threatcrowd.SearchThreatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'threatcrowd')
except Exception as error:
print(error)
print('\033[94m[*] Searching Trello. \033[0m')
from theHarvester.discovery import trello
# Import locally or won't work.
search = trello.SearchTrello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
try:
print('\033[94m[*] Searching Twitter. \033[0m')
from theHarvester.discovery import twittersearch
search = twittersearch.SearchTwitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
print('\nUsers from Twitter:')
print('-------------------')
for user in people:
print(user)
except Exception as error:
print(error)
print('\n[*] Virtual hosts:')
print('------------------')
for l in host_ip:
from theHarvester.discovery import baidusearch
search = bingsearch.SearchBing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print((l + '\t' + x))
vhost.append(l + ':' + x)
full.append(l + ':' + x)
vhost = sorted(set(vhost))
print('\033[94m[*] Searching VirusTotal. \033[0m')
from theHarvester.discovery import virustotal
search = virustotal.SearchVirustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
try:
print('\033[94m[*] Searching Yahoo. \033[0m')
from theHarvester.discovery import yahoosearch
search = yahoosearch.SearchYahoo(word, limit)
search.process()
hosts = search.get_hostnames()
emails = search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
except Exception as e:
print(f'An exception occurred in yahoo: {e}')
else:
print('\033[93m[!] Invalid source.\n\n \033[0m')
sys.exit(1)

View file

@ -68,7 +68,6 @@ def get_supportedengines():
'bingapi',
'censys',
'crtsh',
'cymon',
'dnsdumpster',
'dogpile',
'duckduckgo',
@ -86,7 +85,6 @@ def get_supportedengines():
'vhost',
'virustotal',
'yahoo',
'all'
}
return supportedengines
@ -324,249 +322,3 @@ def get_user_agent():
'Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00'
]
return random.choice(user_agents)
# TODO use this method when -b all is called to replace lines 383-635 in theHarvester.py
# TODO and to find the best approach of getting the word, limit, and start etc vars from
# the arguments and importing libs that are needed.
#
# @staticmethod
# def engine_all_search():
# print(('Full harvest on ' + word))
# all_emails = []
# all_hosts = []
# try:
# print('[*] Searching Baidu.')
# search = baidusearch.SearchBaidu(word, limit)
# search.process()
# all_emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'baidu')
# db.store_all(word, all_emails, 'email', 'baidu')
# except Exception:
# pass
#
# print('[*] Searching Bing.')
# bingapi = 'no'
# search = bingsearch.SearchBing(word, limit, start)
# search.process(bingapi)
# emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'bing')
# all_emails.extend(emails)
# all_emails = sorted(set(all_emails))
# db.store_all(word, all_emails, 'email', 'bing')
#
# print('[*] Searching Censys.')
# from discovery import censys
# search = censys.SearchCensys(word, limit)
# search.process()
# ips = search.get_ipaddresses()
# setips = set(ips)
# uniqueips = list(setips) # Remove duplicates.
# all_ip.extend(uniqueips)
# hosts = filter(search.get_hostnames())
# sethosts = set(hosts)
# uniquehosts = list(sethosts) # Remove duplicates.
# all_hosts.extend(uniquehosts)
# db = stash.stash_manager()
# db.store_all(word, uniquehosts, 'host', 'censys')
# db.store_all(word, uniqueips, 'ip', 'censys')
#
# print('[*] Searching CRT.sh.')
# search = crtsh.SearchCrtsh(word)
# search.process()
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'CRTsh')
#
# # cymon
# print('\033[94m[*] Searching Cymon. \033[0m')
# from discovery import cymon
# # Import locally or won't work.
# search = cymon.search_cymon(word)
# search.process()
# all_ip = search.get_ipaddresses()
# db = stash.stash_manager()
# db.store_all(word, all_ip, 'ip', 'cymon')
#
# print('\033[94m[*] Searching Dogpile. \033[0m')
# search = dogpilesearch.SearchDogpile(word, limit)
# search.process()
# emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# all_emails.extend(emails)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'email', 'dogpile')
# db.store_all(word, all_hosts, 'host', 'dogpile')
#
# print('[*] Searching DuckDuckGo.')
# from discovery import duckduckgosearch
# search = duckduckgosearch.SearchDuckDuckGo(word, limit)
# search.process()
# emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# all_emails.extend(emails)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'email', 'duckduckgo')
# db.store_all(word, all_hosts, 'host', 'duckduckgo')
#
# print('[*] Searching Google.')
# search = googlesearch.SearchGoogle(word, limit, start)
# search.process(google_dorking)
# emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# all_emails.extend(emails)
# db = stash.stash_manager()
# db.store_all(word, all_emails, 'email', 'google')
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'google')
#
# print('[*] Searching Google Certificate transparency report.')
# search = googlecertificates.SearchGoogleCertificates(word, limit, start)
# search.process()
# domains = filter(search.get_domains())
# all_hosts.extend(domains)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'google-certificates')
#
# print('[*] Searching Hunter.')
# from discovery import huntersearch
# # Import locally.
# try:
# search = huntersearch.SearchHunter(word, limit, start)
# search.process()
# emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, hosts, 'host', 'hunter')
# all_emails.extend(emails)
# all_emails = sorted(set(all_emails))
# db.store_all(word, all_emails, 'email', 'hunter')
# except Exception as e:
# if isinstance(e, MissingKey):
# print(e)
# else:
# pass
#
# print('\033[94m[*] Searching Linkedin. \033[0m')
# search = linkedinsearch.SearchLinkedin(word, limit)
# search.process()
# people = search.get_people()
# db = stash.stash_manager()
# db.store_all(word, people, 'name', 'linkedin')
#
# if len(people) == 0:
# print('\n[*] No users found.\n\n')
# else:
# print('\n[*] Users found: ' + str(len(people)))
# print('---------------------')
# for user in sorted(list(set(people))):
# print(user)
#
# print('[*] Searching Netcraft.')
# search = netcraft.SearchNetcraft(word)
# search.process()
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'netcraft')
#
# print('[*] Searching PGP key server.')
# try:
# search = pgpsearch.SearchPgp(word)
# search.process()
# emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# sethosts = set(hosts)
# uniquehosts = list(sethosts) # Remove duplicates.
# all_hosts.extend(uniquehosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'PGP')
# all_emails.extend(emails)
# db = stash.stash_manager()
# db.store_all(word, all_emails, 'email', 'PGP')
# except Exception:
# pass
#
# print('[*] Searching Threatcrowd.')
# try:
# search = threatcrowd.SearchThreatcrowd(word)
# search.process()
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'threatcrowd')
# except Exception:
# pass
#
# print('[*] Searching Trello.')
# from discovery import trello
# # Import locally or won't work.
# search = trello.SearchTrello(word, limit)
# search.process()
# emails = filter(search.get_emails())
# all_emails.extend(emails)
# info = search.get_urls()
# hosts = filter(info[0])
# trello_info = (info[1], True)
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, hosts, 'host', 'trello')
# db.store_all(word, emails, 'email', 'trello')
#
# try:
# print('[*] Searching Twitter.')
# search = twittersearch.SearchTwitter(word, limit)
# search.process()
# people = search.get_people()
# db = stash.stash_manager()
# db.store_all(word, people, 'name', 'twitter')
# print('\nUsers from Twitter:')
# print('-------------------')
# for user in people:
# print(user)
# except Exception:
# pass
#
# print('\n[*] Virtual hosts:')
# print('------------------')
# for l in host_ip:
# search = bingsearch.SearchBing(l, limit, start)
# search.process_vhost()
# res = search.get_allhostnames()
# for x in res:
# x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
# x = re.sub('<', '', x)
# x = re.sub('>', '', x)
# print((l + '\t' + x))
# vhost.append(l + ':' + x)
# full.append(l + ':' + x)
# vhost = sorted(set(vhost))
#
# print('[*] Searching VirusTotal.')
# search = virustotal.search_virustotal(word)
# search.process()
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'virustotal')
#
# print('[*] Searching Yahoo.')
# search = yahoosearch.SearchYahoo(word, limit)
# search.process()
# hosts = search.get_hostnames()
# emails = search.get_emails()
# all_hosts.extend(filter(hosts))
# all_emails.extend(filter(emails))
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'yahoo')
# db.store_all(word, all_emails, 'email', 'yahoo')