mirror of
https://github.com/laramies/theHarvester.git
synced 2024-11-10 17:13:07 +08:00
fix for #88 which allows multiple search engines usage
This commit is contained in:
jzold
parent
4f902a688c
commit
3c23242b44
1 changed files with 244 additions and 236 deletions
480
theHarvester.py
480
theHarvester.py
|
|
@ -115,246 +115,254 @@ def start(argv):
|
|||
elif opt == '-t':
|
||||
dnstld = True
|
||||
elif opt == '-b':
|
||||
engine = arg
|
||||
if engine not in ("baidu", "bing", "crtsh","bingapi","dogpile","google", "googleCSE","virustotal","threatcrowd", "googleplus", "google-profiles","linkedin", "pgp", "twitter", "vhost", "yahoo","netcraft","all"):
|
||||
engines = set(arg.split(','))
|
||||
supportedengines = set(["baidu","bing","crtsh","bingapi","dogpile","google","googleCSE","virustotal","threatcrowd","googleplus","google-profiles","linkedin","pgp","twitter","vhost","yahoo","netcraft","all"])
|
||||
if set(engines).issubset(supportedengines):
|
||||
print "found supported engines"
|
||||
print "[-] Starting harvesting process for domain: " + word + "\n"
|
||||
for engineitem in engines:
|
||||
if engineitem == "google":
|
||||
print "[-] Searching in Google:"
|
||||
search = googlesearch.search_google(word, limit, start)
|
||||
search.process()
|
||||
all_emails = search.get_emails()
|
||||
all_hosts = search.get_hostnames()
|
||||
for x in all_hosts:
|
||||
try:
|
||||
db=stash.stash_manager()
|
||||
db.store(word,x,'host','google')
|
||||
except Exception, e:
|
||||
print e
|
||||
|
||||
if engineitem == "netcraft":
|
||||
print "[-] Searching in Netcraft:"
|
||||
search = netcraft.search_netcraft(word)
|
||||
search.process()
|
||||
all_hosts = search.get_hostnames()
|
||||
all_emails = []
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','netcraft')
|
||||
|
||||
|
||||
if engineitem == "threatcrowd":
|
||||
print "[-] Searching in Threatcrowd:"
|
||||
search = threatcrowd.search_threatcrowd(word)
|
||||
search.process()
|
||||
all_hosts = search.get_hostnames()
|
||||
all_emails = []
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','threatcrowd')
|
||||
|
||||
if engineitem == "virustotal":
|
||||
print "[-] Searching in Virustotal:"
|
||||
search = virustotal.search_virustotal(word)
|
||||
search.process()
|
||||
all_hosts = search.get_hostnames()
|
||||
all_emails = []
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','virustotal')
|
||||
|
||||
|
||||
if engineitem == "crtsh":
|
||||
print "[-] Searching in CRT.sh:"
|
||||
search = crtsh.search_crtsh(word)
|
||||
search.process()
|
||||
all_hosts = search.get_hostnames()
|
||||
all_emails = []
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','CRTsh')
|
||||
|
||||
if engineitem == "googleCSE":
|
||||
print "[-] Searching in Google Custom Search:"
|
||||
search = googleCSE.search_googleCSE(word, limit, start)
|
||||
search.process()
|
||||
search.store_results()
|
||||
all_emails = search.get_emails()
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'email','googleCSE')
|
||||
all_hosts = search.get_hostnames()
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','googleCSE')
|
||||
|
||||
elif engineitem == "bing" or engineitem == "bingapi":
|
||||
print "[-] Searching in Bing:"
|
||||
search = bingsearch.search_bing(word, limit, start)
|
||||
if engineitem == "bingapi":
|
||||
bingapi = "yes"
|
||||
else:
|
||||
bingapi = "no"
|
||||
search.process(bingapi)
|
||||
all_emails = search.get_emails()
|
||||
all_hosts = search.get_hostnames()
|
||||
|
||||
elif engineitem == "dogpile":
|
||||
print "[-] Searching in Dogpilesearch.."
|
||||
search = dogpilesearch.search_dogpile(word, limit)
|
||||
search.process()
|
||||
all_emails = search.get_emails()
|
||||
all_hosts = search.get_hostnames()
|
||||
|
||||
elif engineitem == "pgp":
|
||||
print "[-] Searching in PGP key server.."
|
||||
search = pgpsearch.search_pgp(word)
|
||||
search.process()
|
||||
all_emails = search.get_emails()
|
||||
all_hosts = search.get_hostnames()
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','pgp')
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_emails,'emails','pgp')
|
||||
|
||||
elif engineitem == "yahoo":
|
||||
print "[-] Searching in Yahoo.."
|
||||
search = yahoosearch.search_yahoo(word, limit)
|
||||
search.process()
|
||||
all_emails = search.get_emails()
|
||||
all_hosts = search.get_hostnames()
|
||||
|
||||
# elif engineitem == "baidu":
|
||||
print "[-] Searching in Baidu.."
|
||||
search = baidusearch.search_baidu(word, limit)
|
||||
search.process()
|
||||
all_emails = search.get_emails()
|
||||
all_hosts = search.get_hostnames()
|
||||
|
||||
elif engineitem == "googleplus":
|
||||
print "[-] Searching in Google+ .."
|
||||
search = googleplussearch.search_googleplus(word, limit)
|
||||
search.process()
|
||||
people = search.get_people()
|
||||
print "Users from Google+:"
|
||||
print "===================="
|
||||
for user in people:
|
||||
print user
|
||||
sys.exit()
|
||||
|
||||
elif engineitem == "twitter":
|
||||
print "[-] Searching in Twitter .."
|
||||
search = twittersearch.search_twitter(word, limit)
|
||||
search.process()
|
||||
people = search.get_people()
|
||||
print "Users from Twitter:"
|
||||
print "-------------------"
|
||||
for user in people:
|
||||
print user
|
||||
sys.exit()
|
||||
|
||||
elif engineitem == "linkedin":
|
||||
print "[-] Searching in Linkedin.."
|
||||
search = linkedinsearch.search_linkedin(word, limit)
|
||||
search.process()
|
||||
people = search.get_people()
|
||||
print "Users from Linkedin:"
|
||||
print "-------------------"
|
||||
for user in people:
|
||||
print user
|
||||
sys.exit()
|
||||
|
||||
elif engineitem == "google-profiles":
|
||||
print "[-] Searching in Google profiles.."
|
||||
search = googlesearch.search_google(word, limit, start)
|
||||
search.process_profiles()
|
||||
people = search.get_profiles()
|
||||
print "Users from Google profiles:"
|
||||
print "---------------------------"
|
||||
for users in people:
|
||||
print users
|
||||
sys.exit()
|
||||
|
||||
elif engineitem == "all":
|
||||
print "Full harvest on " + word
|
||||
all_emails = []
|
||||
all_hosts = []
|
||||
|
||||
print "[-] Searching in Google.."
|
||||
search = googlesearch.search_google(word, limit, start)
|
||||
search.process()
|
||||
emails = search.get_emails()
|
||||
hosts = search.get_hostnames()
|
||||
all_emails.extend(emails)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'email','google')
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','google')
|
||||
|
||||
print "[-] Searching in PGP Key server.."
|
||||
search = pgpsearch.search_pgp(word)
|
||||
search.process()
|
||||
emails = search.get_emails()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','PGP')
|
||||
all_emails.extend(emails)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'email','PGP')
|
||||
|
||||
print "[-] Searching in Netcraft server.."
|
||||
search = netcraft.search_netcraft(word)
|
||||
search.process()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','netcraft')
|
||||
|
||||
print "[-] Searching in ThreatCrowd server.."
|
||||
search = threatcrowd.search_threatcrowd(word)
|
||||
search.process()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
all_emails = []
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','threatcrowd')
|
||||
|
||||
search = netcraft.search_netcraft(word)
|
||||
search.process()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','netcraft')
|
||||
|
||||
|
||||
print "[-] Searching in CRTSH server.."
|
||||
search = crtsh.search_crtsh(word)
|
||||
search.process()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','CRTsh')
|
||||
|
||||
print "[-] Searching in Virustotal server.."
|
||||
search = virustotal.search_virustotal(word)
|
||||
search.process()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','virustotal')
|
||||
|
||||
print "[-] Searching in Bing.."
|
||||
bingapi = "no"
|
||||
search = bingsearch.search_bing(word, limit, start)
|
||||
search.process(bingapi)
|
||||
emails = search.get_emails()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','bing')
|
||||
all_emails.extend(emails)
|
||||
#Clean up email list, sort and uniq
|
||||
all_emails=sorted(set(all_emails))
|
||||
else:
|
||||
|
||||
#if engine not in ("baidu", "bing", "crtsh","bingapi","dogpile","google", "googleCSE","virustotal","threatcrowd", "googleplus", "google-profiles","linkedin", "pgp", "twitter", "vhost", "yahoo","netcraft","all"):
|
||||
usage()
|
||||
print "Invalid search engine, try with: baidu, bing, bingapi, crtsh, dogpile, google, googleCSE, virustotal, netcraft, googleplus, google-profiles, linkedin, pgp, twitter, vhost, yahoo, all"
|
||||
sys.exit()
|
||||
else:
|
||||
pass
|
||||
print "[-] Starting harvesting process for domain: " + word + "\n"
|
||||
if engine == "google":
|
||||
print "[-] Searching in Google:"
|
||||
search = googlesearch.search_google(word, limit, start)
|
||||
search.process()
|
||||
all_emails = search.get_emails()
|
||||
all_hosts = search.get_hostnames()
|
||||
for x in all_hosts:
|
||||
try:
|
||||
db=stash.stash_manager()
|
||||
db.store(word,x,'host','google')
|
||||
except Exception, e:
|
||||
print e
|
||||
#else:
|
||||
# pass
|
||||
|
||||
|
||||
if engine == "netcraft":
|
||||
print "[-] Searching in Netcraft:"
|
||||
search = netcraft.search_netcraft(word)
|
||||
search.process()
|
||||
all_hosts = search.get_hostnames()
|
||||
all_emails = []
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','netcraft')
|
||||
|
||||
|
||||
if engine == "threatcrowd":
|
||||
print "[-] Searching in Threatcrowd:"
|
||||
search = threatcrowd.search_threatcrowd(word)
|
||||
search.process()
|
||||
all_hosts = search.get_hostnames()
|
||||
all_emails = []
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','threatcrowd')
|
||||
|
||||
if engine == "virustotal":
|
||||
print "[-] Searching in Virustotal:"
|
||||
search = virustotal.search_virustotal(word)
|
||||
search.process()
|
||||
all_hosts = search.get_hostnames()
|
||||
all_emails = []
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','virustotal')
|
||||
|
||||
|
||||
if engine == "crtsh":
|
||||
print "[-] Searching in CRT.sh:"
|
||||
search = crtsh.search_crtsh(word)
|
||||
search.process()
|
||||
all_hosts = search.get_hostnames()
|
||||
all_emails = []
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','CRTsh')
|
||||
|
||||
if engine == "googleCSE":
|
||||
print "[-] Searching in Google Custom Search:"
|
||||
search = googleCSE.search_googleCSE(word, limit, start)
|
||||
search.process()
|
||||
search.store_results()
|
||||
all_emails = search.get_emails()
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'email','googleCSE')
|
||||
all_hosts = search.get_hostnames()
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','googleCSE')
|
||||
|
||||
elif engine == "bing" or engine == "bingapi":
|
||||
print "[-] Searching in Bing:"
|
||||
search = bingsearch.search_bing(word, limit, start)
|
||||
if engine == "bingapi":
|
||||
bingapi = "yes"
|
||||
else:
|
||||
bingapi = "no"
|
||||
search.process(bingapi)
|
||||
all_emails = search.get_emails()
|
||||
all_hosts = search.get_hostnames()
|
||||
|
||||
elif engine == "dogpile":
|
||||
print "[-] Searching in Dogpilesearch.."
|
||||
search = dogpilesearch.search_dogpile(word, limit)
|
||||
search.process()
|
||||
all_emails = search.get_emails()
|
||||
all_hosts = search.get_hostnames()
|
||||
|
||||
elif engine == "pgp":
|
||||
print "[-] Searching in PGP key server.."
|
||||
search = pgpsearch.search_pgp(word)
|
||||
search.process()
|
||||
all_emails = search.get_emails()
|
||||
all_hosts = search.get_hostnames()
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','pgp')
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_emails,'emails','pgp')
|
||||
|
||||
elif engine == "yahoo":
|
||||
print "[-] Searching in Yahoo.."
|
||||
search = yahoosearch.search_yahoo(word, limit)
|
||||
search.process()
|
||||
all_emails = search.get_emails()
|
||||
all_hosts = search.get_hostnames()
|
||||
|
||||
elif engine == "baidu":
|
||||
print "[-] Searching in Baidu.."
|
||||
search = baidusearch.search_baidu(word, limit)
|
||||
search.process()
|
||||
all_emails = search.get_emails()
|
||||
all_hosts = search.get_hostnames()
|
||||
|
||||
elif engine == "googleplus":
|
||||
print "[-] Searching in Google+ .."
|
||||
search = googleplussearch.search_googleplus(word, limit)
|
||||
search.process()
|
||||
people = search.get_people()
|
||||
print "Users from Google+:"
|
||||
print "===================="
|
||||
for user in people:
|
||||
print user
|
||||
sys.exit()
|
||||
|
||||
elif engine == "twitter":
|
||||
print "[-] Searching in Twitter .."
|
||||
search = twittersearch.search_twitter(word, limit)
|
||||
search.process()
|
||||
people = search.get_people()
|
||||
print "Users from Twitter:"
|
||||
print "-------------------"
|
||||
for user in people:
|
||||
print user
|
||||
sys.exit()
|
||||
|
||||
elif engine == "linkedin":
|
||||
print "[-] Searching in Linkedin.."
|
||||
search = linkedinsearch.search_linkedin(word, limit)
|
||||
search.process()
|
||||
people = search.get_people()
|
||||
print "Users from Linkedin:"
|
||||
print "-------------------"
|
||||
for user in people:
|
||||
print user
|
||||
sys.exit()
|
||||
|
||||
elif engine == "google-profiles":
|
||||
print "[-] Searching in Google profiles.."
|
||||
search = googlesearch.search_google(word, limit, start)
|
||||
search.process_profiles()
|
||||
people = search.get_profiles()
|
||||
print "Users from Google profiles:"
|
||||
print "---------------------------"
|
||||
for users in people:
|
||||
print users
|
||||
sys.exit()
|
||||
|
||||
elif engine == "all":
|
||||
print "Full harvest on " + word
|
||||
all_emails = []
|
||||
all_hosts = []
|
||||
|
||||
print "[-] Searching in Google.."
|
||||
search = googlesearch.search_google(word, limit, start)
|
||||
search.process()
|
||||
emails = search.get_emails()
|
||||
hosts = search.get_hostnames()
|
||||
all_emails.extend(emails)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'email','google')
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','google')
|
||||
|
||||
print "[-] Searching in PGP Key server.."
|
||||
search = pgpsearch.search_pgp(word)
|
||||
search.process()
|
||||
emails = search.get_emails()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','PGP')
|
||||
all_emails.extend(emails)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'email','PGP')
|
||||
|
||||
print "[-] Searching in Netcraft server.."
|
||||
search = netcraft.search_netcraft(word)
|
||||
search.process()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','netcraft')
|
||||
|
||||
print "[-] Searching in ThreatCrowd server.."
|
||||
search = threatcrowd.search_threatcrowd(word)
|
||||
search.process()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
all_emails = []
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','threatcrowd')
|
||||
|
||||
search = netcraft.search_netcraft(word)
|
||||
search.process()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','netcraft')
|
||||
|
||||
|
||||
print "[-] Searching in CRTSH server.."
|
||||
search = crtsh.search_crtsh(word)
|
||||
search.process()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','CRTsh')
|
||||
|
||||
print "[-] Searching in Virustotal server.."
|
||||
search = virustotal.search_virustotal(word)
|
||||
search.process()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','virustotal')
|
||||
|
||||
print "[-] Searching in Bing.."
|
||||
bingapi = "no"
|
||||
search = bingsearch.search_bing(word, limit, start)
|
||||
search.process(bingapi)
|
||||
emails = search.get_emails()
|
||||
hosts = search.get_hostnames()
|
||||
all_hosts.extend(hosts)
|
||||
db=stash.stash_manager()
|
||||
db.store_all(word,all_hosts,'host','bing')
|
||||
all_emails.extend(emails)
|
||||
#Clean up email list, sort and uniq
|
||||
all_emails=sorted(set(all_emails))
|
||||
|
||||
#Results############################################################
|
||||
print("\n\033[1;32;40m Harvesting results")
|
||||
|
|
|
|||
Loading…
Reference in a new issue