laramies/theHarvester
1
1
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2024-09-20 15:26:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

Syncing.

Browse source
This commit is contained in:
NotoriousRebel 2018-12-15 22:07:37 -05:00
parent b1d979d335
commit f3597ee267
2 changed files with 169 additions and 204 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

83
discovery/censys.py
View file

@ -1,89 +1,63 @@
import random import random
import requests import requests
import time
import censysparser import censysparser
class search_censys: class search_censys:
def __init__(self, word, limit): def __init__(self, word):
self.word = word self.word = word
self.limit = int(limit) self.url = ""
self.page = ""
self.results = "" self.results = ""
self.total_results = "" self.total_results = ""
self.server = "https://censys.io/" self.server = "censys.io"
self.userAgent = ["(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", self.userAgent = ["(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36"
,("Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) " + , ("Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) " +
"AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Mobile Safari/537.36"), "AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Mobile Safari/537.36"),
("Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) " + ("Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) " +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254"), "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254"),
"Mozilla/5.0 (SMART-TV; X11; Linux armv7l) AppleWebKit/537.42 (KHTML, like Gecko) Chromium/25.0.1349.2 Chrome/25.0.1349.2 Safari/537.42" "Mozilla/5.0 (SMART-TV; X11; Linux armv7l) AppleWebKit/537.42 (KHTML, like Gecko) Chromium/25.0.1349.2 Chrome/25.0.1349.2 Safari/537.42"
,"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 OPR/43.0.2442.991" ,
,"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 OPR/48.0.2685.52" "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 OPR/43.0.2442.991"
,"Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko" ,
,"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 OPR/48.0.2685.52"
,"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)"] , "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"
, "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko"
, "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)"]
def do_search(self): def do_search(self):
try: try:
self.url = self.server + 'ipv4/_search?q=' + self.word headers = {'user-agent': random.choice(self.userAgent), 'Accept': '*/*', 'Referer': self.url}
headers = {'user-agent': random.choice(self.userAgent),'Accept':'*/*','Referer': self.url}
response = requests.get(self.url, headers=headers) response = requests.get(self.url, headers=headers)
<<<<<<< HEAD
self.results = response.content self.results = response.content
print ('-') print('-')
self.total_results += self.results self.total_results += self.results
print ('-') print('-')
except Exception as e: except Exception as e:
print(e) print(e)
def process(self): def process(self):
self.url="https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=1" self.url = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=1"
self.do_search() self.do_search()
self.counter=2 self.counter = 2
pages = censysparser.parser(self) pages = censysparser.parser(self)
totalpages = pages.search_numberofpages() totalpages = pages.search_numberofpages()
while self.counter <= totalpages: while self.counter <= totalpages:
try: try:
self.page =str(self.counter) self.page = str(self.counter)
self.url="https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page) self.url = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page)
print("\tSearching Censys results page " + self.page + "...") print("\tSearching Censys results page " + self.page + "...")
self.do_search() self.do_search()
except Exception as e: except Exception as e:
print("Error occurred: " + str(e)) print("Error occurred: " + str(e))
self.counter+=1 self.counter += 1
=======
print("\tSearching Censys results..")
self.results = response.text
self.total_results += self.results
pageLimit = self.get_pageLimit(self.total_results)
if pageLimit != -1:
for i in range(2, pageLimit+1):
try:
url = self.server + 'ipv4?q=' + self.word + '&page=' + str(i)
headers = {'user-agent': random.choice(self.userAgent), 'Accept': '*/*', 'Referer': url}
time.sleep(.5)
response = requests.get(url, headers=headers)
self.results = response.text
self.total_results += self.results
except Exception:
continue
except Exception as e:
print(e)
def get_pageLimit(self, first_page_text):
for line in str(first_page_text).strip().splitlines():
if 'Page:' in line:
line = line[18:] #where format is Page:1/# / is at index 18 and want everything after /
return int(line)
return -1
>>>>>>> 8953b4d1006153c1c82cea52d4776c1f87cd42da
def get_hostnames(self): def get_hostnames(self):
try: try:
hostnames = censysparser.parser(self) hostnames = censysparser.parser(self)
return hostnames.search_hostnames(self.total_results) return hostnames.search_hostnames()
except Exception as e: except Exception as e:
print("Error occurred: " + str(e)) print("Error occurred: " + str(e))
@ -93,4 +67,3 @@ def get_ipaddresses(self):
return ips.search_ipaddresses() return ips.search_ipaddresses()
except Exception as e: except Exception as e:
print("Error occurred: " + str(e)) print("Error occurred: " + str(e))

290
theHarvester.py
View file

@ -12,7 +12,7 @@
except: except:
print("Requests library not found, please install it before proceeding\n") print("Requests library not found, please install it before proceeding\n")
sys.exit() sys.exit()
try: try:
import bs4 import bs4
except: except:
@ -38,7 +38,6 @@
def usage(): def usage():
comm = os.path.basename(sys.argv[0]) comm = os.path.basename(sys.argv[0])
if os.path.dirname(sys.argv[0]) == os.getcwd(): if os.path.dirname(sys.argv[0]) == os.getcwd():
@ -72,7 +71,6 @@ def usage():
def start(argv): def start(argv):
if len(sys.argv) < 4: if len(sys.argv) < 4:
usage() usage()
sys.exit() sys.exit()
@ -82,7 +80,7 @@ def start(argv):
usage() usage()
sys.exit() sys.exit()
try: try:
db=stash.stash_manager() db = stash.stash_manager()
db.do_init() db.do_init()
except Exception as e: except Exception as e:
pass pass
@ -100,7 +98,7 @@ def start(argv):
takeover_check = False takeover_check = False
google_dorking = False google_dorking = False
limit = 500 limit = 500
full = [] full = []
dnsserver = "" dnsserver = ""
for value in enumerate(opts): for value in enumerate(opts):
opt = value[1][0] opt = value[1][0]
@ -133,10 +131,13 @@ def start(argv):
dnstld = True dnstld = True
elif opt == '-b': elif opt == '-b':
engines = set(arg.split(',')) engines = set(arg.split(','))
supportedengines = set(["baidu","bing","crtsh","censys","cymon","bingapi","dogpile","google","googleCSE","virustotal","threatcrowd","googleplus","google-profiles",'google-certificates',"linkedin","pgp","twitter","trello","vhost","yahoo","netcraft","hunter","all"]) supportedengines = set(
["baidu", "bing", "crtsh", "censys", "cymon", "bingapi", "dogpile", "google", "googleCSE", "virustotal",
"threatcrowd", "googleplus", "google-profiles", 'google-certificates', "linkedin", "pgp", "twitter",
"trello", "vhost", "yahoo", "netcraft", "hunter", "all"])
if set(engines).issubset(supportedengines): if set(engines).issubset(supportedengines):
print("found supported engines") print("found supported engines")
print(("[-] Starting harvesting process for domain: " + word + "\n")) print(("[-] Starting harvesting process for domain: " + word + "\n"))
for engineitem in engines: for engineitem in engines:
if engineitem == "google": if engineitem == "google":
print("[-] Searching in Google:") print("[-] Searching in Google:")
@ -146,29 +147,28 @@ def start(argv):
all_hosts = search.get_hostnames() all_hosts = search.get_hostnames()
for x in all_hosts: for x in all_hosts:
try: try:
db=stash.stash_manager() db = stash.stash_manager()
db.store(word,x,'host','google') db.store(word, x, 'host', 'google')
except Exception as e: except Exception as e:
print(e) print(e)
if engineitem == "netcraft": if engineitem == "netcraft":
print("[-] Searching in Netcraft:") print("[-] Searching in Netcraft:")
search = netcraft.search_netcraft(word) search = netcraft.search_netcraft(word)
search.process() search.process()
all_hosts = search.get_hostnames() all_hosts = search.get_hostnames()
all_emails = [] all_emails = []
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','netcraft') db.store_all(word, all_hosts, 'host', 'netcraft')
if engineitem == "google-certificates": if engineitem == "google-certificates":
print ("[-] Searching in Google Certificate transparency report..") print("[-] Searching in Google Certificate transparency report..")
search = googlecertificates.search_googlecertificates(word, limit, start) search = googlecertificates.search_googlecertificates(word, limit, start)
search.process() search.process()
all_hosts = search.get_domains() all_hosts = search.get_domains()
all_emails = [] all_emails = []
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','google-certificates') db.store_all(word, all_hosts, 'host', 'google-certificates')
if engineitem == "threatcrowd": if engineitem == "threatcrowd":
print("[-] Searching in Threatcrowd:") print("[-] Searching in Threatcrowd:")
@ -176,18 +176,17 @@ def start(argv):
search.process() search.process()
all_hosts = search.get_hostnames() all_hosts = search.get_hostnames()
all_emails = [] all_emails = []
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','threatcrowd') db.store_all(word, all_hosts, 'host', 'threatcrowd')
if engineitem == "virustotal": if engineitem == "virustotal":
print("[-] Searching in Virustotal:") print("[-] Searching in Virustotal:")
search = virustotal.search_virustotal(word) search = virustotal.search_virustotal(word)
search.process() search.process()
all_hosts = search.get_hostnames() all_hosts = search.get_hostnames()
all_emails = [] all_emails = []
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','virustotal') db.store_all(word, all_hosts, 'host', 'virustotal')
if engineitem == "crtsh": if engineitem == "crtsh":
print("[-] Searching in CRT.sh:") print("[-] Searching in CRT.sh:")
@ -195,8 +194,8 @@ def start(argv):
search.process() search.process()
all_hosts = search.get_hostnames() all_hosts = search.get_hostnames()
all_emails = [] all_emails = []
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','CRTsh') db.store_all(word, all_hosts, 'host', 'CRTsh')
if engineitem == "googleCSE": if engineitem == "googleCSE":
print("[-] Searching in Google Custom Search:") print("[-] Searching in Google Custom Search:")
@ -204,11 +203,11 @@ def start(argv):
search.process() search.process()
search.store_results() search.store_results()
all_emails = search.get_emails() all_emails = search.get_emails()
db=stash.stash_manager() db = stash.stash_manager()
all_hosts = search.get_hostnames() all_hosts = search.get_hostnames()
db.store_all(word,all_hosts,'email','googleCSE') db.store_all(word, all_hosts, 'email', 'googleCSE')
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','googleCSE') db.store_all(word, all_hosts, 'host', 'googleCSE')
elif engineitem == "bing" or engineitem == "bingapi": elif engineitem == "bing" or engineitem == "bingapi":
print("[-] Searching in Bing:") print("[-] Searching in Bing:")
@ -234,10 +233,10 @@ def start(argv):
search.process() search.process()
all_emails = search.get_emails() all_emails = search.get_emails()
all_hosts = search.get_hostnames() all_hosts = search.get_hostnames()
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','pgp') db.store_all(word, all_hosts, 'host', 'pgp')
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_emails,'emails','pgp') db.store_all(word, all_emails, 'emails', 'pgp')
elif engineitem == "yahoo": elif engineitem == "yahoo":
print("[-] Searching in Yahoo..") print("[-] Searching in Yahoo..")
@ -245,9 +244,9 @@ def start(argv):
search.process() search.process()
all_emails = search.get_emails() all_emails = search.get_emails()
all_hosts = search.get_hostnames() all_hosts = search.get_hostnames()
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','yahoo') db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word,all_emails,'emails','yahoo') db.store_all(word, all_emails, 'emails', 'yahoo')
elif engineitem == "baidu": elif engineitem == "baidu":
@ -256,9 +255,9 @@ def start(argv):
search.process() search.process()
all_emails = search.get_emails() all_emails = search.get_emails()
all_hosts = search.get_hostnames() all_hosts = search.get_hostnames()
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','baidu') db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word,all_emails,'emails','baidu') db.store_all(word, all_emails, 'emails', 'baidu')
elif engineitem == "googleplus": elif engineitem == "googleplus":
print("[-] Searching in Google+ ..") print("[-] Searching in Google+ ..")
@ -307,7 +306,7 @@ def start(argv):
elif engineitem == "hunter": elif engineitem == "hunter":
print("[-] Searching in Hunter:") print("[-] Searching in Hunter:")
from discovery import huntersearch from discovery import huntersearch
#import locally or won't work # import locally or won't work
search = huntersearch.search_hunter(word, limit, start) search = huntersearch.search_hunter(word, limit, start)
search.process() search.process()
all_emails = search.get_emails() all_emails = search.get_emails()
@ -316,81 +315,76 @@ def start(argv):
elif engineitem == "censys": elif engineitem == "censys":
print("[-] Searching in Censys:") print("[-] Searching in Censys:")
from discovery import censys from discovery import censys
#import locally or won't work # import locally or won't work
<<<<<<< HEAD
search = censys.search_censys(word) search = censys.search_censys(word)
search.process() search.process()
=======
search = censys.search_censys(word, limit)
search.do_search()
>>>>>>> 8953b4d1006153c1c82cea52d4776c1f87cd42da
all_emails = [] all_emails = []
all_ip = search.get_ipaddresses() all_ip = search.get_ipaddresses()
all_hosts = search.get_hostnames() all_hosts = search.get_hostnames()
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','censys') db.store_all(word, all_hosts, 'host', 'censys')
db.store_all(word,all_ip,'ip','censys') db.store_all(word, all_ip, 'ip', 'censys')
elif engineitem == "cymon": elif engineitem == "cymon":
print("[-] Searching in Cymon:") print("[-] Searching in Cymon:")
from discovery import cymon from discovery import cymon
#import locally or won't work # import locally or won't work
search = cymon.search_cymon(word) search = cymon.search_cymon(word)
search.process() search.process()
all_emails = [] all_emails = []
all_hosts = [] all_hosts = []
all_ip = search.get_ipaddresses() all_ip = search.get_ipaddresses()
db = stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_ip,'ip','cymon') db.store_all(word, all_ip, 'ip', 'cymon')
elif engineitem == "trello": elif engineitem == "trello":
print("[-] Searching in Trello:") print("[-] Searching in Trello:")
from discovery import trello from discovery import trello
#import locally or won't work # import locally or won't work
search = trello.search_trello(word,limit) search = trello.search_trello(word, limit)
search.process() search.process()
all_emails = search.get_emails() all_emails = search.get_emails()
all_hosts = search.get_urls() all_hosts = search.get_urls()
for x in all_hosts: for x in all_hosts:
print (x) print(x)
sys.exit() sys.exit()
elif engineitem == "all": elif engineitem == "all":
print(("Full harvest on " + word)) print(("Full harvest on " + word))
all_emails = [] all_emails = []
all_hosts = [] all_hosts = []
print("[-] Searching in Google..") print("[-] Searching in Google..")
search = googlesearch.search_google(word, limit, start) search = googlesearch.search_google(word, limit, start)
search.process(google_dorking) search.process(google_dorking)
emails = search.get_emails() emails = search.get_emails()
hosts = search.get_hostnames() hosts = search.get_hostnames()
all_emails.extend(emails) all_emails.extend(emails)
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'email','google') db.store_all(word, all_hosts, 'email', 'google')
all_hosts.extend(hosts) all_hosts.extend(hosts)
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','google') db.store_all(word, all_hosts, 'host', 'google')
print("[-] Searching in PGP Key server..") print("[-] Searching in PGP Key server..")
search = pgpsearch.search_pgp(word) search = pgpsearch.search_pgp(word)
search.process() search.process()
emails = search.get_emails() emails = search.get_emails()
hosts = search.get_hostnames() hosts = search.get_hostnames()
all_hosts.extend(hosts) all_hosts.extend(hosts)
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','PGP') db.store_all(word, all_hosts, 'host', 'PGP')
all_emails.extend(emails) all_emails.extend(emails)
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'email','PGP') db.store_all(word, all_hosts, 'email', 'PGP')
print("[-] Searching in Netcraft server..") print("[-] Searching in Netcraft server..")
search = netcraft.search_netcraft(word) search = netcraft.search_netcraft(word)
search.process() search.process()
hosts = search.get_hostnames() hosts = search.get_hostnames()
all_hosts.extend(hosts) all_hosts.extend(hosts)
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','netcraft') db.store_all(word, all_hosts, 'host', 'netcraft')
print("[-] Searching in ThreatCrowd server..") print("[-] Searching in ThreatCrowd server..")
try: try:
@ -399,25 +393,26 @@ def start(argv):
hosts = search.get_hostnames() hosts = search.get_hostnames()
all_hosts.extend(hosts) all_hosts.extend(hosts)
all_emails = [] all_emails = []
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','threatcrowd') db.store_all(word, all_hosts, 'host', 'threatcrowd')
except Exception: pass except Exception:
pass
print("[-] Searching in CRTSH server..") print("[-] Searching in CRTSH server..")
search = crtsh.search_crtsh(word) search = crtsh.search_crtsh(word)
search.process() search.process()
hosts = search.get_hostnames() hosts = search.get_hostnames()
all_hosts.extend(hosts) all_hosts.extend(hosts)
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','CRTsh') db.store_all(word, all_hosts, 'host', 'CRTsh')
print("[-] Searching in Virustotal server..") print("[-] Searching in Virustotal server..")
search = virustotal.search_virustotal(word) search = virustotal.search_virustotal(word)
search.process() search.process()
hosts = search.get_hostnames() hosts = search.get_hostnames()
all_hosts.extend(hosts) all_hosts.extend(hosts)
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','virustotal') db.store_all(word, all_hosts, 'host', 'virustotal')
print("[-] Searching in Bing..") print("[-] Searching in Bing..")
bingapi = "no" bingapi = "no"
@ -426,15 +421,15 @@ def start(argv):
emails = search.get_emails() emails = search.get_emails()
hosts = search.get_hostnames() hosts = search.get_hostnames()
all_hosts.extend(hosts) all_hosts.extend(hosts)
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_hosts,'host','bing') db.store_all(word, all_hosts, 'host', 'bing')
all_emails.extend(emails) all_emails.extend(emails)
#Clean up email list, sort and uniq # Clean up email list, sort and uniq
all_emails=sorted(set(all_emails)) all_emails = sorted(set(all_emails))
print("[-] Searching in Hunter:") print("[-] Searching in Hunter:")
from discovery import huntersearch from discovery import huntersearch
#import locally # import locally
search = huntersearch.search_hunter(word, limit, start) search = huntersearch.search_hunter(word, limit, start)
search.process() search.process()
emails = search.get_emails() emails = search.get_emails()
@ -445,12 +440,12 @@ def start(argv):
all_emails.extend(emails) all_emails.extend(emails)
all_emails = sorted(set(all_emails)) all_emails = sorted(set(all_emails))
print ("[-] Searching in Google Certificate transparency report..") print("[-] Searching in Google Certificate transparency report..")
search = googlecertificates.search_googlecertificates(word, limit, start) search = googlecertificates.search_googlecertificates(word, limit, start)
search.process() search.process()
domains = search.get_domains() domains = search.get_domains()
all_hosts.extend(domains) all_hosts.extend(domains)
print("[-] Searching in Cymon:") print("[-] Searching in Cymon:")
from discovery import cymon from discovery import cymon
search = cymon.search_cymon(word) search = cymon.search_cymon(word)
@ -458,7 +453,7 @@ def start(argv):
all_emails = [] all_emails = []
all_ip = search.get_ipaddresses() all_ip = search.get_ipaddresses()
db = stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_ip,'ip','cymon') db.store_all(word, all_ip, 'ip', 'cymon')
print("[-] Searching in Censys:") print("[-] Searching in Censys:")
from discovery import censys from discovery import censys
@ -468,15 +463,16 @@ def start(argv):
all_ip = search.get_ipaddresses() all_ip = search.get_ipaddresses()
all_hosts = search.get_hostnames() all_hosts = search.get_hostnames()
db = stash.stash_manager() db = stash.stash_manager()
db.store_all(word,all_ip,'ip','censys') db.store_all(word, all_ip, 'ip', 'censys')
db.store_all(word,all_hosts,'host','censys') db.store_all(word, all_hosts, 'host', 'censys')
else: else:
usage() usage()
print("Invalid search engine, try with: baidu, bing, bingapi, crtsh, censys, cymon, dogpile, google, googleCSE, virustotal, netcraft, googleplus, google-profiles, linkedin, pgp, twitter, vhost, yahoo, hunter, all") print(
"Invalid search engine, try with: baidu, bing, bingapi, crtsh, censys, cymon, dogpile, google, googleCSE, virustotal, netcraft, googleplus, google-profiles, linkedin, pgp, twitter, vhost, yahoo, hunter, all")
sys.exit() sys.exit()
#Results############################################################ # Results############################################################
print("\n\033[1;32;40mHarvesting results") print("\n\033[1;32;40mHarvesting results")
if (len(all_ip) == 0): if (len(all_ip) == 0):
print("No IP addresses found") print("No IP addresses found")
@ -488,7 +484,7 @@ def start(argv):
print("\n\n[+] Emails found:") print("\n\n[+] Emails found:")
print("------------------") print("------------------")
#Sanity check to see if all_emails and all_hosts is defined # Sanity check to see if all_emails and all_hosts is defined
try: try:
all_emails all_emails
except NameError: except NameError:
@ -500,24 +496,19 @@ def start(argv):
print('No hosts found as all_hosts is not defined.') print('No hosts found as all_hosts is not defined.')
sys.exit() sys.exit()
<<<<<<< HEAD
if all_emails == []: if all_emails == []:
=======
if all_emails == [] or all_emails is None:
>>>>>>> 8953b4d1006153c1c82cea52d4776c1f87cd42da
print("No emails found") print("No emails found")
else: else:
print(("\n".join(all_emails))) print(("\n".join(all_emails)))
print("\033[1;33;40m \n[+] Hosts found in search engines:") print("\033[1;33;40m \n[+] Hosts found in search engines:")
print("------------------------------------") print("------------------------------------")
if all_hosts == [] or all_hosts is None: if all_hosts == [] or all_emails is None:
print("No hosts found") print("No hosts found")
else: else:
total = len(all_hosts) total = len(all_hosts)
print(("\nTotal hosts: " + str(total) + "\n")) print(("\nTotal hosts: " + str(total) + "\n"))
all_hosts=sorted(set(all_hosts)) all_hosts = sorted(set(all_hosts))
print("\033[94m[-] Resolving hostnames IPs...\033[1;33;40m \n ") print("\033[94m[-] Resolving hostnames IPs...\033[1;33;40m \n ")
full_host = hostchecker.Checker(all_hosts) full_host = hostchecker.Checker(all_hosts)
full = full_host.check() full = full_host.check()
@ -529,8 +520,8 @@ def start(argv):
pass pass
else: else:
host_ip.append(ip.lower()) host_ip.append(ip.lower())
#DNS Brute force#################################################### # DNS Brute force####################################################
dnsres = [] dnsres = []
if dnsbrute == True: if dnsbrute == True:
print("\n\033[94m[-] Starting DNS brute force: \033[1;33;40m") print("\n\033[94m[-] Starting DNS brute force: \033[1;33;40m")
@ -543,33 +534,33 @@ def start(argv):
dnsres.append(y.split(':')[0]) dnsres.append(y.split(':')[0])
if y not in full: if y not in full:
full.append(y) full.append(y)
db=stash.stash_manager() db = stash.stash_manager()
db.store_all(word,dnsres,'host','dns_bruteforce') db.store_all(word, dnsres, 'host', 'dns_bruteforce')
#Port Scanning ################################################# # Port Scanning #################################################
if ports_scanning == True: if ports_scanning == True:
print("\n\n\033[1;32;40m[-] Scanning ports (active):\n") print("\n\n\033[1;32;40m[-] Scanning ports (active):\n")
for x in full: for x in full:
host = x.split(':')[1] host = x.split(':')[1]
domain = x.split(':')[0] domain = x.split(':')[0]
if host != "empty" : if host != "empty":
print(("- Scanning : " + host)) print(("- Scanning : " + host))
ports = [80,443,22,8080,21] ports = [80, 443, 22, 8080, 21]
try: try:
scan = port_scanner.port_scan(host,ports) scan = port_scanner.port_scan(host, ports)
openports = scan.process() openports = scan.process()
if len(openports) > 1: if len(openports) > 1:
print(("\t\033[91m Detected open ports: " + ','.join(str(e) for e in openports) + "\033[1;32;40m")) print(("\t\033[91m Detected open ports: " + ','.join(
takeover_check = 'True' str(e) for e in openports) + "\033[1;32;40m"))
if takeover_check == 'True': takeover_check = 'True'
if len(openports) > 0: if takeover_check == 'True':
search_take = takeover.take_over(domain) if len(openports) > 0:
search_take.process() search_take = takeover.take_over(domain)
except Exception as e: search_take.process()
print(e) except Exception as e:
print(e)
#DNS reverse lookup################################################# # DNS reverse lookup#################################################
dnsrev = [] dnsrev = []
if dnslookup == True: if dnslookup == True:
print("\n[+] Starting active queries:") print("\n[+] Starting active queries:")
@ -598,8 +589,8 @@ def start(argv):
print("---------------------------------") print("---------------------------------")
for xh in dnsrev: for xh in dnsrev:
print(xh) print(xh)
#DNS TLD expansion################################################### # DNS TLD expansion###################################################
dnstldres = [] dnstldres = []
if dnstld == True: if dnstld == True:
print("[-] Starting DNS TLD expansion:") print("[-] Starting DNS TLD expansion:")
@ -613,7 +604,7 @@ def start(argv):
if y not in full: if y not in full:
full.append(y) full.append(y)
#Virtual hosts search############################################### # Virtual hosts search###############################################
if virtual == "basic": if virtual == "basic":
print("\n[+] Virtual hosts:") print("\n[+] Virtual hosts:")
print("------------------") print("------------------")
@ -622,22 +613,22 @@ def start(argv):
search.process_vhost() search.process_vhost()
res = search.get_allhostnames() res = search.get_allhostnames()
for x in res: for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*','',x) x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<','',x) x = re.sub('<', '', x)
x = re.sub('>','',x) x = re.sub('>', '', x)
print((l + "\t" + x)) print((l + "\t" + x))
vhost.append(l + ":" + x) vhost.append(l + ":" + x)
full.append(l + ":" + x) full.append(l + ":" + x)
vhost=sorted(set(vhost)) vhost = sorted(set(vhost))
else: else:
pass pass
#Shodan search#################################################### # Shodan search####################################################
shodanres = [] shodanres = []
shodanvisited = [] shodanvisited = []
if shodan == True: if shodan == True:
print("\n\n\033[1;32;40m[-] Shodan DB search (passive):\n") print("\n\n\033[1;32;40m[-] Shodan DB search (passive):\n")
if full ==[]: if full == []:
print ('No host to search, exiting.') print('No host to search, exiting.')
sys.exit() sys.exit()
for x in full: for x in full:
@ -648,15 +639,16 @@ def start(argv):
a = shodansearch.search_shodan(ip) a = shodansearch.search_shodan(ip)
shodanvisited.append(ip) shodanvisited.append(ip)
results = a.run() results = a.run()
#time.sleep(2) # time.sleep(2)
for res in results['data']: for res in results['data']:
shodanres.append(str("%s:%s - %s - %s - %s," % (res['ip_str'], res['port'],res['os'],res['isp']))) shodanres.append(
str("%s:%s - %s - %s - %s," % (res['ip_str'], res['port'], res['os'], res['isp'])))
except Exception as e: except Exception as e:
pass pass
print("\n [+] Shodan results:") print("\n [+] Shodan results:")
print("------------------") print("------------------")
for x in shodanres: for x in shodanres:
print (x) print(x)
else: else:
pass pass
@ -676,7 +668,7 @@ def start(argv):
else: else:
pass pass
#Reporting####################################################### # Reporting#######################################################
if filename != "": if filename != "":
try: try:
print("[+] Saving files...") print("[+] Saving files...")
@ -704,13 +696,13 @@ def start(argv):
for x in full: for x in full:
x = x.split(":") x = x.split(":")
if len(x) == 2: if len(x) == 2:
file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' + x[0] + '</hostname>' + '</host>') file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' + x[0] + '</hostname>' + '</host>')
else: else:
file.write('<host>' + x + '</host>') file.write('<host>' + x + '</host>')
for x in vhost: for x in vhost:
x = x.split(":") x = x.split(":")
if len(x) == 2: if len(x) == 2:
file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' + x[0] + '</hostname>' + '</vhost>') file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' + x[0] + '</hostname>' + '</vhost>')
else: else:
file.write('<vhost>' + x + '</vhost>') file.write('<vhost>' + x + '</vhost>')
@ -722,28 +714,26 @@ def start(argv):
# print " res[1] " + res[1] # banner/info # print " res[1] " + res[1] # banner/info
# print " res[2] " + res[2] # port # print " res[2] " + res[2] # port
file.write('<shodan>') file.write('<shodan>')
#page.h3(res[0]) # page.h3(res[0])
file.write('<host>' + res[0] + '</host>') file.write('<host>' + res[0] + '</host>')
#page.a("Port :" + res[2]) # page.a("Port :" + res[2])
file.write('<port>' + res[2] + '</port>') file.write('<port>' + res[2] + '</port>')
#page.pre(res[1]) # page.pre(res[1])
file.write('<banner><!--' + res[1] + '--></banner>') file.write('<banner><!--' + res[1] + '--></banner>')
reg_server = re.compile('Server:.*') reg_server = re.compile('Server:.*')
temp = reg_server.findall(res[1]) temp = reg_server.findall(res[1])
if temp != []: if temp != []:
shodanalysis.append(res[0] + ":" + temp[0]) shodanalysis.append(res[0] + ":" + temp[0])
file.write('</shodan>') file.write('</shodan>')
if shodanalysis != []: if shodanalysis != []:
shodanalysis=sorted(set(shodanalysis)) shodanalysis = sorted(set(shodanalysis))
file.write('<servers>') file.write('<servers>')
for x in shodanalysis: for x in shodanalysis:
#page.pre(x) # page.pre(x)
file.write('<server>' + x + '</server>') file.write('<server>' + x + '</server>')
file.write('</servers>') file.write('</servers>')
file.write('</theHarvester>') file.write('</theHarvester>')
file.flush() file.flush()
@ -753,6 +743,7 @@ def start(argv):
print(("Error saving XML file: " + str(er))) print(("Error saving XML file: " + str(er)))
sys.exit() sys.exit()
if __name__ == "__main__": if __name__ == "__main__":
try: try:
start(sys.argv[1:]) start(sys.argv[1:])
@ -760,5 +751,6 @@ def start(argv):
print("Search interrupted by user..") print("Search interrupted by user..")
except Exception: except Exception:
import traceback import traceback
print(traceback.print_exc()) print(traceback.print_exc())
sys.exit() sys.exit()