laramies/theHarvester
1
1
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2025-02-25 23:13:24 +08:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #151 from L1ghtn1ng/dev

Browse source 
Misc changes
This commit is contained in:
Christian Martorella 2019-01-09 00:36:12 +01:00 committed by GitHub
commit 807821f1a7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
43 changed files with 639 additions and 578 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -5,7 +5,5 @@
.vscode
.xml
debug_results.txt
discovery/constants.py
stash.sqlite
tests/myparser.py
venv

2
README.md
View file

@ -6,7 +6,7 @@
* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *
* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *
* *
* theHarvester 3.0.6 v137 *
* theHarvester 3.0.6 v138 *
* Coded by Christian Martorella *
* Edge-Security Research *
* cmartorella@edge-security.com *

1
discovery/__init__.py
View file

@ -1,4 +1,5 @@
__all__ = ["bingsearch",
"duckduckgosearch",
"googlesearch",
"pgpsearch",
"linkedinsearch",

9
discovery/asksearch.py
View file

@ -3,9 +3,10 @@
import requests
import time
from discovery.constants import *
from lib.core import *
class search_ask:
class SearchAsk:
def __init__(self, word, limit):
self.word = word.replace(' ', '%20')
@ -19,9 +20,9 @@ def __init__(self, word, limit):
def do_search(self):
headers = {
'User-agent': getUserAgent()
'User-agent': Core.get_user_agent()
}
url = 'http://' + self.server + '/web?q=%40' + self.word \
url = 'https://' + self.server + '/web?q=%40' + self.word \
+ "&pu=100&page=" + str(self.counter)
h = requests.get(url=url, headers=headers)
time.sleep(getDelay())
@ -38,7 +39,7 @@ def check_next(self):
return nexty
def get_people(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.people_jigsaw()
def process(self):

11
discovery/baidusearch.py
View file

@ -2,9 +2,10 @@
import time
import requests
from discovery.constants import *
from lib.core import *
class search_baidu:
class SearchBaidu:
def __init__(self, word, limit):
self.word = word
@ -18,7 +19,7 @@ def do_search(self):
url = 'http://' + self.server + "/s?wd=%40" + self.word + "&pn=" + str(self.counter) + "&oq=" + self.word
headers = {
'Host': self.hostname,
'User-agent': getUserAgent()
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
time.sleep(getDelay())
@ -27,13 +28,13 @@ def do_search(self):
def process(self):
while self.counter <= self.limit and self.counter <= 1000:
self.do_search()
print("\tSearching " + str(self.counter) + " results...")
print(f'\tSearching {self.counter} results...')
self.counter += 10
def get_emails(self):
rawres = myparser.parser(self.total_results, self.word)
rawres = myparser.Parser(self.total_results, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.parser(self.total_results, self.word)
rawres = myparser.Parser(self.total_results, self.word)
return rawres.hostnames()

23
discovery/bingsearch.py
View file

@ -2,9 +2,10 @@
import time
import requests
from discovery.constants import *
from lib.core import *
class search_bing:
class SearchBing:
def __init__(self, word, limit, start):
self.word = word.replace(' ', '%20')
@ -21,11 +22,11 @@ def __init__(self, word, limit, start):
def do_search(self):
headers = {
'Host': self.hostname,
'Cookie':'SRCHHPGUSR=ADLT=DEMOTE&NRSLT=50',
'Cookie': 'SRCHHPGUSR=ADLT=DEMOTE&NRSLT=50',
'Accept-Language': 'en-us,en',
'User-agent': getUserAgent()
'User-agent': Core.get_user_agent()
}
h = requests.get(url=('http://'+self.server + "/search?q=%40" + self.word + "&count=50&first=" + str(self.counter)),headers=headers)
h = requests.get(url=('http://'+self.server + "/search?q=%40" + self.word + "&count=50&first=" + str(self.counter)), headers=headers)
self.results = h.text
self.totalresults += self.results
@ -38,7 +39,7 @@ def do_search_api(self):
'mkt': 'en-us',
'safesearch': 'Off'
}
headers = {'User-Agent': getUserAgent(), 'Ocp-Apim-Subscription-Key': self.bingApi}
headers = {'User-Agent': Core.get_user_agent(), 'Ocp-Apim-Subscription-Key': self.bingApi}
h = requests.get(url=url, headers=headers, params=params)
self.results = h.text
self.totalresults += self.results
@ -48,7 +49,7 @@ def do_search_vhost(self):
'Host': self.hostname,
'Cookie': 'mkt=en-US;ui=en-US;SRCHHPGUSR=NEWWND=0&ADLT=DEMOTE&NRSLT=50',
'Accept-Language': 'en-us,en',
'User-agent': getUserAgent()
'User-agent': Core.get_user_agent()
}
url = 'http://' + self.server + "/search?q=ip:" + self.word + "&go=&count=50&FORM=QBHL&qs=n&first=" + str(self.counter)
h = requests.get(url=url, headers=headers)
@ -56,15 +57,15 @@ def do_search_vhost(self):
self.totalresults += self.results
def get_emails(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames()
def get_allhostnames(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames_all()
def process(self, api):
@ -79,10 +80,10 @@ def process(self, api):
self.do_search()
time.sleep(getDelay())
self.counter += 50
print("\tSearching " + str(self.counter) + " results...")
print(f'\tSearching {self.counter} results...')
def process_vhost(self):
# Maybe it is good to use other limit for this.
while self.counter < self.limit:
self.do_search_vhost()
self.counter += 50
self.counter += 50

38
discovery/censys.py
View file

@ -1,9 +1,9 @@
import requests
from parsers import censysparser
from discovery.constants import *
from lib.core import *
class search_censys:
class SearchCensys:
def __init__(self, word, limit):
self.word = word
@ -21,21 +21,21 @@ def __init__(self, word, limit):
def do_searchhosturl(self):
try:
headers = {'user-agent': getUserAgent(), 'Accept': '*/*', 'Referer': self.urlhost}
headers = {'user-agent': Core.get_user_agent(), 'Accept': '*/*', 'Referer': self.urlhost}
responsehost = requests.get(self.urlhost, headers=headers)
self.resultshosts = responsehost.text
self.total_resultshosts += self.resultshosts
except Exception as e:
print("Error occurred in the Censys module downloading pages from Censys - IP search: " + str(e))
print(f'Error occurred in the Censys module downloading pages from Censys - IP search: + {e}')
def do_searchcertificateurl(self):
try:
headers = {'user-agent': getUserAgent(), 'Accept': '*/*', 'Referer': self.urlcert}
headers = {'user-agent': Core.get_user_agent(), 'Accept': '*/*', 'Referer': self.urlcert}
responsecert = requests.get(self.urlcert, headers=headers)
self.resultcerts = responsecert.text
self.total_resultscerts += self.resultcerts
except Exception as e:
print("Error occurred in the Censys module downloading pages from Censys - certificates search: " + str(e))
print(f'Error occurred in the Censys module downloading pages from Censys - certificates search: {e}')
def process(self):
try:
@ -44,7 +44,7 @@ def process(self):
self.do_searchhosturl()
self.do_searchcertificateurl()
counter = 2
pages = censysparser.parser(self)
pages = censysparser.Parser(self)
totalpages = pages.search_totalpageshosts()
pagestosearch = int(self.limit / 25) # 25 results/page
if totalpages <= pagestosearch:
@ -57,18 +57,18 @@ def process(self):
self.do_searchhosturl()
counter += 1
except Exception as e:
print("Error occurred in the Censys module requesting the pages: " + str(e))
print(f'Error occurred in the Censys module requesting the pages: {e}')
else:
while counter <= pagestosearch:
try:
self.page = str(counter)
self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(
self.page)
print("\tSearching Censys IP results page " + self.page + "...")
print(f'\tSearching Censys IP results page {self.page} ...')
self.do_searchhosturl()
counter += 1
except Exception as e:
print("Error occurred in the Censys module requesting the pages: " + str(e))
print(f'Error occurred in the Censys module requesting the pages: {e}')
counter = 2
totalpages = pages.search_totalpagescerts()
if totalpages <= pagestosearch:
@ -77,11 +77,11 @@ def process(self):
self.page = str(counter)
self.urlhost = "https://" + self.server + "/certificates/_search?q=" + str(
self.word) + "&page=" + str(self.page)
print("\tSearching Censys certificates results page " + self.page + "...")
print(f'\tSearching Censys certificates results page {self.page} ...')
self.do_searchcertificateurl()
counter += 1
except Exception as e:
print("Error occurred in the Censys module requesting the pages: " + str(e))
print(f'Error occurred in the Censys module requesting the pages: {e}')
else:
while counter <= pagestosearch:
try:
@ -92,15 +92,15 @@ def process(self):
self.do_searchhosturl()
counter += 1
except Exception as e:
print("Error occurred in the Censys module requesting the pages: " + str(e))
print(f'Error occurred in the Censys module requesting the pages: {e}')
except Exception as e:
print("Error occurred in the main Censys module: " + str(e))
print(f'Error occurred in the main Censys module: {e}')
def get_hostnames(self):
try:
ips = self.get_ipaddresses()
headers = {'user-agent': getUserAgent(), 'Accept': '*/*', 'Referer': self.urlcert}
headers = {'user-agent': Core.get_user_agent(), 'Accept': '*/*', 'Referer': self.urlcert}
response = requests.post("https://censys.io/ipv4/getdns", json={"ips": ips}, headers=headers)
responsejson = response.json()
domainsfromcensys = []
@ -111,16 +111,16 @@ def get_hostnames(self):
pass
matchingdomains = [s for s in domainsfromcensys if str(self.word) in s]
self.hostnamesall.extend(matchingdomains)
hostnamesfromcerts = censysparser.parser(self)
hostnamesfromcerts = censysparser.Parser(self)
self.hostnamesall.extend(hostnamesfromcerts.search_hostnamesfromcerts())
return self.hostnamesall
except Exception as e:
print("Error occurred in the Censys module - hostname search: " + str(e))
print(f'Error occurred in the Censys module - hostname search: {e}')
def get_ipaddresses(self):
try:
ips = censysparser.parser(self)
ips = censysparser.Parser(self)
self.ips = ips.search_ipaddresses()
return self.ips
except Exception as e:
print("Error occurred in the main Censys module - IP address search: " + str(e))
print(f'Error occurred in the main Censys module - IP address search: {e}')

235
discovery/constants.py
View file

@ -21,237 +21,6 @@
shodanAPI_key = 'oCiMsgM6rQWqiTvPxFHYcExlZgg7wvTt' # this is the default key
user_agents = [
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_0) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1216.0 Safari/537.2",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)",
"Mozilla/5.0 (Linux; U; Android 2.3; en-us) AppleWebKit/999+ (KHTML, like Gecko) Safari/999.9",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36",
"Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; ja) Opera 11.00",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.17 Safari/537.11",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/536.5 (KHTML like Gecko) Chrome/19.0.1084.56 Safari/1EA69",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10",
"Mozilla/5.0 (Linux; U; Android 2.3.5; zh-cn; HTC_IncredibleS_S710e Build/GRJ90) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36",
"Mozilla/5.0 (X11; CrOS i686 1660.57.0) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.46 Safari/535.19",
"Mozilla/5.0 (Windows NT 6.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01",
"Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/4.0; GTB7.4; InfoPath.3; SV1; .NET CLR 3.1.76908; WOW64; en-US)",
"Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)",
"Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS)",
"Opera/9.80 (Windows NT 5.1; U; en) Presto/2.9.168 Version/11.51",
"Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01",
"Opera/9.80 (Windows NT 5.1; U; MRA 5.5 (build 02842); ru) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Opera/9.80 (Windows NT 6.1; U; cs) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
"Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00",
"Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.7.39 Version/11.00",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)",
"Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; de) Presto/2.9.168 Version/11.52",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)",
"Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/537.11",
"Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10",
"Opera/9.80 (Windows NT 6.1; U; en-US) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)",
"Mozilla/5.0 (Linux; U; Android 2.3.4; fr-fr; HTC Desire Build/GRJ22) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Opera/9.80 (X11; Linux i686; U; fr) Presto/2.7.62 Version/11.01",
"Mozilla/4.0 (compatible; MSIE 8.0; X11; Linux x86_64; pl) Opera 11.00",
"Opera/9.80 (X11; Linux i686; U; hu) Presto/2.9.168 Version/11.50",
"Opera/9.80 (X11; Linux x86_64; U; bg) Presto/2.8.131 Version/11.10",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14",
"Opera/9.80 (X11; Linux i686; U; it) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.11 Safari/535.19",
"Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Opera/9.80 (X11; Linux i686; U; es-ES) Presto/2.8.131 Version/11.11",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; fr) Opera 11.00",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.1; SV1; .NET CLR 2.8.52393; WOW64; en-US)",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13",
"Opera/9.80 (Windows NT 5.1; U; it) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Linux; U; Android 2.3.3; ko-kr; LG-LU3000 Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Opera/9.80 (Windows NT 6.1; U; fi) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02",
"Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; pl) Opera 11.00",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
"Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)",
"Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36",
"Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
"Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1",
"Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00",
"Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; de) Opera 11.01",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; en) Opera 11.00",
"Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Opera/9.80 (Windows NT 6.1; U; ko) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7",
"Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36",
"Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.6.37 Version/11.00",
"Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
"Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13",
"Mozilla/5.0 (Windows NT 6.0; U; ja; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)",
"Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/19.0.1047.0 Safari/535.22",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3)",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
"Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36",
"Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/5.0 Opera 11.11",
"Mozilla/5.0 (Macintosh; AMD Mac OS X 10_8_2) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/18.6.872",
"Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)",
"Mozilla/5.0 (Linux; U; Android 4.0.3; de-ch; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.0) yi; AppleWebKit/345667.12221 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/453667.1221",
"Mozilla/1.22 (compatible; MSIE 10.0; Windows 3.1)",
"Opera/9.80 (Windows NT 5.1; U;) Presto/2.7.62 Version/11.01",
"Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16",
"Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21",
"Mozilla/5.0 (Linux; U; Android 2.3.5; en-us; HTC Vision Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F",
"Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.151 Safari/535.19",
"Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1284.0 Safari/537.13",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)",
"Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0)",
"Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00",
"Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
"Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36",
"Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10",
"Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00",
"Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari",
"Mozilla/5.0 (Windows NT 5.1; U; pl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00",
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; de) Opera 11.51",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11",
"Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19",
"Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/18.6.872.0 Safari/535.2 UNTRUSTED/1.0 3gpp-gba UNTRUSTED/1.0",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36",
"Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Linux; U; Android 2.3.4; en-us; T-Mobile myTouch 3G Slide Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1041.0 Safari/535.21",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.19 (KHTML, like Gecko) Ubuntu/11.10 Chromium/18.0.1025.142 Chrome/18.0.1025.142 Safari/535.19",
"Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00"
]
def filter(lst):
"""
@ -278,10 +47,6 @@ def getDelay():
return random.randint(1, 3) - .5
def getUserAgent():
return random.choice(user_agents)
def search(text):
# helper function to check if google has blocked traffic
for line in text.strip().splitlines():

11
discovery/crtsh.py
View file

@ -2,6 +2,7 @@
from parsers import myparser
import time
from discovery.constants import *
from lib.core import *
class search_crtsh:
@ -20,13 +21,13 @@ def do_search(self):
except Exception as e:
print(e)
try:
params = {'User-Agent': getUserAgent()}
r=requests.get(urly,headers=params)
params = {'User-Agent': Core.get_user_agent()}
r=requests.get(urly, headers=params)
except Exception as e:
print(e)
links = self.get_info(r.text)
for link in links:
params = {'User-Agent': getUserAgent()}
params = {'User-Agent': Core.get_user_agent()}
r = requests.get(link, headers=params)
time.sleep(getDelay())
self.results = r.text
@ -37,7 +38,7 @@ def do_search(self):
@param text requests text
@return list of links
"""
def get_info(self,text):
def get_info(self, text):
lines = []
for line in str(text).splitlines():
line = line.strip()
@ -58,7 +59,7 @@ def get_info(self,text):
return links
def get_hostnames(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames()
def process(self):

11
discovery/cymon.py
View file

@ -1,6 +1,7 @@
import requests
from parsers import cymonparser
from discovery.constants import *
from lib.core import *
import time
@ -14,7 +15,7 @@ def __init__(self, word):
def do_search(self):
try:
headers = {'user-agent': getUserAgent(), 'Accept':'*/*', 'Referer': self.url}
headers = {'user-agent': Core.get_user_agent(), 'Accept': '*/*', 'Referer': self.url}
response = requests.get(self.url, headers=headers)
time.sleep(getDelay())
self.results = response.content
@ -24,14 +25,14 @@ def do_search(self):
def process(self):
try:
self.url = "https://" + self.server + "/domain/" + str(self.word)
print("\tSearching Cymon results...")
print('\tSearching Cymon results...')
self.do_search()
except Exception as e:
print("Error occurred: " + str(e))
print(f'Error occurred: {e}')
def get_ipaddresses(self):
try:
ips = cymonparser.parser(self)
ips = cymonparser.Parser(self)
return ips.search_ipaddresses()
except Exception as e:
print("Error occurred: " + str(e))
print(f'Error occurred: {e}')

12
discovery/dnssearch.py
View file

@ -73,7 +73,7 @@ def __init__(self, domain, dnsserver, verbose=False):
res_path = os.path.join(fileDir,'lib/resolvers.txt')
with open(res_path) as f:
self.resolvers = f.read().splitlines()
except Exception as e:
except Exception:
print("Resolvers file can't be open")
try:
f = open(self.file, "r")
@ -86,7 +86,7 @@ def getdns(self, domain):
DNS.ParseResolvConf("/etc/resolv.conf")
# nameserver=DNS.defaults['server'][0]
dom = domain
if self.subdo == True:
if self.subdo is True:
dom = domain.split(".")
dom.pop(0)
rootdom = ".".join(dom)
@ -139,7 +139,7 @@ def run(self, host):
# TODO FIX test is sometimes not getting answers and leads to an indexing erro
hostip = test.answers[0]['data']
return hostname + ":" + hostip
except Exception as e:
except Exception:
pass
def process(self):
@ -190,13 +190,13 @@ def getdns(self, domain):
# DNS.ParseResolvConf("/etc/resolv.conf")
# nameserver=DNS.defaults['server'][0]
dom = domain
if self.subdo == True:
if self.subdo is True:
dom = domain.split(".")
dom.pop(0)
rootdom = ".".join(dom)
else:
rootdom = dom
if self.nameserver == False:
if self.nameserver is False:
r = DNS.Request(rootdom, qtype='SOA').req()
primary, email, serial, refresh, retry, expire, minimum = r.answers[
0]['data']
@ -225,7 +225,7 @@ def run(self, tld):
)
hostip = test.answers[0]['data']
return hostip + ":" + hostname
except Exception as e:
except Exception:
pass
def process(self):

12
discovery/dogpilesearch.py
View file

@ -2,8 +2,10 @@
import time
import requests
from discovery.constants import *
from lib.core import *
class search_dogpile:
class SearchDogpile:
def __init__(self, word, limit):
self.word = word
@ -19,7 +21,7 @@ def do_search(self):
+ "&q=\"%40" + self.word + "\""
headers = {
'Host': self.hostname,
'User-agent': getUserAgent()
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
self.total_results += h.text
@ -28,13 +30,13 @@ def process(self):
while self.counter <= self.limit and self.counter <= 1000:
self.do_search()
time.sleep(getDelay())
print("\tSearching " + str(self.counter) + " results...")
print(f'\tSearching {self.counter} results...')
self.counter += 10
def get_emails(self):
rawres = myparser.parser(self.total_results, self.word)
rawres = myparser.Parser(self.total_results, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.parser(self.total_results, self.word)
rawres = myparser.Parser(self.total_results, self.word)
return rawres.hostnames()

181
discovery/duckduckgosearch.py
View file

@ -1,90 +1,91 @@
from parsers import myparser
import time
import requests
import json
from discovery.constants import *
class search_duckduckgo:
def __init__(self, word, limit):
self.word = word
self.results = ""
self.totalresults = ""
self.dorks = []
self.links = []
self.database = "https://duckduckgo.com/?q="
self.api = "https://api.duckduckgo.com/?q=x&format=json&pretty=1" # currently using api
self.quantity = "100"
self.limit = limit
def do_search(self):
try: # do normal scraping
url = self.api.replace('x', self.word)
headers = {'User-Agent': googleUA}
r = requests.get(url, headers=headers)
except Exception as e:
print(e)
time.sleep(getDelay())
self.results = r.text
self.totalresults += self.results
urls = self.crawl(self.results)
for url in urls:
try:
self.totalresults += requests.get(url, headers={'User-Agent': getUserAgent()}).text
time.sleep(getDelay())
except Exception:
continue
def crawl(self, text):
"""
function parses json and returns urls
:param text: formatted json
:return: set of urls
"""
urls = set()
try:
load = json.loads(text)
for key in load.keys(): # iterate through keys of dict
val = load.get(key)
if isinstance(val, int) or isinstance(val, dict):
continue
if isinstance(val, list):
val = val[0] # first value should be dict
if isinstance(val, dict): # sanity check
for key in val.keys():
value = val.get(key)
if isinstance(value, str) and value != '' and 'https://' in value or 'http://' in value:
urls.add(value)
if isinstance(val, str) and val != '' and 'https://' in val or 'http://' in val:
urls.add(val)
tmp = set()
for url in urls:
if '<' in url and 'href=' in url: # format is <href="https://www.website.com"/>
equal_index = url.index('=')
true_url = ''
for ch in url[equal_index + 1:]:
if ch == '"':
tmp.add(true_url)
break
true_url += ch
else:
if url != '':
tmp.add(url)
return tmp
except Exception as e:
print('Exception occurred: ' + str(e))
import traceback as t
print(t.print_exc())
return []
def get_emails(self):
rawres = myparser.parser(self.totalresults, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.parser(self.totalresults, self.word)
return rawres.hostnames()
def process(self):
self.do_search() # only need to search once since using API
from parsers import myparser
import time
import requests
import json
from discovery.constants import *
from lib.core import *
class SearchDuckDuckGo:
def __init__(self, word, limit):
self.word = word
self.results = ""
self.totalresults = ""
self.dorks = []
self.links = []
self.database = "https://duckduckgo.com/?q="
self.api = "https://api.duckduckgo.com/?q=x&format=json&pretty=1" # currently using api
self.quantity = "100"
self.limit = limit
def do_search(self):
try: # do normal scraping
url = self.api.replace('x', self.word)
headers = {'User-Agent': googleUA}
r = requests.get(url, headers=headers)
except Exception as e:
print(e)
time.sleep(getDelay())
self.results = r.text
self.totalresults += self.results
urls = self.crawl(self.results)
for url in urls:
try:
self.totalresults += requests.get(url, headers={'User-Agent': Core.get_user_agent()}).text
time.sleep(getDelay())
except Exception:
continue
def crawl(self, text):
"""
function parses json and returns urls
:param text: formatted json
:return: set of urls
"""
urls = set()
try:
load = json.loads(text)
for key in load.keys(): # iterate through keys of dict
val = load.get(key)
if isinstance(val, int) or isinstance(val, dict):
continue
if isinstance(val, list):
val = val[0] # first value should be dict
if isinstance(val, dict): # sanity check
for key in val.keys():
value = val.get(key)
if isinstance(value, str) and value != '' and 'https://' in value or 'http://' in value:
urls.add(value)
if isinstance(val, str) and val != '' and 'https://' in val or 'http://' in val:
urls.add(val)
tmp = set()
for url in urls:
if '<' in url and 'href=' in url: # format is <href="https://www.website.com"/>
equal_index = url.index('=')
true_url = ''
for ch in url[equal_index + 1:]:
if ch == '"':
tmp.add(true_url)
break
true_url += ch
else:
if url != '':
tmp.add(url)
return tmp
except Exception as e:
print(f'Exception occurred: {e}')
import traceback as t
print(t.print_exc())
return []
def get_emails(self):
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames()
def process(self):
self.do_search() # only need to search once since using API

16
discovery/exaleadsearch.py
View file

@ -3,6 +3,8 @@
import time
import requests
from discovery.constants import *
from lib.core import *
class search_exalead:
def __init__(self, word, limit, start):
@ -20,8 +22,8 @@ def do_search(self):
+ "&elements_per_page=50&start_index=" + str(self.counter)
headers = {
'Host': self.hostname,
'Referer': ("http://" +self.hostname +"/search/web/results/?q=%40" +self.word),
'User-agent': getUserAgent()
'Referer': ("http://" + self.hostname + "/search/web/results/?q=%40" + self.word),
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
self.results = h.text
@ -33,7 +35,7 @@ def do_search_files(self, files):
headers = {
'Host': self.hostname,
'Referer': ("http://" + self.hostname + "/search/web/results/?q=%40" + self.word),
'User-agent': getUserAgent()
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
self.results = h.text
@ -50,22 +52,22 @@ def check_next(self):
return nexty
def get_emails(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames()
def get_files(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.fileurls(self.files)
def process(self):
while self.counter <= self.limit:
self.do_search()
self.counter += 50
print("\ tSearching " + str(self.counter) + " results...")
print(f'\tSearching {self.counter} results...')
def process_files(self, files):
while self.counter < self.limit:

8
discovery/googleCSE.py
View file

@ -6,7 +6,7 @@
from discovery.constants import *
class search_googleCSE:
class SearchGoogleCSE:
def __init__(self, word, limit, start):
self.word = word
@ -63,15 +63,15 @@ def check_next(self):
return nexty
def get_emails(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames()
def get_files(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.fileurls(self.files)
def process(self):

7
discovery/googlecertificates.py
View file

@ -1,8 +1,9 @@
import requests
import json
from discovery.constants import *
from lib.core import *
class search_googlecertificates:
class SearchGoogleCertificates:
# https://www.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch?include_expired=true&include_subdomains=true&domain=
def __init__(self, word, limit, start):
self.word = word
@ -19,7 +20,7 @@ def do_search(self):
except Exception as e:
print(e)
try:
headers = {'User-Agent': getUserAgent()}
headers = {'User-Agent': Core.get_user_agent()}
r = requests.get(urly, headers=headers)
except Exception as e:
print(e)

14
discovery/googlesearch.py
View file

@ -55,26 +55,26 @@ def do_search_profiles(self):
self.totalresults += self.results
def get_emails(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames()
def get_files(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.fileurls(self.files)
def get_profiles(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.profiles()
def process(self, google_dorking):
if google_dorking is False:
while self.counter <= self.limit and self.counter <= 1000:
self.do_search()
print("\tSearching " + str(self.counter) + " results...")
print(f'\tSearching {self.counter} results...')
self.counter += 100
else: # google dorking is true
self.counter = 0 # reset counter
@ -82,7 +82,7 @@ def process(self, google_dorking):
print("[-] Searching with Google Dorks: ")
while self.counter <= self.limit and self.counter <= 200: # only 200 dorks in list
self.googledork() # call google dorking method if user wanted it!
print("\tSearching " + str(self.counter) + " results...")
print(f'\tSearching {self.counter} results...')
self.counter += 100
def process_profiles(self):
@ -90,7 +90,7 @@ def process_profiles(self):
self.do_search_profiles()
time.sleep(getDelay())
self.counter += 100
print("\tSearching " + str(self.counter) + " results...")
print(f'\tSearching {self.counter} results...')
def append_dorks(self):
try: # wrap in try-except incase filepaths are messed up

6
discovery/googlesets.py
View file

@ -1,6 +1,6 @@
from parsers import myparser
import requests
from discovery.constants import *
from lib.core import *
class search_google_labs:
@ -23,14 +23,14 @@ def do_search(self):
url = 'http://' + self.server + "/sets?hl-en&" + self.set
headers = {
'Host': self.server,
'User-agent': getUserAgent()
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
self.results = h.text
self.totalresults += self.results
def get_set(self):
rawres = myparser.parser(self.totalresults, list)
rawres = myparser.Parser(self.totalresults, list)
return rawres.set()
def process(self):

12
discovery/huntersearch.py
View file

@ -3,7 +3,7 @@
from discovery.constants import *
class search_hunter:
class SearchHunter:
def __init__(self, word, limit, start):
self.word = word
@ -15,7 +15,7 @@ def __init__(self, word, limit, start):
self.results = ""
self.totalresults = ""
self.counter = start
self.database = "https://api.hunter.io/v2/domain-search?domain=" + word + "&api_key=" + self.key +"&limit=" + str(self.limit)
self.database = "https://api.hunter.io/v2/domain-search?domain=" + word + "&api_key=" + self.key + "&limit=" + str(self.limit)
def do_search(self):
try:
@ -30,13 +30,13 @@ def process(self):
print('\tDone Searching Results')
def get_emails(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames()
def get_profiles(self):
rawres = myparser.parser(self.totalresults, self.word)
return rawres.profiles()
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.profiles()

5
discovery/jigsaw.py
View file

@ -3,6 +3,7 @@
import requests
import time
from discovery.constants import *
from lib.core import *
# http://www.jigsaw.com/SearchAcrossCompanies.xhtml?opCode=refresh&rpage=4&mode=0&cnCountry=&order=0&orderby=0&cmName=accuvant&cnDead=false&cnExOwned=false&count=0&screenNameType=0&screenName=&omitScreenNameType=0&omitScreenName=&companyId=0&estimatedCount=277&rowsPerPage=50
@ -22,7 +23,7 @@ def __init__(self, word, limit):
def do_search(self):
url = 'http://' + self.server + "/FreeTextSearch.xhtml?opCode=search&autoSuggested=True&freeText=" + self.word
headers = {
'User-agent': getUserAgent()
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
self.results = h.text
@ -38,7 +39,7 @@ def check_next(self):
return nexty
def get_people(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.people_jigsaw()
def process(self):

13
discovery/linkedinsearch.py
View file

@ -1,10 +1,11 @@
import requests
from parsers import myparser
from discovery.constants import *
from lib.core import *
import time
class search_linkedin:
class SearchLinkedin:
def __init__(self, word, limit):
self.word = word.replace(' ', '%20')
@ -18,19 +19,19 @@ def __init__(self, word, limit):
def do_search(self):
try:
urly = "http://"+ self.server + "/search?num=100&start=" + str(self.counter) + "&hl=en&meta=&q=site%3Alinkedin.com/in%20" + self.word
urly = "http://" + self.server + "/search?num=100&start=" + str(self.counter) + "&hl=en&meta=&q=site%3Alinkedin.com/in%20" + self.word
except Exception as e:
print(e)
try:
headers = {'User-Agent': getUserAgent()}
r = requests.get(urly,headers=headers)
headers = {'User-Agent': Core.get_user_agent()}
r = requests.get(urly, headers=headers)
except Exception as e:
print(e)
self.results = r.text
self.totalresults += self.results
def get_people(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.people_linkedin()
def process(self):
@ -38,4 +39,4 @@ def process(self):
self.do_search()
time.sleep(getDelay())
self.counter += 100
print("\tSearching " + str(self.counter) + " results..")
print(f'\tSearching {self.counter} results..')

8
discovery/netcraft.py
View file

@ -1,9 +1,9 @@
import requests
from parsers import myparser
from discovery.constants import *
from lib.core import *
class search_netcraft:
class SearchNetcraft:
def __init__(self, word):
self.word = word.replace(' ', '%20')
@ -19,7 +19,7 @@ def do_search(self):
urly = "https://searchdns.netcraft.com/?restriction=site+ends+with&host=" + self.word
except Exception as e:
print(e)
headers = {'User-Agent': getUserAgent()}
headers = {'User-Agent': Core.get_user_agent()}
try:
r=requests.get(urly, headers=headers)
except Exception as e:
@ -28,7 +28,7 @@ def do_search(self):
self.totalresults += self.results
def get_hostnames(self):
rawres = myparser.parser(self.results, self.word)
rawres = myparser.Parser(self.results, self.word)
return rawres.hostnames()
def process(self):

10
discovery/pgpsearch.py
View file

@ -1,9 +1,9 @@
from parsers import myparser
import requests
from discovery.constants import *
from lib.core import *
class search_pgp:
class SearchPgp:
def __init__(self, word):
self.word = word
@ -17,7 +17,7 @@ def process(self):
url = 'http://' + self.server + "/pks/lookup?search=" + self.word + "&op=index"
headers = {
'Host': self.hostname,
'User-agent': getUserAgent()
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
self.results = h.text
@ -26,9 +26,9 @@ def process(self):
print("Unable to connect to PGP server: ", str(e))
def get_emails(self):
rawres = myparser.parser(self.results, self.word)
rawres = myparser.Parser(self.results, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.parser(self.results, self.word)
rawres = myparser.Parser(self.results, self.word)
return rawres.hostnames()

2
discovery/s3_scanner.py
View file

@ -26,7 +26,7 @@ def __check_http(self, bucket_url):
if bucket_response.status_code == 200\
and (not ARGS.only_interesting or
(ARGS.only_interesting and any(keyword in bucket_response.text for keyword in KEYWORDS))):
cprint("Found bucket '{}'".format(new_bucket_url), "green", attrs=["bold"])
print(f"Found bucket '{new_bucket_url}'")
self.__log(new_bucket_url)
def do_s3(self):

4
discovery/securitytrailssearch.py
View file

@ -49,7 +49,7 @@ def do_search(self):
def process(self):
self.authenticate()
self.do_search()
parser = securitytrailsparser.parser(word=self.word, text=self.totalresults)
parser = securitytrailsparser.Parser(word=self.word, text=self.totalresults)
self.info = parser.parse_text()
# create parser and set self.info to tuple returned from parsing text
print('\tDone Searching Results')
@ -58,4 +58,4 @@ def get_ips(self):
return self.info[0]
def get_hostnames(self):
return self.info[1]
return self.info[1]

11
discovery/shodansearch.py
View file

@ -1,6 +1,5 @@
from shodan import Shodan
from shodan import exception
#from discovery.shodan import Shodan
from discovery.constants import *
@ -8,7 +7,7 @@ class search_shodan:
def __init__(self):
self.key = shodanAPI_key
if self.key == "":
if self.key == '':
raise MissingKey(True)
self.api = Shodan(self.key)
self.hostdatarow = []
@ -23,7 +22,7 @@ def search_ip(self, ip):
try:
for key in result['http']['components'].keys():
technologies.append(key)
except KeyError as e:
except KeyError:
pass
port = str(result.get('port'))
product = str(result.get('product'))
@ -34,10 +33,10 @@ def search_ip(self, ip):
str(results.get('org')), str(servicesports).replace('\'', '').strip('[]'),
str(technologies).replace('\'', '').strip('[]')]
except exception.APIError:
print(ipaddress+": Not in Shodan")
print(f'{ipaddress}: Not in Shodan')
self.hostdatarow = [ipaddress, "Not in Shodan", "Not in Shodan", "Not in Shodan", "Not in Shodan"]
except Exception as e:
print("Error occurred in the Shodan IP search module: " + str(e))
print(f'Error occurred in the Shodan IP search module: {e}')
finally:
return self.hostdatarow
return self.hostdatarow

2
discovery/takeover.py
View file

@ -35,7 +35,7 @@ def do_take(self):
take_reg = re.compile(x)
self.temp = take_reg.findall(r.text)
if self.temp != []:
print("\t\033[91m Takeover detected! - " + self.host + "\033[1;32;40m ")
print(f"\t\033[91m Takeover detected! - {self.host} \033[1;32;40m")
except Exception as e:
print(e)

6
discovery/threatcrowd.py
View file

@ -1,6 +1,6 @@
import requests
from parsers import myparser
from discovery.constants import *
from lib.core import *
class search_threatcrowd:
@ -19,7 +19,7 @@ def do_search(self):
urly = "https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=" + self.word
except Exception as e:
print(e)
headers = {'User-Agent': getUserAgent()}
headers = {'User-Agent': Core.get_user_agent()}
try:
r = requests.get(urly, headers=headers)
except Exception as e:
@ -28,7 +28,7 @@ def do_search(self):
self.totalresults += self.results
def get_hostnames(self):
rawres = myparser.parser(self.results, self.word)
rawres = myparser.Parser(self.results, self.word)
return rawres.hostnames()
def process(self):

10
discovery/trello.py
View file

@ -32,13 +32,13 @@ def do_search(self):
self.totalresults += self.results
def get_emails(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.emails()
def get_urls(self):
print('\tSearching Trello Urls..')
try:
rawres = myparser.parser(self.totalresults, "trello.com")
rawres = myparser.Parser(self.totalresults, "trello.com")
trello_urls = rawres.urls()
visited = set()
for url in trello_urls:
@ -46,10 +46,10 @@ def get_urls(self):
if url not in visited: # make sure visiting unique urls
visited.add(url)
self.totalresults += requests.get(url=url, headers={'User-Agent': googleUA}).text
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames(), trello_urls
except Exception as e:
print("Error occurred: " + str(e))
print(f'Error occurred: {e}')
def process(self):
while self.counter < self.limit:
@ -59,4 +59,4 @@ def process(self):
else:
time.sleep(getDelay())
self.counter += 100
print("\tSearching " + str(self.counter) + " results..")
print(f'\tSearching {self.counter} results..')

9
discovery/twittersearch.py
View file

@ -1,6 +1,7 @@
import requests
from parsers import myparser
from discovery.constants import *
from lib.core import *
import time
@ -18,10 +19,10 @@ def __init__(self, word, limit):
def do_search(self):
try:
urly = "https://"+ self.server + "/search?num=100&start=" + str(self.counter) + "&hl=en&meta=&q=site%3Atwitter.com%20intitle%3A%22on+Twitter%22%20" + self.word
urly = "https://" + self.server + "/search?num=100&start=" + str(self.counter) + "&hl=en&meta=&q=site%3Atwitter.com%20intitle%3A%22on+Twitter%22%20" + self.word
except Exception as e:
print(e)
headers = {'User-Agent':getUserAgent()}
headers = {'User-Agent': Core.get_user_agent()}
try:
r=requests.get(urly, headers=headers)
except Exception as e:
@ -30,7 +31,7 @@ def do_search(self):
self.totalresults += self.results
def get_people(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.people_twitter()
def process(self):
@ -38,4 +39,4 @@ def process(self):
self.do_search()
time.sleep(getDelay())
self.counter += 100
print("\tSearching " + str(self.counter) + " results..")
print(f'\tSearching {self.counter} results..')

6
discovery/virustotal.py
View file

@ -1,6 +1,6 @@
import requests
from parsers import myparser
from discovery.constants import *
from lib.core import *
class search_virustotal:
@ -19,7 +19,7 @@ def do_search(self):
urly = "https://www.virustotal.com/en/domain/" + self.word + "/information/"
except Exception as e:
print(e)
headers = {'User-Agent': getUserAgent()}
headers = {'User-Agent': Core.get_user_agent()}
try:
r=requests.get(urly, headers=headers)
except Exception as e:
@ -28,7 +28,7 @@ def do_search(self):
self.totalresults += self.results
def get_hostnames(self):
rawres = myparser.parser(self.results, self.word)
rawres = myparser.Parser(self.results, self.word)
return rawres.hostnames()
def process(self):

9
discovery/yahoosearch.py
View file

@ -2,6 +2,7 @@
import time
import requests
from discovery.constants import *
from lib.core import *
class search_yahoo:
@ -19,7 +20,7 @@ def do_search(self):
+ "\"&b=" + str(self.counter) + "&pz=10"
headers = {
'Host': self.hostname,
'User-agent': getUserAgent()
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
self.total_results += h.text
@ -28,13 +29,13 @@ def process(self):
while self.counter <= self.limit and self.counter <= 1000:
self.do_search()
time.sleep(getDelay())
print("\tSearching " + str(self.counter) + " results...")
print(f'\tSearching {self.counter} results...')
self.counter += 10
def get_emails(self):
rawres = myparser.parser(self.total_results, self.word)
rawres = myparser.Parser(self.total_results, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.parser(self.total_results, self.word)
rawres = myparser.Parser(self.total_results, self.word)
return rawres.hostnames()

13
discovery/yandexsearch.py
View file

@ -3,6 +3,7 @@
import time
import requests
from discovery.constants import *
from lib.core import *
class search_yandex:
@ -20,7 +21,7 @@ def do_search(self):
url = 'http://' + self.server + "/search?text=%40" + self.word + "&numdoc=50&lr=" + str(self.counter)
headers = {
'Host': self.hostname,
'User-agent': getUserAgent()
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
self.results = h.text
@ -31,7 +32,7 @@ def do_search_files(self, files): # TODO
url = 'http://' + self.server + "/search?text=%40" + self.word + "&numdoc=50&lr=" + str(self.counter)
headers = {
'Host': self.hostname,
'User-agent': getUserAgent()
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
self.results = h.text
@ -48,22 +49,22 @@ def check_next(self):
return nexty
def get_emails(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames()
def get_files(self):
rawres = myparser.parser(self.totalresults, self.word)
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.fileurls(self.files) # self.files is not init?
def process(self):
while self.counter <= self.limit:
self.do_search()
self.counter += 50
print("Searching " + str(self.counter) + " results...")
print(f'Searching {self.counter} results...')
def process_files(self, files):
while self.counter < self.limit:

238
lib/core.py
View file

@ -1,8 +1,10 @@
# coding=utf-8
import os
import random
import sys
class Core:
@staticmethod
def banner():
@ -13,7 +15,7 @@ def banner():
print("* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *")
print("* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *")
print("* *")
print("* theHarvester 3.0.6 v137 *")
print("* theHarvester 3.0.6 v138 *")
print("* Coded by Christian Martorella *")
print("* Edge-Security Research *")
print("* cmartorella@edge-security.com *")
@ -50,3 +52,237 @@ def usage():
print((" " + comm + " -d acme.com -l 200 -g -b google"))
print((" " + comm + " -d acme.com -b googleCSE -l 500 -s 300"))
print((" " + comm + " -d acme.edu -l 100 -b bing -h \n"))
@staticmethod
def get_user_agent():
user_agents = [
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_0) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1216.0 Safari/537.2",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)",
"Mozilla/5.0 (Linux; U; Android 2.3; en-us) AppleWebKit/999+ (KHTML, like Gecko) Safari/999.9",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36",
"Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; ja) Opera 11.00",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.17 Safari/537.11",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/536.5 (KHTML like Gecko) Chrome/19.0.1084.56 Safari/1EA69",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10",
"Mozilla/5.0 (Linux; U; Android 2.3.5; zh-cn; HTC_IncredibleS_S710e Build/GRJ90) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36",
"Mozilla/5.0 (X11; CrOS i686 1660.57.0) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.46 Safari/535.19",
"Mozilla/5.0 (Windows NT 6.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01",
"Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/4.0; GTB7.4; InfoPath.3; SV1; .NET CLR 3.1.76908; WOW64; en-US)",
"Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)",
"Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS)",
"Opera/9.80 (Windows NT 5.1; U; en) Presto/2.9.168 Version/11.51",
"Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01",
"Opera/9.80 (Windows NT 5.1; U; MRA 5.5 (build 02842); ru) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Opera/9.80 (Windows NT 6.1; U; cs) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
"Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00",
"Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.7.39 Version/11.00",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)",
"Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; de) Presto/2.9.168 Version/11.52",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)",
"Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/537.11",
"Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10",
"Opera/9.80 (Windows NT 6.1; U; en-US) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)",
"Mozilla/5.0 (Linux; U; Android 2.3.4; fr-fr; HTC Desire Build/GRJ22) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Opera/9.80 (X11; Linux i686; U; fr) Presto/2.7.62 Version/11.01",
"Mozilla/4.0 (compatible; MSIE 8.0; X11; Linux x86_64; pl) Opera 11.00",
"Opera/9.80 (X11; Linux i686; U; hu) Presto/2.9.168 Version/11.50",
"Opera/9.80 (X11; Linux x86_64; U; bg) Presto/2.8.131 Version/11.10",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14",
"Opera/9.80 (X11; Linux i686; U; it) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.11 Safari/535.19",
"Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Opera/9.80 (X11; Linux i686; U; es-ES) Presto/2.8.131 Version/11.11",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; fr) Opera 11.00",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.1; SV1; .NET CLR 2.8.52393; WOW64; en-US)",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13",
"Opera/9.80 (Windows NT 5.1; U; it) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Linux; U; Android 2.3.3; ko-kr; LG-LU3000 Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Opera/9.80 (Windows NT 6.1; U; fi) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02",
"Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; pl) Opera 11.00",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
"Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)",
"Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36",
"Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
"Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1",
"Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00",
"Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; de) Opera 11.01",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; en) Opera 11.00",
"Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Opera/9.80 (Windows NT 6.1; U; ko) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7",
"Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36",
"Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.6.37 Version/11.00",
"Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
"Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13",
"Mozilla/5.0 (Windows NT 6.0; U; ja; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)",
"Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/19.0.1047.0 Safari/535.22",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3)",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
"Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36",
"Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/5.0 Opera 11.11",
"Mozilla/5.0 (Macintosh; AMD Mac OS X 10_8_2) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/18.6.872",
"Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)",
"Mozilla/5.0 (Linux; U; Android 4.0.3; de-ch; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.0) yi; AppleWebKit/345667.12221 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/453667.1221",
"Mozilla/1.22 (compatible; MSIE 10.0; Windows 3.1)",
"Opera/9.80 (Windows NT 5.1; U;) Presto/2.7.62 Version/11.01",
"Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16",
"Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21",
"Mozilla/5.0 (Linux; U; Android 2.3.5; en-us; HTC Vision Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F",
"Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.151 Safari/535.19",
"Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1284.0 Safari/537.13",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)",
"Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0)",
"Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00",
"Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
"Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36",
"Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10",
"Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00",
"Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari",
"Mozilla/5.0 (Windows NT 5.1; U; pl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00",
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; de) Opera 11.51",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11",
"Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19",
"Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/18.6.872.0 Safari/535.2 UNTRUSTED/1.0 3gpp-gba UNTRUSTED/1.0",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36",
"Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00",
"Mozilla/5.0 (Linux; U; Android 2.3.4; en-us; T-Mobile myTouch 3G Slide Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
"Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1041.0 Safari/535.21",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.19 (KHTML, like Gecko) Ubuntu/11.10 Chromium/18.0.1025.142 Chrome/18.0.1025.142 Safari/535.19",
"Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00"
]
return random.choice(user_agents)

0
parsers/__init__.py Normal file
View file

6
parsers/censysparser.py
View file

@ -2,12 +2,12 @@
import re
class parser:
class Parser:
def __init__(self, resultstoparse):
self.ipaddresses = []
self.souphosts = BeautifulSoup(resultstoparse.total_resultshosts, features = "html.parser")
self.soupcerts = BeautifulSoup(resultstoparse.total_resultscerts, features = "html.parser")
self.souphosts = BeautifulSoup(resultstoparse.total_resultshosts, features="html.parser")
self.soupcerts = BeautifulSoup(resultstoparse.total_resultscerts, features="html.parser")
self.hostnames = []
self.hostnamesfromcerts = []
self.urls = []

2
parsers/cymonparser.py
View file

@ -2,7 +2,7 @@
from bs4 import BeautifulSoup
class parser:
class Parser:
def __init__(self, results):
self.results = results

2
parsers/myparser.py
View file

@ -1,7 +1,7 @@
import re
class parser:
class Parser:
def __init__(self, results, word):
self.results = results

75
parsers/securitytrailsparser.py
View file

@ -1,37 +1,38 @@
class parser:
def __init__(self, word, text):
self.word = word
self.text = text
self.hostnames = set()
self.ips = set()
def parse_text(self):
subDomain_flag = 0
self.text = str(self.text).splitlines()
# Split lines to get a list of lines.
for index in range(0, len(self.text)):
line = self.text[index].strip()
if '"ip":' in line:
# Extract IP.
ip = ''
for ch in line[7:]:
if ch == '"':
break
else:
ip += ch
self.ips.add(ip)
elif '"subdomains":' in line:
# subdomains start here so set flag to 1.
subDomain_flag = 1
continue
elif subDomain_flag > 0:
if ']' in line:
subDomain_flag = 0
else:
if 'www' in self.word: self.word = str(self.word).replace('www.', '').replace('www', '')
# Remove www from word if entered.
self.hostnames.add(str(line).replace('"', '').replace(',', '') + '.' + self.word)
else:
continue
return list(self.ips), list(self.hostnames)
class Parser:
def __init__(self, word, text):
self.word = word
self.text = text
self.hostnames = set()
self.ips = set()
def parse_text(self):
sub_domain_flag = 0
self.text = str(self.text).splitlines()
# Split lines to get a list of lines.
for index in range(0, len(self.text)):
line = self.text[index].strip()
if '"ip":' in line:
# Extract IP.
ip = ''
for ch in line[7:]:
if ch == '"':
break
else:
ip += ch
self.ips.add(ip)
elif '"subdomains":' in line:
# subdomains start here so set flag to 1
sub_domain_flag = 1
continue
elif sub_domain_flag > 0:
if ']' in line:
sub_domain_flag = 0
else:
if 'www' in self.word:
self.word = str(self.word).replace('www.', '').replace('www', '')
# Remove www from word if entered
self.hostnames.add(str(line).replace('"', '').replace(',', '') + '.' + self.word)
else:
continue
return list(self.ips), list(self.hostnames)

1
requirements.txt
View file

@ -3,3 +3,4 @@ plotly>=3.4.2
requests>=2.21.0
texttable>=1.4.0
shodan>=1.10.0
pytest>=4.0.2

18
tests/myparser_test.py
View file

@ -1,18 +0,0 @@
#
# Unit tests for myparser.py
#
from parsers import myparser
import unittest
class TestMyParser(unittest.TestCase):
def test_emails(self):
word = 'domain.com'
results = '@domain.com***a@domain***banotherdomain.com***c@domain.com***d@sub.domain.com***'
p = myparser.parser(results, word)
emails = sorted(p.emails())
self.assertEqual(emails, [ 'c@domain.com', 'd@sub.domain.com' ])
if __name__ == '__main__':
unittest.main()

20
tests/test_myparser.py Normal file
View file

@ -0,0 +1,20 @@
#!/usr/bin/env python3
# coding=utf-8
from parsers import myparser
import pytest
class TestMyParser(object):
def test_emails(self):
word = 'domain.com'
results = '@domain.com***a@domain***banotherdomain.com***c@domain.com***d@sub.domain.com***'
parse = myparser.Parser(results, word)
emails = sorted(parse.emails())
assert emails, ['c@domain.com', 'd@sub.domain.com']
if __name__ == '__main__':
pytest.main()

136
theHarvester.py
View file

@ -1,6 +1,5 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import datetime
import getopt
import re
import stash
@ -100,12 +99,12 @@ def start(argv):
'netcraft', 'pgp', 'securityTrails', 'threatcrowd',
'trello', 'twitter', 'vhost', 'virustotal', 'yahoo', 'all'])
if set(engines).issubset(supportedengines):
print(("[-] Target domain: " + word + "\n"))
print(f"[-] Target domain: {word} \n")
for engineitem in engines:
if engineitem == "baidu":
print("[-] Searching in Baidu.")
try:
search = baidusearch.search_baidu(word, limit)
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
@ -119,7 +118,7 @@ def start(argv):
elif engineitem == "bing" or engineitem == "bingapi":
print("[-] Searching in Bing.")
try:
search = bingsearch.search_bing(word, limit, start)
search = bingsearch.SearchBing(word, limit, start)
if engineitem == "bingapi":
bingapi = "yes"
else:
@ -141,7 +140,7 @@ def start(argv):
print("[-] Searching in Censys.")
from discovery import censys
# Import locally or won't work
search = censys.search_censys(word, limit)
search = censys.SearchCensys(word, limit)
search.process()
all_ip = search.get_ipaddresses()
hosts = filter(search.get_hostnames())
@ -171,7 +170,7 @@ def start(argv):
elif engineitem == "dogpile":
print("[-] Searching in Dogpilesearch.")
search = dogpilesearch.search_dogpile(word, limit)
search = dogpilesearch.SearchDogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
@ -184,7 +183,7 @@ def start(argv):
elif engineitem == "duckduckgo":
print("[-] Searching in DuckDuckGo.")
from discovery import duckduckgosearch
search = duckduckgosearch.search_duckduckgo(word, limit)
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
@ -209,7 +208,7 @@ def start(argv):
elif engineitem == "googleCSE":
print("[-] Searching in Google Custom Search.")
try:
search = googleCSE.search_googleCSE(
search = googleCSE.SearchGoogleCSE(
word, limit, start)
search.process()
search.store_results()
@ -227,16 +226,13 @@ def start(argv):
pass
elif engineitem == "google-certificates":
print(
"[-] Searching in Google Certificate transparency report.")
search = googlecertificates.search_googlecertificates(
word, limit, start)
print("[-] Searching in Google Certificate transparency report.")
search = googlecertificates.SearchGoogleCertificates(word, limit, start)
search.process()
hosts = filter(search.get_domains())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
db.store_all(word, all_hosts, 'host', 'google-certificates')
elif engineitem == "google-profiles":
print("[-] Searching in Google profiles.")
@ -256,8 +252,7 @@ def start(argv):
from discovery import huntersearch
# Import locally or won't work.
try:
search = huntersearch.search_hunter(
word, limit, start)
search = huntersearch.SearchHunter(word, limit, start)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
@ -274,7 +269,7 @@ def start(argv):
elif engineitem == "linkedin":
print("[-] Searching in Linkedin.")
search = linkedinsearch.search_linkedin(word, limit)
search = linkedinsearch.SearchLinkedin(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
@ -287,7 +282,7 @@ def start(argv):
elif engineitem == "netcraft":
print("[-] Searching in Netcraft.")
search = netcraft.search_netcraft(word)
search = netcraft.SearchNetcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
@ -297,7 +292,7 @@ def start(argv):
elif engineitem == "pgp":
print("[-] Searching in PGP key server.")
try:
search = pgpsearch.search_pgp(word)
search = pgpsearch.SearchPgp(word)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
@ -337,8 +332,7 @@ def start(argv):
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts,
'host', 'threatcrowd')
db.store_all(word, all_hosts, 'host', 'threatcrowd')
except Exception:
pass
@ -399,11 +393,22 @@ def start(argv):
all_emails = []
all_hosts = []
# baidu
try:
print("[-] Searching in Baidu.")
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
print("[-] Searching in Bing.")
bingapi = "no"
search = bingsearch.search_bing(word, limit, start)
search = bingsearch.SearchBing(word, limit, start)
search.process(bingapi)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
@ -416,7 +421,7 @@ def start(argv):
print("[-] Searching in Censys.")
from discovery import censys
search = censys.search_censys(word, limit)
search = censys.SearchCensys(word, limit)
search.process()
ips = search.get_ipaddresses()
setips = set(ips)
@ -442,6 +447,18 @@ def start(argv):
# dogpile
print("[-] Searching in DuckDuckGo.")
from discovery import duckduckgosearch
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
print("[-] Searching in Google.")
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
@ -454,29 +471,34 @@ def start(argv):
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
print(
"[-] Searching in Google Certificate transparency report.")
search = googlecertificates.search_googlecertificates(
print("[-] Searching in Google Certificate transparency report.")
search = googlecertificates.SearchGoogleCertificates(
word, limit, start)
search.process()
domains = filter(search.get_domains())
all_hosts.extend(domains)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
db.store_all(word, all_hosts, 'host', 'google-certificates')
# googleplus
# google-certificates
# google-profiles
try:
print("[-] Searching in Google profiles.")
search = googlesearch.search_google(word, limit, start)
search.process_profiles()
people = search.get_profiles()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'google-profile')
print("\nUsers from Google profiles:")
print("---------------------------")
for users in people:
print(users)
except Exception:
pass
print("[-] Searching in Hunter.")
from discovery import huntersearch
# Import locally.
try:
search = huntersearch.search_hunter(
word, limit, start)
search = huntersearch.SearchHunter(word, limit, start)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
@ -495,7 +517,7 @@ def start(argv):
# linkedin
print("[-] Searching in Netcraft server.")
search = netcraft.search_netcraft(word)
search = netcraft.SearchNetcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
@ -504,7 +526,7 @@ def start(argv):
print("[-] Searching in PGP key server.")
try:
search = pgpsearch.search_pgp(word)
search = pgpsearch.SearchPgp(word)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
@ -526,8 +548,7 @@ def start(argv):
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts,
'host', 'threatcrowd')
db.store_all(word, all_hosts, 'host', 'threatcrowd')
except Exception:
pass
@ -546,7 +567,19 @@ def start(argv):
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
# twitter
try:
print("[-] Searching in Twitter.")
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
print("\nUsers from Twitter:")
print("-------------------")
for user in people:
print(user)
except Exception:
pass
# vhost
@ -558,7 +591,16 @@ def start(argv):
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
# yahoo
print("[-] Searching in Yahoo.")
search = yahoosearch.search_yahoo(word, limit)
search.process()
hosts = search.get_hostnames()
emails = search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
else:
print("[!] Invalid source.\n\n")
sys.exit(1)
@ -704,7 +746,7 @@ def start(argv):
# DNS TLD expansion
dnstldres = []
if dnstld == True:
if dnstld is True:
print("[-] Starting DNS TLD expansion.")
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
@ -721,7 +763,7 @@ def start(argv):
print("\n[+] Virtual hosts:")
print("------------------")
for l in host_ip:
search = bingsearch.search_bing(l, limit, start)
search = bingsearch.SearchBing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
@ -761,7 +803,7 @@ def start(argv):
print("-------------------")
print(printedtable)
except Exception as e:
print("Error occurred in theHarvester - Shodan search module: " + str(e))
print(f'Error occurred in theHarvester - Shodan search module: {e}')
else:
pass
@ -874,7 +916,7 @@ def start(argv):
file.close()
print("Files saved!")
except Exception as er:
print(("Error saving XML file: " + str(er)))
print(f'Error saving XML file: {er}')
sys.exit()