diff --git a/.gitignore b/.gitignore index 830be271..17977c9c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,5 @@ .vscode .xml debug_results.txt -discovery/constants.py -stash.sqlite tests/myparser.py venv diff --git a/README.md b/README.md index b85915a8..3d14844f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ * | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | * * \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| * * * -* theHarvester 3.0.6 v137 * +* theHarvester 3.0.6 v138 * * Coded by Christian Martorella * * Edge-Security Research * * cmartorella@edge-security.com * diff --git a/discovery/__init__.py b/discovery/__init__.py index 399fe0c3..dc6e3593 100644 --- a/discovery/__init__.py +++ b/discovery/__init__.py @@ -1,4 +1,5 @@ __all__ = ["bingsearch", + "duckduckgosearch", "googlesearch", "pgpsearch", "linkedinsearch", diff --git a/discovery/asksearch.py b/discovery/asksearch.py index 349fd20f..eb1a3a06 100644 --- a/discovery/asksearch.py +++ b/discovery/asksearch.py @@ -3,9 +3,10 @@ import requests import time from discovery.constants import * +from lib.core import * -class search_ask: +class SearchAsk: def __init__(self, word, limit): self.word = word.replace(' ', '%20') @@ -19,9 +20,9 @@ def __init__(self, word, limit): def do_search(self): headers = { - 'User-agent': getUserAgent() + 'User-agent': Core.get_user_agent() } - url = 'http://' + self.server + '/web?q=%40' + self.word \ + url = 'https://' + self.server + '/web?q=%40' + self.word \ + "&pu=100&page=" + str(self.counter) h = requests.get(url=url, headers=headers) time.sleep(getDelay()) @@ -38,7 +39,7 @@ def check_next(self): return nexty def get_people(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.people_jigsaw() def process(self): diff --git a/discovery/baidusearch.py b/discovery/baidusearch.py index ab9a5e97..f3608dde 100644 --- a/discovery/baidusearch.py +++ b/discovery/baidusearch.py @@ -2,9 +2,10 @@ import time import requests from discovery.constants import * +from lib.core import * -class search_baidu: +class SearchBaidu: def __init__(self, word, limit): self.word = word @@ -18,7 +19,7 @@ def do_search(self): url = 'http://' + self.server + "/s?wd=%40" + self.word + "&pn=" + str(self.counter) + "&oq=" + self.word headers = { 'Host': self.hostname, - 'User-agent': getUserAgent() + 'User-agent': Core.get_user_agent() } h = requests.get(url=url, headers=headers) time.sleep(getDelay()) @@ -27,13 +28,13 @@ def do_search(self): def process(self): while self.counter <= self.limit and self.counter <= 1000: self.do_search() - print("\tSearching " + str(self.counter) + " results...") + print(f'\tSearching {self.counter} results...') self.counter += 10 def get_emails(self): - rawres = myparser.parser(self.total_results, self.word) + rawres = myparser.Parser(self.total_results, self.word) return rawres.emails() def get_hostnames(self): - rawres = myparser.parser(self.total_results, self.word) + rawres = myparser.Parser(self.total_results, self.word) return rawres.hostnames() diff --git a/discovery/bingsearch.py b/discovery/bingsearch.py index 84ca8fc1..f7ad4d92 100644 --- a/discovery/bingsearch.py +++ b/discovery/bingsearch.py @@ -2,9 +2,10 @@ import time import requests from discovery.constants import * +from lib.core import * -class search_bing: +class SearchBing: def __init__(self, word, limit, start): self.word = word.replace(' ', '%20') @@ -21,11 +22,11 @@ def __init__(self, word, limit, start): def do_search(self): headers = { 'Host': self.hostname, - 'Cookie':'SRCHHPGUSR=ADLT=DEMOTE&NRSLT=50', + 'Cookie': 'SRCHHPGUSR=ADLT=DEMOTE&NRSLT=50', 'Accept-Language': 'en-us,en', - 'User-agent': getUserAgent() + 'User-agent': Core.get_user_agent() } - h = requests.get(url=('http://'+self.server + "/search?q=%40" + self.word + "&count=50&first=" + str(self.counter)),headers=headers) + h = requests.get(url=('http://'+self.server + "/search?q=%40" + self.word + "&count=50&first=" + str(self.counter)), headers=headers) self.results = h.text self.totalresults += self.results @@ -38,7 +39,7 @@ def do_search_api(self): 'mkt': 'en-us', 'safesearch': 'Off' } - headers = {'User-Agent': getUserAgent(), 'Ocp-Apim-Subscription-Key': self.bingApi} + headers = {'User-Agent': Core.get_user_agent(), 'Ocp-Apim-Subscription-Key': self.bingApi} h = requests.get(url=url, headers=headers, params=params) self.results = h.text self.totalresults += self.results @@ -48,7 +49,7 @@ def do_search_vhost(self): 'Host': self.hostname, 'Cookie': 'mkt=en-US;ui=en-US;SRCHHPGUSR=NEWWND=0&ADLT=DEMOTE&NRSLT=50', 'Accept-Language': 'en-us,en', - 'User-agent': getUserAgent() + 'User-agent': Core.get_user_agent() } url = 'http://' + self.server + "/search?q=ip:" + self.word + "&go=&count=50&FORM=QBHL&qs=n&first=" + str(self.counter) h = requests.get(url=url, headers=headers) @@ -56,15 +57,15 @@ def do_search_vhost(self): self.totalresults += self.results def get_emails(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.emails() def get_hostnames(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.hostnames() def get_allhostnames(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.hostnames_all() def process(self, api): @@ -79,10 +80,10 @@ def process(self, api): self.do_search() time.sleep(getDelay()) self.counter += 50 - print("\tSearching " + str(self.counter) + " results...") + print(f'\tSearching {self.counter} results...') def process_vhost(self): # Maybe it is good to use other limit for this. while self.counter < self.limit: self.do_search_vhost() - self.counter += 50 \ No newline at end of file + self.counter += 50 diff --git a/discovery/censys.py b/discovery/censys.py index d68e1967..8c472b76 100644 --- a/discovery/censys.py +++ b/discovery/censys.py @@ -1,9 +1,9 @@ import requests from parsers import censysparser -from discovery.constants import * +from lib.core import * -class search_censys: +class SearchCensys: def __init__(self, word, limit): self.word = word @@ -21,21 +21,21 @@ def __init__(self, word, limit): def do_searchhosturl(self): try: - headers = {'user-agent': getUserAgent(), 'Accept': '*/*', 'Referer': self.urlhost} + headers = {'user-agent': Core.get_user_agent(), 'Accept': '*/*', 'Referer': self.urlhost} responsehost = requests.get(self.urlhost, headers=headers) self.resultshosts = responsehost.text self.total_resultshosts += self.resultshosts except Exception as e: - print("Error occurred in the Censys module downloading pages from Censys - IP search: " + str(e)) + print(f'Error occurred in the Censys module downloading pages from Censys - IP search: + {e}') def do_searchcertificateurl(self): try: - headers = {'user-agent': getUserAgent(), 'Accept': '*/*', 'Referer': self.urlcert} + headers = {'user-agent': Core.get_user_agent(), 'Accept': '*/*', 'Referer': self.urlcert} responsecert = requests.get(self.urlcert, headers=headers) self.resultcerts = responsecert.text self.total_resultscerts += self.resultcerts except Exception as e: - print("Error occurred in the Censys module downloading pages from Censys - certificates search: " + str(e)) + print(f'Error occurred in the Censys module downloading pages from Censys - certificates search: {e}') def process(self): try: @@ -44,7 +44,7 @@ def process(self): self.do_searchhosturl() self.do_searchcertificateurl() counter = 2 - pages = censysparser.parser(self) + pages = censysparser.Parser(self) totalpages = pages.search_totalpageshosts() pagestosearch = int(self.limit / 25) # 25 results/page if totalpages <= pagestosearch: @@ -57,18 +57,18 @@ def process(self): self.do_searchhosturl() counter += 1 except Exception as e: - print("Error occurred in the Censys module requesting the pages: " + str(e)) + print(f'Error occurred in the Censys module requesting the pages: {e}') else: while counter <= pagestosearch: try: self.page = str(counter) self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str( self.page) - print("\tSearching Censys IP results page " + self.page + "...") + print(f'\tSearching Censys IP results page {self.page} ...') self.do_searchhosturl() counter += 1 except Exception as e: - print("Error occurred in the Censys module requesting the pages: " + str(e)) + print(f'Error occurred in the Censys module requesting the pages: {e}') counter = 2 totalpages = pages.search_totalpagescerts() if totalpages <= pagestosearch: @@ -77,11 +77,11 @@ def process(self): self.page = str(counter) self.urlhost = "https://" + self.server + "/certificates/_search?q=" + str( self.word) + "&page=" + str(self.page) - print("\tSearching Censys certificates results page " + self.page + "...") + print(f'\tSearching Censys certificates results page {self.page} ...') self.do_searchcertificateurl() counter += 1 except Exception as e: - print("Error occurred in the Censys module requesting the pages: " + str(e)) + print(f'Error occurred in the Censys module requesting the pages: {e}') else: while counter <= pagestosearch: try: @@ -92,15 +92,15 @@ def process(self): self.do_searchhosturl() counter += 1 except Exception as e: - print("Error occurred in the Censys module requesting the pages: " + str(e)) + print(f'Error occurred in the Censys module requesting the pages: {e}') except Exception as e: - print("Error occurred in the main Censys module: " + str(e)) + print(f'Error occurred in the main Censys module: {e}') def get_hostnames(self): try: ips = self.get_ipaddresses() - headers = {'user-agent': getUserAgent(), 'Accept': '*/*', 'Referer': self.urlcert} + headers = {'user-agent': Core.get_user_agent(), 'Accept': '*/*', 'Referer': self.urlcert} response = requests.post("https://censys.io/ipv4/getdns", json={"ips": ips}, headers=headers) responsejson = response.json() domainsfromcensys = [] @@ -111,16 +111,16 @@ def get_hostnames(self): pass matchingdomains = [s for s in domainsfromcensys if str(self.word) in s] self.hostnamesall.extend(matchingdomains) - hostnamesfromcerts = censysparser.parser(self) + hostnamesfromcerts = censysparser.Parser(self) self.hostnamesall.extend(hostnamesfromcerts.search_hostnamesfromcerts()) return self.hostnamesall except Exception as e: - print("Error occurred in the Censys module - hostname search: " + str(e)) + print(f'Error occurred in the Censys module - hostname search: {e}') def get_ipaddresses(self): try: - ips = censysparser.parser(self) + ips = censysparser.Parser(self) self.ips = ips.search_ipaddresses() return self.ips except Exception as e: - print("Error occurred in the main Censys module - IP address search: " + str(e)) + print(f'Error occurred in the main Censys module - IP address search: {e}') diff --git a/discovery/constants.py b/discovery/constants.py index 232f09b8..eaf23a74 100644 --- a/discovery/constants.py +++ b/discovery/constants.py @@ -21,237 +21,6 @@ shodanAPI_key = 'oCiMsgM6rQWqiTvPxFHYcExlZgg7wvTt' # this is the default key -user_agents = [ - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_0) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36", - "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1216.0 Safari/537.2", - "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", - "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", - "Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)", - "Mozilla/5.0 (Linux; U; Android 2.3; en-us) AppleWebKit/999+ (KHTML, like Gecko) Safari/999.9", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", - "Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01", - "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", - "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", - "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)", - "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; ja) Opera 11.00", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", - "Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01", - "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)", - "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", - "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.17 Safari/537.11", - "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US)", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", - "Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/536.5 (KHTML like Gecko) Chrome/19.0.1084.56 Safari/1EA69", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", - "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", - "Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10", - "Mozilla/5.0 (Linux; U; Android 2.3.5; zh-cn; HTC_IncredibleS_S710e Build/GRJ90) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)", - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36", - "Mozilla/5.0 (X11; CrOS i686 1660.57.0) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.46 Safari/535.19", - "Mozilla/5.0 (Windows NT 6.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01", - "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/4.0; GTB7.4; InfoPath.3; SV1; .NET CLR 3.1.76908; WOW64; en-US)", - "Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", - "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)", - "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS)", - "Opera/9.80 (Windows NT 5.1; U; en) Presto/2.9.168 Version/11.51", - "Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01", - "Opera/9.80 (Windows NT 5.1; U; MRA 5.5 (build 02842); ru) Presto/2.7.62 Version/11.00", - "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", - "Opera/9.80 (Windows NT 6.1; U; cs) Presto/2.7.62 Version/11.01", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", - "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00", - "Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.7.39 Version/11.00", - "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)", - "Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.7.62 Version/11.01", - "Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01", - "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)", - "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", - "Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62", - "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", - "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; de) Presto/2.9.168 Version/11.52", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36", - "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)", - "Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19", - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/537.11", - "Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10", - "Opera/9.80 (Windows NT 6.1; U; en-US) Presto/2.7.62 Version/11.01", - "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)", - "Mozilla/5.0 (Linux; U; Android 2.3.4; fr-fr; HTC Desire Build/GRJ22) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", - "Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", - "Opera/9.80 (X11; Linux i686; U; fr) Presto/2.7.62 Version/11.01", - "Mozilla/4.0 (compatible; MSIE 8.0; X11; Linux x86_64; pl) Opera 11.00", - "Opera/9.80 (X11; Linux i686; U; hu) Presto/2.9.168 Version/11.50", - "Opera/9.80 (X11; Linux x86_64; U; bg) Presto/2.8.131 Version/11.10", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", - "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14", - "Opera/9.80 (X11; Linux i686; U; it) Presto/2.7.62 Version/11.00", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.11 Safari/535.19", - "Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", - "Opera/9.80 (X11; Linux i686; U; es-ES) Presto/2.8.131 Version/11.11", - "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; fr) Opera 11.00", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.1; SV1; .NET CLR 2.8.52393; WOW64; en-US)", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13", - "Opera/9.80 (Windows NT 5.1; U; it) Presto/2.7.62 Version/11.00", - "Mozilla/5.0 (Linux; U; Android 2.3.3; ko-kr; LG-LU3000 Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", - "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.7.62 Version/11.01", - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", - "Opera/9.80 (Windows NT 6.1; U; fi) Presto/2.7.62 Version/11.00", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", - "Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02", - "Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", - "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; pl) Opera 11.00", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", - "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", - "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)", - "Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36", - "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62", - "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", - "Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", - "Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1", - "Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00", - "Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", - "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; de) Opera 11.01", - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", - "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; en) Opera 11.00", - "Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko", - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3", - "Opera/9.80 (Windows NT 6.1; U; ko) Presto/2.7.62 Version/11.00", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7", - "Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36", - "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36", - "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.6.37 Version/11.00", - "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14", - "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", - "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13", - "Mozilla/5.0 (Windows NT 6.0; U; ja; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)", - "Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/19.0.1047.0 Safari/535.22", - "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3)", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", - "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", - "Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", - "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/5.0 Opera 11.11", - "Mozilla/5.0 (Macintosh; AMD Mac OS X 10_8_2) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/18.6.872", - "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", - "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", - "Mozilla/5.0 (Linux; U; Android 4.0.3; de-ch; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30", - "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.0) yi; AppleWebKit/345667.12221 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/453667.1221", - "Mozilla/1.22 (compatible; MSIE 10.0; Windows 3.1)", - "Opera/9.80 (Windows NT 5.1; U;) Presto/2.7.62 Version/11.01", - "Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16", - "Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21", - "Mozilla/5.0 (Linux; U; Android 2.3.5; en-us; HTC Vision Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", - "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F", - "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.7.62 Version/11.01", - "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.151 Safari/535.19", - "Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36", - "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1284.0 Safari/537.13", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", - "Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E)", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)", - "Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0)", - "Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00", - "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", - "Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36", - "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10", - "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00", - "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari", - "Mozilla/5.0 (Windows NT 5.1; U; pl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00", - "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36", - "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; de) Opera 11.51", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11", - "Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.7.62 Version/11.00", - "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19", - "Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/18.6.872.0 Safari/535.2 UNTRUSTED/1.0 3gpp-gba UNTRUSTED/1.0", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36", - "Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00", - "Mozilla/5.0 (Linux; U; Android 2.3.4; en-us; T-Mobile myTouch 3G Slide Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", - "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1041.0 Safari/535.21", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.19 (KHTML, like Gecko) Ubuntu/11.10 Chromium/18.0.1025.142 Chrome/18.0.1025.142 Safari/535.19", - "Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00" -] - def filter(lst): """ @@ -278,10 +47,6 @@ def getDelay(): return random.randint(1, 3) - .5 -def getUserAgent(): - return random.choice(user_agents) - - def search(text): # helper function to check if google has blocked traffic for line in text.strip().splitlines(): diff --git a/discovery/crtsh.py b/discovery/crtsh.py index 1531afdb..02143cfa 100644 --- a/discovery/crtsh.py +++ b/discovery/crtsh.py @@ -2,6 +2,7 @@ from parsers import myparser import time from discovery.constants import * +from lib.core import * class search_crtsh: @@ -20,13 +21,13 @@ def do_search(self): except Exception as e: print(e) try: - params = {'User-Agent': getUserAgent()} - r=requests.get(urly,headers=params) + params = {'User-Agent': Core.get_user_agent()} + r=requests.get(urly, headers=params) except Exception as e: print(e) links = self.get_info(r.text) for link in links: - params = {'User-Agent': getUserAgent()} + params = {'User-Agent': Core.get_user_agent()} r = requests.get(link, headers=params) time.sleep(getDelay()) self.results = r.text @@ -37,7 +38,7 @@ def do_search(self): @param text requests text @return list of links """ - def get_info(self,text): + def get_info(self, text): lines = [] for line in str(text).splitlines(): line = line.strip() @@ -58,7 +59,7 @@ def get_info(self,text): return links def get_hostnames(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.hostnames() def process(self): diff --git a/discovery/cymon.py b/discovery/cymon.py index 0c601e5e..b1cdad2d 100644 --- a/discovery/cymon.py +++ b/discovery/cymon.py @@ -1,6 +1,7 @@ import requests from parsers import cymonparser from discovery.constants import * +from lib.core import * import time @@ -14,7 +15,7 @@ def __init__(self, word): def do_search(self): try: - headers = {'user-agent': getUserAgent(), 'Accept':'*/*', 'Referer': self.url} + headers = {'user-agent': Core.get_user_agent(), 'Accept': '*/*', 'Referer': self.url} response = requests.get(self.url, headers=headers) time.sleep(getDelay()) self.results = response.content @@ -24,14 +25,14 @@ def do_search(self): def process(self): try: self.url = "https://" + self.server + "/domain/" + str(self.word) - print("\tSearching Cymon results...") + print('\tSearching Cymon results...') self.do_search() except Exception as e: - print("Error occurred: " + str(e)) + print(f'Error occurred: {e}') def get_ipaddresses(self): try: - ips = cymonparser.parser(self) + ips = cymonparser.Parser(self) return ips.search_ipaddresses() except Exception as e: - print("Error occurred: " + str(e)) + print(f'Error occurred: {e}') diff --git a/discovery/dnssearch.py b/discovery/dnssearch.py index 5b78a57e..624cb31b 100644 --- a/discovery/dnssearch.py +++ b/discovery/dnssearch.py @@ -73,7 +73,7 @@ def __init__(self, domain, dnsserver, verbose=False): res_path = os.path.join(fileDir,'lib/resolvers.txt') with open(res_path) as f: self.resolvers = f.read().splitlines() - except Exception as e: + except Exception: print("Resolvers file can't be open") try: f = open(self.file, "r") @@ -86,7 +86,7 @@ def getdns(self, domain): DNS.ParseResolvConf("/etc/resolv.conf") # nameserver=DNS.defaults['server'][0] dom = domain - if self.subdo == True: + if self.subdo is True: dom = domain.split(".") dom.pop(0) rootdom = ".".join(dom) @@ -139,7 +139,7 @@ def run(self, host): # TODO FIX test is sometimes not getting answers and leads to an indexing erro hostip = test.answers[0]['data'] return hostname + ":" + hostip - except Exception as e: + except Exception: pass def process(self): @@ -190,13 +190,13 @@ def getdns(self, domain): # DNS.ParseResolvConf("/etc/resolv.conf") # nameserver=DNS.defaults['server'][0] dom = domain - if self.subdo == True: + if self.subdo is True: dom = domain.split(".") dom.pop(0) rootdom = ".".join(dom) else: rootdom = dom - if self.nameserver == False: + if self.nameserver is False: r = DNS.Request(rootdom, qtype='SOA').req() primary, email, serial, refresh, retry, expire, minimum = r.answers[ 0]['data'] @@ -225,7 +225,7 @@ def run(self, tld): ) hostip = test.answers[0]['data'] return hostip + ":" + hostname - except Exception as e: + except Exception: pass def process(self): diff --git a/discovery/dogpilesearch.py b/discovery/dogpilesearch.py index ebb87a1b..0e718246 100644 --- a/discovery/dogpilesearch.py +++ b/discovery/dogpilesearch.py @@ -2,8 +2,10 @@ import time import requests from discovery.constants import * +from lib.core import * -class search_dogpile: + +class SearchDogpile: def __init__(self, word, limit): self.word = word @@ -19,7 +21,7 @@ def do_search(self): + "&q=\"%40" + self.word + "\"" headers = { 'Host': self.hostname, - 'User-agent': getUserAgent() + 'User-agent': Core.get_user_agent() } h = requests.get(url=url, headers=headers) self.total_results += h.text @@ -28,13 +30,13 @@ def process(self): while self.counter <= self.limit and self.counter <= 1000: self.do_search() time.sleep(getDelay()) - print("\tSearching " + str(self.counter) + " results...") + print(f'\tSearching {self.counter} results...') self.counter += 10 def get_emails(self): - rawres = myparser.parser(self.total_results, self.word) + rawres = myparser.Parser(self.total_results, self.word) return rawres.emails() def get_hostnames(self): - rawres = myparser.parser(self.total_results, self.word) + rawres = myparser.Parser(self.total_results, self.word) return rawres.hostnames() diff --git a/discovery/duckduckgosearch.py b/discovery/duckduckgosearch.py index d110dd2b..a99f597c 100644 --- a/discovery/duckduckgosearch.py +++ b/discovery/duckduckgosearch.py @@ -1,90 +1,91 @@ -from parsers import myparser -import time -import requests -import json -from discovery.constants import * - - -class search_duckduckgo: - - def __init__(self, word, limit): - self.word = word - self.results = "" - self.totalresults = "" - self.dorks = [] - self.links = [] - self.database = "https://duckduckgo.com/?q=" - self.api = "https://api.duckduckgo.com/?q=x&format=json&pretty=1" # currently using api - self.quantity = "100" - self.limit = limit - - def do_search(self): - try: # do normal scraping - url = self.api.replace('x', self.word) - headers = {'User-Agent': googleUA} - r = requests.get(url, headers=headers) - except Exception as e: - print(e) - time.sleep(getDelay()) - self.results = r.text - self.totalresults += self.results - urls = self.crawl(self.results) - for url in urls: - try: - self.totalresults += requests.get(url, headers={'User-Agent': getUserAgent()}).text - time.sleep(getDelay()) - except Exception: - continue - - def crawl(self, text): - """ - function parses json and returns urls - :param text: formatted json - :return: set of urls - """ - urls = set() - try: - load = json.loads(text) - for key in load.keys(): # iterate through keys of dict - val = load.get(key) - if isinstance(val, int) or isinstance(val, dict): - continue - if isinstance(val, list): - val = val[0] # first value should be dict - if isinstance(val, dict): # sanity check - for key in val.keys(): - value = val.get(key) - if isinstance(value, str) and value != '' and 'https://' in value or 'http://' in value: - urls.add(value) - if isinstance(val, str) and val != '' and 'https://' in val or 'http://' in val: - urls.add(val) - tmp = set() - for url in urls: - if '<' in url and 'href=' in url: # format is - equal_index = url.index('=') - true_url = '' - for ch in url[equal_index + 1:]: - if ch == '"': - tmp.add(true_url) - break - true_url += ch - else: - if url != '': - tmp.add(url) - return tmp - except Exception as e: - print('Exception occurred: ' + str(e)) - import traceback as t - print(t.print_exc()) - return [] - - def get_emails(self): - rawres = myparser.parser(self.totalresults, self.word) - return rawres.emails() - - def get_hostnames(self): - rawres = myparser.parser(self.totalresults, self.word) - return rawres.hostnames() - - def process(self): - self.do_search() # only need to search once since using API +from parsers import myparser +import time +import requests +import json +from discovery.constants import * +from lib.core import * + + +class SearchDuckDuckGo: + + def __init__(self, word, limit): + self.word = word + self.results = "" + self.totalresults = "" + self.dorks = [] + self.links = [] + self.database = "https://duckduckgo.com/?q=" + self.api = "https://api.duckduckgo.com/?q=x&format=json&pretty=1" # currently using api + self.quantity = "100" + self.limit = limit + + def do_search(self): + try: # do normal scraping + url = self.api.replace('x', self.word) + headers = {'User-Agent': googleUA} + r = requests.get(url, headers=headers) + except Exception as e: + print(e) + time.sleep(getDelay()) + self.results = r.text + self.totalresults += self.results + urls = self.crawl(self.results) + for url in urls: + try: + self.totalresults += requests.get(url, headers={'User-Agent': Core.get_user_agent()}).text + time.sleep(getDelay()) + except Exception: + continue + + def crawl(self, text): + """ + function parses json and returns urls + :param text: formatted json + :return: set of urls + """ + urls = set() + try: + load = json.loads(text) + for key in load.keys(): # iterate through keys of dict + val = load.get(key) + if isinstance(val, int) or isinstance(val, dict): + continue + if isinstance(val, list): + val = val[0] # first value should be dict + if isinstance(val, dict): # sanity check + for key in val.keys(): + value = val.get(key) + if isinstance(value, str) and value != '' and 'https://' in value or 'http://' in value: + urls.add(value) + if isinstance(val, str) and val != '' and 'https://' in val or 'http://' in val: + urls.add(val) + tmp = set() + for url in urls: + if '<' in url and 'href=' in url: # format is + equal_index = url.index('=') + true_url = '' + for ch in url[equal_index + 1:]: + if ch == '"': + tmp.add(true_url) + break + true_url += ch + else: + if url != '': + tmp.add(url) + return tmp + except Exception as e: + print(f'Exception occurred: {e}') + import traceback as t + print(t.print_exc()) + return [] + + def get_emails(self): + rawres = myparser.Parser(self.totalresults, self.word) + return rawres.emails() + + def get_hostnames(self): + rawres = myparser.Parser(self.totalresults, self.word) + return rawres.hostnames() + + def process(self): + self.do_search() # only need to search once since using API diff --git a/discovery/exaleadsearch.py b/discovery/exaleadsearch.py index d9b8839c..a16f1c8a 100644 --- a/discovery/exaleadsearch.py +++ b/discovery/exaleadsearch.py @@ -3,6 +3,8 @@ import time import requests from discovery.constants import * +from lib.core import * + class search_exalead: def __init__(self, word, limit, start): @@ -20,8 +22,8 @@ def do_search(self): + "&elements_per_page=50&start_index=" + str(self.counter) headers = { 'Host': self.hostname, - 'Referer': ("http://" +self.hostname +"/search/web/results/?q=%40" +self.word), - 'User-agent': getUserAgent() + 'Referer': ("http://" + self.hostname + "/search/web/results/?q=%40" + self.word), + 'User-agent': Core.get_user_agent() } h = requests.get(url=url, headers=headers) self.results = h.text @@ -33,7 +35,7 @@ def do_search_files(self, files): headers = { 'Host': self.hostname, 'Referer': ("http://" + self.hostname + "/search/web/results/?q=%40" + self.word), - 'User-agent': getUserAgent() + 'User-agent': Core.get_user_agent() } h = requests.get(url=url, headers=headers) self.results = h.text @@ -50,22 +52,22 @@ def check_next(self): return nexty def get_emails(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.emails() def get_hostnames(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.hostnames() def get_files(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.fileurls(self.files) def process(self): while self.counter <= self.limit: self.do_search() self.counter += 50 - print("\ tSearching " + str(self.counter) + " results...") + print(f'\tSearching {self.counter} results...') def process_files(self, files): while self.counter < self.limit: diff --git a/discovery/googleCSE.py b/discovery/googleCSE.py index d34b81e5..a626436d 100644 --- a/discovery/googleCSE.py +++ b/discovery/googleCSE.py @@ -6,7 +6,7 @@ from discovery.constants import * -class search_googleCSE: +class SearchGoogleCSE: def __init__(self, word, limit, start): self.word = word @@ -63,15 +63,15 @@ def check_next(self): return nexty def get_emails(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.emails() def get_hostnames(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.hostnames() def get_files(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.fileurls(self.files) def process(self): diff --git a/discovery/googlecertificates.py b/discovery/googlecertificates.py index 9fc07162..fa75787a 100644 --- a/discovery/googlecertificates.py +++ b/discovery/googlecertificates.py @@ -1,8 +1,9 @@ import requests import json -from discovery.constants import * +from lib.core import * -class search_googlecertificates: + +class SearchGoogleCertificates: # https://www.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch?include_expired=true&include_subdomains=true&domain= def __init__(self, word, limit, start): self.word = word @@ -19,7 +20,7 @@ def do_search(self): except Exception as e: print(e) try: - headers = {'User-Agent': getUserAgent()} + headers = {'User-Agent': Core.get_user_agent()} r = requests.get(urly, headers=headers) except Exception as e: print(e) diff --git a/discovery/googlesearch.py b/discovery/googlesearch.py index ab940d1c..82267578 100644 --- a/discovery/googlesearch.py +++ b/discovery/googlesearch.py @@ -55,26 +55,26 @@ def do_search_profiles(self): self.totalresults += self.results def get_emails(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.emails() def get_hostnames(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.hostnames() def get_files(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.fileurls(self.files) def get_profiles(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.profiles() def process(self, google_dorking): if google_dorking is False: while self.counter <= self.limit and self.counter <= 1000: self.do_search() - print("\tSearching " + str(self.counter) + " results...") + print(f'\tSearching {self.counter} results...') self.counter += 100 else: # google dorking is true self.counter = 0 # reset counter @@ -82,7 +82,7 @@ def process(self, google_dorking): print("[-] Searching with Google Dorks: ") while self.counter <= self.limit and self.counter <= 200: # only 200 dorks in list self.googledork() # call google dorking method if user wanted it! - print("\tSearching " + str(self.counter) + " results...") + print(f'\tSearching {self.counter} results...') self.counter += 100 def process_profiles(self): @@ -90,7 +90,7 @@ def process_profiles(self): self.do_search_profiles() time.sleep(getDelay()) self.counter += 100 - print("\tSearching " + str(self.counter) + " results...") + print(f'\tSearching {self.counter} results...') def append_dorks(self): try: # wrap in try-except incase filepaths are messed up diff --git a/discovery/googlesets.py b/discovery/googlesets.py index a89440d5..24665c92 100644 --- a/discovery/googlesets.py +++ b/discovery/googlesets.py @@ -1,6 +1,6 @@ from parsers import myparser import requests -from discovery.constants import * +from lib.core import * class search_google_labs: @@ -23,14 +23,14 @@ def do_search(self): url = 'http://' + self.server + "/sets?hl-en&" + self.set headers = { 'Host': self.server, - 'User-agent': getUserAgent() + 'User-agent': Core.get_user_agent() } h = requests.get(url=url, headers=headers) self.results = h.text self.totalresults += self.results def get_set(self): - rawres = myparser.parser(self.totalresults, list) + rawres = myparser.Parser(self.totalresults, list) return rawres.set() def process(self): diff --git a/discovery/huntersearch.py b/discovery/huntersearch.py index 09655db0..7767d653 100644 --- a/discovery/huntersearch.py +++ b/discovery/huntersearch.py @@ -3,7 +3,7 @@ from discovery.constants import * -class search_hunter: +class SearchHunter: def __init__(self, word, limit, start): self.word = word @@ -15,7 +15,7 @@ def __init__(self, word, limit, start): self.results = "" self.totalresults = "" self.counter = start - self.database = "https://api.hunter.io/v2/domain-search?domain=" + word + "&api_key=" + self.key +"&limit=" + str(self.limit) + self.database = "https://api.hunter.io/v2/domain-search?domain=" + word + "&api_key=" + self.key + "&limit=" + str(self.limit) def do_search(self): try: @@ -30,13 +30,13 @@ def process(self): print('\tDone Searching Results') def get_emails(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.emails() def get_hostnames(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.hostnames() def get_profiles(self): - rawres = myparser.parser(self.totalresults, self.word) - return rawres.profiles() \ No newline at end of file + rawres = myparser.Parser(self.totalresults, self.word) + return rawres.profiles() diff --git a/discovery/jigsaw.py b/discovery/jigsaw.py index 5e574393..552378fb 100644 --- a/discovery/jigsaw.py +++ b/discovery/jigsaw.py @@ -3,6 +3,7 @@ import requests import time from discovery.constants import * +from lib.core import * # http://www.jigsaw.com/SearchAcrossCompanies.xhtml?opCode=refresh&rpage=4&mode=0&cnCountry=&order=0&orderby=0&cmName=accuvant&cnDead=false&cnExOwned=false&count=0&screenNameType=0&screenName=&omitScreenNameType=0&omitScreenName=&companyId=0&estimatedCount=277&rowsPerPage=50 @@ -22,7 +23,7 @@ def __init__(self, word, limit): def do_search(self): url = 'http://' + self.server + "/FreeTextSearch.xhtml?opCode=search&autoSuggested=True&freeText=" + self.word headers = { - 'User-agent': getUserAgent() + 'User-agent': Core.get_user_agent() } h = requests.get(url=url, headers=headers) self.results = h.text @@ -38,7 +39,7 @@ def check_next(self): return nexty def get_people(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.people_jigsaw() def process(self): diff --git a/discovery/linkedinsearch.py b/discovery/linkedinsearch.py index 29e9d50c..827efac1 100644 --- a/discovery/linkedinsearch.py +++ b/discovery/linkedinsearch.py @@ -1,10 +1,11 @@ import requests from parsers import myparser from discovery.constants import * +from lib.core import * import time -class search_linkedin: +class SearchLinkedin: def __init__(self, word, limit): self.word = word.replace(' ', '%20') @@ -18,19 +19,19 @@ def __init__(self, word, limit): def do_search(self): try: - urly = "http://"+ self.server + "/search?num=100&start=" + str(self.counter) + "&hl=en&meta=&q=site%3Alinkedin.com/in%20" + self.word + urly = "http://" + self.server + "/search?num=100&start=" + str(self.counter) + "&hl=en&meta=&q=site%3Alinkedin.com/in%20" + self.word except Exception as e: print(e) try: - headers = {'User-Agent': getUserAgent()} - r = requests.get(urly,headers=headers) + headers = {'User-Agent': Core.get_user_agent()} + r = requests.get(urly, headers=headers) except Exception as e: print(e) self.results = r.text self.totalresults += self.results def get_people(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.people_linkedin() def process(self): @@ -38,4 +39,4 @@ def process(self): self.do_search() time.sleep(getDelay()) self.counter += 100 - print("\tSearching " + str(self.counter) + " results..") + print(f'\tSearching {self.counter} results..') diff --git a/discovery/netcraft.py b/discovery/netcraft.py index 632732b0..66a5a1f3 100644 --- a/discovery/netcraft.py +++ b/discovery/netcraft.py @@ -1,9 +1,9 @@ import requests from parsers import myparser -from discovery.constants import * +from lib.core import * -class search_netcraft: +class SearchNetcraft: def __init__(self, word): self.word = word.replace(' ', '%20') @@ -19,7 +19,7 @@ def do_search(self): urly = "https://searchdns.netcraft.com/?restriction=site+ends+with&host=" + self.word except Exception as e: print(e) - headers = {'User-Agent': getUserAgent()} + headers = {'User-Agent': Core.get_user_agent()} try: r=requests.get(urly, headers=headers) except Exception as e: @@ -28,7 +28,7 @@ def do_search(self): self.totalresults += self.results def get_hostnames(self): - rawres = myparser.parser(self.results, self.word) + rawres = myparser.Parser(self.results, self.word) return rawres.hostnames() def process(self): diff --git a/discovery/pgpsearch.py b/discovery/pgpsearch.py index 1d6a5ed9..5e8e7b94 100644 --- a/discovery/pgpsearch.py +++ b/discovery/pgpsearch.py @@ -1,9 +1,9 @@ from parsers import myparser import requests -from discovery.constants import * +from lib.core import * -class search_pgp: +class SearchPgp: def __init__(self, word): self.word = word @@ -17,7 +17,7 @@ def process(self): url = 'http://' + self.server + "/pks/lookup?search=" + self.word + "&op=index" headers = { 'Host': self.hostname, - 'User-agent': getUserAgent() + 'User-agent': Core.get_user_agent() } h = requests.get(url=url, headers=headers) self.results = h.text @@ -26,9 +26,9 @@ def process(self): print("Unable to connect to PGP server: ", str(e)) def get_emails(self): - rawres = myparser.parser(self.results, self.word) + rawres = myparser.Parser(self.results, self.word) return rawres.emails() def get_hostnames(self): - rawres = myparser.parser(self.results, self.word) + rawres = myparser.Parser(self.results, self.word) return rawres.hostnames() diff --git a/discovery/s3_scanner.py b/discovery/s3_scanner.py index f039ed30..a66a63da 100644 --- a/discovery/s3_scanner.py +++ b/discovery/s3_scanner.py @@ -26,7 +26,7 @@ def __check_http(self, bucket_url): if bucket_response.status_code == 200\ and (not ARGS.only_interesting or (ARGS.only_interesting and any(keyword in bucket_response.text for keyword in KEYWORDS))): - cprint("Found bucket '{}'".format(new_bucket_url), "green", attrs=["bold"]) + print(f"Found bucket '{new_bucket_url}'") self.__log(new_bucket_url) def do_s3(self): diff --git a/discovery/securitytrailssearch.py b/discovery/securitytrailssearch.py index 6944c68e..a2840e56 100644 --- a/discovery/securitytrailssearch.py +++ b/discovery/securitytrailssearch.py @@ -49,7 +49,7 @@ def do_search(self): def process(self): self.authenticate() self.do_search() - parser = securitytrailsparser.parser(word=self.word, text=self.totalresults) + parser = securitytrailsparser.Parser(word=self.word, text=self.totalresults) self.info = parser.parse_text() # create parser and set self.info to tuple returned from parsing text print('\tDone Searching Results') @@ -58,4 +58,4 @@ def get_ips(self): return self.info[0] def get_hostnames(self): - return self.info[1] \ No newline at end of file + return self.info[1] diff --git a/discovery/shodansearch.py b/discovery/shodansearch.py index 9750e7db..5bd9b302 100644 --- a/discovery/shodansearch.py +++ b/discovery/shodansearch.py @@ -1,6 +1,5 @@ from shodan import Shodan from shodan import exception -#from discovery.shodan import Shodan from discovery.constants import * @@ -8,7 +7,7 @@ class search_shodan: def __init__(self): self.key = shodanAPI_key - if self.key == "": + if self.key == '': raise MissingKey(True) self.api = Shodan(self.key) self.hostdatarow = [] @@ -23,7 +22,7 @@ def search_ip(self, ip): try: for key in result['http']['components'].keys(): technologies.append(key) - except KeyError as e: + except KeyError: pass port = str(result.get('port')) product = str(result.get('product')) @@ -34,10 +33,10 @@ def search_ip(self, ip): str(results.get('org')), str(servicesports).replace('\'', '').strip('[]'), str(technologies).replace('\'', '').strip('[]')] except exception.APIError: - print(ipaddress+": Not in Shodan") + print(f'{ipaddress}: Not in Shodan') self.hostdatarow = [ipaddress, "Not in Shodan", "Not in Shodan", "Not in Shodan", "Not in Shodan"] except Exception as e: - print("Error occurred in the Shodan IP search module: " + str(e)) + print(f'Error occurred in the Shodan IP search module: {e}') finally: - return self.hostdatarow \ No newline at end of file + return self.hostdatarow diff --git a/discovery/takeover.py b/discovery/takeover.py index e9f9e2aa..f3f5998d 100644 --- a/discovery/takeover.py +++ b/discovery/takeover.py @@ -35,7 +35,7 @@ def do_take(self): take_reg = re.compile(x) self.temp = take_reg.findall(r.text) if self.temp != []: - print("\t\033[91m Takeover detected! - " + self.host + "\033[1;32;40m ") + print(f"\t\033[91m Takeover detected! - {self.host} \033[1;32;40m") except Exception as e: print(e) diff --git a/discovery/threatcrowd.py b/discovery/threatcrowd.py index dc956e49..2d7f141f 100644 --- a/discovery/threatcrowd.py +++ b/discovery/threatcrowd.py @@ -1,6 +1,6 @@ import requests from parsers import myparser -from discovery.constants import * +from lib.core import * class search_threatcrowd: @@ -19,7 +19,7 @@ def do_search(self): urly = "https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=" + self.word except Exception as e: print(e) - headers = {'User-Agent': getUserAgent()} + headers = {'User-Agent': Core.get_user_agent()} try: r = requests.get(urly, headers=headers) except Exception as e: @@ -28,7 +28,7 @@ def do_search(self): self.totalresults += self.results def get_hostnames(self): - rawres = myparser.parser(self.results, self.word) + rawres = myparser.Parser(self.results, self.word) return rawres.hostnames() def process(self): diff --git a/discovery/trello.py b/discovery/trello.py index 9bd99b60..d10c7866 100644 --- a/discovery/trello.py +++ b/discovery/trello.py @@ -32,13 +32,13 @@ def do_search(self): self.totalresults += self.results def get_emails(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.emails() def get_urls(self): print('\tSearching Trello Urls..') try: - rawres = myparser.parser(self.totalresults, "trello.com") + rawres = myparser.Parser(self.totalresults, "trello.com") trello_urls = rawres.urls() visited = set() for url in trello_urls: @@ -46,10 +46,10 @@ def get_urls(self): if url not in visited: # make sure visiting unique urls visited.add(url) self.totalresults += requests.get(url=url, headers={'User-Agent': googleUA}).text - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.hostnames(), trello_urls except Exception as e: - print("Error occurred: " + str(e)) + print(f'Error occurred: {e}') def process(self): while self.counter < self.limit: @@ -59,4 +59,4 @@ def process(self): else: time.sleep(getDelay()) self.counter += 100 - print("\tSearching " + str(self.counter) + " results..") + print(f'\tSearching {self.counter} results..') diff --git a/discovery/twittersearch.py b/discovery/twittersearch.py index 36aa324b..768ffed2 100644 --- a/discovery/twittersearch.py +++ b/discovery/twittersearch.py @@ -1,6 +1,7 @@ import requests from parsers import myparser from discovery.constants import * +from lib.core import * import time @@ -18,10 +19,10 @@ def __init__(self, word, limit): def do_search(self): try: - urly = "https://"+ self.server + "/search?num=100&start=" + str(self.counter) + "&hl=en&meta=&q=site%3Atwitter.com%20intitle%3A%22on+Twitter%22%20" + self.word + urly = "https://" + self.server + "/search?num=100&start=" + str(self.counter) + "&hl=en&meta=&q=site%3Atwitter.com%20intitle%3A%22on+Twitter%22%20" + self.word except Exception as e: print(e) - headers = {'User-Agent':getUserAgent()} + headers = {'User-Agent': Core.get_user_agent()} try: r=requests.get(urly, headers=headers) except Exception as e: @@ -30,7 +31,7 @@ def do_search(self): self.totalresults += self.results def get_people(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.people_twitter() def process(self): @@ -38,4 +39,4 @@ def process(self): self.do_search() time.sleep(getDelay()) self.counter += 100 - print("\tSearching " + str(self.counter) + " results..") + print(f'\tSearching {self.counter} results..') diff --git a/discovery/virustotal.py b/discovery/virustotal.py index a2c5cc42..0c883669 100644 --- a/discovery/virustotal.py +++ b/discovery/virustotal.py @@ -1,6 +1,6 @@ import requests from parsers import myparser -from discovery.constants import * +from lib.core import * class search_virustotal: @@ -19,7 +19,7 @@ def do_search(self): urly = "https://www.virustotal.com/en/domain/" + self.word + "/information/" except Exception as e: print(e) - headers = {'User-Agent': getUserAgent()} + headers = {'User-Agent': Core.get_user_agent()} try: r=requests.get(urly, headers=headers) except Exception as e: @@ -28,7 +28,7 @@ def do_search(self): self.totalresults += self.results def get_hostnames(self): - rawres = myparser.parser(self.results, self.word) + rawres = myparser.Parser(self.results, self.word) return rawres.hostnames() def process(self): diff --git a/discovery/yahoosearch.py b/discovery/yahoosearch.py index 1a765b33..03ed650e 100644 --- a/discovery/yahoosearch.py +++ b/discovery/yahoosearch.py @@ -2,6 +2,7 @@ import time import requests from discovery.constants import * +from lib.core import * class search_yahoo: @@ -19,7 +20,7 @@ def do_search(self): + "\"&b=" + str(self.counter) + "&pz=10" headers = { 'Host': self.hostname, - 'User-agent': getUserAgent() + 'User-agent': Core.get_user_agent() } h = requests.get(url=url, headers=headers) self.total_results += h.text @@ -28,13 +29,13 @@ def process(self): while self.counter <= self.limit and self.counter <= 1000: self.do_search() time.sleep(getDelay()) - print("\tSearching " + str(self.counter) + " results...") + print(f'\tSearching {self.counter} results...') self.counter += 10 def get_emails(self): - rawres = myparser.parser(self.total_results, self.word) + rawres = myparser.Parser(self.total_results, self.word) return rawres.emails() def get_hostnames(self): - rawres = myparser.parser(self.total_results, self.word) + rawres = myparser.Parser(self.total_results, self.word) return rawres.hostnames() diff --git a/discovery/yandexsearch.py b/discovery/yandexsearch.py index e2b91936..7cc03b7e 100644 --- a/discovery/yandexsearch.py +++ b/discovery/yandexsearch.py @@ -3,6 +3,7 @@ import time import requests from discovery.constants import * +from lib.core import * class search_yandex: @@ -20,7 +21,7 @@ def do_search(self): url = 'http://' + self.server + "/search?text=%40" + self.word + "&numdoc=50&lr=" + str(self.counter) headers = { 'Host': self.hostname, - 'User-agent': getUserAgent() + 'User-agent': Core.get_user_agent() } h = requests.get(url=url, headers=headers) self.results = h.text @@ -31,7 +32,7 @@ def do_search_files(self, files): # TODO url = 'http://' + self.server + "/search?text=%40" + self.word + "&numdoc=50&lr=" + str(self.counter) headers = { 'Host': self.hostname, - 'User-agent': getUserAgent() + 'User-agent': Core.get_user_agent() } h = requests.get(url=url, headers=headers) self.results = h.text @@ -48,22 +49,22 @@ def check_next(self): return nexty def get_emails(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.emails() def get_hostnames(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.hostnames() def get_files(self): - rawres = myparser.parser(self.totalresults, self.word) + rawres = myparser.Parser(self.totalresults, self.word) return rawres.fileurls(self.files) # self.files is not init? def process(self): while self.counter <= self.limit: self.do_search() self.counter += 50 - print("Searching " + str(self.counter) + " results...") + print(f'Searching {self.counter} results...') def process_files(self, files): while self.counter < self.limit: diff --git a/lib/core.py b/lib/core.py index 6bb7dcea..e22704c5 100644 --- a/lib/core.py +++ b/lib/core.py @@ -1,8 +1,10 @@ # coding=utf-8 import os +import random import sys + class Core: @staticmethod def banner(): @@ -13,7 +15,7 @@ def banner(): print("* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *") print("* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *") print("* *") - print("* theHarvester 3.0.6 v137 *") + print("* theHarvester 3.0.6 v138 *") print("* Coded by Christian Martorella *") print("* Edge-Security Research *") print("* cmartorella@edge-security.com *") @@ -50,3 +52,237 @@ def usage(): print((" " + comm + " -d acme.com -l 200 -g -b google")) print((" " + comm + " -d acme.com -b googleCSE -l 500 -s 300")) print((" " + comm + " -d acme.edu -l 100 -b bing -h \n")) + + @staticmethod + def get_user_agent(): + user_agents = [ + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_0) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1216.0 Safari/537.2", + "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", + "Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)", + "Mozilla/5.0 (Linux; U; Android 2.3; en-us) AppleWebKit/999+ (KHTML, like Gecko) Safari/999.9", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", + "Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; ja) Opera 11.00", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", + "Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.17 Safari/537.11", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", + "Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/536.5 (KHTML like Gecko) Chrome/19.0.1084.56 Safari/1EA69", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10", + "Mozilla/5.0 (Linux; U; Android 2.3.5; zh-cn; HTC_IncredibleS_S710e Build/GRJ90) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36", + "Mozilla/5.0 (X11; CrOS i686 1660.57.0) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.46 Safari/535.19", + "Mozilla/5.0 (Windows NT 6.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01", + "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/4.0; GTB7.4; InfoPath.3; SV1; .NET CLR 3.1.76908; WOW64; en-US)", + "Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)", + "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS)", + "Opera/9.80 (Windows NT 5.1; U; en) Presto/2.9.168 Version/11.51", + "Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 5.1; U; MRA 5.5 (build 02842); ru) Presto/2.7.62 Version/11.00", + "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", + "Opera/9.80 (Windows NT 6.1; U; cs) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00", + "Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.7.39 Version/11.00", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)", + "Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)", + "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", + "Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; de) Presto/2.9.168 Version/11.52", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)", + "Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/537.11", + "Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10", + "Opera/9.80 (Windows NT 6.1; U; en-US) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)", + "Mozilla/5.0 (Linux; U; Android 2.3.4; fr-fr; HTC Desire Build/GRJ22) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", + "Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", + "Opera/9.80 (X11; Linux i686; U; fr) Presto/2.7.62 Version/11.01", + "Mozilla/4.0 (compatible; MSIE 8.0; X11; Linux x86_64; pl) Opera 11.00", + "Opera/9.80 (X11; Linux i686; U; hu) Presto/2.9.168 Version/11.50", + "Opera/9.80 (X11; Linux x86_64; U; bg) Presto/2.8.131 Version/11.10", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", + "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14", + "Opera/9.80 (X11; Linux i686; U; it) Presto/2.7.62 Version/11.00", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.11 Safari/535.19", + "Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", + "Opera/9.80 (X11; Linux i686; U; es-ES) Presto/2.8.131 Version/11.11", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; fr) Opera 11.00", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.1; SV1; .NET CLR 2.8.52393; WOW64; en-US)", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13", + "Opera/9.80 (Windows NT 5.1; U; it) Presto/2.7.62 Version/11.00", + "Mozilla/5.0 (Linux; U; Android 2.3.3; ko-kr; LG-LU3000 Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", + "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", + "Opera/9.80 (Windows NT 6.1; U; fi) Presto/2.7.62 Version/11.00", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", + "Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02", + "Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; pl) Opera 11.00", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)", + "Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36", + "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", + "Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", + "Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1", + "Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00", + "Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; de) Opera 11.01", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; en) Opera 11.00", + "Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3", + "Opera/9.80 (Windows NT 6.1; U; ko) Presto/2.7.62 Version/11.00", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7", + "Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36", + "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.6.37 Version/11.00", + "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", + "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13", + "Mozilla/5.0 (Windows NT 6.0; U; ja; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)", + "Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/19.0.1047.0 Safari/535.22", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3)", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/5.0 Opera 11.11", + "Mozilla/5.0 (Macintosh; AMD Mac OS X 10_8_2) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/18.6.872", + "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", + "Mozilla/5.0 (Linux; U; Android 4.0.3; de-ch; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.0) yi; AppleWebKit/345667.12221 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/453667.1221", + "Mozilla/1.22 (compatible; MSIE 10.0; Windows 3.1)", + "Opera/9.80 (Windows NT 5.1; U;) Presto/2.7.62 Version/11.01", + "Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16", + "Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21", + "Mozilla/5.0 (Linux; U; Android 2.3.5; en-us; HTC Vision Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F", + "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.151 Safari/535.19", + "Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1284.0 Safari/537.13", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", + "Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)", + "Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0)", + "Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00", + "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36", + "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10", + "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00", + "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari", + "Mozilla/5.0 (Windows NT 5.1; U; pl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00", + "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; de) Opera 11.51", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11", + "Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.7.62 Version/11.00", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19", + "Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/18.6.872.0 Safari/535.2 UNTRUSTED/1.0 3gpp-gba UNTRUSTED/1.0", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36", + "Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00", + "Mozilla/5.0 (Linux; U; Android 2.3.4; en-us; T-Mobile myTouch 3G Slide Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", + "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1041.0 Safari/535.21", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.19 (KHTML, like Gecko) Ubuntu/11.10 Chromium/18.0.1025.142 Chrome/18.0.1025.142 Safari/535.19", + "Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00" + ] + return random.choice(user_agents) diff --git a/parsers/__init__.py b/parsers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/parsers/censysparser.py b/parsers/censysparser.py index 5c1e64d9..87b77796 100755 --- a/parsers/censysparser.py +++ b/parsers/censysparser.py @@ -2,12 +2,12 @@ import re -class parser: +class Parser: def __init__(self, resultstoparse): self.ipaddresses = [] - self.souphosts = BeautifulSoup(resultstoparse.total_resultshosts, features = "html.parser") - self.soupcerts = BeautifulSoup(resultstoparse.total_resultscerts, features = "html.parser") + self.souphosts = BeautifulSoup(resultstoparse.total_resultshosts, features="html.parser") + self.soupcerts = BeautifulSoup(resultstoparse.total_resultscerts, features="html.parser") self.hostnames = [] self.hostnamesfromcerts = [] self.urls = [] diff --git a/parsers/cymonparser.py b/parsers/cymonparser.py index 5b45fb23..38703f9a 100755 --- a/parsers/cymonparser.py +++ b/parsers/cymonparser.py @@ -2,7 +2,7 @@ from bs4 import BeautifulSoup -class parser: +class Parser: def __init__(self, results): self.results = results diff --git a/parsers/myparser.py b/parsers/myparser.py index e241fad4..d4c58424 100755 --- a/parsers/myparser.py +++ b/parsers/myparser.py @@ -1,7 +1,7 @@ import re -class parser: +class Parser: def __init__(self, results, word): self.results = results diff --git a/parsers/securitytrailsparser.py b/parsers/securitytrailsparser.py index 6d2f7d05..b96239c0 100755 --- a/parsers/securitytrailsparser.py +++ b/parsers/securitytrailsparser.py @@ -1,37 +1,38 @@ -class parser: - - def __init__(self, word, text): - self.word = word - self.text = text - self.hostnames = set() - self.ips = set() - - def parse_text(self): - subDomain_flag = 0 - self.text = str(self.text).splitlines() - # Split lines to get a list of lines. - for index in range(0, len(self.text)): - line = self.text[index].strip() - if '"ip":' in line: - # Extract IP. - ip = '' - for ch in line[7:]: - if ch == '"': - break - else: - ip += ch - self.ips.add(ip) - elif '"subdomains":' in line: - # subdomains start here so set flag to 1. - subDomain_flag = 1 - continue - elif subDomain_flag > 0: - if ']' in line: - subDomain_flag = 0 - else: - if 'www' in self.word: self.word = str(self.word).replace('www.', '').replace('www', '') - # Remove www from word if entered. - self.hostnames.add(str(line).replace('"', '').replace(',', '') + '.' + self.word) - else: - continue - return list(self.ips), list(self.hostnames) +class Parser: + + def __init__(self, word, text): + self.word = word + self.text = text + self.hostnames = set() + self.ips = set() + + def parse_text(self): + sub_domain_flag = 0 + self.text = str(self.text).splitlines() + # Split lines to get a list of lines. + for index in range(0, len(self.text)): + line = self.text[index].strip() + if '"ip":' in line: + # Extract IP. + ip = '' + for ch in line[7:]: + if ch == '"': + break + else: + ip += ch + self.ips.add(ip) + elif '"subdomains":' in line: + # subdomains start here so set flag to 1 + sub_domain_flag = 1 + continue + elif sub_domain_flag > 0: + if ']' in line: + sub_domain_flag = 0 + else: + if 'www' in self.word: + self.word = str(self.word).replace('www.', '').replace('www', '') + # Remove www from word if entered + self.hostnames.add(str(line).replace('"', '').replace(',', '') + '.' + self.word) + else: + continue + return list(self.ips), list(self.hostnames) diff --git a/requirements.txt b/requirements.txt index adb75b15..8c31c250 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ plotly>=3.4.2 requests>=2.21.0 texttable>=1.4.0 shodan>=1.10.0 +pytest>=4.0.2 \ No newline at end of file diff --git a/tests/myparser_test.py b/tests/myparser_test.py deleted file mode 100644 index 79362469..00000000 --- a/tests/myparser_test.py +++ /dev/null @@ -1,18 +0,0 @@ -# -# Unit tests for myparser.py -# -from parsers import myparser - -import unittest - -class TestMyParser(unittest.TestCase): - - def test_emails(self): - word = 'domain.com' - results = '@domain.com***a@domain***banotherdomain.com***c@domain.com***d@sub.domain.com***' - p = myparser.parser(results, word) - emails = sorted(p.emails()) - self.assertEqual(emails, [ 'c@domain.com', 'd@sub.domain.com' ]) - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_myparser.py b/tests/test_myparser.py new file mode 100644 index 00000000..090469fb --- /dev/null +++ b/tests/test_myparser.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +from parsers import myparser +import pytest + + +class TestMyParser(object): + + def test_emails(self): + word = 'domain.com' + results = '@domain.com***a@domain***banotherdomain.com***c@domain.com***d@sub.domain.com***' + parse = myparser.Parser(results, word) + emails = sorted(parse.emails()) + assert emails, ['c@domain.com', 'd@sub.domain.com'] + + +if __name__ == '__main__': + pytest.main() + diff --git a/theHarvester.py b/theHarvester.py index 6dccb7e9..3f3d752e 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -import datetime import getopt import re import stash @@ -100,12 +99,12 @@ def start(argv): 'netcraft', 'pgp', 'securityTrails', 'threatcrowd', 'trello', 'twitter', 'vhost', 'virustotal', 'yahoo', 'all']) if set(engines).issubset(supportedengines): - print(("[-] Target domain: " + word + "\n")) + print(f"[-] Target domain: {word} \n") for engineitem in engines: if engineitem == "baidu": print("[-] Searching in Baidu.") try: - search = baidusearch.search_baidu(word, limit) + search = baidusearch.SearchBaidu(word, limit) search.process() all_emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) @@ -119,7 +118,7 @@ def start(argv): elif engineitem == "bing" or engineitem == "bingapi": print("[-] Searching in Bing.") try: - search = bingsearch.search_bing(word, limit, start) + search = bingsearch.SearchBing(word, limit, start) if engineitem == "bingapi": bingapi = "yes" else: @@ -141,7 +140,7 @@ def start(argv): print("[-] Searching in Censys.") from discovery import censys # Import locally or won't work - search = censys.search_censys(word, limit) + search = censys.SearchCensys(word, limit) search.process() all_ip = search.get_ipaddresses() hosts = filter(search.get_hostnames()) @@ -171,7 +170,7 @@ def start(argv): elif engineitem == "dogpile": print("[-] Searching in Dogpilesearch.") - search = dogpilesearch.search_dogpile(word, limit) + search = dogpilesearch.SearchDogpile(word, limit) search.process() emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) @@ -184,7 +183,7 @@ def start(argv): elif engineitem == "duckduckgo": print("[-] Searching in DuckDuckGo.") from discovery import duckduckgosearch - search = duckduckgosearch.search_duckduckgo(word, limit) + search = duckduckgosearch.SearchDuckDuckGo(word, limit) search.process() emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) @@ -209,7 +208,7 @@ def start(argv): elif engineitem == "googleCSE": print("[-] Searching in Google Custom Search.") try: - search = googleCSE.search_googleCSE( + search = googleCSE.SearchGoogleCSE( word, limit, start) search.process() search.store_results() @@ -227,16 +226,13 @@ def start(argv): pass elif engineitem == "google-certificates": - print( - "[-] Searching in Google Certificate transparency report.") - search = googlecertificates.search_googlecertificates( - word, limit, start) + print("[-] Searching in Google Certificate transparency report.") + search = googlecertificates.SearchGoogleCertificates(word, limit, start) search.process() hosts = filter(search.get_domains()) all_hosts.extend(hosts) db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', - 'google-certificates') + db.store_all(word, all_hosts, 'host', 'google-certificates') elif engineitem == "google-profiles": print("[-] Searching in Google profiles.") @@ -256,8 +252,7 @@ def start(argv): from discovery import huntersearch # Import locally or won't work. try: - search = huntersearch.search_hunter( - word, limit, start) + search = huntersearch.SearchHunter(word, limit, start) search.process() emails = filter(search.get_emails()) all_emails.extend(emails) @@ -274,7 +269,7 @@ def start(argv): elif engineitem == "linkedin": print("[-] Searching in Linkedin.") - search = linkedinsearch.search_linkedin(word, limit) + search = linkedinsearch.SearchLinkedin(word, limit) search.process() people = search.get_people() db = stash.stash_manager() @@ -287,7 +282,7 @@ def start(argv): elif engineitem == "netcraft": print("[-] Searching in Netcraft.") - search = netcraft.search_netcraft(word) + search = netcraft.SearchNetcraft(word) search.process() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) @@ -297,7 +292,7 @@ def start(argv): elif engineitem == "pgp": print("[-] Searching in PGP key server.") try: - search = pgpsearch.search_pgp(word) + search = pgpsearch.SearchPgp(word) search.process() all_emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) @@ -337,8 +332,7 @@ def start(argv): hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() - db.store_all(word, all_hosts, - 'host', 'threatcrowd') + db.store_all(word, all_hosts, 'host', 'threatcrowd') except Exception: pass @@ -399,11 +393,22 @@ def start(argv): all_emails = [] all_hosts = [] - # baidu + try: + print("[-] Searching in Baidu.") + search = baidusearch.SearchBaidu(word, limit) + search.process() + all_emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'baidu') + db.store_all(word, all_emails, 'email', 'baidu') + except Exception: + pass print("[-] Searching in Bing.") bingapi = "no" - search = bingsearch.search_bing(word, limit, start) + search = bingsearch.SearchBing(word, limit, start) search.process(bingapi) emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) @@ -416,7 +421,7 @@ def start(argv): print("[-] Searching in Censys.") from discovery import censys - search = censys.search_censys(word, limit) + search = censys.SearchCensys(word, limit) search.process() ips = search.get_ipaddresses() setips = set(ips) @@ -442,6 +447,18 @@ def start(argv): # dogpile + print("[-] Searching in DuckDuckGo.") + from discovery import duckduckgosearch + search = duckduckgosearch.SearchDuckDuckGo(word, limit) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'duckduckgo') + db.store_all(word, all_hosts, 'host', 'duckduckgo') + print("[-] Searching in Google.") search = googlesearch.search_google(word, limit, start) search.process(google_dorking) @@ -454,29 +471,34 @@ def start(argv): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'google') - print( - "[-] Searching in Google Certificate transparency report.") - search = googlecertificates.search_googlecertificates( + print("[-] Searching in Google Certificate transparency report.") + search = googlecertificates.SearchGoogleCertificates( word, limit, start) search.process() domains = filter(search.get_domains()) all_hosts.extend(domains) db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', - 'google-certificates') + db.store_all(word, all_hosts, 'host', 'google-certificates') - # googleplus - - # google-certificates - - # google-profiles + try: + print("[-] Searching in Google profiles.") + search = googlesearch.search_google(word, limit, start) + search.process_profiles() + people = search.get_profiles() + db = stash.stash_manager() + db.store_all(word, people, 'name', 'google-profile') + print("\nUsers from Google profiles:") + print("---------------------------") + for users in people: + print(users) + except Exception: + pass print("[-] Searching in Hunter.") from discovery import huntersearch # Import locally. try: - search = huntersearch.search_hunter( - word, limit, start) + search = huntersearch.SearchHunter(word, limit, start) search.process() emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) @@ -495,7 +517,7 @@ def start(argv): # linkedin print("[-] Searching in Netcraft server.") - search = netcraft.search_netcraft(word) + search = netcraft.SearchNetcraft(word) search.process() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) @@ -504,7 +526,7 @@ def start(argv): print("[-] Searching in PGP key server.") try: - search = pgpsearch.search_pgp(word) + search = pgpsearch.SearchPgp(word) search.process() emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) @@ -526,8 +548,7 @@ def start(argv): hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() - db.store_all(word, all_hosts, - 'host', 'threatcrowd') + db.store_all(word, all_hosts, 'host', 'threatcrowd') except Exception: pass @@ -546,7 +567,19 @@ def start(argv): db.store_all(word, hosts, 'host', 'trello') db.store_all(word, emails, 'email', 'trello') - # twitter + try: + print("[-] Searching in Twitter.") + search = twittersearch.search_twitter(word, limit) + search.process() + people = search.get_people() + db = stash.stash_manager() + db.store_all(word, people, 'name', 'twitter') + print("\nUsers from Twitter:") + print("-------------------") + for user in people: + print(user) + except Exception: + pass # vhost @@ -558,7 +591,16 @@ def start(argv): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'virustotal') - # yahoo + print("[-] Searching in Yahoo.") + search = yahoosearch.search_yahoo(word, limit) + search.process() + hosts = search.get_hostnames() + emails = search.get_emails() + all_hosts.extend(filter(hosts)) + all_emails.extend(filter(emails)) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'yahoo') + db.store_all(word, all_emails, 'email', 'yahoo') else: print("[!] Invalid source.\n\n") sys.exit(1) @@ -704,7 +746,7 @@ def start(argv): # DNS TLD expansion dnstldres = [] - if dnstld == True: + if dnstld is True: print("[-] Starting DNS TLD expansion.") a = dnssearch.dns_tld(word, dnsserver, verbose=True) res = a.process() @@ -721,7 +763,7 @@ def start(argv): print("\n[+] Virtual hosts:") print("------------------") for l in host_ip: - search = bingsearch.search_bing(l, limit, start) + search = bingsearch.SearchBing(l, limit, start) search.process_vhost() res = search.get_allhostnames() for x in res: @@ -761,7 +803,7 @@ def start(argv): print("-------------------") print(printedtable) except Exception as e: - print("Error occurred in theHarvester - Shodan search module: " + str(e)) + print(f'Error occurred in theHarvester - Shodan search module: {e}') else: pass @@ -874,7 +916,7 @@ def start(argv): file.close() print("Files saved!") except Exception as er: - print(("Error saving XML file: " + str(er))) + print(f'Error saving XML file: {er}') sys.exit()