From 52b7a7d427f02b94966923baf18ae4a115db40f8 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Fri, 21 Dec 2018 17:47:15 -0500 Subject: [PATCH] Integrated security trails into all mode, and touched up minor cosmetics. --- discovery/censys.py | 2 +- discovery/crtsh.py | 2 +- discovery/securitytrailssearch.py | 3 +-- securitytrailsparser.py | 3 +-- theHarvester.py | 17 ++++++++++++++--- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/discovery/censys.py b/discovery/censys.py index 497d1a01..55ab916c 100644 --- a/discovery/censys.py +++ b/discovery/censys.py @@ -32,7 +32,7 @@ def process(self): try: self.page = str(self.counter) self.url = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page) - print("\t - Searching Censys results page " + self.page + "...") + print("\t -Searching Censys results page " + self.page + "...") self.do_search() time.sleep(getDelay()) except Exception as e: diff --git a/discovery/crtsh.py b/discovery/crtsh.py index c0ab9249..412fd3a5 100644 --- a/discovery/crtsh.py +++ b/discovery/crtsh.py @@ -26,7 +26,7 @@ def do_search(self): links = self.get_info(r.text) for link in links: params = {'User-Agent': getUserAgent()} - print ("\t\tSearching " + link) + #print("\tSearching " + link) r = requests.get(link, headers=params) time.sleep(getDelay()) self.results = r.text diff --git a/discovery/securitytrailssearch.py b/discovery/securitytrailssearch.py index db8a8bc9..e052f5d0 100644 --- a/discovery/securitytrailssearch.py +++ b/discovery/securitytrailssearch.py @@ -7,7 +7,6 @@ class search_securitytrail: def __init__(self, word): self.word = word - self.limit = 100 self.key = "" if self.key == "": print("You need an API key in order to use the SecurityTrails search engine. You can get one here: https://securitytrails.com/") @@ -35,7 +34,7 @@ def do_search(self): url = self.database + 'domain/' + self.word headers = {'APIKEY': self.key} r = requests.get(url, headers=headers) - time.sleep(2) + time.sleep(2) #not random delay because 2 seconds is required due to rate limit except Exception as e: print(e) self.results = r.text diff --git a/securitytrailsparser.py b/securitytrailsparser.py index 971b422b..93ef5b8e 100644 --- a/securitytrailsparser.py +++ b/securitytrailsparser.py @@ -4,7 +4,6 @@ def __init__(self, word, text): self.word = word self.text = text self.hostnames = set() - self.urls = [] self.ips = set() def parse_text(self): @@ -35,4 +34,4 @@ def parse_text(self): self.hostnames.add(str(line).replace('"', '').replace(',', '') + '.' + self.word) else: continue - return list(self.ips), list(self.hostnames) + return list(self.ips), list(self.hostnames) \ No newline at end of file diff --git a/theHarvester.py b/theHarvester.py index 3fd98f56..27b547f7 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -489,6 +489,19 @@ def start(argv): # all_emails = sorted(set(all_emails)) db.store_all(word, emails, 'email', 'hunter') + print("[-] Searching in securityTrails:") + from discovery import securitytrailssearch + search = securitytrailssearch.search_securitytrail(word) + search.process() + hosts = search.get_hostnames() + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, hosts, 'host', 'securityTrails') + ips = search.get_ips() + all_ip.extend(ips) + db = stash.stash_manager() + db.store_all(word, ips, 'ip', 'securityTrails') + print("[-] Searching in Google Certificate transparency report..") search = googlecertificates.search_googlecertificates(word, limit, start) search.process() @@ -769,7 +782,6 @@ def start(argv): file.write('') for x in all_emails: file.write('' + x + '') - for x in full: x = x.split(":") if len(x) == 2: @@ -796,13 +808,12 @@ def start(argv): file.write('' + res[2] + '') # page.pre(res[1]) file.write('') - reg_server = re.compile('Server:.*') temp = reg_server.findall(res[1]) if temp != []: shodanalysis.append(res[0] + ":" + temp[0]) - file.write('') + if shodanalysis != []: shodanalysis = sorted(set(shodanalysis)) file.write('')