Integrated security trails into all mode, and touched up minor cosmetics.

discovery/censys.py

@@ -32,7 +32,7 @@ def process(self):
             try:
                 self.page = str(self.counter)
                 self.url = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page)
-                print("\t - Searching Censys results page " + self.page + "...")
+                print("\t -Searching Censys results page " + self.page + "...")
                 self.do_search()
                 time.sleep(getDelay())
             except Exception as e:

discovery/crtsh.py

@@ -26,7 +26,7 @@ def do_search(self):
         links = self.get_info(r.text)
         for link in links:
             params = {'User-Agent': getUserAgent()}
-            print ("\t\tSearching " + link)
+            #print("\tSearching " + link)
             r = requests.get(link, headers=params)
             time.sleep(getDelay())
             self.results = r.text

discovery/securitytrailssearch.py

@@ -7,7 +7,6 @@ class search_securitytrail:
 
     def __init__(self, word):
         self.word = word
-        self.limit = 100
         self.key = ""
         if self.key == "":
             print("You need an API key in order to use the SecurityTrails search engine. You can get one here: https://securitytrails.com/")
@@ -35,7 +34,7 @@ def do_search(self):
             url = self.database + 'domain/' + self.word
             headers = {'APIKEY': self.key}
             r = requests.get(url, headers=headers)
-            time.sleep(2)
+            time.sleep(2) #not random delay because 2 seconds is required due to rate limit
         except Exception as e:
             print(e)
         self.results = r.text

securitytrailsparser.py

@@ -4,7 +4,6 @@ def __init__(self, word, text):
         self.word = word
         self.text = text
         self.hostnames = set()
-        self.urls = []
         self.ips = set()
 
     def parse_text(self):
@@ -35,4 +34,4 @@ def parse_text(self):
                     self.hostnames.add(str(line).replace('"', '').replace(',', '') + '.' + self.word)
             else:
                 continue
-        return list(self.ips), list(self.hostnames)
+        return list(self.ips), list(self.hostnames)

theHarvester.py

@@ -489,6 +489,19 @@ def start(argv):
                         # all_emails = sorted(set(all_emails))
                         db.store_all(word, emails, 'email', 'hunter')
 
+                        print("[-] Searching in securityTrails:")
+                        from discovery import securitytrailssearch
+                        search = securitytrailssearch.search_securitytrail(word)
+                        search.process()
+                        hosts = search.get_hostnames()
+                        all_hosts.extend(hosts)
+                        db = stash.stash_manager()
+                        db.store_all(word, hosts, 'host', 'securityTrails')
+                        ips = search.get_ips()
+                        all_ip.extend(ips)
+                        db = stash.stash_manager()
+                        db.store_all(word, ips, 'ip', 'securityTrails')
+
                         print("[-] Searching in Google Certificate transparency report..")
                         search = googlecertificates.search_googlecertificates(word, limit, start)
                         search.process()
@@ -769,7 +782,6 @@ def start(argv):
             file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>')
             for x in all_emails:
                 file.write('<email>' + x + '</email>')
-
             for x in full:
                 x = x.split(":")
                 if len(x) == 2:
@@ -796,13 +808,12 @@ def start(argv):
                     file.write('<port>' + res[2] + '</port>')
                     # page.pre(res[1])
                     file.write('<banner><!--' + res[1] + '--></banner>')
-
                     reg_server = re.compile('Server:.*')
                     temp = reg_server.findall(res[1])
                     if temp != []:
                         shodanalysis.append(res[0] + ":" + temp[0])
-
                     file.write('</shodan>')
+
                 if shodanalysis != []:
                     shodanalysis = sorted(set(shodanalysis))
                     file.write('<servers>')