Fixed and expanded Censys

Expanded Censys to search certificates,hosts, ips.

censysparser.py

@@ -3,39 +3,59 @@
 
 
 class parser:
-
+    
-    def __init__(self, results):
+    def __init__(self, resultstoparse):
-        self.results = results
         self.ipaddresses = []
-        self.soup = BeautifulSoup(results.results, features="html.parser")
+        self.souphosts = BeautifulSoup(resultstoparse.total_resultshosts,features="html.parser")
+        self.soupcerts = BeautifulSoup(resultstoparse.total_resultscerts,features="html.parser")
         self.hostnames = []
+        self.hostnamesfromcerts = []
         self.urls = []
-        self.numberofpages = 0
+        self.numberofpageshosts = 0
+        self.numberofpagescerts = 0
 
-    def search_hostnames(self):
+    def search_hostnamesfromcerts(self):
         try:
-            hostnamelist = self.soup.findAll('tt')
+            hostnamelist = self.soupcerts.findAll("i", "fa fa-fw fa-home")
             for hostnameitem in hostnamelist:
-                self.hostnames.append(hostnameitem.text)
+                hostitems = hostnameitem.next_sibling
-            return self.hostnames
+                hostnames = str(hostitems)
+                hostnamesclean = re.sub('[ \'\[\]]','', hostnames)
+                hostnamesclean = re.sub(r'\.\.\.',r'',hostnamesclean)
+                self.hostnamesfromcerts.extend(hostnamesclean.split(","))
+            self.hostnamesfromcerts = list(filter(None, self.hostnamesfromcerts))   #filter out duplicates
+            return self.hostnamesfromcerts
         except Exception as e:
-            print("Error occurred: " + str(e))
+            print("Error occurred in the Censys module: certificate hostname parser: " + str(e))
 
     def search_ipaddresses(self):
         try:
-            ipaddresslist = self.soup.findAll('a', 'SearchResult__title-text')
+            ipaddresslist = self.souphosts.findAll('a','SearchResult__title-text')
             for ipaddressitem in ipaddresslist:
                 self.ipaddresses.append(ipaddressitem.text.strip())
             return self.ipaddresses
         except Exception as e:
-            print("Error occurred: " + str(e))
+            print("Error occurred in the Censys module: IP address parser: " + str(e))
 
-    def search_numberofpages(self):
+    def search_numberofpageshosts(self):
         try:
-            items = self.soup.findAll(href=re.compile("page"))
+            items = self.souphosts.findAll(href=re.compile("page"))
             for item in items:
-                if (item.text != 'next'):  # to filter out pagination
+                if (item.text !='next'):            #to filter out pagination
-                    self.numberofpages += 1
+                    self.numberofpageshosts+=1
-            return self.numberofpages
+            return self.numberofpageshosts
         except Exception as e:
-            print("Error occurred: " + str(e))
+            print("Error occurred in the Censys module IP search: page parser: " + str(e))
+
+    def search_numberofpagescerts(self):
+        try:
+            items = self.soupcerts.findAll(href=re.compile("page"))
+            for item in items:
+                if (item.text != 'next'):            #to filter out pagination
+                    self.numberofpagescerts += 1
+            return self.numberofpagescerts
+        except Exception as e:
+            print("Error occurred in the Censys module certificate search: page parser: " + str(e))
+
+              
+   

discovery/censys.py

@@ -7,48 +7,89 @@ class search_censys:
 
     def __init__(self, word):
         self.word = word
-        self.url = ""
+        self.urlhost = ""
+        self.urlcert = ""
         self.page = ""
-        self.results = ""
+        self.resultshosts = ""
-        self.total_results = ""
+        self.resultcerts = ""
+        self.total_resultshosts = ""
+        self.total_resultscerts = ""
         self.server = "censys.io"
-
+        self.ips = []
-    def do_search(self):
+        self.hostnamesall = []
+        
+    def do_searchhosturl(self):
         try:
-            headers = {'user-agent': getUserAgent(), 'Accept': '*/*', 'Referer': self.url}
+            headers = {'user-agent': getUserAgent(), 'Accept':'*/*','Referer': self.urlhost}       
-            response = requests.get(self.url, headers=headers)
+            responsehost = requests.get(self.urlhost, headers=headers)
-            self.results = response.text
+            self.resultshosts = responsehost.text
-            self.total_results += self.results
+            self.total_resultshosts += self.resultshosts
         except Exception as e:
-            print(e)
+            print("Error occurred in the Censys module downloading pages from Censys - IP search: " + str(e))
+
+    def do_searchcertificateurl(self):
+        try:
+            headers = {'user-agent': getUserAgent(), 'Accept':'*/*','Referer': self.urlcert}       
+            responsecert = requests.get(self.urlcert, headers=headers)
+            self.resultcerts = responsecert.text
+            self.total_resultscerts += self.resultcerts
+        except Exception as e:
+            print("Error occurred in the Censys module downloading pages from Censys - certificates search: " + str(e))
 
     def process(self):
-        self.url = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=1"
+        try:
-        self.do_search()
+            self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=1"
-        self.counter = 2
+            self.urlcert = "https://"+ self.server + "/certificates/_search?q=" + str(self.word) + "&page=1"
-        pages = censysparser.parser(self)
+            self.do_searchhosturl()
-        totalpages = pages.search_numberofpages()
+            self.do_searchcertificateurl()
-        while self.counter <= totalpages:
+            counter = 2
-            try:
+            pages = censysparser.parser(self)
-                self.page = str(self.counter)
+            totalpages = pages.search_numberofpageshosts()
-                self.url = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page)
+            while counter <= totalpages:
-                print("\t - Searching Censys results page " + self.page + "...")
+                try:
-                self.do_search()
+                    self.page =str(counter)
-                time.sleep(getDelay())
+                    self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page)                   
-            except Exception as e:
+                    print("\tSearching Censys IP results page " + self.page + "...")
-                print("Error occurred: " + str(e))
+                    self.do_searchhosturl()
-            self.counter += 1
+                    counter+= 1
+                except Exception as e:
+                    print("Error occurred in the Censys module requesting the pages: " + str(e))
+            counter = 2
+            totalpages = pages.search_numberofpagescerts()
+            while counter <= totalpages:
+                try:
+                    self.page = str(counter)
+                    self.urlhost = "https://" + self.server + "/certificates/_search?q=" + str(self.word) + "&page=" + str(self.page)                   
+                    print("\tSearching Censys certificates results page " + self.page + "...")
+                    self.do_searchcertificateurl()
+                    counter += 1
+                except Exception as e:
+                    print("Error occurred in the Censys module requesting the pages: " + str(e))
+        except Exception as e:
+            print("Error occurred in the main Censys module: " + str(e))
 
     def get_hostnames(self):
         try:
-            hostnames = censysparser.parser(self)
+            ips = self.get_ipaddresses()
-            return hostnames.search_hostnames()
+            headers = {'user-agent': getUserAgent(), 'Accept':'*/*','Referer': self.urlcert}
+            response = requests.post("https://censys.io/ipv4/getdns", json={"ips": ips}, headers=headers)
+            responsejson = response.json()
+            for key, jdata in responsejson.items():
+                    if jdata is not None:
+                        self.hostnamesall.append(jdata)
+                    else:
+                        pass
+            hostnamesfromcerts = censysparser.parser(self)
+            self.hostnamesall.extend(hostnamesfromcerts.search_hostnamesfromcerts())
+            return self.hostnamesall
         except Exception as e:
-            print("Error occurred: " + str(e))
+            print("Error occurred in the Censys module - hostname search: " + str(e))
 
     def get_ipaddresses(self):
         try:
             ips = censysparser.parser(self)
-            return ips.search_ipaddresses()
+            self.ips = ips.search_ipaddresses()
+            return self.ips
         except Exception as e:
-            print("Error occurred: " + str(e))
+            print("Error occurred in the main Censys module - IP address search: " + str(e))
+

theHarvester.py

@@ -396,11 +396,18 @@ def start(argv):
                         from discovery import censys
                         search = censys.search_censys(word)
                         search.process()
-                        all_ip = search.get_ipaddresses()
+                        ips = search.get_ipaddresses()
-                        all_hosts = search.get_hostnames()
+                        setips = set(ips)               
+                        uniqueips = list(setips)            #remove duplicates
+                        all_ip.extend(uniqueips)
+                        hosts = search.get_hostnames()
+                        sethosts = set(hosts)
+                        uniquehosts = list(sethosts)        #remove duplicates
+                        all_hosts.extend(uniquehosts)
                         db = stash.stash_manager()
-                        db.store_all(word, all_ip, 'ip', 'censys')
+                        db.store_all(word,uniquehosts,'host','censys')
-                        db.store_all(word, all_hosts, 'host', 'censys')
+                        db.store_all(word,uniqueips,'ip','censys')           
+
 
                         print("[-] Searching in CRTSH server..")
                         search = crtsh.search_crtsh(word)
@@ -469,7 +476,9 @@ def start(argv):
                         search.process()
                         emails = search.get_emails()
                         hosts = search.get_hostnames()
-                        all_hosts.extend(hosts)
+                        sethosts = set(hosts)
+                        uniquehosts = list(sethosts)        #remove duplicates
+                        all_hosts.extend(uniquehosts)
                         db = stash.stash_manager()
                         db.store_all(word, all_hosts, 'host', 'PGP')
                         all_emails.extend(emails)
@@ -513,6 +522,7 @@ def start(argv):
     else:
         print("\033[1;33;40m \n[+] IP addresses found in search engines:")
         print("------------------------------------")
+        print("Total IP addresses: "+ str(len(all_ip)) + "\n")
         for i in all_ip:
             print(i)
     print("\n\n[+] Emails found:")
@@ -533,6 +543,7 @@ def start(argv):
     if all_emails == []:
         print("No emails found.")
     else:
+        print("Total emails: "+ str(len(all_emails)) + "\n")
         print(("\n".join(all_emails)))
 
     print("\033[1;33;40m \n[+] Hosts found in search engines:")
@@ -543,6 +554,8 @@ def start(argv):
         total = len(all_hosts)
         print(("\nTotal hosts: " + str(total) + "\n"))
         all_hosts = sorted(set(all_hosts))
+        for host in all_hosts:
+            print(host)
         print("\033[94m[-] Resolving hostnames IPs...\033[1;33;40m \n ")
         full_host = hostchecker.Checker(all_hosts)
         full = full_host.check()