shodan parsing fix

Dockerfile

@@ -1,6 +1,6 @@
 FROM python:2-alpine
 RUN mkdir /app
-RUN pip install requests beautifulsoup4
+RUN pip install requests beautifulsoup4 texttable
 WORKDIR /app
 COPY . /app
 RUN chmod +x *.py

discovery/shodansearch.py

@@ -2,23 +2,39 @@
 import sys
 from discovery.constants import *
 
+
 class search_shodan():
 
-    def __init__(self, host):
-        self.host = host
+    def __init__(self, ip):
+        self.ip = ip
         self.key = shodanAPI_key
         if self.key == "":
             raise MissingKey(True)
         self.api = Shodan(self.key)
+        self.hostdatarow = []
         
-    def run(self):
+    def search_host(self):
         try:
-            result = self.api.host(self.host)
-            #for service in result['data']:
-            #    print ("%s:%s" % (service['ip_str'], service['port']))
-            #   print ("%s" % (service['product']))
-            #    print ("%s" % (service['hostnames']))
-            return result
+            results = self.api.host(self.ip)
+            for result in results['data']:
+                technologies = []
+                if ('components' in result) and ('http' in result): #not always getting "http" key in the response
+                    for key in result['http']['components'].keys():
+                        technologies.append(key)
+                else:
+                    technologies.append("None")
+                if 'product' not in result: #not always getting "product" key in the response
+                    product = 'None'
+                else:
+                    product = str(result['product'])
+                if 'tags' not in result: #not always getting "tags": key in the response
+                    tags = 'None'
+                else:
+                    tags = str(result['tags']).strip('[]')
+                self.hostdatarow = [str(result['ip_str']), str(result['hostnames']).strip('[]'), str(result['location']['country_name']),
+                            str(result['location']['city']), str(result['org']), str(result['isp']), str(result['asn']),
+                            str(result['port']), str(result['os']), product, str(technologies).strip('[]'), tags]
         except Exception as e:
-            print("SHODAN empty reply or error in the call")
-            return "error"
+            print("Error occurred in the Shodan host search module: " + str(e))
+        finally:
+            return self.hostdatarow

requirements.txt

@@ -1,3 +1,4 @@
 requests==2.18.4
 beautifulsoup4==4.6.3
 plotly==3.4.2
+texttable==1.5.0

theHarvester.py

@@ -760,35 +760,35 @@ def start(argv):
     else:
         pass
     # Shodan search ####################################################
-    shodanres = []
-    shodanvisited = []
-    if shodan == True:
-        print("\n\n\033[1;32;40m[-] Shodan DB search (passive):\n")
-        if full == []:
-            print('No host to search, exiting.')
-            sys.exit()
-
-        for x in full:
-            try:
-                ip = x.split(":")[1]
-                if not shodanvisited.count(ip):
-                    print(("\tSearching for: " + ip))
-                    a = shodansearch.search_shodan(ip)
-                    shodanvisited.append(ip)
-                    results = a.run()
-                    # time.sleep(2)
-                    for res in results['data']:
-                        shodanres.append(
-                            str("%s:%s - %s - %s - %s," % (res['ip_str'], res['port'], res['os'], res['isp'])))
-            except Exception as e:
-                pass
-        print("\n [+] Shodan results:")
-        print("------------------")
-        for x in shodanres:
-            print(x)
-    else:
-        pass
-
+    try:
+        shodanres = []
+        import texttable
+        tab = texttable.Texttable()
+        header = ["IP address", "HostShodan", "Country", "City", "Org", "ISP", "ASN", "Port", "OS", "Product", "Stack", "Tags"]
+        tab.header(header)
+        tab.set_cols_align(["c", "c", "c", "c", "c", "c", "c", "c", "c", "c", "c", "c"])
+        tab.set_cols_valign(["m", "m", "m", "m", "m", "m", "m", "m", "m", "m", "m", "m"])
+        tab.set_chars(['-', '|', '+', '#'])
+        tab.set_cols_width([15, 15, 8, 8, 10, 15, 7, 5, 8, 15, 10, 10])
+        host_ip = list(set(host_ip))
+        shodan = True
+        if shodan:
+            print("\n\n\033[1;32;40m[-] Shodan DB search (passive):\n")
+            for ip in host_ip:
+                try:
+                    print("\tSearching for: " + ip)
+                    shodanhostquery = shodansearch.search_shodan(ip)
+                    rowdata = shodanhostquery.search_host()
+                    time.sleep(2)
+                    tab.add_row(rowdata)
+                except Exception as e:
+                    print("No information available in Shodan for: " + str(ip))
+            print("\n [+] Shodan results:")
+            print("------------------")
+            printedtable = tab.draw()
+            print(printedtable)
+    except Exception as e:
+        print("Error occurred in the Shodan main module: " + str(e))
     ###################################################################
     # Here we need to add explosion mode.
     # Tengo que sacar los TLD para hacer esto.