Merge pull request #5 from NotoriousRebel/dev

Modified otx module to work properly with set comprehensions.

requirements.txt

@@ -5,6 +5,7 @@ dnspython==1.16.0
 flake8==3.7.8
 grequests==0.4.0
 mypy==0.720
+netaddr==0.7.19
 plotly==4.1.1
 pytest==5.1.3
 PyYaml==5.1.2

tests/discovery/test_otx.py

@@ -20,7 +20,7 @@ def test_api(self):
     def test_search(self):
         search = otxsearch.SearchOtx(TestOtx.domain())
         search.process()
-        assert type(search.get_hostnames()) == list
+        assert isinstance(search.get_hostnames(), set)
 
     def test_search_no_results(self):
         search = otxsearch.SearchOtx('radiant.eu')

theHarvester/__main__.py

@@ -9,7 +9,7 @@
 from theHarvester.lib.core import *
 import argparse
 import datetime
-import ipaddress
+import netaddr
 import re
 import sys
 import time
@@ -315,9 +315,14 @@ def start():
                         otxsearch_search = otxsearch.SearchOtx(word)
                         otxsearch_search.process()
                         hosts = filter(otxsearch_search.get_hostnames())
-                        all_hosts.extend(hosts)
+                        print('hosts: ', hosts)
+                        all_hosts.extend(list(hosts))
+                        ips = filter(otxsearch_search.get_ips())
+                        print('ips: ', ips)
+                        all_ip.extend(list(ips))
                         db = stash.stash_manager()
                         db.store_all(word, all_hosts, 'host', 'otx')
+                        db.store_all(word, all_ip, 'ip', 'otx')
                     except Exception as e:
                         print(e)
 
@@ -430,8 +435,11 @@ def start():
     else:
         print('\n[*] IPs found: ' + str(len(all_ip)))
         print('-------------------')
-        ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip))
-        print('\n'.join(map(str, ips)))
+        # ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip))
+        # print('\n'.join(map(str, ips)))
+        ip_list = sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)])
+        # use netaddr as the list may contain ipv4 and ipv6 addresses
+        print('\n'.join(map(str, ip_list)))
 
     if len(all_emails) == 0:
         print('\n[*] No emails found.')

theHarvester/discovery/otxsearch.py

@@ -1,5 +1,5 @@
 from theHarvester.lib.core import *
-from theHarvester.parsers import myparser
+import json
 import grequests
 
 
@@ -9,6 +9,8 @@ def __init__(self, word):
         self.word = word
         self.results = ''
         self.totalresults = ''
+        self.totalhosts = set()
+        self.totalips = set()
 
     def do_search(self):
         base_url = f'https://otx.alienvault.com/api/v1/indicators/domain/{self.word}/passive_dns'
@@ -19,12 +21,19 @@ def do_search(self):
             self.results = data[0].content.decode('UTF-8')
         except Exception as e:
             print(e)
-        self.totalresults += self.results
 
-    def get_hostnames(self) -> Set:
-        return myparser.Parser(self.totalresults, self.word).hostnames()
+        self.totalresults += self.results
+        dct = json.loads(self.totalresults)
+        self.totalhosts: set = {host['hostname'] for host in dct['passive_dns']}
+        self.totalips: set = {ip['address'] for ip in dct['passive_dns'] if 'NXDOMAIN' not in ip['address']}
+        # filter out ips that are just called NXDOMAIN
+
+    def get_hostnames(self) -> set:
+        return self.totalhosts
+
+    def get_ips(self) -> set:
+        return self.totalips
 
     def process(self):
         self.do_search()
-        self.get_hostnames()
         print('\tSearching results.')