From cdd6b9020cf0713e883e647792d54ed96fbba70b Mon Sep 17 00:00:00 2001
From: NotoriousRebel <matt12299@yahoo.com>
Date: Wed, 25 Sep 2019 15:09:11 -0400
Subject: [PATCH 1/2] Modified otx module to work properly with set
 comprehensions and using netaddr to sort ipv4 and ipv6 in same list.

---
 requirements.txt                    |  1 +
 theHarvester/__main__.py            | 17 +++++++++++++----
 theHarvester/discovery/otxsearch.py | 19 ++++++++++++++-----
 3 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 44940d3b..2bd9cb06 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,6 +5,7 @@ dnspython==1.16.0
 flake8==3.7.8
 grequests==0.4.0
 mypy==0.720
+netaddr==0.7.19
 plotly==4.1.1
 pytest==5.1.3
 PyYaml==5.1.2
diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py
index ed69b793..cf7b920c 100644
--- a/theHarvester/__main__.py
+++ b/theHarvester/__main__.py
@@ -9,7 +9,7 @@
 from theHarvester.lib.core import *
 import argparse
 import datetime
-import ipaddress
+import netaddr
 import re
 import sys
 import time
@@ -315,9 +315,14 @@ def start():
                         otxsearch_search = otxsearch.SearchOtx(word)
                         otxsearch_search.process()
                         hosts = filter(otxsearch_search.get_hostnames())
-                        all_hosts.extend(hosts)
+                        print('hosts: ', hosts)
+                        all_hosts.extend(list(hosts))
+                        ips = filter(otxsearch_search.get_ips())
+                        print('ips: ', ips)
+                        all_ip.extend(list(ips))
                         db = stash.stash_manager()
                         db.store_all(word, all_hosts, 'host', 'otx')
+                        db.store_all(word, all_ip, 'ip', 'otx')
                     except Exception as e:
                         print(e)
 
@@ -430,8 +435,12 @@ def start():
     else:
         print('\n[*] IPs found: ' + str(len(all_ip)))
         print('-------------------')
-        ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip))
-        print('\n'.join(map(str, ips)))
+        #ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip))
+        #print('\n'.join(map(str, ips)))
+        ip_list = [netaddr.IPAddress(ip.strip()) for ip in set(all_ip)]
+        # use netaddr as the list may contain ipv4 and ipv6 addresses
+        ip_list.sort()
+        print('\n'.join(map(str, ip_list)))
 
     if len(all_emails) == 0:
         print('\n[*] No emails found.')
diff --git a/theHarvester/discovery/otxsearch.py b/theHarvester/discovery/otxsearch.py
index 03c1dbc2..b9323513 100644
--- a/theHarvester/discovery/otxsearch.py
+++ b/theHarvester/discovery/otxsearch.py
@@ -1,5 +1,5 @@
 from theHarvester.lib.core import *
-from theHarvester.parsers import myparser
+import json
 import grequests
 
 
@@ -9,6 +9,8 @@ def __init__(self, word):
         self.word = word
         self.results = ''
         self.totalresults = ''
+        self.totalhosts = set()
+        self.totalips = set()
 
     def do_search(self):
         base_url = f'https://otx.alienvault.com/api/v1/indicators/domain/{self.word}/passive_dns'
@@ -19,12 +21,19 @@ def do_search(self):
             self.results = data[0].content.decode('UTF-8')
         except Exception as e:
             print(e)
-        self.totalresults += self.results
 
-    def get_hostnames(self) -> Set:
-        return myparser.Parser(self.totalresults, self.word).hostnames()
+        self.totalresults += self.results
+        dct = json.loads(self.totalresults)
+        self.totalhosts: set = {host['hostname'] for host in dct['passive_dns']}
+        self.totalips: set = {ip['address'] for ip in dct['passive_dns'] if 'NXDOMAIN' not in ip['address']}
+        # filter out ips that are just called NXDOMAIN
+
+    def get_hostnames(self) -> set:
+        return self.totalhosts
+
+    def get_ips(self) -> set:
+        return self.totalips
 
     def process(self):
         self.do_search()
-        self.get_hostnames()
         print('\tSearching results.')

From 1286ba4b756c4627a8ccefbb37442e58992dd2d3 Mon Sep 17 00:00:00 2001
From: NotoriousRebel <matt12299@yahoo.com>
Date: Wed, 25 Sep 2019 15:17:24 -0400
Subject: [PATCH 2/2] Fixed test in test_otx and fixed pep8 error in main.py

---
 tests/discovery/test_otx.py | 2 +-
 theHarvester/__main__.py    | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/tests/discovery/test_otx.py b/tests/discovery/test_otx.py
index 67d54d10..58d6d018 100644
--- a/tests/discovery/test_otx.py
+++ b/tests/discovery/test_otx.py
@@ -20,7 +20,7 @@ def test_api(self):
     def test_search(self):
         search = otxsearch.SearchOtx(TestOtx.domain())
         search.process()
-        assert type(search.get_hostnames()) == list
+        assert isinstance(search.get_hostnames(), set)
 
     def test_search_no_results(self):
         search = otxsearch.SearchOtx('radiant.eu')
diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py
index cf7b920c..99d606a1 100644
--- a/theHarvester/__main__.py
+++ b/theHarvester/__main__.py
@@ -435,11 +435,10 @@ def start():
     else:
         print('\n[*] IPs found: ' + str(len(all_ip)))
         print('-------------------')
-        #ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip))
-        #print('\n'.join(map(str, ips)))
-        ip_list = [netaddr.IPAddress(ip.strip()) for ip in set(all_ip)]
+        # ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip))
+        # print('\n'.join(map(str, ips)))
+        ip_list = sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)])
         # use netaddr as the list may contain ipv4 and ipv6 addresses
-        ip_list.sort()
         print('\n'.join(map(str, ip_list)))
 
     if len(all_emails) == 0: