Merge branch 'master' of https://github.com/laramies/theHarvester

README.md

@@ -6,7 +6,7 @@
 * | |_| | | |  __/ / __  / (_| | |   \ V /  __/\__ \ ||  __/ |    *
 *  \__|_| |_|\___| \/ /_/ \__,_|_|    \_/ \___||___/\__\___|_|    *
 *                                                                 *
-* theHarvester 3.0.6 v247                                         *
+* theHarvester 3.0.6 v260                                         *
 * Coded by Christian Martorella                                   *
 * Edge-Security Research                                          *
 * cmartorella@edge-security.com                                   *
@@ -46,6 +46,8 @@ Passive:
 
 * hunter: Hunter search engine (Requires API key, see below.) - www.hunter.io
 
+* intelx: Intelx search engine (Requires API key, see below.) - www.intelx.io
+
 * linkedin: Google search engine, specific search for Linkedin users
 
 * netcraft: Netcraft Data Mining
@@ -84,6 +86,7 @@ Add your keys to api-keys.yaml
 
 * googleCSE: API key and CSE ID
 * hunter: API key
+* intelx: API key
 * securityTrails: API key
 * shodan: API key
 

api-keys.yaml

@@ -1,16 +1,19 @@
-apikeys:	
-  bing:	
-    key:	
+apikeys:
+  bing:
+    key:
 
-   googleCSE:	
-    key:	
-    id:	
+  googleCSE:
+    key:
+    id:
 
-   hunter:	
-    key:	
+  hunter:
+    key:
 
-   securityTrails:	
-    key:	
+  intelx:
+    key: 9df61df0-84f7-4dc7-b34c-8ccfb8646ace
 
-   shodan:	
+  securityTrails:
+    key: 
+
+  shodan:
     key: oCiMsgM6rQWqiTvPxFHYcExlZgg7wvTt

discovery/__init__.py

@@ -9,6 +9,8 @@
            'googleCSE',
            'googlecertificates',
            'googlesearch',
+           'huntersearch',
+           'intelxsearch',
            'linkedinsearch',
            'netcraft',
            'pgpsearch',

discovery/intelxsearch.py

@@ -0,0 +1,53 @@
+from discovery.constants import *
+from lib.core import *
+from parsers import intelxparser
+import requests
+import time
+
+class search_intelx:
+
+    def __init__(self, word, limit):
+        self.word = word
+        # default key is public key
+        self.key = Core.intelx_key()
+        if self.key is None:
+            raise MissingKey(True)
+        self.database = 'https://public.intelx.io/'
+        self.results = None
+        self.info = ()
+        self.limit = limit
+
+    def do_search(self):
+        try:
+            user_agent = Core.get_user_agent()
+            headers = {'User-Agent': user_agent, 'x-key': self.key}
+            data = f'{{"term": "{self.word}", "maxresults": {self.limit}, "media": 0, "sort": 2 , "terminate": []}}'
+            # data is json that corresponds to what we are searching for, sort:2 means sort by most relevant
+            r = requests.post(f'{self.database}phonebook/search', data=data, headers=headers)
+            if r.status_code == 400:
+                raise Exception('Invalid json was passed in.')
+            time.sleep(1)
+            # grab uuid to send get request to fetch data
+            uuid = r.json()['id']
+            url = f'{self.database}phonebook/search/result?id={uuid}&offset=0&limit={self.limit}'
+            r = requests.get(url, headers=headers)
+            time.sleep(1)
+            # to add in future grab status from r.text and check if more results can be gathered
+            if r.status_code != 200:
+                raise Exception('Error occurred while searching intelx.')
+            self.results = r.json()
+        except Exception as e:
+            print(f'An exception has occurred: {e}')
+
+    def process(self):
+        self.do_search()
+        intelx_parser = intelxparser.Parser()
+        self.info = intelx_parser.parse_dictionaries(self.results)
+        # Create parser and set self.info to tuple returned from parsing text.
+
+    def get_emails(self):
+        return self.info[0]
+
+    def get_hostnames(self):
+        return self.info[1]
+

lib/core.py

@@ -35,6 +35,12 @@ def shodan_key():
             keys = yaml.safe_load(api_keys)
             return keys['apikeys']['shodan']['key']
 
+    @staticmethod
+    def intelx_key():
+        with open('api-keys.yaml', 'r') as api_keys:
+            keys = yaml.safe_load(api_keys)
+            return keys['apikeys']['intelx']['key']
+
     @staticmethod
     def banner():
         print('\n\033[93m*******************************************************************')
@@ -44,7 +50,7 @@ def banner():
         print("* | |_| | | |  __/ / __  / (_| | |   \ V /  __/\__ \ ||  __/ |    *")
         print("*  \__|_| |_|\___| \/ /_/ \__,_|_|    \_/ \___||___/\__\___|_|    *")
         print('*                                                                 *')
-        print('* theHarvester 3.0.6 v247                                         *')
+        print('* theHarvester 3.0.6 v260                                         *')
         print('* Coded by Christian Martorella                                   *')
         print('* Edge-Security Research                                          *')
         print('* cmartorella@edge-security.com                                   *')
@@ -65,6 +71,7 @@ def get_supportedengines():
                             'googleCSE',
                             'google-certificates',
                             'hunter',
+                            'intelx',
                             'linkedin',
                             'netcraft',
                             'pgp',

parsers/intelxparser.py

@@ -0,0 +1,27 @@
+class Parser:
+
+    def __init__(self):
+        self.emails = set()
+        self.hosts = set()
+
+    def parse_dictionaries(self, results):
+        """
+        Parse method to parse json results
+        :param results: Dictionary containing a list of dictionaries known as selectors
+        :return: tuple of emails and hosts
+        """
+        if results is not None:
+            for dictionary in results["selectors"]:
+                field = dictionary['selectorvalue']
+                if '@' in field:
+                    self.emails.add(field)
+                else:
+                    field = str(field)
+                    if 'http' in field or 'https' in field:
+                        if field[:5] == 'https':
+                            field = field[8:]
+                        else:
+                            field = field[7:]
+                    self.hosts.add(field.replace(')', '').replace(',', ''))
+            return self.emails, self.hosts
+        return None, None