Merge pull request #323 from thehappydinoa/dev

Update code to pass CI and Censys Module

.github/workflows/theHarvester.yml

@@ -8,6 +8,7 @@ on:
   pull_request:
     branches:
     - '*'
+
 jobs:
   Python:
     runs-on: ${{ matrix.os }}
@@ -146,6 +147,7 @@ jobs:
     - name: Test with pytest
       run: |
         pytest
+
     - name: Static type checking with mypy
       run: |
-        mypy --pretty theHarvester/*/*.py
+        mypy --pretty theHarvester/*/*.py

README.md

@@ -20,6 +20,8 @@ Passive:
 
 * bufferoverun: Uses data from Rapid7's Project Sonar - www.rapid7.com/research/project-sonar/
 
+* censys: Censys search engine, will use certificates searches to enumerate subdomains (Requires an API key, see below.) - [censys.io](https://censys.io/)
+
 * certspotter: Cert Spotter monitors Certificate Transparency logs - https://sslmate.com/certspotter/
 
 * crtsh: Comodo Certificate search - https://crt.sh
@@ -63,7 +65,7 @@ to enhance research and analyse changes around DNS for better insights - https:/
 * securityTrails: Security Trails search engine, the world's largest repository of historical DNS data<br>
   (Requires an API key, see below.) - www.securitytrails.com
 
-* shodan: Shodan search engine, will search for ports and banners from discovered hosts - www.shodanhq.com
+* shodan: Shodan search engine, will search for ports and banners from discovered hosts (Requires an API key, see below.) - www.shodanhq.com
 
 * spyse: Web research tools for professionals (Requires an API key.) - https://spyse.com
 
@@ -104,7 +106,7 @@ Documentation to setup API keys can be found at - https://github.com/laramies/th
 * securityTrails
 * shodan
 * spyse - need to have a paid account be able to use the api now
-
+* censys
 
 Install and dependencies:
 -------------------------

theHarvester/__main__.py

@@ -328,7 +328,7 @@ async def store(search_engine: Any, source: str, process_param: Any = None, stor
                         if isinstance(e, MissingKey):
                             print(e)
                         else:
-                            print(f'An exception has occurred in ProjectDiscovery')
+                            print('An exception has occurred in ProjectDiscovery')
 
                 elif engineitem == 'qwant':
                     from theHarvester.discovery import qwantsearch

theHarvester/discovery/bingsearch.py

@@ -72,7 +72,7 @@ async def process(self, api, proxy=False):
         self.proxy = proxy
         if api == 'yes':
             if self.bingApi is None:
-                raise MissingKey(True, 'BingAPI')
+                raise MissingKey('BingAPI')
         else:
             if api == 'yes':
                 await self.do_search_api()

theHarvester/discovery/censys.py

@@ -1,29 +1,34 @@
-from theHarvester.discovery.constants import *
-from theHarvester.lib.core import *
-import censys.certificates
-import censys.base
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import Core
+from censys.certificates import CensysCertificates
+from censys.exceptions import (
+    CensysRateLimitExceededException,
+    CensysUnauthorizedException,
+)
 
 
 class SearchCensys:
-
-    def __init__(self, word):
-        self.word = word
+    def __init__(self, domain):
+        self.word = domain
         self.key = Core.censys_key()
         if self.key[0] is None or self.key[1] is None:
-            raise MissingKey(True, 'Censys ID or Secret')
+            raise MissingKey("Censys ID and/or Secret")
         self.totalhosts = set()
         self.proxy = False
 
     async def do_search(self):
-        cert = censys.certificates.CensysCertificates(api_id=self.key[0], api_secret=self.key[1])
-        query = f'parsed.names: {self.word}'
         try:
-            response = cert.search(query=query, fields=['parsed.names'], page=1)
-        except censys.base.CensysRateLimitExceededException:
-            print('Censys rate limit exceeded')
+            c = CensysCertificates(api_id=self.key[0], api_secret=self.key[1])
+        except CensysUnauthorizedException:
+            raise MissingKey("Censys ID and/or Secret")
 
-        for hosts in response:
-            self.totalhosts.update(hosts['parsed.names'])
+        query = f"parsed.names: {self.word}"
+        try:
+            response = c.search(query=query, fields=["parsed.names", "metadata"])
+            for cert in response:
+                self.totalhosts.update(cert["parsed.names"])
+        except CensysRateLimitExceededException:
+            print("Censys rate limit exceeded")
 
     async def get_hostnames(self) -> set:
         return self.totalhosts

theHarvester/discovery/constants.py

@@ -1,5 +1,5 @@
 from theHarvester.lib.core import *
-from typing import Union
+from typing import Union, Optional
 import random
 
 googleUA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 ' \
@@ -107,8 +107,8 @@ class MissingKey(Exception):
     """
     :raise: When there is a module that has not been provided its API key
     """
-    def __init__(self, identity_flag: bool, source: str):
-        if identity_flag:
+    def __init__(self, source: Optional[str]):
+        if source:
             self.message = f'\n\033[93m[!] Missing API key for {source}. \033[0m'
         else:
             self.message = '\n\033[93m[!] Missing CSE id. \033[0m'

theHarvester/discovery/githubcode.py

@@ -37,7 +37,7 @@ def __init__(self, word, limit):
         # rate limits you more severely
         # https://developer.github.com/v3/search/#rate-limit
         if self.key is None:
-            raise MissingKey(True, 'Github')
+            raise MissingKey('Github')
         self.proxy = False
 
     @staticmethod

theHarvester/discovery/huntersearch.py

@@ -11,7 +11,7 @@ def __init__(self, word, limit, start):
         self.start = start
         self.key = Core.hunter_key()
         if self.key is None:
-            raise MissingKey(True, 'Hunter')
+            raise MissingKey('Hunter')
         self.total_results = ""
         self.counter = start
         self.database = f'https://api.hunter.io/v2/domain-search?domain={self.word}&api_key={self.key}&limit={self.limit}'

theHarvester/discovery/intelxsearch.py

@@ -11,7 +11,7 @@ def __init__(self, word, limit):
         # default key is public key
         self.key = Core.intelx_key()
         if self.key is None:
-            raise MissingKey(True, 'Intelx')
+            raise MissingKey('Intelx')
         self.database = 'https://public.intelx.io/'
         self.results = None
         self.info = ()

theHarvester/discovery/pentesttools.py

@@ -11,7 +11,7 @@ def __init__(self, word):
         self.word = word
         self.key = Core.pentest_tools_key()
         if self.key is None:
-            raise MissingKey(True, 'PentestTools')
+            raise MissingKey('PentestTools')
         self.total_results = []
         self.api = f'https://pentest-tools.com/api?key={self.key}'
         self.proxy = False

theHarvester/discovery/projectdiscovery.py

@@ -8,7 +8,7 @@ def __init__(self, word):
         self.word = word
         self.key = Core.projectdiscovery_key()
         if self.key is None:
-            raise MissingKey(True, 'ProjectDiscovery')
+            raise MissingKey('ProjectDiscovery')
         self.total_results = None
         self.proxy = False
 

theHarvester/discovery/securitytrailssearch.py

@@ -10,7 +10,7 @@ def __init__(self, word):
         self.word = word
         self.key = Core.security_trails_key()
         if self.key is None:
-            raise MissingKey(True, 'Securitytrail')
+            raise MissingKey('Securitytrail')
         self.results = ""
         self.totalresults = ""
         self.api = 'https://api.securitytrails.com/v1/'

theHarvester/discovery/shodansearch.py

@@ -9,7 +9,7 @@ class SearchShodan:
     def __init__(self):
         self.key = Core.shodan_key()
         if self.key is None:
-            raise MissingKey(True, 'Shodan')
+            raise MissingKey('Shodan')
         self.api = Shodan(self.key)
         self.hostdatarow = []
 

theHarvester/discovery/spyse.py

@@ -9,7 +9,7 @@ def __init__(self, word):
         self.word = word
         self.key = Core.spyse_key()
         if self.key is None:
-            raise MissingKey(True, 'Spyse')
+            raise MissingKey('Spyse')
         self.results = ''
         self.hosts = set()
         self.proxy = False