Add certspotter module

.github/workflows/theHarvester.yml

@@ -35,6 +35,10 @@ jobs:
       run: |
         python theHarvester.py -d metasploit.com -b bing
 
+    - name: Run theHarvester module certspotter
+      run: |
+        python theHarvester.py -d metasploit.com -b certspotter
+
     - name: Run theHarvester module crtsh
       run: |
         python theHarvester.py -d metasploit.com -b crtsh

.travis.yml

@@ -12,7 +12,7 @@ before_install:
 install:
 - python setup.py test
 script:
-- python theHarvester.py -d metasploit.com -b baidu,bing,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,otx,intelx,threatcrowd,trello,twitter,virustotal,yahoo -l 200
+- python theHarvester.py -d metasploit.com -b baidu,bing,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,otx,intelx,threatcrowd,trello,twitter,virustotal,yahoo -l 200
 - pytest
 - flake8 . --count --show-source --statistics
 #- mypy --pretty *.py

README.md

@@ -19,6 +19,8 @@ Passive:
 
 * bingapi: Microsoft search engine, through the API (Requires API key, see below.)
 
+* CertSporter: Cert Spotter monitors Certificate Transparency logs - https://sslmate.com/certspotter/
+
 * crtsh: Comodo Certificate search - www.crt.sh
 
 * dnsdumpster: DNSdumpster search engine - dnsdumpster.com

tests/discovery/test_certspotter.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+# coding=utf-8
+from theHarvester.lib.core import *
+from theHarvester.discovery import certspottersearch
+import requests
+import pytest
+
+
+class TestCertspotter(object):
+    @staticmethod
+    def domain() -> str:
+        return 'metasploit.com'
+
+    def test_api(self):
+        base_url = f'https://api.certspotter.com/v1/issuances?domain={TestCertspotter.domain()}&expand=dns_names'
+        headers = {'User-Agent': Core.get_user_agent()}
+        request = requests.get(base_url, headers=headers)
+        assert request.status_code == 200
+
+    def test_search(self):
+        search = certspottersearch.SearchCertspoter(TestCertspotter.domain())
+        search.process()
+        assert isinstance(search.get_hostnames(), set)
+
+    def test_search_no_results(self):
+        search = certspottersearch.SearchCertspoter('radiant.eu')
+        search.process()
+        assert len(search.get_hostnames()) == 0
+
+
+if __name__ == '__main__':
+    pytest.main()

theHarvester/__main__.py

@@ -34,7 +34,7 @@ def start():
     parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default False', default=False, action='store_true')
     parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true')
     parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str)
-    parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, crtsh, dnsdumpster,
+    parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, certspotter, crtsh, dnsdumpster,
                         dogpile, duckduckgo, github-code, google,
                         hunter, intelx,
                         linkedin, linkedin_links, netcraft, otx, securityTrails, spyse(disabled for now), threatcrowd,
@@ -117,6 +117,20 @@ def start():
                         else:
                             pass
 
+                elif engineitem == 'certspotter':
+                    print('\033[94m[*] Searching CertSpotter. \033[0m')
+                    from theHarvester.discovery import certspottersearch
+                    try:
+                        certspotter_search = certspottersearch.SearchCertspoter(word)
+                        certspotter_search.process()
+                        hosts = filter(certspotter_search.get_hostnames())
+                        all_hosts.extend(list(hosts))
+                        all_hosts.extend(hosts)
+                        db = stash.stash_manager()
+                        db.store_all(word, all_hosts, 'host', 'certspotter')
+                    except Exception as e:
+                        print(e)
+
                 elif engineitem == 'crtsh':
                     try:
                         print('\033[94m[*] Searching CRT.sh. \033[0m')

theHarvester/discovery/__init__.py

@@ -1,6 +1,7 @@
 __all__ = ['baidusearch',
            'bingsearch',
            'crtsh',
+           'certspottersearch',
            'dnssearch',
            'dogpilesearch',
            'duckduckgosearch',

theHarvester/discovery/certspottersearch.py

@@ -0,0 +1,29 @@
+from theHarvester.lib.core import *
+import requests
+
+
+class SearchCertspoter:
+
+    def __init__(self, word):
+        self.word = word
+        self.totalhosts = set()
+
+    def do_search(self) -> None:
+        base_url = f'https://api.certspotter.com/v1/issuances?domain={self.word}&expand=dns_names'
+        headers = {'User-Agent': Core.get_user_agent()}
+        try:
+            request = requests.get(base_url, headers=headers)
+            response = request.json()
+            for dct in response:
+                for key, value in dct.items():
+                    if key == 'dns_names':
+                        self.totalhosts.update({name for name in value if name})
+        except Exception as e:
+            print(e)
+
+    def get_hostnames(self) -> set:
+        return self.totalhosts
+
+    def process(self):
+        self.do_search()
+        print('\tSearching results.')

theHarvester/lib/core.py

@@ -73,6 +73,7 @@ def get_supportedengines() -> Set[Union[str, Any]]:
         supportedengines = {'baidu',
                             'bing',
                             'bingapi',
+                            'certspotter',
                             'crtsh',
                             'dnsdumpster',
                             'dogpile',