Syncing.

Merge branch 'master' of https://github.com/laramies/theHarvester into dev

.github/workflows/theHarvester.yml

@@ -63,6 +63,10 @@ jobs:
       run: |
         python theHarvester.py -d metasploit.com -b google
 
+    - name: Run theHarvester module Intelx
+      run: |
+        python theHarvester.py -d metasploit.com -b intelx
+
     - name: Run theHarvester module linkedin
       run: |
         python theHarvester.py -d metasploit.com -b linkedin

.lgtm.yml

@@ -1,3 +1,8 @@
 queries:
   - exclude: py/import-and-import-from
-  - exclude: py/polluting-import
+  - exclude: py/polluting-import
+
+extraction:
+  python:
+    python_setup:
+      version: 3

.travis.yml

@@ -14,7 +14,7 @@ before_install:
 install:
 - python setup.py test
 script:
-- python theHarvester.py -d metasploit.com -b baidu,bing,censys,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,threatcrowd,trello,twitter,virustotal,yahoo -l 200
+- python theHarvester.py -d metasploit.com -b baidu,bing,censys,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,otx,intelx,threatcrowd,trello,twitter,virustotal,yahoo -l 200
 - pytest
 - flake8 . --count --show-source --statistics
 #- mypy *.py

Dockerfile

@@ -1,8 +1,9 @@
-FROM python:3.6-alpine3.7
+FROM kalilinux/kali-linux-docker
 RUN mkdir /app
 WORKDIR /app
 COPY . /app
-RUN apk add build-base
+RUN apt-get -qq update
+RUN apt-get install -yqq python3-pip
 RUN pip3 install -r requirements.txt
 RUN chmod +x *.py
 ENTRYPOINT ["/app/theHarvester.py"]

README.md

@@ -43,6 +43,8 @@ Passive:
 
 * netcraft: Netcraft Data Mining - www.netcraft.com
 
+* otx: AlienVault Open Threat Exchange - https://otx.alienvault.com
+
 * securityTrails: Security Trails search engine, the world's largest repository<br>
   of historical DNS data (Requires API key, see below.) - www.securitytrails.com
 

theHarvester/__main__.py

@@ -52,6 +52,7 @@ def start():
     dnslookup = args.dns_lookup
     dnsserver = args.dns_server
     dnstld = args.dns_tld
+    engines = []
     filename = args.filename  # type: str
     full = []
     google_dorking = args.google_dork
@@ -315,11 +316,10 @@ def start():
                         otxsearch_search = otxsearch.SearchOtx(word)
                         otxsearch_search.process()
                         hosts = filter(otxsearch_search.get_hostnames())
-                        print('hosts: ', hosts)
                         all_hosts.extend(list(hosts))
                         ips = filter(otxsearch_search.get_ips())
-                        print('ips: ', ips)
                         all_ip.extend(list(ips))
+                        all_hosts.extend(hosts)
                         db = stash.stash_manager()
                         db.store_all(word, all_hosts, 'host', 'otx')
                         db.store_all(word, all_ip, 'ip', 'otx')
@@ -448,10 +448,8 @@ def start():
     else:
         print('\n[*] IPs found: ' + str(len(all_ip)))
         print('-------------------')
-        # ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip))
-        # print('\n'.join(map(str, ips)))
-        ip_list = sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)])
         # use netaddr as the list may contain ipv4 and ipv6 addresses
+        ip_list = sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)])
         print('\n'.join(map(str, ip_list)))
 
     if len(all_emails) == 0:
@@ -617,7 +615,7 @@ def start():
 
     # Here we need to add explosion mode.
     # We have to take out the TLDs to do this.
-    recursion = None
+    recursion = False
     if recursion:
         counter = 0
         for word in vhost:

theHarvester/discovery/otxsearch.py

@@ -25,8 +25,8 @@ def do_search(self):
         self.totalresults += self.results
         dct = json.loads(self.totalresults)
         self.totalhosts: set = {host['hostname'] for host in dct['passive_dns']}
-        self.totalips: set = {ip['address'] for ip in dct['passive_dns'] if 'NXDOMAIN' not in ip['address']}
         # filter out ips that are just called NXDOMAIN
+        self.totalips: set = {ip['address'] for ip in dct['passive_dns'] if 'NXDOMAIN' not in ip['address']}
 
     def get_hostnames(self) -> set:
         return self.totalhosts