diff --git a/README.md b/README.md index a9987715..7a17b90a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ * | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | * * \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| * * * -* theHarvester 3.0.6 v247 * +* theHarvester 3.0.6 v260 * * Coded by Christian Martorella * * Edge-Security Research * * cmartorella@edge-security.com * @@ -46,6 +46,8 @@ Passive: * hunter: Hunter search engine (Requires API key, see below.) - www.hunter.io +* intelx: Intelx search engine (Requires API key, see below.) - www.intelx.io + * linkedin: Google search engine, specific search for Linkedin users * netcraft: Netcraft Data Mining @@ -84,6 +86,7 @@ Add your keys to api-keys.yaml * googleCSE: API key and CSE ID * hunter: API key +* intelx: API key * securityTrails: API key * shodan: API key diff --git a/api-keys.yaml b/api-keys.yaml index 55b6d2ef..d037a1e4 100644 --- a/api-keys.yaml +++ b/api-keys.yaml @@ -1,16 +1,19 @@ -apikeys: - bing: - key: +apikeys: + bing: + key: - googleCSE: - key: - id: + googleCSE: + key: + id: - hunter: - key: + hunter: + key: - securityTrails: - key: + intelx: + key: 9df61df0-84f7-4dc7-b34c-8ccfb8646ace - shodan: + securityTrails: + key: + + shodan: key: oCiMsgM6rQWqiTvPxFHYcExlZgg7wvTt \ No newline at end of file diff --git a/discovery/__init__.py b/discovery/__init__.py index b25adf7b..1ed84950 100644 --- a/discovery/__init__.py +++ b/discovery/__init__.py @@ -9,6 +9,8 @@ 'googleCSE', 'googlecertificates', 'googlesearch', + 'huntersearch', + 'intelxsearch', 'linkedinsearch', 'netcraft', 'pgpsearch', diff --git a/discovery/intelxsearch.py b/discovery/intelxsearch.py new file mode 100644 index 00000000..03d9fb93 --- /dev/null +++ b/discovery/intelxsearch.py @@ -0,0 +1,53 @@ +from discovery.constants import * +from lib.core import * +from parsers import intelxparser +import requests +import time + +class search_intelx: + + def __init__(self, word, limit): + self.word = word + # default key is public key + self.key = Core.intelx_key() + if self.key is None: + raise MissingKey(True) + self.database = 'https://public.intelx.io/' + self.results = None + self.info = () + self.limit = limit + + def do_search(self): + try: + user_agent = Core.get_user_agent() + headers = {'User-Agent': user_agent, 'x-key': self.key} + data = f'{{"term": "{self.word}", "maxresults": {self.limit}, "media": 0, "sort": 2 , "terminate": []}}' + # data is json that corresponds to what we are searching for, sort:2 means sort by most relevant + r = requests.post(f'{self.database}phonebook/search', data=data, headers=headers) + if r.status_code == 400: + raise Exception('Invalid json was passed in.') + time.sleep(1) + # grab uuid to send get request to fetch data + uuid = r.json()['id'] + url = f'{self.database}phonebook/search/result?id={uuid}&offset=0&limit={self.limit}' + r = requests.get(url, headers=headers) + time.sleep(1) + # to add in future grab status from r.text and check if more results can be gathered + if r.status_code != 200: + raise Exception('Error occurred while searching intelx.') + self.results = r.json() + except Exception as e: + print(f'An exception has occurred: {e}') + + def process(self): + self.do_search() + intelx_parser = intelxparser.Parser() + self.info = intelx_parser.parse_dictionaries(self.results) + # Create parser and set self.info to tuple returned from parsing text. + + def get_emails(self): + return self.info[0] + + def get_hostnames(self): + return self.info[1] + diff --git a/lib/core.py b/lib/core.py index 6999401c..6c9450cd 100644 --- a/lib/core.py +++ b/lib/core.py @@ -35,6 +35,12 @@ def shodan_key(): keys = yaml.safe_load(api_keys) return keys['apikeys']['shodan']['key'] + @staticmethod + def intelx_key(): + with open('api-keys.yaml', 'r') as api_keys: + keys = yaml.safe_load(api_keys) + return keys['apikeys']['intelx']['key'] + @staticmethod def banner(): print('\n\033[93m*******************************************************************') @@ -44,7 +50,7 @@ def banner(): print("* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *") print("* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *") print('* *') - print('* theHarvester 3.0.6 v247 *') + print('* theHarvester 3.0.6 v260 *') print('* Coded by Christian Martorella *') print('* Edge-Security Research *') print('* cmartorella@edge-security.com *') @@ -65,6 +71,7 @@ def get_supportedengines(): 'googleCSE', 'google-certificates', 'hunter', + 'intelx', 'linkedin', 'netcraft', 'pgp', diff --git a/parsers/intelxparser.py b/parsers/intelxparser.py new file mode 100644 index 00000000..8d0dbb39 --- /dev/null +++ b/parsers/intelxparser.py @@ -0,0 +1,27 @@ +class Parser: + + def __init__(self): + self.emails = set() + self.hosts = set() + + def parse_dictionaries(self, results): + """ + Parse method to parse json results + :param results: Dictionary containing a list of dictionaries known as selectors + :return: tuple of emails and hosts + """ + if results is not None: + for dictionary in results["selectors"]: + field = dictionary['selectorvalue'] + if '@' in field: + self.emails.add(field) + else: + field = str(field) + if 'http' in field or 'https' in field: + if field[:5] == 'https': + field = field[8:] + else: + field = field[7:] + self.hosts.add(field.replace(')', '').replace(',', '')) + return self.emails, self.hosts + return None, None diff --git a/theHarvester.py b/theHarvester.py index 0839773b..d7784cf2 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -1,921 +1,1015 @@ -#!/usr/bin/env python3 - -from discovery import * -from discovery.constants import * -from lib import hostchecker -from lib import htmlExport -from lib import reportgraph -from lib import statichtmlgenerator -from lib import stash -from lib.core import * -from platform import python_version -import argparse -import datetime -import ipaddress -import re -import time -import sys - -try: - import bs4 -except ImportError: - print('\n\033[93m[!] BeautifulSoup library not found, please install before proceeding.\n\n \033[0m') - sys.exit(1) - -try: - import requests -except ImportError: - print('\n\033[93m[!] Requests library not found, please install before proceeding.\n\n \033[0m') - sys.exit(1) - -Core.banner() - - -def start(): - parser = argparse.ArgumentParser(description='theHarvester is used to gather open source intelligence on a company or domain.') - parser.add_argument('-d', '--domain', help='Company or domain to search', required=True) - parser.add_argument('-l', '--limit', help='Limit the number of search results, default=500', default=500, type=int) - parser.add_argument('-S', '--start', help='Start with result number X, default=0', default=0, type=int) - parser.add_argument('-g', '--google-dork', help='Use Google Dorks for Google search', default=False, action='store_true') - parser.add_argument('-p', '--port-scan', help='Scan discovered hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True', default=False) - parser.add_argument('-s', '--shodan', help='Use Shodan to query discovered hosts', default=False, action='store_true') - parser.add_argument('-v', '--virtual-host', help='Verify host name via DNS resolution and search for virtual hosts params=basic, default=False', default=False) - parser.add_argument('-e', '--dns-server', help='Specify a DNS server to use for lookup') - parser.add_argument('-c', '--dns-brute', help='Perform a DNS brute force on a domain', default=False, action='store_true') - parser.add_argument('-n', '--dns-lookup', help='Enable a DNS server lookup, default=False, params=True', default=False) - parser.add_argument('-t', '--dns-tld', help='Perform a DNS TLD expansion discovery, default False', default=False) - parser.add_argument('-f', '--filename', help='Save the results to an HTML and/or XML file', default='', type=str) - parser.add_argument('-b', '--source', help='''Source: baidu, bing, bingapi, censys, crtsh, cymon, - dogpile, duckduckgo, google, googleCSE, - google-certificates, hunter, linkedin, - netcraft, pgp, securityTrails, threatcrowd, - trello, twitter, vhost, virustotal, yahoo, all''') - args = parser.parse_args() - - try: - db = stash.stash_manager() - db.do_init() - except Exception: - pass - - all_emails = [] - all_hosts = [] - all_ip = [] - dnsbrute = args.dns_brute - dnslookup = args.dns_lookup - dnsserver = args.dns_server - dnstld = args.dns_tld - filename = args.filename - full = [] - google_dorking = args.google_dork - host_ip = [] - limit = args.limit - ports_scanning = args.port_scan - shodan = args.shodan - start = args.start - takeover_check = False - trello_info = ([], False) - vhost = [] - virtual = args.virtual_host - word = args.domain - if args.source is not None: - engines = set(args.source.split(',')) - if set(engines).issubset(Core.get_supportedengines()): - print(f'\033[94m[*] Target: {word} \n \033[0m') - for engineitem in engines: - if engineitem == 'baidu': - print('\033[94m[*] Searching Baidu. \033[0m') - try: - search = baidusearch.SearchBaidu(word, limit) - search.process() - all_emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'baidu') - db.store_all(word, all_emails, 'email', 'baidu') - except Exception: - pass - - elif engineitem == 'bing' or engineitem == 'bingapi': - print('\033[94m[*] Searching Bing. \033[0m') - try: - search = bingsearch.SearchBing(word, limit, start) - bingapi = '' - if engineitem == 'bingapi': - bingapi += 'yes' - else: - bingapi += 'no' - search.process(bingapi) - all_emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'email', 'bing') - db.store_all(word, all_hosts, 'host', 'bing') - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - pass - - elif engineitem == 'censys': - print('\033[94m[*] Searching Censys. \033[0m') - from discovery import censys - # Import locally or won't work - search = censys.SearchCensys(word, limit) - search.process() - all_ip = search.get_ipaddresses() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'censys') - db.store_all(word, all_ip, 'ip', 'censys') - - elif engineitem == 'crtsh': - print('\033[94m[*] Searching CRT.sh. \033[0m') - search = crtsh.search_crtsh(word) - search.process() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'CRTsh') - - elif engineitem == 'cymon': - print('\033[94m[*] Searching Cymon. \033[0m') - from discovery import cymon - # Import locally or won't work. - search = cymon.search_cymon(word) - search.process() - all_ip = search.get_ipaddresses() - db = stash.stash_manager() - db.store_all(word, all_ip, 'ip', 'cymon') - - elif engineitem == 'dogpile': - try: - print('\033[94m[*] Searching Dogpile. \033[0m') - search = dogpilesearch.SearchDogpile(word, limit) - search.process() - emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - all_emails.extend(emails) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'email', 'dogpile') - db.store_all(word, all_hosts, 'host', 'dogpile') - except Exception as e: - print(f'\033[93m[!] A error occurred in Dogpile: {e} \033[0m') - - elif engineitem == 'duckduckgo': - print('\033[94m[*] Searching DuckDuckGo. \033[0m') - from discovery import duckduckgosearch - search = duckduckgosearch.SearchDuckDuckGo(word, limit) - search.process() - emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - all_emails.extend(emails) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'email', 'duckduckgo') - db.store_all(word, all_hosts, 'host', 'duckduckgo') - - elif engineitem == 'google': - print('\033[94m[*] Searching Google. \033[0m') - search = googlesearch.search_google(word, limit, start) - search.process(google_dorking) - emails = filter(search.get_emails()) - all_emails.extend(emails) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'google') - db.store_all(word, all_emails, 'email', 'google') - - elif engineitem == 'googleCSE': - print('\033[94m[*] Searching Google Custom Search. \033[0m') - try: - search = googleCSE.SearchGoogleCSE(word, limit, start) - search.process() - search.store_results() - all_emails = filter(search.get_emails()) - db = stash.stash_manager() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db.store_all(word, all_hosts, 'email', 'googleCSE') - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'googleCSE') - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - pass - - elif engineitem == 'google-certificates': - print('\033[94m[*] Searching Google Certificate transparency report. \033[0m') - search = googlecertificates.SearchGoogleCertificates(word, limit, start) - search.process() - hosts = filter(search.get_domains()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'google-certificates') - - elif engineitem == 'hunter': - print('\033[94m[*] Searching Hunter. \033[0m') - from discovery import huntersearch - # Import locally or won't work. - try: - search = huntersearch.SearchHunter(word, limit, start) - search.process() - emails = filter(search.get_emails()) - all_emails.extend(emails) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'hunter') - db.store_all(word, all_emails, 'email', 'hunter') - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - pass - - elif engineitem == 'linkedin': - print('\033[94m[*] Searching Linkedin. \033[0m') - search = linkedinsearch.SearchLinkedin(word, limit) - search.process() - people = search.get_people() - db = stash.stash_manager() - db.store_all(word, people, 'name', 'linkedin') - - if len(people) == 0: - print('\n[*] No users found Linkedin.\n\n') - else: - print(f'\n[*] Users found: {len(people)}') - print('---------------------') - for user in sorted(list(set(people))): - print(user) - sys.exit(0) - - elif engineitem == 'netcraft': - print('\033[94m[*] Searching Netcraft. \033[0m') - search = netcraft.SearchNetcraft(word) - search.process() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'netcraft') - - elif engineitem == 'pgp': - print('\033[94m[*] Searching PGP key server. \033[0m') - try: - search = pgpsearch.SearchPgp(word) - search.process() - all_emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'pgp') - db.store_all(word, all_emails, 'email', 'pgp') - except Exception: - pass - - elif engineitem == 'securityTrails': - print('\033[94m[*] Searching SecurityTrails. \033[0m') - from discovery import securitytrailssearch - try: - search = securitytrailssearch.search_securitytrail(word) - search.process() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, hosts, 'host', 'securityTrails') - ips = search.get_ips() - all_ip.extend(ips) - db = stash.stash_manager() - db.store_all(word, ips, 'ip', 'securityTrails') - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - pass - - elif engineitem == 'threatcrowd': - print('\033[94m[*] Searching Threatcrowd. \033[0m') - try: - search = threatcrowd.search_threatcrowd(word) - search.process() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'threatcrowd') - except Exception: - pass - - elif engineitem == 'trello': - print('\033[94m[*] Searching Trello. \033[0m') - from discovery import trello - # Import locally or won't work. - search = trello.search_trello(word, limit) - search.process() - emails = filter(search.get_emails()) - all_emails.extend(emails) - info = search.get_urls() - hosts = filter(info[0]) - trello_info = (info[1], True) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, hosts, 'host', 'trello') - db.store_all(word, emails, 'email', 'trello') - - elif engineitem == 'twitter': - print('\033[94m[*] Searching Twitter. \033[0m') - search = twittersearch.search_twitter(word, limit) - search.process() - people = search.get_people() - db = stash.stash_manager() - db.store_all(word, people, 'name', 'twitter') - - if len(people) == 0: - print('\n[*] No users found.\n\n') - else: - print('\n[*] Users found: ' + str(len(people))) - print('---------------------') - for user in sorted(list(set(people))): - print(user) - - elif engineitem == 'virustotal': - print('\033[94m[*] Searching VirusTotal. \033[0m') - search = virustotal.search_virustotal(word) - search.process() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'virustotal') - - elif engineitem == 'yahoo': - print('\033[94m[*] Searching Yahoo. \033[0m') - search = yahoosearch.search_yahoo(word, limit) - search.process() - hosts = search.get_hostnames() - emails = search.get_emails() - all_hosts.extend(filter(hosts)) - all_emails.extend(filter(emails)) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'yahoo') - db.store_all(word, all_emails, 'email', 'yahoo') - - elif engineitem == 'all': - print('\033[94m[*] Running a full harvest. \033[0m') - all_emails = [] - all_hosts = [] - try: - print('\033[94m\n[*] Searching Baidu. \033[0m') - search = baidusearch.SearchBaidu(word, limit) - search.process() - all_emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'baidu') - db.store_all(word, all_emails, 'email', 'baidu') - except Exception: - pass - - print('\033[94m\n[*] Searching Bing. \033[0m') - bingapi = 'no' - search = bingsearch.SearchBing(word, limit, start) - search.process(bingapi) - emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'bing') - all_emails.extend(emails) - all_emails = sorted(set(all_emails)) - db.store_all(word, all_emails, 'email', 'bing') - - print('\033[94m\n[*] Searching Censys. \033[0m') - from discovery import censys - search = censys.SearchCensys(word, limit) - search.process() - ips = search.get_ipaddresses() - setips = set(ips) - uniqueips = list(setips) # Remove duplicates. - all_ip.extend(uniqueips) - hosts = filter(search.get_hostnames()) - sethosts = set(hosts) - uniquehosts = list(sethosts) # Remove duplicates. - all_hosts.extend(uniquehosts) - db = stash.stash_manager() - db.store_all(word, uniquehosts, 'host', 'censys') - db.store_all(word, uniqueips, 'ip', 'censys') - - print('\033[94m\n[*] Searching CRT.sh. \033[0m') - search = crtsh.search_crtsh(word) - search.process() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'CRTsh') - - # cymon - print('\033[94m\n[*] Searching Cymon. \033[0m') - from discovery import cymon - # Import locally or won't work. - search = cymon.search_cymon(word) - search.process() - all_ip = search.get_ipaddresses() - db = stash.stash_manager() - db.store_all(word, all_ip, 'ip', 'cymon') - - print('\033[94m\n[*] Searching Dogpile. \033[0m') - search = dogpilesearch.SearchDogpile(word, limit) - search.process() - emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - all_emails.extend(emails) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'email', 'dogpile') - db.store_all(word, all_hosts, 'host', 'dogpile') - - print('\033[94m\n[*] Searching DuckDuckGo. \033[0m') - from discovery import duckduckgosearch - search = duckduckgosearch.SearchDuckDuckGo(word, limit) - search.process() - emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - all_emails.extend(emails) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'email', 'duckduckgo') - db.store_all(word, all_hosts, 'host', 'duckduckgo') - - print('\033[94m\n[*] Searching Google. \033[0m') - search = googlesearch.search_google(word, limit, start) - search.process(google_dorking) - emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_emails.extend(emails) - db = stash.stash_manager() - db.store_all(word, all_emails, 'email', 'google') - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'google') - - print('\033[94m\n[*] Searching Google Certificate transparency report. \033[0m') - search = googlecertificates.SearchGoogleCertificates(word, limit, start) - search.process() - domains = filter(search.get_domains()) - all_hosts.extend(domains) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'google-certificates') - - print('\033[94m\n[*] Searching Hunter. \033[0m') - from discovery import huntersearch - # Import locally. - try: - search = huntersearch.SearchHunter(word, limit, start) - search.process() - emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, hosts, 'host', 'hunter') - all_emails.extend(emails) - all_emails = sorted(set(all_emails)) - db.store_all(word, all_emails, 'email', 'hunter') - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - pass - - print('\033[94m\n[*] Searching Linkedin. \033[0m') - search = linkedinsearch.SearchLinkedin(word, limit) - search.process() - people = search.get_people() - db = stash.stash_manager() - db.store_all(word, people, 'name', 'linkedin') - - if len(people) == 0: - print('\n[*] No users found.\n\n') - else: - print('\n[*] Users found: ' + str(len(people))) - print('---------------------') - for user in sorted(list(set(people))): - print(user) - - print('\033[94m\n[*] Searching Netcraft. \033[0m') - search = netcraft.SearchNetcraft(word) - search.process() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'netcraft') - - print('\033[94m\n[*] Searching PGP key server. \033[0m') - try: - search = pgpsearch.SearchPgp(word) - search.process() - emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - sethosts = set(hosts) - uniquehosts = list(sethosts) # Remove duplicates. - all_hosts.extend(uniquehosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'PGP') - all_emails.extend(emails) - db = stash.stash_manager() - db.store_all(word, all_emails, 'email', 'PGP') - except Exception: - pass - - print('\033[94m\n[*] Searching Threatcrowd. \033[0m') - try: - search = threatcrowd.search_threatcrowd(word) - search.process() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'threatcrowd') - except Exception: - pass - - print('\033[94m\n[*] Searching Trello. \033[0m') - from discovery import trello - # Import locally or won't work. - search = trello.search_trello(word, limit) - search.process() - emails = filter(search.get_emails()) - all_emails.extend(emails) - info = search.get_urls() - hosts = filter(info[0]) - trello_info = (info[1], True) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, hosts, 'host', 'trello') - db.store_all(word, emails, 'email', 'trello') - - try: - print('\033[94m\n[*] Searching Twitter. \033[0m') - search = twittersearch.search_twitter(word, limit) - search.process() - people = search.get_people() - db = stash.stash_manager() - db.store_all(word, people, 'name', 'twitter') - print('\nUsers from Twitter:') - print('-------------------') - for user in people: - print(user) - except Exception: - pass - - print('\n[*] Virtual hosts:') - print('------------------') - for l in host_ip: - search = bingsearch.SearchBing(l, limit, start) - search.process_vhost() - res = search.get_allhostnames() - for x in res: - x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x) - x = re.sub('<', '', x) - x = re.sub('>', '', x) - print((l + '\t' + x)) - vhost.append(l + ':' + x) - full.append(l + ':' + x) - vhost = sorted(set(vhost)) - - print('\033[94m\n[*] Searching VirusTotal. \033[0m') - search = virustotal.search_virustotal(word) - search.process() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'virustotal') - - print('\033[94m\n[*] Searching Yahoo. \033[0m') - search = yahoosearch.search_yahoo(word, limit) - search.process() - hosts = search.get_hostnames() - emails = search.get_emails() - all_hosts.extend(filter(hosts)) - all_emails.extend(filter(emails)) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'yahoo') - db.store_all(word, all_emails, 'email', 'yahoo') - else: - print('\033[93m[!] Invalid source.\n\n \033[0m') - sys.exit(1) - - # Sanity check to see if all_emails and all_hosts are defined. - try: - all_emails - except NameError: - print('\n\n\033[93m[!] No emails found because all_emails is not defined.\n\n \033[0m') - sys.exit(1) - try: - all_hosts - except NameError: - print('\n\n\033[93m[!] No hosts found because all_hosts is not defined.\n\n \033[0m') - sys.exit(1) - - # Results - if len(all_ip) == 0: - print('\n[*] No IPs found.') - else: - print('\n[*] IPs found: ' + str(len(all_ip))) - print('-------------------') - ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip)) - print('\n'.join(map(str, ips))) - - if len(all_emails) == 0: - print('\n[*] No emails found.') - else: - print('\n[*] Emails found: ' + str(len(all_emails))) - print('----------------------') - print(('\n'.join(sorted(list(set(all_emails)))))) - - if len(all_hosts) == 0: - print('\n[*] No hosts found.\n\n') - else: - print('\n[*] Hosts found: ' + str(len(all_hosts))) - print('---------------------') - all_hosts = sorted(list(set(all_hosts))) - full_host = hostchecker.Checker(all_hosts) - full = full_host.check() - for host in full: - ip = host.split(':')[1] - print(host) - if ip != 'empty': - if host_ip.count(ip.lower()): - pass - else: - host_ip.append(ip.lower()) - - db = stash.stash_manager() - db.store_all(word, host_ip, 'ip', 'DNS-resolver') - - if trello_info[1] is True: - trello_urls = trello_info[0] - if trello_urls == []: - print('\n[*] No URLs found.') - else: - total = len(trello_urls) - print('\n[*] URLs found: ' + str(total)) - print('--------------------') - for url in sorted(list(set(trello_urls))): - print(url) - - # DNS brute force - dnsres = [] - if dnsbrute is True: - print('\n[*] Starting DNS brute force.') - a = dnssearch.dns_force(word, dnsserver, verbose=True) - res = a.process() - print('\n[*] Hosts found after DNS brute force:') - print('-------------------------------------') - for y in res: - print(y) - dnsres.append(y.split(':')[0]) - if y not in full: - full.append(y) - db = stash.stash_manager() - db.store_all(word, dnsres, 'host', 'dns_bruteforce') - - # Port scanning - if ports_scanning is True: - print('\n\n[*] Scanning ports (active).\n') - for x in full: - host = x.split(':')[1] - domain = x.split(':')[0] - if host != 'empty': - print(('[*] Scanning ' + host)) - ports = [21, 22, 80, 443, 8080] - try: - scan = port_scanner.PortScan(host, ports) - openports = scan.process() - if len(openports) > 1: - print(('\t[*] Detected open ports: ' + ','.join(str(e) for e in openports))) - takeover_check = 'True' - if takeover_check == 'True': - if len(openports) > 0: - search_take = takeover.take_over(domain) - search_take.process() - except Exception as e: - print(e) - - # DNS reverse lookup - dnsrev = [] - if dnslookup is True: - print('\n[*] Starting active queries.') - analyzed_ranges = [] - for x in host_ip: - print(x) - ip = x.split(':')[0] - range = ip.split('.') - range[3] = '0/24' - s = '.' - range = s.join(range) - if not analyzed_ranges.count(range): - print('[*] Performing reverse lookup in ' + range) - a = dnssearch.dns_reverse(range, True) - a.list() - res = a.process() - analyzed_ranges.append(range) - else: - continue - for x in res: - if x.count(word): - dnsrev.append(x) - if x not in full: - full.append(x) - print('[*] Hosts found after reverse lookup (in target domain):') - print('--------------------------------------------------------') - for xh in dnsrev: - print(xh) - - # DNS TLD expansion - dnstldres = [] - if dnstld is True: - print('[*] Starting DNS TLD expansion.') - a = dnssearch.dns_tld(word, dnsserver, verbose=True) - res = a.process() - print('\n[*] Hosts found after DNS TLD expansion:') - print('----------------------------------------') - for y in res: - print(y) - dnstldres.append(y) - if y not in full: - full.append(y) - - # Virtual hosts search - if virtual == 'basic': - print('\n[*] Virtual hosts:') - print('------------------') - for l in host_ip: - search = bingsearch.SearchBing(l, limit, start) - search.process_vhost() - res = search.get_allhostnames() - for x in res: - x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x) - x = re.sub('<', '', x) - x = re.sub('>', '', x) - print((l + '\t' + x)) - vhost.append(l + ':' + x) - full.append(l + ':' + x) - vhost = sorted(set(vhost)) - else: - pass - - # Shodan - shodanres = [] - if shodan is True: - import texttable - tab = texttable.Texttable() - header = ['IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies'] - tab.header(header) - tab.set_cols_align(['c', 'c', 'c', 'c', 'c']) - tab.set_cols_valign(['m', 'm', 'm', 'm', 'm']) - tab.set_chars(['-', '|', '+', '#']) - tab.set_cols_width([15, 20, 15, 15, 18]) - host_ip = list(set(host_ip)) - print('\033[94m[*] Searching Shodan. \033[0m') - try: - for ip in host_ip: - print(('\tSearching for ' + ip)) - shodan = shodansearch.search_shodan() - rowdata = shodan.search_ip(ip) - time.sleep(2) - tab.add_row(rowdata) - printedtable = tab.draw() - print(printedtable) - except Exception as e: - print(f'\033[93m[!] Error occurred in the Shodan search module: {e} \033[0m') - else: - pass - - # Here we need to add explosion mode. - # We have to take out the TLDs to do this. - recursion = None - if recursion: - start = 0 - for word in vhost: - search = googlesearch.search_google(word, limit, start) - search.process(google_dorking) - emails = search.get_emails() - hosts = search.get_hostnames() - print(emails) - print(hosts) - else: - pass - - # Reporting - if filename != "": - try: - print('\nNEW REPORTING BEGINS.') - db = stash.stash_manager() - scanboarddata = db.getscanboarddata() - latestscanresults = db.getlatestscanresults(word) - previousscanresults = db.getlatestscanresults(word, previousday=True) - latestscanchartdata = db.latestscanchartdata(word) - scanhistorydomain = db.getscanhistorydomain(word) - pluginscanstatistics = db.getpluginscanstatistics() - generator = statichtmlgenerator.htmlgenerator(word) - HTMLcode = generator.beginhtml() - HTMLcode += generator.generatelatestscanresults(latestscanresults) - HTMLcode += generator.generatepreviousscanresults(previousscanresults) - graph = reportgraph.GraphGenerator(word) - HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata) - HTMLcode += graph.drawscattergraphscanhistory(word, scanhistorydomain) - HTMLcode += generator.generatepluginscanstatistics(pluginscanstatistics) - HTMLcode += generator.generatedashboardcode(scanboarddata) - HTMLcode += '

Report generated on ' + str( - datetime.datetime.now()) + '

' - HTMLcode += ''' - - - ''' - Html_file = open('report.html', 'w') - Html_file.write(HTMLcode) - Html_file.close() - print('NEW REPORTING FINISHED!') - print('[*] Saving files.') - html = htmlExport.htmlExport( - all_emails, - full, - vhost, - dnsres, - dnsrev, - filename, - word, - shodanres, - dnstldres) - save = html.writehtml() - except Exception as e: - print(e) - print('\n\033[93m[!] An error occurred creating the file.\033[0m') - - try: - filename = filename.split('.')[0] + '.xml' - file = open(filename, 'w') - file.write('') - for x in all_emails: - file.write('' + x + '') - for x in full: - x = x.split(':') - if len(x) == 2: - file.write( - '' + '' + x[1] + '' + x[0] + '' + '') - else: - file.write('' + x + '') - for x in vhost: - x = x.split(':') - if len(x) == 2: - file.write( - '' + '' + x[1] + '' + x[0] + '' + '') - else: - file.write('' + x + '') - if shodanres != []: - shodanalysis = [] - for x in shodanres: - res = x.split('SAPO') - file.write('') - file.write('' + res[0] + '') - file.write('' + res[2] + '') - file.write('') - reg_server = re.compile('Server:.*') - temp = reg_server.findall(res[1]) - if temp != []: - shodanalysis.append(res[0] + ':' + temp[0]) - file.write('') - if shodanalysis != []: - shodanalysis = sorted(set(shodanalysis)) - file.write('') - for x in shodanalysis: - file.write('' + x + '') - file.write('') - - file.write('') - file.flush() - file.close() - print('[*] Files saved.') - except Exception as er: - print(f'\033[93m[!] An error occurred saving XML file: {er} \033[0m') - print('\n\n') - sys.exit(0) - - -if __name__ == '__main__': - if python_version()[0:3] < '3.6': - print('\033[93m[!] Make sure you have Python 3.6+ installed, quitting. \033[0m') - sys.exit(1) - try: - start() - except KeyboardInterrupt: - print('\n\n\033[93m[!] ctrl+c detected from user, quitting.\n\n \033[0m') - except Exception: - import traceback - - print(traceback.print_exc()) - sys.exit(1) +#!/usr/bin/env python3 + +from discovery import * +from discovery.constants import * +from lib import hostchecker +from lib import htmlExport +from lib import reportgraph +from lib import statichtmlgenerator +from lib import stash +from lib.core import * +from platform import python_version +import argparse +import datetime +import ipaddress +import re +import time +import sys + +try: + import bs4 +except ImportError: + print('\n\033[93m[!] BeautifulSoup library not found, please install before proceeding.\n\n \033[0m') + sys.exit(1) + +try: + import requests +except ImportError: + print('\n\033[93m[!] Requests library not found, please install before proceeding.\n\n \033[0m') + sys.exit(1) + +Core.banner() + + +def start(): + parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon') + parser.add_argument('-d', '--domain', help='company name or domain to search', required=True) + parser.add_argument('-l', '--limit', help='limit the number of search results, default=500', default=500, type=int) + parser.add_argument('-S', '--start', help='start with result number X, default=0', default=0, type=int) + parser.add_argument('-g', '--google-dork', help='use Google Dorks for google search', default=False, action='store_true') + parser.add_argument('-p', '--port-scan', help='scan the detected hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True', default=False) + parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=False, action='store_true') + parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts params=basic, default=False', default=False) + parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup') + parser.add_argument('-t', '--dns-tld', help='perform a DNS TLD expansion discovery, default False', default=False) + parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default=False, params=True', default=False) + parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true') + parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) + parser.add_argument('-b', '--source', help='''source: baidu, bing, bingapi, censys, crtsh, cymon, + dogpile, duckduckgo, google, googleCSE, + google-certificates, google-profiles, hunter, intelx, + linkedin, netcraft, pgp, securityTrails, threatcrowd, + trello, twitter, vhost, virustotal, yahoo, all''') + args = parser.parse_args() + + try: + db = stash.stash_manager() + db.do_init() + except Exception: + pass + + all_emails = [] + all_hosts = [] + all_ip = [] + dnsbrute = args.dns_brute + dnslookup = args.dns_lookup + dnsserver = args.dns_server + dnstld = args.dns_tld + filename = args.filename + full = [] + google_dorking = args.google_dork + host_ip = [] + limit = args.limit + ports_scanning = args.port_scan + shodan = args.shodan + start = args.start + takeover_check = False + trello_info = ([], False) + vhost = [] + virtual = args.virtual_host + word = args.domain + if args.source is not None: + engines = set(args.source.split(',')) + if set(engines).issubset(Core.get_supportedengines()): + print(f'\033[94m[*] Target: {word} \n \033[0m') + for engineitem in engines: + if engineitem == 'baidu': + print('\033[94m[*] Searching Baidu. \033[0m') + try: + search = baidusearch.SearchBaidu(word, limit) + search.process() + all_emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'baidu') + db.store_all(word, all_emails, 'email', 'baidu') + except Exception: + pass + + elif engineitem == 'bing' or engineitem == 'bingapi': + print('\033[94m[*] Searching Bing. \033[0m') + try: + search = bingsearch.SearchBing(word, limit, start) + bingapi = '' + if engineitem == 'bingapi': + bingapi += 'yes' + else: + bingapi += 'no' + search.process(bingapi) + all_emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'bing') + db.store_all(word, all_hosts, 'host', 'bing') + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + pass + + elif engineitem == 'censys': + print('\033[94m[*] Searching Censys. \033[0m') + from discovery import censys + # Import locally or won't work + search = censys.SearchCensys(word, limit) + search.process() + all_ip = search.get_ipaddresses() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'censys') + db.store_all(word, all_ip, 'ip', 'censys') + + elif engineitem == 'crtsh': + print('\033[94m[*] Searching CRT.sh. \033[0m') + search = crtsh.search_crtsh(word) + search.process() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'CRTsh') + + elif engineitem == 'cymon': + print('\033[94m[*] Searching Cymon. \033[0m') + from discovery import cymon + # Import locally or won't work. + search = cymon.search_cymon(word) + search.process() + all_ip = search.get_ipaddresses() + db = stash.stash_manager() + db.store_all(word, all_ip, 'ip', 'cymon') + + elif engineitem == 'dogpile': + try: + print('\033[94m[*] Searching Dogpile. \033[0m') + search = dogpilesearch.SearchDogpile(word, limit) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'dogpile') + db.store_all(word, all_hosts, 'host', 'dogpile') + except Exception as e: + print(f'\033[93m[!] A error occurred in Dogpile: {e} \033[0m') + + elif engineitem == 'duckduckgo': + print('\033[94m[*] Searching DuckDuckGo. \033[0m') + from discovery import duckduckgosearch + search = duckduckgosearch.SearchDuckDuckGo(word, limit) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'duckduckgo') + db.store_all(word, all_hosts, 'host', 'duckduckgo') + + elif engineitem == 'google': + print('\033[94m[*] Searching Google. \033[0m') + search = googlesearch.search_google(word, limit, start) + search.process(google_dorking) + emails = filter(search.get_emails()) + all_emails.extend(emails) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'google') + db.store_all(word, all_emails, 'email', 'google') + + elif engineitem == 'googleCSE': + print('\033[94m[*] Searching Google Custom Search. \033[0m') + try: + search = googleCSE.SearchGoogleCSE(word, limit, start) + search.process() + search.store_results() + all_emails = filter(search.get_emails()) + db = stash.stash_manager() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db.store_all(word, all_hosts, 'email', 'googleCSE') + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'googleCSE') + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + pass + + elif engineitem == 'google-certificates': + print('\033[94m[*] Searching Google Certificate transparency report. \033[0m') + search = googlecertificates.SearchGoogleCertificates(word, limit, start) + search.process() + hosts = filter(search.get_domains()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'google-certificates') + + elif engineitem == 'google-profiles': + print('\033[94m[*] Searching Google profiles. \033[0m') + search = googlesearch.search_google(word, limit, start) + search.process_profiles() + people = search.get_profiles() + db = stash.stash_manager() + db.store_all(word, people, 'name', 'google-profile') + + if len(people) == 0: + print('\n[*] No users found.\n\n') + else: + print('\n[*] Users found: ' + str(len(people))) + print('---------------------') + for user in sorted(list(set(people))): + print(user) + sys.exit(0) + + elif engineitem == 'hunter': + print('\033[94m[*] Searching Hunter. \033[0m') + from discovery import huntersearch + # Import locally or won't work. + try: + search = huntersearch.SearchHunter(word, limit, start) + search.process() + emails = filter(search.get_emails()) + all_emails.extend(emails) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'hunter') + db.store_all(word, all_emails, 'email', 'hunter') + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + pass + + elif engineitem == 'intelx': + print('\033[94m[*] Searching Intelx. \033[0m') + from discovery import intelxsearch + # Import locally or won't work. + try: + search = intelxsearch.search_intelx(word, limit) + search.process() + emails = filter(search.get_emails()) + all_emails.extend(emails) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'intelx') + db.store_all(word, all_emails, 'email', 'intelx') + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print(e) + + elif engineitem == 'linkedin': + print('\033[94m[*] Searching Linkedin. \033[0m') + search = linkedinsearch.SearchLinkedin(word, limit) + search.process() + people = search.get_people() + db = stash.stash_manager() + db.store_all(word, people, 'name', 'linkedin') + + if len(people) == 0: + print('\n[*] No users found Linkedin.\n\n') + else: + print(f'\n[*] Users found: {len(people)}') + print('---------------------') + for user in sorted(list(set(people))): + print(user) + sys.exit(0) + + elif engineitem == 'netcraft': + print('\033[94m[*] Searching Netcraft. \033[0m') + search = netcraft.SearchNetcraft(word) + search.process() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'netcraft') + + elif engineitem == 'pgp': + print('\033[94m[*] Searching PGP key server. \033[0m') + try: + search = pgpsearch.SearchPgp(word) + search.process() + all_emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'pgp') + db.store_all(word, all_emails, 'email', 'pgp') + except Exception: + pass + + elif engineitem == 'securityTrails': + print('\033[94m[*] Searching SecurityTrails. \033[0m') + from discovery import securitytrailssearch + try: + search = securitytrailssearch.search_securitytrail(word) + search.process() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, hosts, 'host', 'securityTrails') + ips = search.get_ips() + all_ip.extend(ips) + db = stash.stash_manager() + db.store_all(word, ips, 'ip', 'securityTrails') + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + pass + + elif engineitem == 'threatcrowd': + print('\033[94m[*] Searching Threatcrowd. \033[0m') + try: + search = threatcrowd.search_threatcrowd(word) + search.process() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'threatcrowd') + except Exception: + pass + + elif engineitem == 'trello': + print('\033[94m[*] Searching Trello. \033[0m') + from discovery import trello + # Import locally or won't work. + search = trello.search_trello(word, limit) + search.process() + emails = filter(search.get_emails()) + all_emails.extend(emails) + info = search.get_urls() + hosts = filter(info[0]) + trello_info = (info[1], True) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, hosts, 'host', 'trello') + db.store_all(word, emails, 'email', 'trello') + + elif engineitem == 'twitter': + print('\033[94m[*] Searching Twitter. \033[0m') + search = twittersearch.search_twitter(word, limit) + search.process() + people = search.get_people() + db = stash.stash_manager() + db.store_all(word, people, 'name', 'twitter') + + if len(people) == 0: + print('\n[*] No users found.\n\n') + else: + print('\n[*] Users found: ' + str(len(people))) + print('---------------------') + for user in sorted(list(set(people))): + print(user) + + elif engineitem == 'virustotal': + print('\033[94m[*] Searching VirusTotal. \033[0m') + search = virustotal.search_virustotal(word) + search.process() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'virustotal') + + elif engineitem == 'yahoo': + print('\033[94m[*] Searching Yahoo. \033[0m') + search = yahoosearch.search_yahoo(word, limit) + search.process() + hosts = search.get_hostnames() + emails = search.get_emails() + all_hosts.extend(filter(hosts)) + all_emails.extend(filter(emails)) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'yahoo') + db.store_all(word, all_emails, 'email', 'yahoo') + + elif engineitem == 'all': + print(('[*] Full harvest on ' + word)) + all_emails = [] + all_hosts = [] + try: + print('\033[94m[*] Searching Baidu. \033[0m') + search = baidusearch.SearchBaidu(word, limit) + search.process() + all_emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'baidu') + db.store_all(word, all_emails, 'email', 'baidu') + except Exception: + pass + try: + print('\033[94m[*] Searching Bing. \033[0m') + bingapi = 'no' + search = bingsearch.SearchBing(word, limit, start) + search.process(bingapi) + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'bing') + all_emails.extend(emails) + all_emails = sorted(set(all_emails)) + db.store_all(word, all_emails, 'email', 'bing') + except Exception: + pass + + print('\033[94m[*] Searching Censys. \033[0m') + from discovery import censys + search = censys.SearchCensys(word, limit) + search.process() + ips = search.get_ipaddresses() + setips = set(ips) + uniqueips = list(setips) # Remove duplicates. + all_ip.extend(uniqueips) + hosts = filter(search.get_hostnames()) + sethosts = set(hosts) + uniquehosts = list(sethosts) # Remove duplicates. + all_hosts.extend(uniquehosts) + db = stash.stash_manager() + db.store_all(word, uniquehosts, 'host', 'censys') + db.store_all(word, uniqueips, 'ip', 'censys') + + print('\033[94m[*] Searching CRT.sh. \033[0m') + search = crtsh.search_crtsh(word) + search.process() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'CRTsh') + + # cymon + print('\033[94m[*] Searching Cymon. \033[0m') + from discovery import cymon + # Import locally or won't work. + search = cymon.search_cymon(word) + search.process() + all_ip = search.get_ipaddresses() + db = stash.stash_manager() + db.store_all(word, all_ip, 'ip', 'cymon') + + print('\033[94m[*] Searching Dogpile. \033[0m') + search = dogpilesearch.SearchDogpile(word, limit) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'dogpile') + db.store_all(word, all_hosts, 'host', 'dogpile') + + print('\033[94m[*] Searching DuckDuckGo. \033[0m') + from discovery import duckduckgosearch + search = duckduckgosearch.SearchDuckDuckGo(word, limit) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'duckduckgo') + db.store_all(word, all_hosts, 'host', 'duckduckgo') + + print('\033[94m[*] Searching Google. \033[0m') + search = googlesearch.search_google(word, limit, start) + search.process(google_dorking) + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_emails, 'email', 'google') + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'google') + + print('\033[94m[*] Searching Google Certificate transparency report. \033[0m') + search = googlecertificates.SearchGoogleCertificates(word, limit, start) + search.process() + domains = filter(search.get_domains()) + all_hosts.extend(domains) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'google-certificates') + + try: + print('\033[94m[*] Searching Google profiles. \033[0m') + search = googlesearch.search_google(word, limit, start) + search.process_profiles() + people = search.get_profiles() + db = stash.stash_manager() + db.store_all(word, people, 'name', 'google-profile') + print('\nUsers from Google profiles:') + print('---------------------------') + for users in people: + print(users) + except Exception: + pass + + print('\033[94m[*] Searching Hunter. \033[0m') + from discovery import huntersearch + # Import locally. + try: + search = huntersearch.SearchHunter(word, limit, start) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, hosts, 'host', 'hunter') + all_emails.extend(emails) + all_emails = sorted(set(all_emails)) + db.store_all(word, all_emails, 'email', 'hunter') + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + pass + + print('\033[94m[*] Searching Intelx. \033[0m') + from discovery import intelxsearch + # Import locally or won't work. + try: + search = intelxsearch.search_intelx(word, limit) + search.process() + emails = filter(search.get_emails()) + all_emails.extend(emails) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'intelx') + db.store_all(word, all_emails, 'email', 'intelx') + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print(e) + + print('\033[94m[*] Searching Linkedin. \033[0m') + search = linkedinsearch.SearchLinkedin(word, limit) + search.process() + people = search.get_people() + db = stash.stash_manager() + db.store_all(word, people, 'name', 'linkedin') + + if len(people) == 0: + print('\n[*] No users found.\n\n') + else: + print('\n[*] Users found: ' + str(len(people))) + print('---------------------') + for user in sorted(list(set(people))): + print(user) + + print('\033[94m[*] Searching Netcraft. \033[0m') + search = netcraft.SearchNetcraft(word) + search.process() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'netcraft') + + print('\033[94m[*] Searching PGP key server. \033[0m') + try: + search = pgpsearch.SearchPgp(word) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + sethosts = set(hosts) + uniquehosts = list(sethosts) # Remove duplicates. + all_hosts.extend(uniquehosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'PGP') + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_emails, 'email', 'PGP') + except Exception: + pass + + print('\033[94m[*] Searching SecurityTrails. \033[0m') + from discovery import securitytrailssearch + try: + search = securitytrailssearch.search_securitytrail(word) + search.process() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, hosts, 'host', 'securityTrails') + ips = search.get_ips() + all_ip.extend(ips) + db = stash.stash_manager() + db.store_all(word, ips, 'ip', 'securityTrails') + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + pass + + print('\033[94m[*] Searching Threatcrowd. \033[0m') + try: + search = threatcrowd.search_threatcrowd(word) + search.process() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'threatcrowd') + except Exception: + pass + + print('\033[94m[*] Searching Trello. \033[0m') + from discovery import trello + # Import locally or won't work. + search = trello.search_trello(word, limit) + search.process() + emails = filter(search.get_emails()) + all_emails.extend(emails) + info = search.get_urls() + hosts = filter(info[0]) + trello_info = (info[1], True) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, hosts, 'host', 'trello') + db.store_all(word, emails, 'email', 'trello') + + try: + print('\033[94m[*] Searching Twitter. \033[0m') + search = twittersearch.search_twitter(word, limit) + search.process() + people = search.get_people() + db = stash.stash_manager() + db.store_all(word, people, 'name', 'twitter') + print('\nUsers from Twitter:') + print('-------------------') + for user in people: + print(user) + except Exception: + pass + + print('\n[*] Virtual hosts:') + print('------------------') + for l in host_ip: + search = bingsearch.SearchBing(l, limit, start) + search.process_vhost() + res = search.get_allhostnames() + for x in res: + x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x) + x = re.sub('<', '', x) + x = re.sub('>', '', x) + print((l + '\t' + x)) + vhost.append(l + ':' + x) + full.append(l + ':' + x) + vhost = sorted(set(vhost)) + + print('\033[94m[*] Searching VirusTotal. \033[0m') + search = virustotal.search_virustotal(word) + search.process() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'virustotal') + + try: + print('\033[94m[*] Searching Yahoo. \033[0m') + search = yahoosearch.search_yahoo(word, limit) + search.process() + hosts = search.get_hostnames() + emails = search.get_emails() + all_hosts.extend(filter(hosts)) + all_emails.extend(filter(emails)) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'yahoo') + db.store_all(word, all_emails, 'email', 'yahoo') + except Exception as e: + print(f'An exception occurred in yahoo: {e}') + else: + print('\033[93m[!] Invalid source.\n\n \033[0m') + sys.exit(1) + + # Sanity check to see if all_emails and all_hosts are defined. + try: + all_emails + except NameError: + print('\n\n\033[93m[!] No emails found because all_emails is not defined.\n\n \033[0m') + sys.exit(1) + try: + all_hosts + except NameError: + print('\n\n\033[93m[!] No hosts found because all_hosts is not defined.\n\n \033[0m') + sys.exit(1) + + # Results + if len(all_ip) == 0: + print('\n[*] No IPs found.') + else: + print('\n[*] IPs found: ' + str(len(all_ip))) + print('-------------------') + ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip)) + print('\n'.join(map(str, ips))) + + if len(all_emails) == 0: + print('\n[*] No emails found.') + else: + print('\n[*] Emails found: ' + str(len(all_emails))) + print('----------------------') + print(('\n'.join(sorted(list(set(all_emails)))))) + + if len(all_hosts) == 0: + print('\n[*] No hosts found.\n\n') + else: + print('\n[*] Hosts found: ' + str(len(all_hosts))) + print('---------------------') + all_hosts = sorted(list(set(all_hosts))) + full_host = hostchecker.Checker(all_hosts) + full = full_host.check() + for host in full: + ip = host.split(':')[1] + print(host) + if ip != 'empty': + if host_ip.count(ip.lower()): + pass + else: + host_ip.append(ip.lower()) + + db = stash.stash_manager() + db.store_all(word, host_ip, 'ip', 'DNS-resolver') + + if trello_info[1] is True: + trello_urls = trello_info[0] + if trello_urls == []: + print('\n[*] No URLs found.') + else: + total = len(trello_urls) + print('\n[*] URLs found: ' + str(total)) + print('--------------------') + for url in sorted(list(set(trello_urls))): + print(url) + + # DNS brute force + dnsres = [] + if dnsbrute is True: + print('\n[*] Starting DNS brute force.') + a = dnssearch.dns_force(word, dnsserver, verbose=True) + res = a.process() + print('\n[*] Hosts found after DNS brute force:') + print('-------------------------------------') + for y in res: + print(y) + dnsres.append(y.split(':')[0]) + if y not in full: + full.append(y) + db = stash.stash_manager() + db.store_all(word, dnsres, 'host', 'dns_bruteforce') + + # Port scanning + if ports_scanning is True: + print('\n\n[*] Scanning ports (active).\n') + for x in full: + host = x.split(':')[1] + domain = x.split(':')[0] + if host != 'empty': + print(('[*] Scanning ' + host)) + ports = [21, 22, 80, 443, 8080] + try: + scan = port_scanner.PortScan(host, ports) + openports = scan.process() + if len(openports) > 1: + print(('\t[*] Detected open ports: ' + ','.join(str(e) for e in openports))) + takeover_check = 'True' + if takeover_check == 'True': + if len(openports) > 0: + search_take = takeover.take_over(domain) + search_take.process() + except Exception as e: + print(e) + + # DNS reverse lookup + dnsrev = [] + if dnslookup is True: + print('\n[*] Starting active queries.') + analyzed_ranges = [] + for x in host_ip: + print(x) + ip = x.split(':')[0] + range = ip.split('.') + range[3] = '0/24' + s = '.' + range = s.join(range) + if not analyzed_ranges.count(range): + print('[*] Performing reverse lookup in ' + range) + a = dnssearch.dns_reverse(range, True) + a.list() + res = a.process() + analyzed_ranges.append(range) + else: + continue + for x in res: + if x.count(word): + dnsrev.append(x) + if x not in full: + full.append(x) + print('[*] Hosts found after reverse lookup (in target domain):') + print('--------------------------------------------------------') + for xh in dnsrev: + print(xh) + + # DNS TLD expansion + dnstldres = [] + if dnstld is True: + print('[*] Starting DNS TLD expansion.') + a = dnssearch.dns_tld(word, dnsserver, verbose=True) + res = a.process() + print('\n[*] Hosts found after DNS TLD expansion:') + print('----------------------------------------') + for y in res: + print(y) + dnstldres.append(y) + if y not in full: + full.append(y) + + # Virtual hosts search + if virtual == 'basic': + print('\n[*] Virtual hosts:') + print('------------------') + for l in host_ip: + search = bingsearch.SearchBing(l, limit, start) + search.process_vhost() + res = search.get_allhostnames() + for x in res: + x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x) + x = re.sub('<', '', x) + x = re.sub('>', '', x) + print((l + '\t' + x)) + vhost.append(l + ':' + x) + full.append(l + ':' + x) + vhost = sorted(set(vhost)) + else: + pass + + # Shodan + shodanres = [] + if shodan is True: + import texttable + tab = texttable.Texttable() + header = ['IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies'] + tab.header(header) + tab.set_cols_align(['c', 'c', 'c', 'c', 'c']) + tab.set_cols_valign(['m', 'm', 'm', 'm', 'm']) + tab.set_chars(['-', '|', '+', '#']) + tab.set_cols_width([15, 20, 15, 15, 18]) + host_ip = list(set(host_ip)) + print('\033[94m[*] Searching Shodan. \033[0m') + try: + for ip in host_ip: + print(('\tSearching for ' + ip)) + shodan = shodansearch.search_shodan() + rowdata = shodan.search_ip(ip) + time.sleep(2) + tab.add_row(rowdata) + printedtable = tab.draw() + print(printedtable) + except Exception as e: + print(f'\033[93m[!] Error occurred in the Shodan search module: {e} \033[0m') + else: + pass + + # Here we need to add explosion mode. + # We have to take out the TLDs to do this. + recursion = None + if recursion: + start = 0 + for word in vhost: + search = googlesearch.search_google(word, limit, start) + search.process(google_dorking) + emails = search.get_emails() + hosts = search.get_hostnames() + print(emails) + print(hosts) + else: + pass + + # Reporting + if filename != "": + try: + print('\nNEW REPORTING BEGINS.') + db = stash.stash_manager() + scanboarddata = db.getscanboarddata() + latestscanresults = db.getlatestscanresults(word) + previousscanresults = db.getlatestscanresults(word, previousday=True) + latestscanchartdata = db.latestscanchartdata(word) + scanhistorydomain = db.getscanhistorydomain(word) + pluginscanstatistics = db.getpluginscanstatistics() + generator = statichtmlgenerator.htmlgenerator(word) + HTMLcode = generator.beginhtml() + HTMLcode += generator.generatelatestscanresults(latestscanresults) + HTMLcode += generator.generatepreviousscanresults(previousscanresults) + graph = reportgraph.GraphGenerator(word) + HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata) + HTMLcode += graph.drawscattergraphscanhistory(word, scanhistorydomain) + HTMLcode += generator.generatepluginscanstatistics(pluginscanstatistics) + HTMLcode += generator.generatedashboardcode(scanboarddata) + HTMLcode += '

Report generated on ' + str( + datetime.datetime.now()) + '

' + HTMLcode += ''' + + + ''' + Html_file = open('report.html', 'w') + Html_file.write(HTMLcode) + Html_file.close() + print('NEW REPORTING FINISHED!') + print('[*] Saving files.') + html = htmlExport.htmlExport( + all_emails, + full, + vhost, + dnsres, + dnsrev, + filename, + word, + shodanres, + dnstldres) + save = html.writehtml() + except Exception as e: + print(e) + print('\n\033[93m[!] An error occurred creating the file.\033[0m') + + try: + filename = filename.split('.')[0] + '.xml' + file = open(filename, 'w') + file.write('') + for x in all_emails: + file.write('' + x + '') + for x in full: + x = x.split(':') + if len(x) == 2: + file.write( + '' + '' + x[1] + '' + x[0] + '' + '') + else: + file.write('' + x + '') + for x in vhost: + x = x.split(':') + if len(x) == 2: + file.write( + '' + '' + x[1] + '' + x[0] + '' + '') + else: + file.write('' + x + '') + if shodanres != []: + shodanalysis = [] + for x in shodanres: + res = x.split('SAPO') + file.write('') + file.write('' + res[0] + '') + file.write('' + res[2] + '') + file.write('') + reg_server = re.compile('Server:.*') + temp = reg_server.findall(res[1]) + if temp != []: + shodanalysis.append(res[0] + ':' + temp[0]) + file.write('') + if shodanalysis != []: + shodanalysis = sorted(set(shodanalysis)) + file.write('') + for x in shodanalysis: + file.write('' + x + '') + file.write('') + + file.write('') + file.flush() + file.close() + print('[*] Files saved.') + except Exception as er: + print(f'\033[93m[!] An error occurred saving XML file: {er} \033[0m') + print('\n\n') + sys.exit(0) + + +if __name__ == '__main__': + if python_version()[0:3] < '3.6': + print('\033[93m[!] Make sure you have Python 3.6+ installed, quitting. \033[0m') + sys.exit(1) + try: + start() + except KeyboardInterrupt: + print('\n\n\033[93m[!] ctrl+c detected from user, quitting.\n\n \033[0m') + except Exception: + import traceback + + print(traceback.print_exc()) + sys.exit(1) \ No newline at end of file