From d963ee9904600bfc62a3996df0f6b21875eea5ca Mon Sep 17 00:00:00 2001 From: Lee Baird Date: Wed, 13 Feb 2019 23:05:52 -0600 Subject: [PATCH] Removed google-profiles and clean up. --- .gitattributes | 5 ++--- .gitignore | 2 +- theHarvester.py | 57 ++++++++++++------------------------------------- 3 files changed, 17 insertions(+), 47 deletions(-) diff --git a/.gitattributes b/.gitattributes index 29da73ba..e268d554 100644 --- a/.gitattributes +++ b/.gitattributes @@ -22,7 +22,6 @@ *.pyd binary *.pyo binary -# Note: .db, .p, and .pkl files are associated -# with the python modules ``pickle``, ``dbm.*``, -# ``shelve``, ``marshal``, ``anydbm``, & ``bsddb`` +# Note: .db, .p, and .pkl files are associated with the python modules +# ``pickle``, ``dbm.*``, # ``shelve``, ``marshal``, ``anydbm``, & ``bsddb`` # (among others). diff --git a/.gitignore b/.gitignore index 473116a5..83c17fa5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ .html .vscode .xml +api-keys.yaml debug_results.txt tests/myparser.py venv -api-keys.yaml diff --git a/theHarvester.py b/theHarvester.py index 0cfb21e5..2d65f0e6 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -5,16 +5,16 @@ from lib import hostchecker from lib import htmlExport from lib import reportgraph -from lib import statichtmlgenerator from lib import stash +from lib import statichtmlgenerator from lib.core import * from platform import python_version import argparse import datetime import ipaddress import re -import time import sys +import time try: import bs4 @@ -32,11 +32,12 @@ def start(): - parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon') + parser = argparse.ArgumentParser(description='theHarvester is used to gather open source intelligence (OSINT) on a\n' + 'company or domain.') parser.add_argument('-d', '--domain', help='company name or domain to search', required=True) parser.add_argument('-l', '--limit', help='limit the number of search results, default=500', default=500, type=int) parser.add_argument('-S', '--start', help='start with result number X, default=0', default=0, type=int) - parser.add_argument('-g', '--google-dork', help='use Google Dorks for google search', default=False, action='store_true') + parser.add_argument('-g', '--google-dork', help='use Google Dorks for Google search', default=False, action='store_true') parser.add_argument('-p', '--port-scan', help='scan the detected hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True', default=False) parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=False, action='store_true') parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts params=basic, default=False', default=False) @@ -45,9 +46,9 @@ def start(): parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default=False, params=True', default=False) parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true') parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) - parser.add_argument('-b', '--source', help='''source: baidu, bing, bingapi, censys, crtsh, cymon, + parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, censys, crtsh, cymon, dogpile, duckduckgo, google, googleCSE, - google-certificates, google-profiles, hunter, intelx, + google-certificates, hunter, intelx, linkedin, netcraft, pgp, securityTrails, threatcrowd, trello, twitter, vhost, virustotal, yahoo, all''') args = parser.parse_args() @@ -78,6 +79,7 @@ def start(): vhost = [] virtual = args.virtual_host word = args.domain + if args.source is not None: engines = set(args.source.split(',')) if set(engines).issubset(Core.get_supportedengines()): @@ -219,23 +221,6 @@ def start(): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'google-certificates') - elif engineitem == 'google-profiles': - print('\033[94m[*] Searching Google profiles. \033[0m') - search = googlesearch.search_google(word, limit, start) - search.process_profiles() - people = search.get_profiles() - db = stash.stash_manager() - db.store_all(word, people, 'name', 'google-profile') - - if len(people) == 0: - print('\n[*] No users found.\n\n') - else: - print('\n[*] Users found: ' + str(len(people))) - print('---------------------') - for user in sorted(list(set(people))): - print(user) - sys.exit(0) - elif engineitem == 'hunter': print('\033[94m[*] Searching Hunter. \033[0m') from discovery import huntersearch @@ -457,7 +442,6 @@ def start(): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'CRTsh') - # cymon print('\033[94m[*] Searching Cymon. \033[0m') from discovery import cymon # Import locally or won't work. @@ -510,20 +494,6 @@ def start(): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'google-certificates') - try: - print('\033[94m[*] Searching Google profiles. \033[0m') - search = googlesearch.search_google(word, limit, start) - search.process_profiles() - people = search.get_profiles() - db = stash.stash_manager() - db.store_all(word, people, 'name', 'google-profile') - print('\nUsers from Google profiles:') - print('---------------------------') - for users in people: - print(users) - except Exception: - pass - print('\033[94m[*] Searching Hunter. \033[0m') from discovery import huntersearch # Import locally. @@ -907,7 +877,7 @@ def start(): # Reporting if filename != "": try: - print('\nNEW REPORTING BEGINS.') + print('\n[*] Reporting started.') db = stash.stash_manager() scanboarddata = db.getscanboarddata() latestscanresults = db.getlatestscanresults(word) @@ -933,7 +903,7 @@ def start(): Html_file = open('report.html', 'w') Html_file.write(HTMLcode) Html_file.close() - print('NEW REPORTING FINISHED!') + print('[*] Reporting finished.') print('[*] Saving files.') html = htmlExport.htmlExport( all_emails, @@ -948,7 +918,8 @@ def start(): save = html.writehtml() except Exception as e: print(e) - print('\n\033[93m[!] An error occurred creating the file.\033[0m') + print('\n\033[93m[!] An error occurred creating the output file.\n\n \033[0m') + sys.exit(1) try: filename = filename.split('.')[0] + '.xml' @@ -1002,7 +973,7 @@ def start(): if __name__ == '__main__': if python_version()[0:3] < '3.6': - print('\033[93m[!] Make sure you have Python 3.6+ installed, quitting. \033[0m') + print('\033[93m[!] Make sure you have Python 3.6+ installed, quitting.\n\n \033[0m') sys.exit(1) try: start() @@ -1012,4 +983,4 @@ def start(): import traceback print(traceback.print_exc()) - sys.exit(1) \ No newline at end of file + sys.exit(1)