From e822f1c3ffa5f15b22ba84117e847db1dcea08d4 Mon Sep 17 00:00:00 2001 From: Lee Baird Date: Wed, 27 Feb 2019 21:31:45 -0600 Subject: [PATCH] Updated README and removed stale code. --- README.md | 26 ++++------- changelog.txt | 8 ++++ discovery/googleCSE.py | 104 ----------------------------------------- lib/core.py | 14 +++--- 4 files changed, 25 insertions(+), 127 deletions(-) delete mode 100644 discovery/googleCSE.py diff --git a/README.md b/README.md index fd2df399..6296d51c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ * | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | * * \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| * * * -* theHarvester 3.0.6 v322 * +* theHarvester 3.0.6 v332 * * Coded by Christian Martorella * * Edge-Security Research * * cmartorella@edge-security.com * @@ -17,9 +17,10 @@ What is this? ------------- theHarvester is a very simple, yet effective tool designed to be used in the early
-stages of a penetration test. Use it for open source intelligence gathering and helping
-to determine a company's external threat landscape on the internet. The tool gathers
-emails, names, subdomains, IPs, and URLs using multiple public data sources that include: +stages of a penetration test. Use it for open source intelligence gathering and
+helping to determine a company's external threat landscape on the internet. The
+tool gathers emails, names, subdomains, IPs, and URLs using multiple public data
+sources that include: Passive: -------- @@ -81,24 +82,17 @@ Modules that require an API key: -------------------------------- Add your keys to api-keys.yaml -* hunter: API key -* intelx: API key -* securityTrails: API key -* shodan: API key +* bingapi +* hunter +* intelx +* securityTrails +* shodan Dependencies: ------------- * Python 3.6 * python3 -m pip install -r requirements.txt -Changelog in 3.0: ------------------ -* Subdomain takeover checks. -* Port scanning (basic). -* Improved DNS dictionary. -* Shodan DB search fixed. -* Result storage in Sqlite. - Comments, bugs, or requests? ---------------------------- cmartorella@edge-security.com diff --git a/changelog.txt b/changelog.txt index 00621dda..b03f71d3 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,12 @@ +Changelog in 3.0: +----------------- +Subdomain takeover checks. +Port scanning (basic). +Improved DNS dictionary. +Shodan DB search fixed. +Result storage in Sqlite. + Changelog in 2.6: ----------------- - usage() improvement, CameronNemo. diff --git a/discovery/googleCSE.py b/discovery/googleCSE.py deleted file mode 100644 index 1b8ca61c..00000000 --- a/discovery/googleCSE.py +++ /dev/null @@ -1,104 +0,0 @@ -from discovery.constants import * -from lib.core import * -from parsers import myparser -import re -import requests -import sys -import time - - -class SearchGoogleCSE: - - def __init__(self, word, limit, start): - self.word = word - self.files = 'pdf' - self.results = "" - self.totalresults = "" - self.server = 'www.googleapis.com' - self.hostname = 'www.googleapis.com' - self.userAgent = '(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6' - self.quantity = '10' - self.limit = limit - self.counter = 1 - self.api_key = Core.google_cse_key()['key'] - if self.api_key is None: - raise MissingKey(True) - self.cse_id = Core.google_cse_key()['id'] - if self.cse_id is None: - raise MissingKey(False) - self.lowRange = start - self.highRange = start + 100 - - def do_search(self): - url = 'https://' + self.server + '/customsearch/v1?key=' + self.api_key + '&highrange=' + str(self.highRange) \ - + '&lowrange=' + str(self.lowRange) + '&cx=' + self.cse_id + '&start=' + str(self.counter) + \ - '&q=' + self.word - headers = { - 'Host': self.server, - 'User-agent': self.userAgent - } - - h = requests.get(url=url, headers=headers) - self.results = h.text - self.totalresults += self.results - - def do_search_files(self, files): - url = 'https://' + self.server + '/customsearch/v1?key=' + self.api_key + '&highRange=' + str(self.highRange) \ - + '&lowRange=' + str(self.lowRange) + '&cx=' + self.cse_id + '&start=' + str(self.counter) + \ - '&q=filetype:' + files + '%20site:' + self.word - headers = { - 'Host': self.server, - 'User-agent': self.userAgent - } - h = requests.get(url=url, headers=headers) - self.results = h.text - self.totalresults += self.results - - def check_next(self): - renext = re.compile('> Next <') - nextres = renext.findall(self.results) - if nextres != []: - nexty = '1' - else: - nexty = '0' - return nexty - - def get_emails(self): - rawres = myparser.Parser(self.totalresults, self.word) - return rawres.emails() - - def get_hostnames(self): - rawres = myparser.Parser(self.totalresults, self.word) - return rawres.hostnames() - - def get_files(self): - rawres = myparser.Parser(self.totalresults, self.word) - return rawres.fileurls(self.files) - - def process(self): - tracker = self.counter + self.lowRange - while tracker <= self.limit: - self.do_search() - ESC = chr(27) - sys.stdout.write(ESC + '[2K' + ESC + '[G') - sys.stdout.write('\r\t' + 'Searching ' + str(self.counter + self.lowRange) + ' results.') - sys.stdout.flush() - if self.counter == 101: - self.counter = 1 - self.lowRange += 100 - self.highRange += 100 - else: - self.counter += 10 - tracker = self.counter + self.lowRange - - def store_results(self): - filename = 'debug_results.txt' - file = open(filename, 'w') - file.write(self.totalresults) - - def process_files(self, files): - while self.counter <= self.limit: - self.do_search_files(files) - time.sleep(1) - self.counter += 100 - print('\tSearching ' + str(self.counter) + ' results.') diff --git a/lib/core.py b/lib/core.py index 5b83f8e5..d791b5ab 100644 --- a/lib/core.py +++ b/lib/core.py @@ -17,6 +17,12 @@ def hunter_key(): keys = yaml.safe_load(api_keys) return keys['apikeys']['hunter']['key'] + @staticmethod + def intelx_key(): + with open('api-keys.yaml', 'r') as api_keys: + keys = yaml.safe_load(api_keys) + return keys['apikeys']['intelx']['key'] + @staticmethod def security_trails_key(): with open('api-keys.yaml', 'r') as api_keys: @@ -29,12 +35,6 @@ def shodan_key(): keys = yaml.safe_load(api_keys) return keys['apikeys']['shodan']['key'] - @staticmethod - def intelx_key(): - with open('api-keys.yaml', 'r') as api_keys: - keys = yaml.safe_load(api_keys) - return keys['apikeys']['intelx']['key'] - @staticmethod def banner(): print('\n\033[93m*******************************************************************') @@ -44,7 +44,7 @@ def banner(): print(r"* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *") print(r"* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *") print('* *') - print('* theHarvester 3.0.6 v322 *') + print('* theHarvester 3.0.6 v332 *') print('* Coded by Christian Martorella *') print('* Edge-Security Research *') print('* cmartorella@edge-security.com *')