laramies/theHarvester
1
1
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2024-09-20 15:26:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #145 from L1ghtn1ng/dev

Browse source 
More misc changes
This commit is contained in:
Christian Martorella 2019-01-01 12:51:13 +01:00 committed by GitHub
parent 5724f9f087 5b6037f93d
commit d174dc846c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 11080 additions and 9774 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -6,7 +6,7 @@
* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *
* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *
* *
* theHarvester 3.0.6 v110 *
* theHarvester 3.0.6 v111 *
* Coded by Christian Martorella *
* Edge-Security Research *
* cmartorella@edge-security.com *
@ -40,8 +40,6 @@ Passive:
* googleCSE: Google custom search engine
* googleplus: Users that work in target company (Uses Google search.)
* google-certificates: Google Certificate Transparency report
* google-profiles: Google search engine, specific search for Google profiles

3
discovery/__init__.py
View file

@ -1,6 +1,5 @@
__all__ = ["bingsearch",
"googlesearch",
"googleplussearch",
"pgpsearch",
"linkedinsearch",
"exaleadsearch",
@ -24,4 +23,4 @@
"googleCSE",
"censys",
"securitytrailssearch",
"trello"]
"trello"]

41
discovery/googleplussearch.py
View file

@ -1,41 +0,0 @@
import requests
from parsers import myparser
import time
from discovery.constants import *
class search_googleplus:
def __init__(self, word, limit):
self.word = word.replace(' ', '%20')
self.results = ""
self.totalresults = ""
self.server = "www.google.com"
self.hostname = "www.google.com"
self.quantity = "100"
self.limit = int(limit)
self.counter = 0
def do_search(self):
try:
urly = "https://" + self.server + "/search?num=100&start=" + str(self.counter) + "&hl=en&meta=&q=site%3Aplus.google.com%20intext%3A%22Works%20at%22%20" + self.word + "%20-inurl%3Aphotos%20-inurl%3Aabout%20-inurl%3Aposts%20-inurl%3Aplusones"
except Exception as e:
print(e)
try:
headers = {'User-Agent': getUserAgent()}
r = requests.get(urly, headers=headers)
except Exception as e:
print(e)
self.results = r.text
self.totalresults += self.results
def get_people(self):
rawres = myparser.parser(self.totalresults, self.word)
return rawres.people_googleplus()
def process(self):
while self.counter < self.limit:
self.do_search()
time.sleep(getDelay())
self.counter += 100
print("\tSearching " + str(self.counter) + " results..")

12
discovery/port_scanner.py
View file

@ -2,7 +2,7 @@
import socket
class port_scan():
class PortScan:
def __init__(self, host, ports):
self.threads = 25
self.host = host
@ -14,12 +14,12 @@ def port_scanner(self, host, ports):
self.lock.acquire()
for port in ports:
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(2)
result = s.connect_ex((host, int(port)))
connect = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connect.settimeout(2)
result = connect.connect_ex((host, int(port)))
if result == 0:
openports.append(port)
s.close()
connect.close()
except Exception as e:
print(e)
pass
@ -27,5 +27,5 @@ def port_scanner(self, host, ports):
return openports
def process(self):
ports = self.port_scanner (self.host, self.ports)
ports = self.port_scanner(self.host, self.ports)
return ports

53
lib/core.py Normal file
View file

@ -0,0 +1,53 @@
# coding=utf-8
import os
import sys
class Core:
@staticmethod
def banner():
print("\n\033[92m*******************************************************************")
print("* *")
print("* | |_| |__ ___ /\ /\__ _ _ ____ _____ ___| |_ ___ _ __ *")
print("* | __| '_ \ / _ \ / /_/ / _` | '__\ \ / / _ \/ __| __/ _ \ '__| *")
print("* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *")
print("* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *")
print("* *")
print("* theHarvester 3.0.6 v111 *")
print("* Coded by Christian Martorella *")
print("* Edge-Security Research *")
print("* cmartorella@edge-security.com *")
print("*******************************************************************\033[94m\n\n")
@staticmethod
def usage():
comm = os.path.basename(sys.argv[0])
if os.path.dirname(sys.argv[0]) == os.getcwd():
comm = "./" + comm
print("Usage: theHarvester.py <options> \n")
print(" -d: company name or domain to search")
print(""" -b: source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile, google,
googleCSE, google-certificates, google-profiles,
hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd,
trello, twitter, vhost, virustotal, yahoo, all""")
print(" -g: use Google Dorking instead of normal Google search")
print(" -s: start with result number X (default: 0)")
print(" -v: verify host name via DNS resolution and search for virtual hosts")
print(" -f: save the results into an HTML and/or XML file")
print(" -n: perform a DNS reverse query on all ranges discovered")
print(" -c: perform a DNS brute force on the domain")
print(" -t: perform a DNS TLD expansion discovery")
print(" -e: specify DNS server")
print(" -p: port scan the detected hosts and check for Takeovers (21,22,80,443,8080)")
print(" -l: limit the number of results (Bing goes from 50 to 50 results,")
print(" Google 100 to 100, and PGP doesn't use this option)")
print(" -h: use Shodan to query discovered hosts")
print("\nExamples:")
print((" " + comm + " -d acme.com -l 500 -b google -f myresults.html"))
print((" " + comm + " -d acme.com -b pgp, virustotal"))
print((" " + comm + " -d acme -l 200 -b linkedin"))
print((" " + comm + " -d acme.com -l 200 -g -b google"))
print((" " + comm + " -d acme.com -b googleCSE -l 500 -s 300"))
print((" " + comm + " -d acme.edu -l 100 -b bing -h \n"))

16599
lib/ip-ranges.json
View file

File diff suppressed because it is too large Load diff

4028
lib/resolvers.txt
View file

File diff suppressed because it is too large Load diff

2
requirements.txt
View file

@ -1,3 +1,3 @@
beautifulsoup4==4.6.3
beautifulsoup4==4.7.0
plotly==3.4.2
requests==2.21.0

112
theHarvester.py
View file

@ -1,11 +1,14 @@
#!/usr/bin/env python
import getopt
import os
import re
import stash
import sys
import datetime
from discovery import *
from discovery.constants import *
from lib.core import *
from lib import hostchecker
from lib import htmlExport
from lib import reportgraph
from lib import statichtmlgenerator
@ -21,66 +24,17 @@
print("Requests library not found, please install before proceeding.\n\n")
sys.exit(1)
from discovery import *
from discovery.constants import *
from lib import hostchecker
from lib import htmlExport
print("\n\033[92m*******************************************************************")
print("* *")
print("* | |_| |__ ___ /\ /\__ _ _ ____ _____ ___| |_ ___ _ __ *")
print("* | __| '_ \ / _ \ / /_/ / _` | '__\ \ / / _ \/ __| __/ _ \ '__| *")
print("* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *")
print("* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *")
print("* *")
print("* theHarvester 3.0.6 v110 *")
print("* Coded by Christian Martorella *")
print("* Edge-Security Research *")
print("* cmartorella@edge-security.com *")
print("*******************************************************************\033[94m\n\n")
def usage():
comm = os.path.basename(sys.argv[0])
if os.path.dirname(sys.argv[0]) == os.getcwd():
comm = "./" + comm
print("Usage: theHarvester.py <options> \n")
print(" -d: company name or domain to search")
print(""" -b: source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile, google,
googleCSE, googleplus, google-certificates, google-profiles,
hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd,
trello, twitter, vhost, virustotal, yahoo, all""")
print(" -g: use Google Dorking instead of normal Google search")
print(" -s: start with result number X (default: 0)")
print(" -v: verify host name via DNS resolution and search for virtual hosts")
print(" -f: save the results into an HTML and/or XML file")
print(" -n: perform a DNS reverse query on all ranges discovered")
print(" -c: perform a DNS brute force on the domain")
print(" -t: perform a DNS TLD expansion discovery")
print(" -e: specify DNS server")
print(" -p: port scan the detected hosts and check for Takeovers (21,22,80,443,8080)")
print(" -l: limit the number of results (Bing goes from 50 to 50 results,")
print(" Google 100 to 100, and PGP doesn't use this option)")
print(" -h: use Shodan to query discovered hosts")
print("\nExamples:")
print((" " + comm + " -d acme.com -l 500 -b google -f myresults.html"))
print((" " + comm + " -d acme.com -b pgp, virustotal"))
print((" " + comm + " -d acme -l 200 -b linkedin"))
print((" " + comm + " -d acme.com -l 200 -g -b google"))
print((" " + comm + " -d acme.com -b googleCSE -l 500 -s 300"))
print((" " + comm + " -d acme.edu -l 100 -b bing -h \n"))
Core.banner()
def start(argv):
if len(sys.argv) < 4:
usage()
Core.usage()
sys.exit(1)
try:
opts, args = getopt.getopt(argv, "l:d:b:s:u:vf:nhcgpte:")
except getopt.GetoptError:
usage()
Core.usage()
sys.exit(1)
try:
db = stash.stash_manager()
@ -138,8 +92,8 @@ def start(argv):
dnstld = True
elif opt == '-b':
engines = set(arg.split(','))
supportedengines = set(['baidu', 'bing', 'bingapi', 'censys', 'crtsh', 'cymon', 'dogpile',
'google', 'googleCSE', 'googleplus', 'google-certificates',
supportedengines = set(['baidu', 'bing', 'bingapi', 'censys', 'crtsh',
'cymon', 'dogpile', 'google', 'googleCSE', 'google-certificates',
'google-profiles', 'hunter', 'linkedin',
'netcraft', 'pgp', 'securityTrails', 'threatcrowd',
'trello', 'twitter', 'vhost', 'virustotal', 'yahoo', 'all'])
@ -256,19 +210,6 @@ def start(argv):
else:
pass
elif engineitem == "googleplus":
print("[-] Searching in Google+.")
search = googleplussearch.search_googleplus(word, limit)
search.process()
people = search.get_people()
print("\nUsers from Google+:")
print("===================")
db = stash.stash_manager()
db.store_all(word, people, 'name', 'googleplus')
for user in people:
print(user)
sys.exit()
elif engineitem == "google-certificates":
print("[-] Searching in Google Certificate transparency report.")
search = googlecertificates.search_googlecertificates(word, limit, start)
@ -595,7 +536,7 @@ def start(argv):
print("[!] Invalid source.\n\n")
sys.exit(1)
# Results ############################################################
# Results
print("\n\033[1;32;40mHarvesting results")
if len(all_ip) == 0:
print("No IP addresses found.")
@ -652,7 +593,7 @@ def start(argv):
db = stash.stash_manager()
db.store_all(word, host_ip, 'ip', 'DNS-resolver')
if trello_info[1] == True: # Indicates user selected Trello.
if trello_info[1] is True: # Indicates user selected Trello.
print("\033[1;33;40m \n[+] URLs found from Trello:")
print("--------------------------")
trello_urls = trello_info[0]
@ -664,9 +605,9 @@ def start(argv):
for url in sorted(list(set(trello_urls))):
print(url)
# DNS Brute force ################################################
# DNS Brute force
dnsres = []
if dnsbrute == True:
if dnsbrute is True:
print("\n\033[94m[-] Starting DNS brute force. \033[1;33;40m")
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
@ -680,8 +621,8 @@ def start(argv):
db = stash.stash_manager()
db.store_all(word, dnsres, 'host', 'dns_bruteforce')
# Port Scanning #################################################
if ports_scanning == True:
# Port Scanning
if ports_scanning is True:
print("\n\n\033[1;32;40m[-] Scanning ports (active).\n")
for x in full:
host = x.split(':')[1]
@ -690,7 +631,7 @@ def start(argv):
print(("- Scanning " + host))
ports = [21, 22, 80, 443, 8080]
try:
scan = port_scanner.port_scan(host, ports)
scan = port_scanner.PortScan(host, ports)
openports = scan.process()
if len(openports) > 1:
print(("\t\033[91m Detected open ports: " + ','.join(
@ -703,9 +644,9 @@ def start(argv):
except Exception as e:
print(e)
# DNS reverse lookup #############################################
# DNS reverse lookup
dnsrev = []
if dnslookup == True:
if dnslookup is True:
print("\n[+] Starting active queries.")
analyzed_ranges = []
for x in host_ip:
@ -733,7 +674,7 @@ def start(argv):
for xh in dnsrev:
print(xh)
# DNS TLD expansion ##############################################
# DNS TLD expansion
dnstldres = []
if dnstld == True:
print("[-] Starting DNS TLD expansion.")
@ -747,7 +688,7 @@ def start(argv):
if y not in full:
full.append(y)
# Virtual hosts search ###########################################
# Virtual hosts search
if virtual == "basic":
print("\n[+] Virtual hosts:")
print("------------------")
@ -766,10 +707,10 @@ def start(argv):
else:
pass
# Shodan search ##################################################
# Shodan search
shodanres = []
shodanvisited = []
if shodan == True:
if shodan is True:
print("\n\n\033[1;32;40m[-] Shodan DB search (passive):\n")
if full == []:
print('No host to search, exiting.')
@ -794,7 +735,6 @@ def start(argv):
else:
pass
##################################################################
# Here we need to add explosion mode.
# Tengo que sacar los TLD para hacer esto.
recursion = None
@ -810,7 +750,7 @@ def start(argv):
else:
pass
# Reporting ######################################################
# Reporting
if filename != "":
try:
print("NEW REPORTING BEGINS.")
@ -906,8 +846,8 @@ def start(argv):
try:
start(sys.argv[1:])
except KeyboardInterrupt:
print("\n[!] Search interrupted by user.\n\n")
print("\n[!] ctrl+c detected, quitting.....\n\n")
except Exception:
import traceback
print(traceback.print_exc())
sys.exit()
sys.exit(1)