laramies/theHarvester
1
1
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2025-02-25 06:53:05 +08:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse source
This commit is contained in:
Lee Baird 2018-12-28 12:09:01 -06:00
commit 8e9adcf77b
4 changed files with 67 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
censysparser.py
View file

@ -40,22 +40,23 @@ def search_ipaddresses(self):
except Exception as e:
print("An error occurred in the Censys module: IP address parser: " + str(e))
def search_numberofpageshosts(self):
def search_totalpageshosts(self):
try:
items = self.souphosts.findAll(href=re.compile("page"))
for item in items:
if item.text != 'next': # filter out pagination
self.numberofpageshosts += 1
items = self.souphosts.findAll('span','SearchResultSectionHeader__statistic')
numbers = re.findall(r"/\d*",items[0].text)
pagenumber = numbers[0].replace('/','')
self.numberofpageshosts = int(pagenumber)
return self.numberofpageshosts
except Exception as e:
print("An error occurred in the Censys module IP search: page parser: " + str(e))
def search_numberofpagescerts(self):
def search_totalpagescerts(self):
try:
items = self.soupcerts.findAll(href=re.compile("page"))
for item in items:
if item.text != 'next': # filter out pagination
self.numberofpagescerts += 1
items = self.soupcerts.findAll('span','SearchResultSectionHeader__statistic')
numbers = re.findall(r"/\d*",items[0].text)
pagenumber = numbers[0].replace('/','')
self.numberofpagescerts = int(pagenumber)
return self.numberofpagescerts
except Exception as e:
print("An error occurred in the Censys module certificate search: page parser: " + str(e))
print("Error occurred in the Censys module IP search: page parser: " + str(e))

75
discovery/censys.py
View file

@ -5,7 +5,7 @@
class search_censys:
def __init__(self, word):
def __init__(self, word, limit):
self.word = word
self.urlhost = ""
self.urlcert = ""
@ -17,6 +17,7 @@ def __init__(self, word):
self.server = "censys.io"
self.ips = []
self.hostnamesall = []
self.limit = limit
def do_searchhosturl(self):
try:
@ -44,27 +45,51 @@ def process(self):
self.do_searchcertificateurl()
counter = 2
pages = censysparser.parser(self)
totalpages = pages.search_numberofpageshosts()
while counter <= totalpages:
try:
self.page =str(counter)
self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page)
print("\tSearching Censys IP results page " + self.page + "...")
self.do_searchhosturl()
counter+= 1
except Exception as e:
print("Error occurred in the Censys module requesting the pages: " + str(e))
totalpages = pages.search_totalpageshosts()
pagestosearch = int(self.limit/25) #25 results/page
if totalpages <= pagestosearch:
while counter <= totalpages:
try:
self.page =str(counter)
self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page)
print("\tSearching Censys IP results page " + self.page + "...")
self.do_searchhosturl()
counter+= 1
except Exception as e:
print("Error occurred in the Censys module requesting the pages: " + str(e))
else:
while counter <= pagestosearch:
try:
self.page =str(counter)
self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page)
print("\tSearching Censys IP results page " + self.page + "...")
self.do_searchhosturl()
counter+= 1
except Exception as e:
print("Error occurred in the Censys module requesting the pages: " + str(e))
counter = 2
totalpages = pages.search_numberofpagescerts()
while counter <= totalpages:
try:
self.page = str(counter)
self.urlhost = "https://" + self.server + "/certificates/_search?q=" + str(self.word) + "&page=" + str(self.page)
print("\tSearching Censys certificates results page " + self.page + "...")
self.do_searchcertificateurl()
counter += 1
except Exception as e:
print("Error occurred in the Censys module requesting the pages: " + str(e))
totalpages = pages.search_totalpagescerts()
if totalpages <= pagestosearch:
while counter <= totalpages:
try:
self.page = str(counter)
self.urlhost = "https://" + self.server + "/certificates/_search?q=" + str(self.word) + "&page=" + str(self.page)
print("\tSearching Censys certificates results page " + self.page + "...")
self.do_searchcertificateurl()
counter += 1
except Exception as e:
print("Error occurred in the Censys module requesting the pages: " + str(e))
else:
while counter <= pagestosearch:
try:
self.page =str(counter)
self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page)
print("\tSearching Censys IP results page " + self.page + "...")
self.do_searchhosturl()
counter+= 1
except Exception as e:
print("Error occurred in the Censys module requesting the pages: " + str(e))
except Exception as e:
print("Error occurred in the main Censys module: " + str(e))
@ -74,11 +99,14 @@ def get_hostnames(self):
headers = {'user-agent': getUserAgent(), 'Accept':'*/*','Referer': self.urlcert}
response = requests.post("https://censys.io/ipv4/getdns", json={"ips": ips}, headers=headers)
responsejson = response.json()
domainsfromcensys = []
for key, jdata in responsejson.items():
if jdata is not None:
self.hostnamesall.append(jdata)
domainsfromcensys.append(jdata)
else:
pass
matchingdomains = [s for s in domainsfromcensys if str(self.word) in s]
self.hostnamesall.extend(matchingdomains)
hostnamesfromcerts = censysparser.parser(self)
self.hostnamesall.extend(hostnamesfromcerts.search_hostnamesfromcerts())
return self.hostnamesall
@ -91,5 +119,4 @@ def get_ipaddresses(self):
self.ips = ips.search_ipaddresses()
return self.ips
except Exception as e:
print("Error occurred in the main Censys module - IP address search: " + str(e))
print("Error occurred in the main Censys module - IP address search: " + str(e))

2
requirements.txt
View file

@ -1,3 +1,3 @@
beautifulsoup4==4.6.3
plotly==3.4.2
requests==2.18.4
requests==2.21.0

6
theHarvester.py
View file

@ -176,8 +176,8 @@ def start(argv):
elif engineitem == "censys":
print("[-] Searching in Censys.")
from discovery import censys
# Import locally or won't work.
search = censys.search_censys(word)
# Import locally or won't work
search = censys.search_censys(word,limit)
search.process()
all_ip = search.get_ipaddresses()
hosts = filter(search.get_hostnames())
@ -439,7 +439,7 @@ def start(argv):
print("[-] Searching in Censys.")
from discovery import censys
search = censys.search_censys(word)
search = censys.search_censys(word,limit)
search.process()
ips = search.get_ipaddresses()
setips = set(ips)