Merge pull request #116 from jzold/report

Saving more data in the DB
This commit is contained in:
Christian Martorella 2018-12-16 15:59:19 +01:00 committed by GitHub
commit 9940892c50
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 844 additions and 29 deletions

61
oldreport.html Normal file
View file

@ -0,0 +1,61 @@
<html><head><style type='text/css'>body {
background: #FFFFFF top no-repeat;
}
h1 { font-family: arial, Times New Roman, times-roman, georgia, serif;
color: #680000;
margin: 0;
padding: 0px 0px 6px 0px;
font-size: 51px;
line-height: 44px;
letter-spacing: -2px;
font-weight: bold;
}
h3 { font-family: arial, Times New Roman, times-roman, georgia, serif;
color: #444;
margin: 0;
padding: 0px 0px 6px 0px;
font-size: 30px;
line-height: 44px;
letter-spacing: -2px;
font-weight: bold;
}
li { font-family: arial, Times New Roman, times-roman, georgia, serif;
color: #444;
margin: 0;
padding: 0px 0px 6px 0px;
font-size: 15px;
line-height: 15px;
letter-spacing: 0.4px;
}
h2{
font-family: arial, Times New Roman, times-roman, georgia, serif;
font-size: 48px;
line-height: 40px;
letter-spacing: -1px;
color: #680000 ;
margin: 0 0 0 0;
padding: 0 0 0 0;
font-weight: 100;
}
pre {
overflow: auto;
padding-left: 15px;
padding-right: 15px;
font-size: 11px;
line-height: 15px;
margin-top: 10px;
width: 93%;
display: block;
background-color: #eeeeee;
color: #000000;
max-height: 300px;
}
</style>
</head><body><h1>theHarvester results</h1><h2>for :google.com</h2><h3>Dashboard:</h3><body><table border=0 cellspacing=0 cellpadding=0><tr><td><table border=0 cellspacing=2 cellpadding=0><tr align=center valign=bottom><td><table border=0 cellspacing=0 cellpadding=0 width=100%><tr align=center><td style="color:black;font-family:Arial, Helvetica;font-size:12px;" valign=bottom height=114 nowrap>0%</td></tr><tr><td width=24 height=1></td></tr></table></td><td><table border=0 cellspacing=0 cellpadding=0 width=100%><tr align=center><td style="color:black;font-family:Arial, Helvetica;font-size:12px;" valign=bottom height=14 nowrap>100%</td></tr><tr align=center valign=bottom><td><table border=0 cellspacing=0 cellpadding=0><tr><td style="border:2px outset white;" bgcolor="#0000FF" title="2"><div style="width:20px; height:100px; line-height:1px; font-size:1px;"></div></td></tr></table></td></tr></table></td><td><table border=0 cellspacing=0 cellpadding=0 width=100%><tr align=center><td style="color:black;font-family:Arial, Helvetica;font-size:12px;" valign=bottom height=114 nowrap>0%</td></tr><tr><td width=24 height=1></td></tr></table></td><td><table border=0 cellspacing=0 cellpadding=0 width=100%><tr align=center><td style="color:black;font-family:Arial, Helvetica;font-size:12px;" valign=bottom height=114 nowrap>0%</td></tr><tr><td width=24 height=1></td></tr></table></td><td><table border=0 cellspacing=0 cellpadding=0 width=100%><tr align=center><td style="color:black;font-family:Arial, Helvetica;font-size:12px;" valign=bottom height=114 nowrap>0%</td></tr><tr><td width=24 height=1></td></tr></table></td></tr><tr align=center><td style="color:black;background-color:#C0E0FF;border:2px groove white;font-family:Arial, Helvetica;font-size:12px;" nowrap>&nbsp;0&nbsp;</td><td style="color:black;background-color:#C0E0FF;border:2px groove white;font-family:Arial, Helvetica;font-size:12px;" nowrap>&nbsp;2&nbsp;</td><td style="color:black;background-color:#C0E0FF;border:2px groove white;font-family:Arial, Helvetica;font-size:12px;" nowrap>&nbsp;0&nbsp;</td><td style="color:black;background-color:#C0E0FF;border:2px groove white;font-family:Arial, Helvetica;font-size:12px;" nowrap>&nbsp;0&nbsp;</td><td style="color:black;background-color:#C0E0FF;border:2px groove white;font-family:Arial, Helvetica;font-size:12px;" nowrap>&nbsp;0&nbsp;</td></tr><tr><td style="color:black;background-color:#C0E0FF;border:2px groove white;font-family:Arial, Helvetica;font-size:12px;text-align:center;">&nbsp;Emails&nbsp;</td><td style="color:black;background-color:#C0E0FF;border:2px groove white;font-family:Arial, Helvetica;font-size:12px;text-align:center;">&nbsp;hosts&nbsp;</td><td style="color:black;background-color:#C0E0FF;border:2px groove white;font-family:Arial, Helvetica;font-size:12px;text-align:center;">&nbsp;Vhost&nbsp;</td><td style="color:black;background-color:#C0E0FF;border:2px groove white;font-family:Arial, Helvetica;font-size:12px;text-align:center;">&nbsp;TLD&nbsp;</td><td style="color:black;background-color:#C0E0FF;border:2px groove white;font-family:Arial, Helvetica;font-size:12px;text-align:center;">&nbsp;Shodan&nbsp;</td></tr></table></td></tr></table></body><h3>E-mails names found:</h3><h2>No emails found</h2><h3>Hosts found:</h3><ul class="softlist"><li class="softitem">support.google.com:216.58.198.238</li><li class="softitem">www.google.com:216.58.204.4</li></ul></body></html>

1
oldreport.xml Normal file
View file

@ -0,0 +1 @@
<?xml version="1.0" encoding="UTF-8"?><theHarvester><host><ip>216.58.198.238</ip><hostname>support.google.com</hostname></host><host><ip>216.58.204.4</ip><hostname>www.google.com</hostname></host></theHarvester>

View file

@ -1,7 +1,7 @@
FROM python:2-alpine
RUN mkdir /app
RUN pip install requests
RUN pip install requests beautifulsoup4
WORKDIR /app
COPY . /app
RUN chmod +x *.py
ENTRYPOINT ["/app/theHarvester.py"]
ENTRYPOINT ["/app/theHarvester.py"]

88
lib/reportgraph.py Normal file
View file

@ -0,0 +1,88 @@
try:
import plotly.graph_objs as go
import plotly.plotly as py
import plotly
import stash
try:
db=stash.stash_manager()
db.do_init()
except Exception as e:
pass
class graphgenerator:
def __init__(self, domain):
self.domain = domain
self.bardata = []
self.barcolumns = []
self.scatterxhosts = []
self.scatteryhosts = []
def drawlatestscangraph(self,domain,latestscandata):
self.barcolumns= ['email','host','ip','shodan','vhost']
self.bardata.append(latestscandata['email'])
self.bardata.append(latestscandata['host'])
self.bardata.append(latestscandata['ip'])
self.bardata.append(latestscandata['shodan'])
self.bardata.append(latestscandata['vhost'])
# for i in scandata:
# self.bardata.append(scandata[i])
layout = dict(title = "Last scan - number of targets identified for "+ domain +" on "+str(latestscandata["latestdate"]),
xaxis = dict(title = 'Targets'),
yaxis = dict(title = 'Hits'),)
barchartcode = plotly.offline.plot({
"data": [go.Bar(x=self.barcolumns,y=self.bardata)],
"layout": layout,
}, auto_open=False,include_plotlyjs=False,filename='report.html', output_type='div')
return barchartcode
def drawscattergraph(self,domain,latestscandata):
scandata = latestscandata
for i in scandata['scandetails']:
self.scatterxhosts.append(i)
self.scatteryhosts.append(scandata[i])
trace0 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[3, 10, 9, 17,10],
mode = 'lines+markers',
name = 'hosts')
trace1 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[2, 6, 9, 10, 5],
mode = 'lines+markers',
name = 'IP address')
trace2 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[1, 2, 4, 6, 2],
mode = 'lines+markers',
name = 'vhost')
trace3 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[2, 3, 2, 5, 7],
mode = 'lines+markers',
name = 'shodan')
trace4 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[12, 14, 20, 24, 20],
mode = 'lines+markers',
name = 'email')
data = [trace0, trace1, trace2, trace3, trace4]
layout = dict(title = "Scanning history for " + domain,
xaxis = dict(title = 'Date'),
yaxis = dict(title = 'Results'),
)
scatterchartcode = plotly.offline.plot({
"data": data,
"layout": layout}, auto_open=False,include_plotlyjs=False,filename='report.html', output_type='div')
return scatterchartcode
except Exception as e:
print(e)

View file

@ -0,0 +1,91 @@
class htmlgenerator:
def __init__(self,word):
self.domain = word
def generatedashboardcode(self, scanboarddata):
totalnumberofdomains = scanboarddata['domains']
totalnumberofhosts = scanboarddata['host']
totalnumberofip = scanboarddata['ip']
totalnumberofvhost= scanboarddata['vhost']
totalnumberofemail= scanboarddata['email']
totalnumberofshodan= scanboarddata['shodan']
html='''
<head><script src="https://cdn.plot.ly/plotly-latest.min.js"></script></head>
<html>
<body>
<h1 style="text-align: center;"><span style="color: #ff0000;">theHarvester Scan Report</span></h1>
<h2><span style="color: #000000;"><strong>TheHarvester scanning dashboard</strong></span></h2>
<table align="left" style="height: 108px; border-color: #000000; margin-left: auto; margin-right: auto;" border=" #000000" width="713">
<tbody>
<tr>
<td style="width: 113px; text-align: center;background: #ffff38"><h2><strong>Domains</strong></h2></td>
<td style="width: 108px; text-align: center;background: #1f77b4"><h2><strong>Hosts</strong></h2></td>
<td style="width: 119px; text-align: center;background: #ff7f0e"><h2><strong>IP Addresses</strong></h2></td>
<td style="width: 111px; text-align: center;background: #2ca02c"><h2><strong>Vhosts</strong></h2></td>
<td style="width: 110px; text-align: center;background: #9467bd"><h2><strong>Emails</strong></h2></td>
<td style="width: 110px; text-align: center;background: #d62728"><h2><strong>Shodan</strong></h2></td>
</tr>
<tr>
<td style="width: 113px; text-align: center;background: #ffff38"><h2><strong>'''+str(totalnumberofdomains)+'''</strong></h2></td>
<td style="width: 108px; text-align: center;background: #1f77b4"><h2><strong>'''+str(totalnumberofhosts)+'''</strong></h2></td>
<td style="width: 119px; text-align: center;background: #ff7f0e"><h2><strong>'''+str(totalnumberofip)+'''</strong></h2></td>
<td style="width: 111px; text-align: center;background: #2ca02c"><h2><strong>'''+str(totalnumberofvhost)+'''</strong></h2></td>
<td style="width: 110px; text-align: center;background: #9467bd"><h2><strong>'''+str(totalnumberofemail)+'''</strong></h2></td>
<td style="width: 110px; text-align: center;background: #d62728"><h2><strong>'''+str(totalnumberofshodan)+'''</strong></h2></td>
</tr>
</tbody>
</table>
<p>&nbsp;</p>
<p>&nbsp;</p>
<p>&nbsp;</p>
<p>&nbsp;</p>
'''
return html
def generatescandetailsdomain(self, word, latestscandomain):
emails = latestscandomain['scandetailsemail']
hosts = latestscandomain['scandetailshost']
ips = latestscandomain['scandetailsip']
vhosts = latestscandomain['scandetailsvhost']
shodans = latestscandomain['scandetailsshodan']
html='''
<p>&nbsp;</p>
<p>&nbsp;</p>
<h2><span style="color: #000000;">Latest scan details for '''+ word + ''' on: '''+str(latestscandomain['latestdate'])+'''</span></h2>
<h3><strong><span style="color: #0000ff;">Emails found:</span></strong></h3>
<ul>
'''
for email in emails:
html += '<li><span style="color: #000000;">'+ str(email[1]) + "</span></li>"
html +='''
</ul>
<h3><span style="color: #0000ff;">Hosts found:</span></h3>
<ul>
'''
for host in hosts:
html += '<li><span style="color: #000000;">'+ str(host[1]) + "</span></li>"
html +='''
</ul>
<h3><span style="color: #0000ff;">IP found:</span></h3>
<ul>
'''
for ip in ips:
html += '<li><span style="color: #000000;">'+str(ip[1])+"</span></li>"
html +='''
</ul>
<h3><span style="color: #0000ff;">vhosts found:</span></h3>
<ul>
'''
for vhost in vhosts:
html +='<li><span style="color: #000000;">'+str(vhost[1])+"</span></li>"
html +='''
</ul>
<h3><span style="color: #0000ff;">Shodan results:</span></h3>
<ul>
'''
for shodan in shodans:
html +='<li><span style="color: #000000;">'+str(shodan[1])+"</span></li>"
html +='''
</ul>
'''
return html

58
report.html Normal file

File diff suppressed because one or more lines are too long

View file

@ -1,2 +1,3 @@
requests==2.18.4
bs4==0.0.1
beautifulsoup4==4.6.3
plotly==3.4.2

107
stash.py
View file

@ -8,6 +8,9 @@ def __init__(self):
self.db = "stash.sqlite"
self.results = ""
self.totalresults = ""
self.latestscandomain = {}
self.domainscanhistory = {}
self.scanboarddata = {}
def do_init(self):
conn = sqlite3.connect(self.db)
@ -48,4 +51,106 @@ def store_all(self,domain,all,res_type,source):
conn.close()
except Exception as e:
print(e)
return
return
def getlatestscandomain(self,domain):
try:
self.latestscandomain["domain"] = domain
conn = sqlite3.connect(self.db)
c = conn.cursor()
c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="host"''',(domain,))
data = c.fetchone()
self.latestscandomain["host"] = data[0]
c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="email"''',(domain,))
data = c.fetchone()
self.latestscandomain["email"] = data[0]
c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="ip"''',(domain,))
data = c.fetchone()
self.latestscandomain["ip"] = data[0]
c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="vhost"''',(domain,))
data = c.fetchone()
self.latestscandomain["vhost"] = data[0]
c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="shodan"''',(domain,))
data = c.fetchone()
self.latestscandomain["shodan"] = data[0]
c.execute('''SELECT MAX(find_date) FROM results WHERE domain=?''',(domain,))
data = c.fetchone()
self.latestscandomain["latestdate"] = data[0]
latestdate = data [0]
c.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="host"''',(domain,latestdate,))
scandetailshost = c.fetchall()
self.latestscandomain["scandetailshost"] = scandetailshost
c.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="email"''',(domain,latestdate,))
scandetailsemail = c.fetchall()
self.latestscandomain["scandetailsemail"] = scandetailsemail
c.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="ip"''',(domain,latestdate,))
scandetailsip = c.fetchall()
self.latestscandomain["scandetailsip"] = scandetailsip
c.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="vhost"''',(domain,latestdate,))
scandetailsvhost = c.fetchall()
self.latestscandomain["scandetailsvhost"] = scandetailsvhost
c.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="shodan"''',(domain,latestdate,))
scandetailsshodan = c.fetchall()
self.latestscandomain["scandetailsshodan"] = scandetailsshodan
return self.latestscandomain
except Exception as e:
print(e)
finally:
conn.close()
def getscanboarddata(self):
try:
conn = sqlite3.connect(self.db)
c = conn.cursor()
c.execute('''SELECT COUNT(*) from results WHERE type="host"''')
data = c.fetchone()
self.scanboarddata["host"] = data[0]
c.execute('''SELECT COUNT(*) from results WHERE type="email"''')
data = c.fetchone()
self.scanboarddata["email"] = data[0]
c.execute('''SELECT COUNT(*) from results WHERE type="ip"''')
data = c.fetchone()
self.scanboarddata["ip"] = data[0]
c.execute('''SELECT COUNT(*) from results WHERE type="vhost"''')
data = c.fetchone()
self.scanboarddata["vhost"] = data[0]
c.execute('''SELECT COUNT(*) from results WHERE type="shodan"''')
data = c.fetchone()
self.scanboarddata["shodan"] = data[0]
c.execute('''SELECT COUNT(DISTINCT(domain)) FROM results ''')
data = c.fetchone()
self.scanboarddata["domains"] = data[0]
return self.scanboarddata
except Exception as e:
print(e)
finally:
conn.close()
def getscanhistory(self,domain):
'''dis needs fixing; minden datumhoz kell a count of hosts, emails, ip vhost for a specific domain'''
try:
self.getscanhistory["domain"] = domain
conn = sqlite3.connect(self.db)
c = conn.cursor()
c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="host"''',(domain,))
data = c.fetchone()
self.domainscanhistory["host"] = data[0]
c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="email"''',(domain,))
data = c.fetchone()
self.domainscanhistory["email"] = data[0]
c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="ip"''',(domain,))
data = c.fetchone()
self.domainscanhistory["ip"] = data[0]
c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="vhost"''',(domain,))
data = c.fetchone()
self.domainscanhistory["vhost"] = data[0]
c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="shodan"''',(domain,))
data = c.fetchone()
self.domainscanhistory["shodan"] = data[0]
c.execute('''SELECT find_date FROM results WHERE domain=?''',(domain,))
data = c.fetchone()
return self.domainscanhistory
except Exception as e:
print(e)
finally:
conn.close()

188
testreport.py Normal file
View file

@ -0,0 +1,188 @@
try:
import plotly
import plotly.graph_objs as go
import plotly.plotly as py
import datetime
scanneddomain='google.com'
totalnumberofdomains = 4
totalnumberofhosts = 14
totalnumberofip = 10
totalnumberofvhost=3
totalnumberofemail=15
totalnumberofshodan=7
date1=datetime.date(2018,12,1)
date2=datetime.date(2018,12,3)
date3=datetime.date(2018,12,5)
date4=datetime.date(2018,12,8)
date5=datetime.date(2018,12,10)
HTML='''
<head><script src="https://cdn.plot.ly/plotly-latest.min.js"></script></head>
<html>
<body>
<h1 style="text-align: center;"><span style="color: #ff0000;">theHarvester Scan Report</span></h1>
<h2><span style="color: #000000;"><strong>TheHarvester scanning dashboard</strong></span></h2>
<table align="left" style="height: 108px; border-color: #000000; margin-left: auto; margin-right: auto;" border=" #000000" width="713">
<tbody>
<tr>
<td style="width: 113px; text-align: center;background: #ffff38"><h2><strong>Domains</strong></h2></td>
<td style="width: 108px; text-align: center;background: #1f77b4"><h2><strong>Hosts</strong></h2></td>
<td style="width: 119px; text-align: center;background: #ff7f0e"><h2><strong>IP Addresses</strong></h2></td>
<td style="width: 111px; text-align: center;background: #2ca02c"><h2><strong>Vhosts</strong></h2></td>
<td style="width: 110px; text-align: center;background: #9467bd"><h2><strong>Emails</strong></h2></td>
<td style="width: 110px; text-align: center;background: #d62728"><h2><strong>Shodan</strong></h2></td>
</tr>
'''+'''<tr>
<td style="width: 113px; text-align: center;background: #ffff38"><h2><strong>'''+str(totalnumberofdomains)+'''</strong></h2></td>
<td style="width: 108px; text-align: center;background: #1f77b4"><h2><strong>'''+str(totalnumberofhosts)+'''</strong></h2></td>
<td style="width: 119px; text-align: center;background: #ff7f0e"><h2><strong>'''+str(totalnumberofip)+'''</strong></h2></td>
<td style="width: 111px; text-align: center;background: #2ca02c"><h2><strong>'''+str(totalnumberofvhost)+'''</strong></h2></td>
<td style="width: 110px; text-align: center;background: #9467bd"><h2><strong>'''+str(totalnumberofemail)+'''</strong></h2></td>
<td style="width: 110px; text-align: center;background: #d62728"><h2><strong>'''+str(totalnumberofshodan)+'''</strong></h2></td>
'''+'''
</tr>
</tbody>
</table>
<p>&nbsp;</p>
<p>&nbsp;</p>
<p>&nbsp;</p>
<p>&nbsp;</p>
<h2><span style="color: #000000;">Latest scan summary for '''+scanneddomain+'''</span></h2>
<h3><strong><span style="color: #0000ff;">Emails found:</span></strong></h3>
<ul>
<li><span style="color: #000000;">email1@google.com</span></li>
<li><span style="color: #000000;">email2@google.com</span></li>
</ul>
<h3><span style="color: #0000ff;">Hosts found:</span></h3>
<ul>
<li><span style="color: #000000;">host1.google.com</span></li>
<li><span style="color: #000000;">host2.google.com</span></li>
<li><span style="color: #000000;">host3.google.com</span></li>
</ul>
<h3><strong><span style="color: #0000ff;">IP addresses found:</span></strong></h3>
<ul>
<li><span style="color: #000000;">87.12.42.12</span></li>
<li><span style="color: #000000;">87.12.42.11</span></li>
<li><span style="color: #000000;">87.12.43.11</span></li>
<li><span style="color: #000000;">87.12.44.11</span></li>
</ul>
<h3><span style="color: #0000ff;"><strong>Shodan results:</strong></span></h3>
<ul>
<li><span style="color: #000000;">NONE</span></li>
</ul>
'''
barcolumns = ["host","ip","vhost","shodan","email"]
bardata = [totalnumberofhosts,totalnumberofip,totalnumberofvhost,totalnumberofshodan,totalnumberofemail]
layout = dict(title = "Last scan - number of targets identified for "+scanneddomain+" on "+str(date5),
xaxis = dict(title = 'Targets'),
yaxis = dict(title = 'Hits'),
)
barchart=plotly.offline.plot({
"data": [go.Bar(x=barcolumns,y=bardata)],
"layout": layout,
}, auto_open=False,include_plotlyjs=False,filename='report.html', output_type='div')
HTML+=barchart
trace0 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[3, 10, 9, 17,10],
mode = 'lines+markers',
name = 'hosts')
trace1 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[2, 6, 9, 10, 5],
mode = 'lines+markers',
name = 'IP address')
trace2 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[1, 2, 4, 6, 2],
mode = 'lines+markers',
name = 'vhost')
trace3 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[2, 3, 2, 5, 7],
mode = 'lines+markers',
name = 'shodan')
trace4 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[12, 14, 20, 24, 20],
mode = 'lines+markers',
name = 'email')
data = [trace0, trace1, trace2, trace3, trace4]
layout = dict(title = "Scanning history for "+scanneddomain,
xaxis = dict(title = 'Date'),
yaxis = dict(title = 'Results'),
)
scatterchart = plotly.offline.plot({
"data": data,
"layout": layout}, auto_open=False,include_plotlyjs=False,filename='report.html', output_type='div')
HTML+=scatterchart
trace1 = go.Bar(
x=[date1,date2,date3,date4,date5],
y=[20, 21, 23, 20, 17],
text=['domain1.com', 'domain2.com', 'domain3.com', 'domain4.com', 'domain5.com'],
name='hosts'
)
trace2 = go.Bar(
x=[date1,date2,date3,date4,date5],
y=[24, 23, 29, 30, 25],
text=['domain1.com', 'domain2.com', 'domain3.com', 'domain4.com', 'domain5.com'],
name='IP addresses'
)
trace3 = go.Bar(
x=[date1,date2,date3,date4,date5],
y=[3, 5, 2, 1, 3],
text=['domain1.com', 'domain2.com', 'domain3.com', 'domain4.com', 'domain5.com'],
name='vhosts'
)
trace4 = go.Bar(
x=[date1,date2,date3,date4,date5],
y=[13, 10, 12, 13, 12],
text=['domain1.com', 'domain2.com', 'domain3.com', 'domain4.com', 'domain5.com'],
name='shodan'
)
trace5 = go.Bar(
x=[date1,date2,date3,date4,date5],
y=[30, 34, 35, 41, 31],
text=['domain1.com', 'domain2.com', 'domain3.com', 'domain4.com', 'domain5.com'],
name='email'
)
data = [trace1, trace2, trace3, trace4, trace5]
layout = go.Layout(
barmode='stack'
)
layout = dict(title = "theHarvester scan history",
xaxis = dict(title = 'Date'),
yaxis = dict(title = 'Hits'),
)
barchart=plotly.offline.plot({
"data": data,
"layout": layout,
}, auto_open=False,include_plotlyjs=False,filename='report.html', output_type='div')
HTML+=barchart
HTML+='<p><span style="color: #000000;">Report generated on '+ str(datetime.datetime.now())+'</span></p>'
HTML+='''
</body>
</html>
'''
Html_file= open("report.html","w")
Html_file.write(HTML)
Html_file.close()
except Exception as e:
print("ERROR: "+str(e))

186
testreportstack.py Normal file
View file

@ -0,0 +1,186 @@
try:
import plotly
import plotly.graph_objs as go
import plotly.plotly as py
import datetime
scanneddomain='google.com'
totalnumberofdomains = 4
totalnumberofhosts = 14
totalnumberofip = 10
totalnumberofvhost=3
totalnumberofemail=15
totalnumberofshodan=7
date1=datetime.date(2018,12,1)
date2=datetime.date(2018,12,3)
date3=datetime.date(2018,12,5)
date4=datetime.date(2018,12,8)
date5=datetime.date(2018,12,10)
HTML='''
<head><script src="https://cdn.plot.ly/plotly-latest.min.js"></script></head>
<html>
<body>
<h1 style="text-align: center;"><span style="color: #ff0000;">theHarvester Scan Report</span></h1>
<h2><span style="color: #000000;"><strong>TheHarvester scanning dashboard</strong></span></h2>
<table align="left" style="height: 108px; border-color: #000000; margin-left: auto; margin-right: auto;" border=" #000000" width="713">
<tbody>
<tr>
<td style="width: 113px; text-align: center;background: #ffff38"><h2><strong>Domains</strong></h2></td>
<td style="width: 108px; text-align: center;background: #1f77b4"><h2><strong>Hosts</strong></h2></td>
<td style="width: 119px; text-align: center;background: #ff7f0e"><h2><strong>IP Addresses</strong></h2></td>
<td style="width: 111px; text-align: center;background: #2ca02c"><h2><strong>Vhosts</strong></h2></td>
<td style="width: 110px; text-align: center;background: #9467bd"><h2><strong>Emails</strong></h2></td>
<td style="width: 110px; text-align: center;background: #d62728"><h2><strong>Shodan</strong></h2></td>
</tr>
'''+'''<tr>
<td style="width: 113px; text-align: center;background: #ffff38"><h2><strong>'''+str(totalnumberofdomains)+'''</strong></h2></td>
<td style="width: 108px; text-align: center;background: #1f77b4"><h2><strong>'''+str(totalnumberofhosts)+'''</strong></h2></td>
<td style="width: 119px; text-align: center;background: #ff7f0e"><h2><strong>'''+str(totalnumberofip)+'''</strong></h2></td>
<td style="width: 111px; text-align: center;background: #2ca02c"><h2><strong>'''+str(totalnumberofvhost)+'''</strong></h2></td>
<td style="width: 110px; text-align: center;background: #9467bd"><h2><strong>'''+str(totalnumberofemail)+'''</strong></h2></td>
<td style="width: 110px; text-align: center;background: #d62728"><h2><strong>'''+str(totalnumberofshodan)+'''</strong></h2></td>
'''+'''
</tr>
</tbody>
</table>
<p>&nbsp;</p>
<p>&nbsp;</p>
<p>&nbsp;</p>
<p>&nbsp;</p>
<h2><span style="color: #000000;">Latest scan summary for '''+scanneddomain+'''</span></h2>
<h3><strong><span style="color: #0000ff;">Emails found:</span></strong></h3>
<ul>
<li><span style="color: #000000;">email1@google.com</span></li>
<li><span style="color: #000000;">email2@google.com</span></li>
</ul>
<h3><span style="color: #0000ff;">Hosts found:</span></h3>
<ul>
<li><span style="color: #000000;">host1.google.com</span></li>
<li><span style="color: #000000;">host2.google.com</span></li>
<li><span style="color: #000000;">host3.google.com</span></li>
</ul>
<h3><strong><span style="color: #0000ff;">IP addresses found:</span></strong></h3>
<ul>
<li><span style="color: #000000;">87.12.42.12</span></li>
<li><span style="color: #000000;">87.12.42.11</span></li>
<li><span style="color: #000000;">87.12.43.11</span></li>
<li><span style="color: #000000;">87.12.44.11</span></li>
</ul>
<h3><span style="color: #0000ff;"><strong>Shodan results:</strong></span></h3>
<ul>
<li><span style="color: #000000;">NONE</span></li>
</ul>
'''
barcolumns = ["host","ip","vhost","shodan","email"]
bardata = [totalnumberofhosts,totalnumberofip,totalnumberofvhost,totalnumberofshodan,totalnumberofemail]
layout = dict(title = "Last scan - number of targets identified for "+scanneddomain+" on "+str(date5),
xaxis = dict(title = 'Targets'),
yaxis = dict(title = 'Hits'),
)
barchart=plotly.offline.plot({
"data": [go.Bar(x=barcolumns,y=bardata)],
"layout": layout,
}, auto_open=False,include_plotlyjs=False,filename='report.html', output_type='div')
HTML+=barchart
trace0 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[3, 10, 9, 17,10],
mode = 'lines+markers',
name = 'hosts')
trace1 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[2, 6, 9, 10, 5],
mode = 'lines+markers',
name = 'IP address')
trace2 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[1, 2, 4, 6, 2],
mode = 'lines+markers',
name = 'vhost')
trace3 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[2, 3, 2, 5, 7],
mode = 'lines+markers',
name = 'shodan')
trace4 = go.Scatter(
x=[date1,date2,date3,date4,date5],
y=[12, 14, 20, 24, 20],
mode = 'lines+markers',
name = 'email')
data = [trace0, trace1, trace2, trace3, trace4]
layout = dict(title = "Scanning history for "+scanneddomain,
xaxis = dict(title = 'Date'),
yaxis = dict(title = 'Results'),
)
scatterchart = plotly.offline.plot({
"data": data,
"layout": layout}, auto_open=False,include_plotlyjs=False,filename='report.html', output_type='div')
HTML+=scatterchart
trace1 = go.Bar(
x=[date1,date2,date3,date4,date5],
y=[20, 21, 23, 20, 17],
text=['domain1.com', 'domain1.com', 'domain1.com', 'domain1.com', 'domain1.com'],
name='hosts'
)
trace2 = go.Bar(
x=[date1,date2,date3,date4,date5],
y=[24, 23, 29, 30, 25],
text=['domain2.com', 'domain2.com', 'domain2.com', 'domain2.com', 'domain2.com'],
name='IP addresses'
)
trace3 = go.Bar(
x=[date1,date2,date3,date4,date5],
y=[3, 5, 2, 1, 3],
text=['domain3.com', 'domain3.com', 'domain3.com', 'domain3.com', 'domain3.com'],
name='vhosts'
)
trace4 = go.Bar(
x=[date1,date2,date3,date4,date5],
y=[13, 10, 12, 13, 12],
text=['domain4.com', 'domain4.com', 'domain4.com', 'domain4.com', 'domain4.com'],
name='shodan'
)
trace5 = go.Bar(
x=[date1,date2,date3,date4,date5],
y=[30, 34, 35, 41, 31],
text=['domain5.com', 'domain5.com', 'domain5.com', 'domain5.com', 'domain5.com'],
name='email'
)
data = [trace1, trace2, trace3, trace4, trace5]
layout = dict(title = "theHarvester scan history",
xaxis = dict(title = 'Date'),
yaxis = dict(title = 'Hits'),
barmode='stack'
)
barchart=plotly.offline.plot({
"data": data,
"layout": layout,
}, auto_open=False,include_plotlyjs=False,filename='report.html', output_type='div')
HTML+=barchart
HTML+='<p><span style="color: #000000;">Report generated on '+ str(datetime.datetime.now())+'</span></p>'
HTML+='''
</body>
</html>
'''
Html_file= open("report.html","w")
Html_file.write(HTML)
Html_file.close()
except Exception as e:
print("ERROR: "+str(e))

View file

@ -144,14 +144,10 @@ def start(argv):
search.process(google_dorking)
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
for x in all_hosts:
try:
db=stash.stash_manager()
db.store(word,x,'host','google')
except Exception as e:
print(e)
db=stash.stash_manager()
db.store_all(word,all_hosts,'host','google')
db.store_all(word,all_emails,'email','google')
if engineitem == "netcraft":
print("[-] Searching in Netcraft:")
search = netcraft.search_netcraft(word)
@ -188,7 +184,6 @@ def start(argv):
db=stash.stash_manager()
db.store_all(word,all_hosts,'host','virustotal')
if engineitem == "crtsh":
print("[-] Searching in CRT.sh:")
search = crtsh.search_crtsh(word)
@ -220,6 +215,9 @@ def start(argv):
search.process(bingapi)
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db=stash.stash_manager()
db.store_all(word,all_hosts,'email','bing')
db.store_all(word,all_hosts,'host','bing')
elif engineitem == "dogpile":
print("[-] Searching in Dogpilesearch..")
@ -227,6 +225,8 @@ def start(argv):
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db.store_all(word,all_hosts,'email','dogpile')
db.store_all(word,all_hosts,'host','dogpile')
elif engineitem == "pgp":
print("[-] Searching in PGP key server..")
@ -236,8 +236,7 @@ def start(argv):
all_hosts = search.get_hostnames()
db=stash.stash_manager()
db.store_all(word,all_hosts,'host','pgp')
db=stash.stash_manager()
db.store_all(word,all_emails,'emails','pgp')
db.store_all(word,all_emails,'email','pgp')
elif engineitem == "yahoo":
print("[-] Searching in Yahoo..")
@ -247,8 +246,7 @@ def start(argv):
all_hosts = search.get_hostnames()
db=stash.stash_manager()
db.store_all(word,all_hosts,'host','yahoo')
db.store_all(word,all_emails,'emails','yahoo')
db.store_all(word,all_emails,'email','yahoo')
elif engineitem == "baidu":
print("[-] Searching in Baidu..")
@ -258,7 +256,7 @@ def start(argv):
all_hosts = search.get_hostnames()
db=stash.stash_manager()
db.store_all(word,all_hosts,'host','baidu')
db.store_all(word,all_emails,'emails','baidu')
db.store_all(word,all_emails,'email','baidu')
elif engineitem == "googleplus":
print("[-] Searching in Google+ ..")
@ -267,6 +265,8 @@ def start(argv):
people = search.get_people()
print("Users from Google+:")
print("====================")
db=stash.stash_manager()
db.store_all(word,people,'name','googleplus')
for user in people:
print(user)
sys.exit()
@ -276,6 +276,8 @@ def start(argv):
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db=stash.stash_manager()
db.store_all(word,people,'name','twitter')
print("Users from Twitter:")
print("-------------------")
for user in people:
@ -287,6 +289,8 @@ def start(argv):
search = linkedinsearch.search_linkedin(word, limit)
search.process()
people = search.get_people()
db=stash.stash_manager()
db.store_all(word,people,'name','linkedin')
print("Users from Linkedin:")
print("-------------------")
for user in people:
@ -298,6 +302,8 @@ def start(argv):
search = googlesearch.search_google(word, limit, start)
search.process_profiles()
people = search.get_profiles()
db=stash.stash_manager()
db.store_all(word,people,'name','google-profile')
print("Users from Google profiles:")
print("---------------------------")
for users in people:
@ -312,6 +318,9 @@ def start(argv):
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db=stash.stash_manager()
db.store_all(word,all_hosts,'host','hunter')
db.store_all(word,all_emails,'email','hunter')
elif engineitem == "censys":
print("[-] Searching in Censys:")
@ -337,6 +346,7 @@ def start(argv):
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word,all_ip,'ip','cymon')
elif engineitem == "trello":
print("[-] Searching in Trello:")
@ -346,6 +356,9 @@ def start(argv):
search.process()
all_emails = search.get_emails()
all_hosts = search.get_urls()
db=stash.stash_manager()
db.store_all(word,all_hosts,'host','trello')
db.store_all(word,all_emails,'email','trello')
for x in all_hosts:
print (x)
sys.exit()
@ -362,7 +375,7 @@ def start(argv):
hosts = search.get_hostnames()
all_emails.extend(emails)
db=stash.stash_manager()
db.store_all(word,all_hosts,'email','google')
db.store_all(word,all_emails,'email','google')
all_hosts.extend(hosts)
db=stash.stash_manager()
db.store_all(word,all_hosts,'host','google')
@ -377,7 +390,7 @@ def start(argv):
db.store_all(word,all_hosts,'host','PGP')
all_emails.extend(emails)
db=stash.stash_manager()
db.store_all(word,all_hosts,'email','PGP')
db.store_all(word,all_emails,'email','PGP')
print("[-] Searching in Netcraft server..")
search = netcraft.search_netcraft(word)
@ -426,6 +439,7 @@ def start(argv):
all_emails.extend(emails)
#Clean up email list, sort and uniq
all_emails=sorted(set(all_emails))
db.store_all(word,all_emails,'email','bing')
print("[-] Searching in Hunter:")
from discovery import huntersearch
@ -439,22 +453,16 @@ def start(argv):
db.store_all(word, all_hosts, 'host', 'hunter')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word,all_emails,'email','hunter')
print ("[-] Searching in Google Certificate transparency report..")
search = googlecertificates.search_googlecertificates(word, limit, start)
search.process()
domains = search.get_domains()
all_hosts.extend(domains)
print("[-] Searching in Cymon:")
from discovery import cymon
search = cymon.search_cymon(word)
search.process()
all_emails = []
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word,all_ip,'ip','cymon')
db.store_all(word, all_hosts, 'host', 'google-certificates')
print("[-] Searching in Censys:")
from discovery import censys
search = censys.search_censys(word)
@ -519,7 +527,10 @@ def start(argv):
pass
else:
host_ip.append(ip.lower())
db=stash.stash_manager()
db.store_all(word,host_ip,'ip','DNS-resolver')
#DNS Brute force####################################################
dnsres = []
if dnsbrute == True:
@ -669,6 +680,31 @@ def start(argv):
#Reporting#######################################################
if filename != "":
try:
#NEW REPORT BEGINS
print("NEW REPORTING BEGINS:")
db = stash.stash_manager()
scanboarddata = db.getscanboarddata()
latestscandomain = db.getlatestscandomain(word)
from lib import statichtmlgenerator
generator = statichtmlgenerator.htmlgenerator(word)
HTMLcode = generator.generatedashboardcode(scanboarddata)
HTMLcode += generator.generatescandetailsdomain(word, latestscandomain)
from lib import reportgraph
import datetime
graph = reportgraph.graphgenerator(word)
HTMLcode += graph.drawlatestscangraph(word, latestscandomain)
#HTMLcode += graph.drawscattergraph(word, latestscandata)
HTMLcode += '<p><span style="color: #000000;">Report generated on '+ str(datetime.datetime.now())+'</span></p>'
HTMLcode +='''
</body>
</html>
'''
Html_file= open("report.html","w")
Html_file.write(HTMLcode)
Html_file.close()
print("NEW REPORTING FINISHED:")
#NEW REPORT FINISHED
print("[+] Saving files...")
html = htmlExport.htmlExport(
all_emails,