laramies/theHarvester
1
1
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2024-11-10 17:13:07 +08:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #108 from jzold/master

Browse source 
censys pagination fix; only unique items are added to SQLite from the censys search results
This commit is contained in:
Christian Martorella 2018-12-01 10:35:19 +01:00 committed by GitHub
commit 8746c2ed25
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 22 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -4,3 +4,4 @@ tests/myparser.py
stash.sqlite
*.sqlite
.vscode
venv

24
theHarvester.py
View file

@ -310,11 +310,29 @@ def start(argv):
from discovery import censys
#import locally or won't work
search = censys.search_censys(word)
search.process(5)
search.process()
totalnumberofpages = search.get_totalnumberofpages()
all_emails = []
all_ip = search.get_ipaddresses()
all_hosts = search.get_hostnames()
pagecounter = 1
while pagecounter < totalnumberofpages and pagecounter < 5: #pagecounter < 5: search 4 pages = 100 results
pagecounter += 1
search.process(pagecounter)
moreips = search.get_ipaddresses()
for moreipitem in moreips:
all_ip.append(moreipitem)
morehostnames = search.get_hostnames()
for morehostnameitem in morehostnames:
all_hosts.append(morehostnameitem)
print('')
ipset = set(all_ip)
hostset = set(all_hosts)
for ipitem in ipset:
db.store(word,ipitem,'ipaddress','censys')
for hostitem in hostset:
db.store(word,hostitem,'hostname','censys')
elif engineitem == "all":
print(("Full harvest on " + word))
all_emails = []