mirror of
https://github.com/laramies/theHarvester.git
synced 2024-09-21 07:46:32 +08:00
improvement: bump Spyse API version & add pagination support
This commit is contained in:
parent
ddacfcfb71
commit
bb8894ebde
|
@ -421,7 +421,7 @@ async def store(search_engine: Any, source: str, process_param: Any = None, stor
|
||||||
elif engineitem == 'spyse':
|
elif engineitem == 'spyse':
|
||||||
from theHarvester.discovery import spyse
|
from theHarvester.discovery import spyse
|
||||||
try:
|
try:
|
||||||
spyse_search = spyse.SearchSpyse(word)
|
spyse_search = spyse.SearchSpyse(word, limit)
|
||||||
stor_lst.append(store(spyse_search, engineitem, store_host=True, store_ip=True))
|
stor_lst.append(store(spyse_search, engineitem, store_host=True, store_ip=True))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(e)
|
print(e)
|
||||||
|
|
|
@ -1,10 +1,11 @@
|
||||||
from theHarvester.discovery.constants import *
|
from theHarvester.discovery.constants import *
|
||||||
from theHarvester.lib.core import *
|
from theHarvester.lib.core import *
|
||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
class SearchSpyse:
|
class SearchSpyse:
|
||||||
|
|
||||||
def __init__(self, word):
|
def __init__(self, word, limit):
|
||||||
self.ips = set()
|
self.ips = set()
|
||||||
self.word = word
|
self.word = word
|
||||||
self.key = Core.spyse_key()
|
self.key = Core.spyse_key()
|
||||||
|
@ -13,19 +14,52 @@ def __init__(self, word):
|
||||||
self.results = ''
|
self.results = ''
|
||||||
self.hosts = set()
|
self.hosts = set()
|
||||||
self.proxy = False
|
self.proxy = False
|
||||||
|
self.limit = limit
|
||||||
|
|
||||||
async def do_search(self):
|
async def do_search(self):
|
||||||
try:
|
# Spyse allows to get up to 100 results per one request
|
||||||
headers = {
|
max_limit = 100
|
||||||
'accept': 'application/json',
|
# Spyse "search" methods allows to fetch up to 10 000 first results
|
||||||
'Authorization': f'Bearer {self.key}',
|
max_offset = 9900
|
||||||
}
|
offset = 0
|
||||||
base_url = f'https://api.spyse.com/v3/data/domain/subdomain?limit=100&domain={self.word}'
|
|
||||||
results = await AsyncFetcher.fetch_all([base_url], json=True, proxy=self.proxy, headers=headers)
|
while True:
|
||||||
results = results[0]
|
try:
|
||||||
self.hosts = {domain['name'] for domain in results['data']['items']}
|
headers = {
|
||||||
except Exception as e:
|
'accept': 'application/json',
|
||||||
print(f'An exception has occurred: {e}')
|
'Authorization': f'Bearer {self.key}',
|
||||||
|
}
|
||||||
|
|
||||||
|
base_url = 'https://api.spyse.com/v4/data/domain/search'
|
||||||
|
|
||||||
|
query = {
|
||||||
|
'search_params': [
|
||||||
|
{
|
||||||
|
'name': {
|
||||||
|
'operator': 'ends',
|
||||||
|
'value': '.' + self.word,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
'limit': max_limit if self.limit > max_limit else self.limit,
|
||||||
|
'offset': offset,
|
||||||
|
}
|
||||||
|
|
||||||
|
results = await AsyncFetcher.post_fetch(base_url, json=True, headers=headers, data=json.dumps(query))
|
||||||
|
|
||||||
|
if len(results.get('data').get('items')) > 0:
|
||||||
|
for domain in results['data']['items']:
|
||||||
|
self.hosts.add(domain['name'])
|
||||||
|
|
||||||
|
else:
|
||||||
|
break
|
||||||
|
|
||||||
|
offset += max_limit
|
||||||
|
if offset > max_offset or offset + max_limit > self.limit:
|
||||||
|
break
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f'An exception has occurred: {e}')
|
||||||
|
|
||||||
async def get_hostnames(self):
|
async def get_hostnames(self):
|
||||||
return self.hosts
|
return self.hosts
|
||||||
|
|
Loading…
Reference in a new issue