From ea71c71b39a40a35016c6bef7b64fa113d388111 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Sat, 16 Mar 2019 23:47:55 -0400 Subject: [PATCH] Made misc. change to dnsdumpster and fixed netcraft. --- discovery/dnsdumpster.py | 2 +- discovery/netcraft.py | 95 ++++++++++++++++++++-------------------- 2 files changed, 48 insertions(+), 49 deletions(-) diff --git a/discovery/dnsdumpster.py b/discovery/dnsdumpster.py index e6fe0886..61d0a616 100644 --- a/discovery/dnsdumpster.py +++ b/discovery/dnsdumpster.py @@ -27,7 +27,7 @@ def do_search(self): break csrftoken += ch data = { - 'Cookie': f'csfrtoken={csrftoken}', 'csrfmiddlewaretoken': {csrftoken}, 'targetip': self.word} + 'Cookie': f'csfrtoken={csrftoken}', 'csrfmiddlewaretoken': csrftoken, 'targetip': self.word} headers['Referer'] = url post_req = session.post(url, headers=headers, data=data) self.results = post_req.text diff --git a/discovery/netcraft.py b/discovery/netcraft.py index 38b18859..036336b7 100644 --- a/discovery/netcraft.py +++ b/discovery/netcraft.py @@ -1,73 +1,72 @@ -#from lib.core import * +from lib.core import * from parsers import myparser import requests import hashlib -import urllib +import urllib.parse as urllib import re class SearchNetcraft: + # this module was inspired by sublist3r's netcraft module def __init__(self, word): self.word = word.replace(' ', '%20') - self.results = "" self.totalresults = "" self.server = 'netcraft.com' - self.quantity = '100' - self.counter = 0 + self.base_url = 'https://searchdns.netcraft.com/?restriction=site+ends+with&host={domain}' + self.session = requests.session() + self.headers = { + 'User-Agent': Core.get_user_agent() + } + self.timeout = 25 + self.domain = f"https://searchdns.netcraft.com/?restriction=site+ends+with&host={self.word}" - def do_search(self): - # Module inspired by sublist3r - session = requests.session() + def request(self, url, cookies=None): + cookies = cookies or {} try: - headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.8', - 'Accept-Encoding': 'gzip', - } - base_url = 'https://searchdns.netcraft.com/?restriction=site+ends+with&host=example.com' - cookies = {} - r = session.get(base_url, headers=headers, cookies=cookies) - cookie = str(r.cookies).split(" ")[1] - cookies_list = cookie[0:cookie.find(';')].split("=") - cookies[cookies_list[0]] = cookies_list[1] - cookies['netcraft_js_verification_response'] = hashlib.sha1(urllib.unquote(cookies_list[1]).encode('utf-8')).hexdigest() - #{'netcraft_js_verification_challenge': 'djF8TW9tb3crWWkyaWZWb1hySDU4VVJvWnByZ0NXbHcrQzhVTXVKc2UyeGpmeTlXdXpxWlA1TEdW%0AZjJHMndxVnE2SE5VeExoY1JubmdFOQpmQ2VwZG5EKzBnPT0KfDE1NTI3MDYwNjA%3D%0A%7C5a13284199e4dc7c260a16ae81ae2a717f4a274c', 'netcraft_js_verification_response': '9b87eaabe14a56f873e2212a9ab1cd846a5d1592'} - print(cookies) - #print(cookies); import sys;sys.exit(-2) - search_url = 'https://searchdns.netcraft.com/?restriction=site+ends+with&host=yale.edu' - #r = session.get(search_url, cookies=cookies, headers=headers, timeout=25) - while True: - r = session.get(search_url, cookies=cookies, headers=headers, timeout=25) - self.totalresults += r.text - if 'Next page' not in r.text: - break - search_url = self.get_next(r) - print(r.text) + resp = self.session.get(url, headers=self.headers, timeout=self.timeout, cookies=cookies) except Exception as e: - print('An exception has occured in netcraft: ' +str(e)) + print(e) + resp = None + return resp def get_next(self, resp): link_regx = re.compile('Next page') link = link_regx.findall(resp) - link = re.sub('host=.*?%s' % self.word, 'host=%s' % self.word, link[0]) - url = 'http://searchdns.netcraft.com' + link - url = url.replace(' ', '%20') + link = re.sub(f'host=.*?{self.word}', f'host={self.domain}', link[0]) + url = f'http://searchdns.netcraft.com{link}' return url + def create_cookies(self, cookie): + cookies = dict() + cookies_list = cookie[0:cookie.find(';')].split("=") + cookies[cookies_list[0]] = cookies_list[1] + # get js verification response + cookies['netcraft_js_verification_response'] = hashlib.sha1( + urllib.unquote(cookies_list[1]).encode('utf-8')).hexdigest() + return cookies + + def get_cookies(self, headers): + if 'set-cookie' in headers: + cookies = self.create_cookies(headers['set-cookie']) + else: + cookies = {} + return cookies + + def do_search(self): + start_url = self.base_url + resp = self.request(start_url) + cookies = self.get_cookies(resp.headers) + url = self.base_url.format(domain="yale.edu") + while True: + resp = self.request(url, cookies).text + self.totalresults += resp + if 'Next page' not in resp or resp is None: + break + url = self.get_next(resp) + def get_hostnames(self): rawres = myparser.Parser(self.totalresults, self.word) return rawres.hostnames() def process(self): - self.do_search() - print('\tSearching results.') - -def main(): - n = SearchNetcraft(word="yale.edu") - n.do_search() - y = n.get_hostnames() - for x in y: - print(x) - -main() + self.do_search() \ No newline at end of file