update deps and pep8 fixes and yahoosearch to grequests

requirements.txt

@@ -1,6 +1,7 @@
 beautifulsoup4==4.8.0
 censys==0.0.8
 chart-studio==1.0.0
+grequests==0.4.0
 plotly==4.1.0
 pytest==5.0.1
 PyYaml==5.1.2

theHarvester/__main__.py

@@ -338,7 +338,7 @@ def start():
 
                 elif engineitem == 'twitter':
                     print('\033[94m[*] Searching Twitter usernames using Google. \033[0m')
-                    search = twittersearch.search_twitter(word, limit)
+                    search = twittersearch.SearchTwitter(word, limit)
                     search.process()
                     people = search.get_people()
                     db = stash.stash_manager()
@@ -363,7 +363,7 @@ def start():
 
                 elif engineitem == 'yahoo':
                     print('\033[94m[*] Searching Yahoo. \033[0m')
-                    search = yahoosearch.search_yahoo(word, limit)
+                    search = yahoosearch.SearchYahoo(word, limit)
                     search.process()
                     hosts = search.get_hostnames()
                     emails = search.get_emails()
@@ -588,7 +588,7 @@ def start():
 
                     try:
                         print('\033[94m[*] Searching Twitter. \033[0m')
-                        search = twittersearch.search_twitter(word, limit)
+                        search = twittersearch.SearchTwitter(word, limit)
                         search.process()
                         people = search.get_people()
                         db = stash.stash_manager()
@@ -625,7 +625,7 @@ def start():
 
                     try:
                         print('\033[94m[*] Searching Yahoo. \033[0m')
-                        search = yahoosearch.search_yahoo(word, limit)
+                        search = yahoosearch.SearchYahoo(word, limit)
                         search.process()
                         hosts = search.get_hostnames()
                         emails = search.get_emails()

theHarvester/discovery/__init__.py

@@ -1,3 +1,5 @@
+import gevent.monkey
+gevent.monkey.patch_all()
 __all__ = ['baidusearch',
            'bingsearch',
            'censys',

theHarvester/discovery/twittersearch.py

@@ -6,14 +6,13 @@
 import re
 
 
-class search_twitter:
+class SearchTwitter:
 
     def __init__(self, word, limit):
         self.word = word.replace(' ', '%20')
         self.results = ""
         self.totalresults = ""
         self.server = 'www.google.com'
-        self.hostname = 'www.google.com'
         self.quantity = '100'
         self.limit = int(limit)
         self.counter = 0

theHarvester/discovery/wfuzz_search.py

@@ -18,7 +18,7 @@ def do_search(self):
                 print(r)
                 self.results += r
         except Exception as e:
-                print(e)
+            print(e)
         self.totalresults += self.results
 
     def get_results(self):

theHarvester/discovery/yahoosearch.py

@@ -1,35 +1,30 @@
-from theHarvester.discovery.constants import *
+import grequests
 from theHarvester.lib.core import *
 from theHarvester.parsers import myparser
-import requests
-import time
 
 
-class search_yahoo:
+class SearchYahoo:
 
     def __init__(self, word, limit):
         self.word = word
         self.total_results = ""
         self.server = 'search.yahoo.com'
-        self.hostname = 'search.yahoo.com'
         self.limit = limit
-        self.counter = 0
 
     def do_search(self):
-        url = 'http://' + self.server + '/search?p=\"%40' + self.word + '\"&b=' + str(self.counter) + '&pz=10'
+        base_url = f'https://{self.server}/search?p=%40{self.word}&b=xx&pz=10'
         headers = {
-            'Host': self.hostname,
+            'Host': self.server,
             'User-agent': Core.get_user_agent()
         }
-        h = requests.get(url=url, headers=headers)
-        self.total_results += h.text
+        urls = [base_url.replace("xx", str(num)) for num in range(0, self.limit, 10) if num <= self.limit]
+        request = (grequests.get(url, headers=headers) for url in urls)
+        response = grequests.imap(request, size=5)
+        for entry in response:
+            self.total_results += entry.content.decode('UTF-8')
 
     def process(self):
-        while self.counter <= self.limit and self.counter <= 1000:
-            self.do_search()
-            time.sleep(getDelay())
-            print(f'\tSearching {self.counter} results.')
-            self.counter += 10
+        self.do_search()
 
     def get_emails(self):
         rawres = myparser.Parser(self.total_results, self.word)

theHarvester/discovery/yandexsearch.py

@@ -6,7 +6,7 @@
 import time
 
 
-class search_yandex:
+class SearchYandex:
 
     def __init__(self, word, limit, start):
         self.word = word

theHarvester/lib/core.py

@@ -524,7 +524,7 @@ def get_user_agent():
     #
     #     try:
     #         print('[*] Searching Twitter.')
-    #         search = twittersearch.search_twitter(word, limit)
+    #         search = twittersearch.SearchTwitter(word, limit)
     #         search.process()
     #         people = search.get_people()
     #         db = stash.stash_manager()
@@ -560,7 +560,7 @@ def get_user_agent():
     #     db.store_all(word, all_hosts, 'host', 'virustotal')
     #
     #     print('[*] Searching Yahoo.')
-    #     search = yahoosearch.search_yahoo(word, limit)
+    #     search = yahoosearch.SearchYahoo(word, limit)
     #     search.process()
     #     hosts = search.get_hostnames()
     #     emails = search.get_emails()