Merge branch 'master' into master

.gitignore

@@ -4,8 +4,8 @@
 .html
 .vscode
 .xml
+debug_results.txt
+discovery/constants.py
 stash.sqlite
 tests/myparser.py
 venv
-debug_results.txt
-discovery/constants.py

README.md

@@ -6,7 +6,7 @@
 * | |_| | | |  __/ / __  / (_| | |   \ V /  __/\__ \ ||  __/ |    *
 *  \__|_| |_|\___| \/ /_/ \__,_|_|    \_/ \___||___/\__\___|_|    *
 *                                                                 *
-* theHarvester 3.0.6 v111                                          *
+* theHarvester 3.0.6 v137                                         *
 * Coded by Christian Martorella                                   *
 * Edge-Security Research                                          *
 * cmartorella@edge-security.com                                   *
@@ -15,10 +15,10 @@
 
 What is this?
 -------------
-theHarvester is a very simple, yet effective tool designed to be used in the early stages<br>
-of a penetration test. Use it for open source intelligence gathering and helping to determine<br>
-a company's external threat landscape on the internet. It gathers names, emails, subdomains,<br>
-and virtual hosts using multiple public data sources that include:
+theHarvester is a very simple, yet effective tool designed to be used in the early<br>
+stages of a penetration test. Use it for open source intelligence gathering and helping<br>
+to determine a company's external threat landscape on the internet. The tool gathers<br>
+emails, names, subdomains, IPs, and URLs using multiple public data sources that include:
 
 Passive:
 --------
@@ -36,6 +36,8 @@ Passive:
 
 * dogpile: Dogpile search engine - www.dogpile.com
 
+* duckduckgo:
+
 * google: Google search engine (Optional Google dorking.) - www.google.com
 
 * googleCSE: Google custom search engine
@@ -89,8 +91,8 @@ Add your keys to discovery/constants.py
 
 Dependencies:
 -------------
-* Do ```pip3 install -r requirements.txt``` 
-* Pip3 requires Python3 
+* Python 3.6
+* pip3 install -r requirements.txt
 
 Changelog in 3.0:
 -----------------

lib/core.py

@@ -13,7 +13,7 @@ def banner():
         print("* | |_| | | |  __/ / __  / (_| | |   \ V /  __/\__ \ ||  __/ |    *")
         print("*  \__|_| |_|\___| \/ /_/ \__,_|_|    \_/ \___||___/\__\___|_|    *")
         print("*                                                                 *")
-        print("* theHarvester 3.0.6 v111                                         *")
+        print("* theHarvester 3.0.6 v137                                         *")
         print("* Coded by Christian Martorella                                   *")
         print("* Edge-Security Research                                          *")
         print("* cmartorella@edge-security.com                                   *")
@@ -28,10 +28,11 @@ def usage():
 
         print("Usage: theHarvester.py <options> \n")
         print("   -d: company name or domain to search")
-        print("""   -b: source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile, google,
-                   googleCSE, google-certificates, google-profiles,
-                   hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd,
-                   trello, twitter, vhost, virustotal, yahoo, all""")
+        print("""   -b: source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile,
+               google, googleCSE, google-certificates, google-profiles,
+               hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd,
+               trello, twitter, vhost, virustotal, yahoo, all""")
+        print("   -l: limit the number of search results")
         print("   -g: use Google Dorking instead of normal Google search")
         print("   -s: start with result number X (default: 0)")
         print("   -v: verify host name via DNS resolution and search for virtual hosts")
@@ -41,8 +42,6 @@ def usage():
         print("   -t: perform a DNS TLD expansion discovery")
         print("   -e: specify DNS server")
         print("   -p: port scan the detected hosts and check for Takeovers (21,22,80,443,8080)")
-        print("   -l: limit the number of results (Bing goes from 50 to 50 results,")
-        print("       Google 100 to 100, and PGP doesn't use this option)")
         print("   -h: use Shodan to query discovered hosts")
         print("\nExamples:")
         print(("       " + comm + " -d acme.com -l 500 -b google -f myresults.html"))

theHarvester.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import datetime
 import getopt
 import re
 import stash