Merge branch 'master' into dev

2025-02-25 06:53:05 +08:00 · 2019-12-29 22:39:39 -05:00 · 2019-12-29 22:39:39 -05:00 · 2599a905b8
commit 2599a905b8
parent c33b409f8a 96072583fa
8 changed files with 46 additions and 73 deletions
--- a/.github/workflows/theHarvester.yml
+++ b/.github/workflows/theHarvester.yml
@ -19,7 +19,7 @@ jobs:
    steps:
    - uses: actions/checkout@v1
    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v1.1.1
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
@ -79,9 +79,9 @@ jobs:
      run: |
        python theHarvester.py -d apple.com -b netcraft

-#    - name: Run theHarvester module Otx
-#      run: |
-#        python theHarvester.py -d apple.com -b otx
+    - name: Run theHarvester module Otx
+      run: |
+        python theHarvester.py -d apple.com -b otx

    - name: Run theHarvester module threatcrowd
      run: |
--- a/6
+++ b/6
@ -5,8 +5,8 @@ name = "pypi"

 [packages]
 aiodns = "==2.0.0"
-aiohttp = "==3.5.4"
-beautifulsoup4 = "==4.8.1"
+aiohttp = "==3.6.2"
+beautifulsoup4 = "==4.8.2"
 dnspython = "==1.16.0"
 grequests = "==0.4.0"
 netaddr = "==0.7.19"
@ -14,7 +14,7 @@ plotly = "==4.4.1"
 pyyaml = "==5.2"
 requests = "==2.22.0"
 retrying = "==1.3.3"
-shodan = "==1.21.0"
+shodan = "==1.21.1"
 texttable = "==1.6.2"
 lxml = "==4.4.2"

--- a/Pipfile.lock
+++ b/Pipfile.lock
@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "e7b478160ebf3a0fa9593cd3e7481ce0c2befdf21759f18e812a8be5408a7200"
+            "sha256": "8faa576933b0672fbd2a872fdfe8477f8a4a92f1b55f98610bddecda702f9022"
        },
        "pipfile-spec": 6,
        "requires": {},
@ -24,31 +24,21 @@
        },
        "aiohttp": {
            "hashes": [
-                "sha256:00d198585474299c9c3b4f1d5de1a576cc230d562abc5e4a0e81d71a20a6ca55",
-                "sha256:0155af66de8c21b8dba4992aaeeabf55503caefae00067a3b1139f86d0ec50ed",
-                "sha256:09654a9eca62d1bd6d64aa44db2498f60a5c1e0ac4750953fdd79d5c88955e10",
-                "sha256:199f1d106e2b44b6dacdf6f9245493c7d716b01d0b7fbe1959318ba4dc64d1f5",
-                "sha256:296f30dedc9f4b9e7a301e5cc963012264112d78a1d3094cd83ef148fdf33ca1",
-                "sha256:368ed312550bd663ce84dc4b032a962fcb3c7cae099dbbd48663afc305e3b939",
-                "sha256:40d7ea570b88db017c51392349cf99b7aefaaddd19d2c78368aeb0bddde9d390",
-                "sha256:629102a193162e37102c50713e2e31dc9a2fe7ac5e481da83e5bb3c0cee700aa",
-                "sha256:6d5ec9b8948c3d957e75ea14d41e9330e1ac3fed24ec53766c780f82805140dc",
-                "sha256:87331d1d6810214085a50749160196391a712a13336cd02ce1c3ea3d05bcf8d5",
-                "sha256:9a02a04bbe581c8605ac423ba3a74999ec9d8bce7ae37977a3d38680f5780b6d",
-                "sha256:9c4c83f4fa1938377da32bc2d59379025ceeee8e24b89f72fcbccd8ca22dc9bf",
-                "sha256:9cddaff94c0135ee627213ac6ca6d05724bfe6e7a356e5e09ec57bd3249510f6",
-                "sha256:a25237abf327530d9561ef751eef9511ab56fd9431023ca6f4803f1994104d72",
-                "sha256:a5cbd7157b0e383738b8e29d6e556fde8726823dae0e348952a61742b21aeb12",
-                "sha256:a97a516e02b726e089cffcde2eea0d3258450389bbac48cbe89e0f0b6e7b0366",
-                "sha256:acc89b29b5f4e2332d65cd1b7d10c609a75b88ef8925d487a611ca788432dfa4",
-                "sha256:b05bd85cc99b06740aad3629c2585bda7b83bd86e080b44ba47faf905fdf1300",
-                "sha256:c2bec436a2b5dafe5eaeb297c03711074d46b6eb236d002c13c42f25c4a8ce9d",
-                "sha256:cc619d974c8c11fe84527e4b5e1c07238799a8c29ea1c1285149170524ba9303",
-                "sha256:d4392defd4648badaa42b3e101080ae3313e8f4787cb517efd3f5b8157eaefd6",
-                "sha256:e1c3c582ee11af7f63a34a46f0448fca58e59889396ffdae1f482085061a2889"
+                "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e",
+                "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326",
+                "sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a",
+                "sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654",
+                "sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a",
+                "sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4",
+                "sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17",
+                "sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec",
+                "sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd",
+                "sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48",
+                "sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59",
+                "sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965"
            ],
            "index": "pypi",
-            "version": "==3.5.4"
+            "version": "==3.6.2"
        },
        "async-timeout": {
            "hashes": [
@ -66,12 +56,12 @@
        },
        "beautifulsoup4": {
            "hashes": [
-                "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
-                "sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931",
-                "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
+                "sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a",
+                "sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887",
+                "sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae"
            ],
            "index": "pypi",
-            "version": "==4.8.1"
+            "version": "==4.8.2"
        },
        "certifi": {
            "hashes": [
@ -366,10 +356,10 @@
        },
        "shodan": {
            "hashes": [
-                "sha256:eab999bca9d3b30e6fc549e609194ff2d6fac3caea252414e1d8d735efab8342"
+                "sha256:834dfd084fed290b2b445545b0d1cac7822f3c0ed6ba09707efb1716bb485ede"
            ],
            "index": "pypi",
-            "version": "==1.21.0"
+            "version": "==1.21.1"
        },
        "six": {
            "hashes": [
@ -452,14 +442,6 @@
            "index": "pypi",
            "version": "==3.7.9"
        },
-        "importlib-metadata": {
-            "hashes": [
-                "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45",
-                "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==1.3.0"
-        },
        "mccabe": {
            "hashes": [
                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
@ -539,10 +521,10 @@
        },
        "pyparsing": {
            "hashes": [
-                "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f",
-                "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a"
+                "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
+                "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
            ],
-            "version": "==2.4.5"
+            "version": "==2.4.6"
        },
        "pytest": {
            "hashes": [
@ -598,13 +580,6 @@
                "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
            ],
            "version": "==0.1.7"
-        },
-        "zipp": {
-            "hashes": [
-                "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
-                "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"
-            ],
-            "version": "==0.6.0"
        }
    }
 }
--- a/requirements.txt
+++ b/requirements.txt
@ -1,7 +1,7 @@
 aiodns==2.0.0
-aiohttp==3.5.4
+aiohttp==3.6.2
 aiosqlite==0.11.0
-beautifulsoup4==4.8.1
+beautifulsoup4==4.8.2
 dnspython==1.16.0
 flake8==3.7.9
 gevent==1.4.0 
--- a/theHarvester/main.py
+++ b/theHarvester/main.py
@ -25,20 +25,14 @@ async def start():
    parser.add_argument('-d', '--domain', help='company name or domain to search', required=True)
    parser.add_argument('-l', '--limit', help='limit the number of search results, default=500', default=500, type=int)
    parser.add_argument('-S', '--start', help='start with result number X, default=0', default=0, type=int)
-    parser.add_argument('-g', '--google-dork', help='use Google Dorks for Google search', default=False,
-                        action='store_true')
-    parser.add_argument('-p', '--port-scan', help='scan the detected hosts and check for Takeovers (21,22,80,443,8080)',
-                        default=False, action='store_true')
-    parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=False,
-                        action='store_true')
-    parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts',
-                        action='store_const', const='basic', default=False)
+    parser.add_argument('-g', '--google-dork', help='use Google Dorks for Google search', default=False, action='store_true')
+    parser.add_argument('-p', '--port-scan', help='scan the detected hosts and check for Takeovers (21,22,80,443,8080)', default=False, action='store_true')
+    parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=False, action='store_true')
+    parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts', action='store_const', const='basic', default=False)
    parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup')
    parser.add_argument('-t', '--dns-tld', help='perform a DNS TLD expansion discovery, default False', default=False)
-    parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default False', default=False,
-                        action='store_true')
-    parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False,
-                        action='store_true')
+    parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default False', default=False, action='store_true')
+    parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true')
    parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str)
    parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, certspotter, crtsh, dnsdumpster,
                        dogpile, duckduckgo, goofile, github-code, google,
--- a/theHarvester/discovery/googlesearch.py
+++ b/theHarvester/discovery/googlesearch.py
@ -155,4 +155,4 @@ def send_dorks(self):  # Helper function to minimize code reusability.
                time.sleep(getDelay())
                self.totalresults += self.results
            except Exception as e:
-                print(f'\tException Occurred {e}')
+                print(f'\tException Occurred {e}')
--- a/theHarvester/discovery/shodansearch.py
+++ b/theHarvester/discovery/shodansearch.py
@ -40,4 +40,4 @@ def search_ip(self, ip):
        except Exception as e:
            print(f'Error occurred in the Shodan IP search module: {e}')
        finally:
-            return self.hostdatarow
+            return self.hostdatarow
--- a/theHarvester/lib/core.py
+++ b/theHarvester/lib/core.py
@ -1,7 +1,7 @@
 # coding=utf-8

 import random
-from typing import Set, Union, Any, Tuple
+from typing import Set, Union, Any
 import yaml
 import asyncio
 import aiohttp
@ -380,6 +380,9 @@ async def fetch(session, url, params='', json=False) -> Union[str, dict, list]:
        # This fetch method solely focuses on get requests
        # TODO determine if method for post requests is necessary
        try:
+            # Wrap in try except due to 0x89 png/jpg files
+            # This fetch method solely focuses on get requests
+            # TODO determine if method for post requests is necessary
            if params != '':
                async with session.get(url, params=params) as response:
                    await asyncio.sleep(2)
@ -393,8 +396,9 @@ async def fetch(session, url, params='', json=False) -> Union[str, dict, list]:

    @staticmethod
    async def fetch_all(urls, headers='', params='') -> list:
-        timeout = aiohttp.ClientTimeout(total=30)
        # By default timeout is 5 minutes, 30 seconds should suffice
+        timeout = aiohttp.ClientTimeout(total=30)
+
        if len(headers) == 0:
            headers = {'User-Agent': Core.get_user_agent()}
        if len(params) == 0: