mirror of
https://github.com/laramies/theHarvester.git
synced 2024-09-22 00:06:30 +08:00
Merge pull request #27 from NotoriousRebel/dev
Added more asynchronization and added suip module
This commit is contained in:
J.Townsend
GitHub
commit
ad489865b4
32
.github/ISSUE_TEMPLATE/issue-template.md
vendored
Normal file
32
.github/ISSUE_TEMPLATE/issue-template.md
vendored
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
---
|
||||
name: Issue Template
|
||||
about: A template for new issues.
|
||||
title: "[Bug|Feature Request|Other] Short Description of Issue"
|
||||
labels: ''
|
||||
|
||||
---
|
||||
|
||||
**Feature Request or Bug or Other**
|
||||
Feature Request | Bug | Other
|
||||
|
||||
**Describe the feature request or bug or other**
|
||||
A clear and concise description of what the bug, feature request,
|
||||
or other request is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
1. Run tool like this: '...'
|
||||
2. See error
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots**
|
||||
If possible please add screenshots to help explain your problem.
|
||||
|
||||
**System Information (System that tool is running on):**
|
||||
- OS: [e.g. Windows10]
|
||||
- Version [e.g. 2.7]
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
10
Pipfile
10
Pipfile
|
|
@ -5,21 +5,21 @@ name = "pypi"
|
|||
|
||||
[packages]
|
||||
aiodns = "==2.0.0"
|
||||
beautifulsoup4 = "==4.8.1"
|
||||
beautifulsoup4 = "==4.8.0"
|
||||
dnspython = "==1.16.0"
|
||||
flake8 = "==3.7.8"
|
||||
grequests = "==0.4.0"
|
||||
mypy = "==0.730"
|
||||
netaddr = "==0.7.19"
|
||||
plotly = "==4.1.1"
|
||||
pytest = "==5.2.0"
|
||||
pyyaml = "==5.1.2"
|
||||
requests = "==2.22.0"
|
||||
retrying = "==1.3.3"
|
||||
shodan = "==1.19.0"
|
||||
texttable = "==1.6.2"
|
||||
retrying = "==1.3.3"
|
||||
|
||||
[dev-packages]
|
||||
flake8 = "==3.7.8"
|
||||
mypy = "==0.730"
|
||||
pytest = "==5.2.1"
|
||||
|
||||
[requires]
|
||||
python_version = "3.7"
|
||||
300
Pipfile.lock
generated
300
Pipfile.lock
generated
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "73ae281f2aa10357b5b1e7360f654b31ba3a3bf4a32a4b2335f3f35258e18a30"
|
||||
"sha256": "af937a3b86c64e37abd6f44f3ff30f08f9c2660a270206637f8cc1cb87416fbb"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
|
|
@ -24,14 +24,28 @@
|
|||
"index": "pypi",
|
||||
"version": "==2.0.0"
|
||||
},
|
||||
"atomicwrites": {
|
||||
"hashes": [
|
||||
"sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
|
||||
"sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
|
||||
],
|
||||
"version": "==1.3.0"
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2",
|
||||
"sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396"
|
||||
],
|
||||
"version": "==19.2.0"
|
||||
},
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
|
||||
"sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931",
|
||||
"sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
|
||||
"sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612",
|
||||
"sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b",
|
||||
"sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.8.1"
|
||||
"version": "==4.8.0"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
|
|
@ -109,6 +123,21 @@
|
|||
"index": "pypi",
|
||||
"version": "==1.16.0"
|
||||
},
|
||||
"entrypoints": {
|
||||
"hashes": [
|
||||
"sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19",
|
||||
"sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"
|
||||
],
|
||||
"version": "==0.3"
|
||||
},
|
||||
"flake8": {
|
||||
"hashes": [
|
||||
"sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548",
|
||||
"sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.7.8"
|
||||
},
|
||||
"gevent": {
|
||||
"hashes": [
|
||||
"sha256:0774babec518a24d9a7231d4e689931f31b332c4517a771e532002614e270a64",
|
||||
|
|
@ -177,6 +206,51 @@
|
|||
],
|
||||
"version": "==2.8"
|
||||
},
|
||||
"importlib-metadata": {
|
||||
"hashes": [
|
||||
"sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26",
|
||||
"sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af"
|
||||
],
|
||||
"markers": "python_version < '3.8'",
|
||||
"version": "==0.23"
|
||||
},
|
||||
"mccabe": {
|
||||
"hashes": [
|
||||
"sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
|
||||
"sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
|
||||
],
|
||||
"version": "==0.6.1"
|
||||
},
|
||||
"more-itertools": {
|
||||
"hashes": [
|
||||
"sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832",
|
||||
"sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"
|
||||
],
|
||||
"version": "==7.2.0"
|
||||
},
|
||||
"mypy": {
|
||||
"hashes": [
|
||||
"sha256:1d98fd818ad3128a5408148c9e4a5edce6ed6b58cc314283e631dd5d9216527b",
|
||||
"sha256:22ee018e8fc212fe601aba65d3699689dd29a26410ef0d2cc1943de7bec7e3ac",
|
||||
"sha256:3a24f80776edc706ec8d05329e854d5b9e464cd332e25cde10c8da2da0a0db6c",
|
||||
"sha256:42a78944e80770f21609f504ca6c8173f7768043205b5ac51c9144e057dcf879",
|
||||
"sha256:4b2b20106973548975f0c0b1112eceb4d77ed0cafe0a231a1318f3b3a22fc795",
|
||||
"sha256:591a9625b4d285f3ba69f541c84c0ad9e7bffa7794da3fa0585ef13cf95cb021",
|
||||
"sha256:5b4b70da3d8bae73b908a90bb2c387b977e59d484d22c604a2131f6f4397c1a3",
|
||||
"sha256:84edda1ffeda0941b2ab38ecf49302326df79947fa33d98cdcfbf8ca9cf0bb23",
|
||||
"sha256:b2b83d29babd61b876ae375786960a5374bba0e4aba3c293328ca6ca5dc448dd",
|
||||
"sha256:cc4502f84c37223a1a5ab700649b5ab1b5e4d2bf2d426907161f20672a21930b",
|
||||
"sha256:e29e24dd6e7f39f200a5bb55dcaa645d38a397dd5a6674f6042ef02df5795046"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.730"
|
||||
},
|
||||
"mypy-extensions": {
|
||||
"hashes": [
|
||||
"sha256:a161e3b917053de87dbe469987e173e49fb454eca10ef28b48b384538cc11458"
|
||||
],
|
||||
"version": "==0.4.2"
|
||||
},
|
||||
"netaddr": {
|
||||
"hashes": [
|
||||
"sha256:38aeec7cdd035081d3a4c306394b19d677623bf76fa0913f6695127c7753aefd",
|
||||
|
|
@ -185,6 +259,13 @@
|
|||
"index": "pypi",
|
||||
"version": "==0.7.19"
|
||||
},
|
||||
"packaging": {
|
||||
"hashes": [
|
||||
"sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47",
|
||||
"sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"
|
||||
],
|
||||
"version": "==19.2"
|
||||
},
|
||||
"plotly": {
|
||||
"hashes": [
|
||||
"sha256:0f0cd50a8bad5acadfc95af367820da8ed3e1569d83ef13bb218cc4259dcf219",
|
||||
|
|
@ -193,6 +274,20 @@
|
|||
"index": "pypi",
|
||||
"version": "==4.1.1"
|
||||
},
|
||||
"pluggy": {
|
||||
"hashes": [
|
||||
"sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6",
|
||||
"sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34"
|
||||
],
|
||||
"version": "==0.13.0"
|
||||
},
|
||||
"py": {
|
||||
"hashes": [
|
||||
"sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
|
||||
"sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
|
||||
],
|
||||
"version": "==1.8.0"
|
||||
},
|
||||
"pycares": {
|
||||
"hashes": [
|
||||
"sha256:2ca080db265ea238dc45f997f94effb62b979a617569889e265c26a839ed6305",
|
||||
|
|
@ -211,12 +306,41 @@
|
|||
],
|
||||
"version": "==3.0.0"
|
||||
},
|
||||
"pycodestyle": {
|
||||
"hashes": [
|
||||
"sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56",
|
||||
"sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"
|
||||
],
|
||||
"version": "==2.5.0"
|
||||
},
|
||||
"pycparser": {
|
||||
"hashes": [
|
||||
"sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3"
|
||||
],
|
||||
"version": "==2.19"
|
||||
},
|
||||
"pyflakes": {
|
||||
"hashes": [
|
||||
"sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0",
|
||||
"sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"
|
||||
],
|
||||
"version": "==2.1.1"
|
||||
},
|
||||
"pyparsing": {
|
||||
"hashes": [
|
||||
"sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80",
|
||||
"sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4"
|
||||
],
|
||||
"version": "==2.4.2"
|
||||
},
|
||||
"pytest": {
|
||||
"hashes": [
|
||||
"sha256:13c1c9b22127a77fc684eee24791efafcef343335d855e3573791c68588fe1a5",
|
||||
"sha256:d8ba7be9466f55ef96ba203fc0f90d0cf212f2f927e69186e1353e30bc7f62e5"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==5.2.0"
|
||||
},
|
||||
"pyyaml": {
|
||||
"hashes": [
|
||||
"sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9",
|
||||
|
|
@ -279,153 +403,6 @@
|
|||
"index": "pypi",
|
||||
"version": "==1.6.2"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
|
||||
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
|
||||
],
|
||||
"version": "==1.25.6"
|
||||
},
|
||||
"xlsxwriter": {
|
||||
"hashes": [
|
||||
"sha256:48c04e6cec449a562455d7cf40305de1190adfc821e834e023425461f345dfc2",
|
||||
"sha256:7f41b1dbd6b38dd1ce243156cac54d1dae50a41f4a45871596ca67b0abcfb3e8"
|
||||
],
|
||||
"version": "==1.2.1"
|
||||
}
|
||||
},
|
||||
"develop": {
|
||||
"atomicwrites": {
|
||||
"hashes": [
|
||||
"sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
|
||||
"sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
|
||||
],
|
||||
"version": "==1.3.0"
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2",
|
||||
"sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396"
|
||||
],
|
||||
"version": "==19.2.0"
|
||||
},
|
||||
"entrypoints": {
|
||||
"hashes": [
|
||||
"sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19",
|
||||
"sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"
|
||||
],
|
||||
"version": "==0.3"
|
||||
},
|
||||
"flake8": {
|
||||
"hashes": [
|
||||
"sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548",
|
||||
"sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.7.8"
|
||||
},
|
||||
"importlib-metadata": {
|
||||
"hashes": [
|
||||
"sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26",
|
||||
"sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af"
|
||||
],
|
||||
"markers": "python_version < '3.8'",
|
||||
"version": "==0.23"
|
||||
},
|
||||
"mccabe": {
|
||||
"hashes": [
|
||||
"sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
|
||||
"sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
|
||||
],
|
||||
"version": "==0.6.1"
|
||||
},
|
||||
"more-itertools": {
|
||||
"hashes": [
|
||||
"sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832",
|
||||
"sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"
|
||||
],
|
||||
"version": "==7.2.0"
|
||||
},
|
||||
"mypy": {
|
||||
"hashes": [
|
||||
"sha256:1d98fd818ad3128a5408148c9e4a5edce6ed6b58cc314283e631dd5d9216527b",
|
||||
"sha256:22ee018e8fc212fe601aba65d3699689dd29a26410ef0d2cc1943de7bec7e3ac",
|
||||
"sha256:3a24f80776edc706ec8d05329e854d5b9e464cd332e25cde10c8da2da0a0db6c",
|
||||
"sha256:42a78944e80770f21609f504ca6c8173f7768043205b5ac51c9144e057dcf879",
|
||||
"sha256:4b2b20106973548975f0c0b1112eceb4d77ed0cafe0a231a1318f3b3a22fc795",
|
||||
"sha256:591a9625b4d285f3ba69f541c84c0ad9e7bffa7794da3fa0585ef13cf95cb021",
|
||||
"sha256:5b4b70da3d8bae73b908a90bb2c387b977e59d484d22c604a2131f6f4397c1a3",
|
||||
"sha256:84edda1ffeda0941b2ab38ecf49302326df79947fa33d98cdcfbf8ca9cf0bb23",
|
||||
"sha256:b2b83d29babd61b876ae375786960a5374bba0e4aba3c293328ca6ca5dc448dd",
|
||||
"sha256:cc4502f84c37223a1a5ab700649b5ab1b5e4d2bf2d426907161f20672a21930b",
|
||||
"sha256:e29e24dd6e7f39f200a5bb55dcaa645d38a397dd5a6674f6042ef02df5795046"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.730"
|
||||
},
|
||||
"mypy-extensions": {
|
||||
"hashes": [
|
||||
"sha256:a161e3b917053de87dbe469987e173e49fb454eca10ef28b48b384538cc11458"
|
||||
],
|
||||
"version": "==0.4.2"
|
||||
},
|
||||
"packaging": {
|
||||
"hashes": [
|
||||
"sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47",
|
||||
"sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"
|
||||
],
|
||||
"version": "==19.2"
|
||||
},
|
||||
"pluggy": {
|
||||
"hashes": [
|
||||
"sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6",
|
||||
"sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34"
|
||||
],
|
||||
"version": "==0.13.0"
|
||||
},
|
||||
"py": {
|
||||
"hashes": [
|
||||
"sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
|
||||
"sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
|
||||
],
|
||||
"version": "==1.8.0"
|
||||
},
|
||||
"pycodestyle": {
|
||||
"hashes": [
|
||||
"sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56",
|
||||
"sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"
|
||||
],
|
||||
"version": "==2.5.0"
|
||||
},
|
||||
"pyflakes": {
|
||||
"hashes": [
|
||||
"sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0",
|
||||
"sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"
|
||||
],
|
||||
"version": "==2.1.1"
|
||||
},
|
||||
"pyparsing": {
|
||||
"hashes": [
|
||||
"sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80",
|
||||
"sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4"
|
||||
],
|
||||
"version": "==2.4.2"
|
||||
},
|
||||
"pytest": {
|
||||
"hashes": [
|
||||
"sha256:7e4800063ccfc306a53c461442526c5571e1462f61583506ce97e4da6a1d88c8",
|
||||
"sha256:ca563435f4941d0cb34767301c27bc65c510cb82e90b9ecf9cb52dc2c63caaa0"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==5.2.1"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
||||
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
||||
],
|
||||
"version": "==1.12.0"
|
||||
},
|
||||
"typed-ast": {
|
||||
"hashes": [
|
||||
"sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e",
|
||||
|
|
@ -454,6 +431,13 @@
|
|||
],
|
||||
"version": "==3.7.4"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
|
||||
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
|
||||
],
|
||||
"version": "==1.25.6"
|
||||
},
|
||||
"wcwidth": {
|
||||
"hashes": [
|
||||
"sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
|
||||
|
|
@ -461,6 +445,13 @@
|
|||
],
|
||||
"version": "==0.1.7"
|
||||
},
|
||||
"xlsxwriter": {
|
||||
"hashes": [
|
||||
"sha256:48c04e6cec449a562455d7cf40305de1190adfc821e834e023425461f345dfc2",
|
||||
"sha256:7f41b1dbd6b38dd1ce243156cac54d1dae50a41f4a45871596ca67b0abcfb3e8"
|
||||
],
|
||||
"version": "==1.2.1"
|
||||
},
|
||||
"zipp": {
|
||||
"hashes": [
|
||||
"sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
|
||||
|
|
@ -468,5 +459,6 @@
|
|||
],
|
||||
"version": "==0.6.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@
|
|||
from theHarvester.lib import statichtmlgenerator
|
||||
from theHarvester.lib.core import *
|
||||
import argparse
|
||||
import asyncio
|
||||
import datetime
|
||||
import netaddr
|
||||
import re
|
||||
|
|
@ -18,19 +19,26 @@
|
|||
|
||||
|
||||
def start():
|
||||
parser = argparse.ArgumentParser(description='theHarvester is used to gather open source intelligence (OSINT) on a\n'
|
||||
'company or domain.')
|
||||
parser = argparse.ArgumentParser(
|
||||
description='theHarvester is used to gather open source intelligence (OSINT) on a\n'
|
||||
'company or domain.')
|
||||
parser.add_argument('-d', '--domain', help='company name or domain to search', required=True)
|
||||
parser.add_argument('-l', '--limit', help='limit the number of search results, default=500', default=500, type=int)
|
||||
parser.add_argument('-S', '--start', help='start with result number X, default=0', default=0, type=int)
|
||||
parser.add_argument('-g', '--google-dork', help='use Google Dorks for Google search', default=False, action='store_true')
|
||||
parser.add_argument('-p', '--port-scan', help='scan the detected hosts and check for Takeovers (21,22,80,443,8080)', default=False, action='store_true')
|
||||
parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=False, action='store_true')
|
||||
parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts', action='store_const', const='basic', default=False)
|
||||
parser.add_argument('-g', '--google-dork', help='use Google Dorks for Google search', default=False,
|
||||
action='store_true')
|
||||
parser.add_argument('-p', '--port-scan', help='scan the detected hosts and check for Takeovers (21,22,80,443,8080)',
|
||||
default=False, action='store_true')
|
||||
parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=False,
|
||||
action='store_true')
|
||||
parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts',
|
||||
action='store_const', const='basic', default=False)
|
||||
parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup')
|
||||
parser.add_argument('-t', '--dns-tld', help='perform a DNS TLD expansion discovery, default False', default=False)
|
||||
parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default False', default=False, action='store_true')
|
||||
parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true')
|
||||
parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default False', default=False,
|
||||
action='store_true')
|
||||
parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False,
|
||||
action='store_true')
|
||||
parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str)
|
||||
parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, crtsh, dnsdumpster,
|
||||
dogpile, duckduckgo, github-code, google,
|
||||
|
|
@ -333,6 +341,19 @@ def start():
|
|||
else:
|
||||
pass
|
||||
|
||||
elif engineitem == 'suip':
|
||||
print('\033[94m[*] Searching suip. \033[0m')
|
||||
from theHarvester.discovery import suip
|
||||
try:
|
||||
suip_search = suip.SearchSuip(word)
|
||||
suip_search.process()
|
||||
hosts = filter(suip_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'suip')
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
elif engineitem == 'spyse':
|
||||
print('\033[94m[*] Searching Spyse. \033[0m')
|
||||
from theHarvester.discovery import spyse
|
||||
|
|
@ -457,14 +478,13 @@ def start():
|
|||
print('---------------------')
|
||||
all_hosts = sorted(list(set(all_hosts)))
|
||||
full_host = hostchecker.Checker(all_hosts)
|
||||
full = full_host.check()
|
||||
full, ips = asyncio.run(full_host.check())
|
||||
db = stash.stash_manager()
|
||||
for host in full:
|
||||
host = str(host)
|
||||
print(host.lower())
|
||||
|
||||
db = stash.stash_manager()
|
||||
print(host)
|
||||
host_ip = [netaddr_ip.format() for netaddr_ip in sorted([netaddr.IPAddress(ip) for ip in ips])]
|
||||
db.store_all(word, host_ip, 'ip', 'DNS-resolver')
|
||||
|
||||
length_urls = len(trello_urls)
|
||||
if length_urls == 0:
|
||||
if len(engines) >= 1 and 'trello' in engines:
|
||||
|
|
@ -587,7 +607,6 @@ def start():
|
|||
tab.set_cols_valign(['m', 'm', 'm', 'm', 'm'])
|
||||
tab.set_chars(['-', '|', '+', '#'])
|
||||
tab.set_cols_width([15, 20, 15, 15, 18])
|
||||
host_ip = list(set(host_ip))
|
||||
print('\033[94m[*] Searching Shodan. \033[0m')
|
||||
try:
|
||||
for ip in host_ip:
|
||||
|
|
|
|||
52
theHarvester/discovery/suip.py
Normal file
52
theHarvester/discovery/suip.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
from theHarvester.lib.core import *
|
||||
from bs4 import BeautifulSoup
|
||||
import requests
|
||||
|
||||
|
||||
class SearchSuip:
|
||||
|
||||
def __init__(self, word: str):
|
||||
self.word: str = word
|
||||
self.results: str = ''
|
||||
self.totalresults: str = ''
|
||||
self.totalhosts: set = set()
|
||||
self.totalips: set = set()
|
||||
|
||||
def do_search(self):
|
||||
headers = {'User-Agent': Core.get_user_agent()}
|
||||
params = (
|
||||
('act', 'subfinder'),
|
||||
)
|
||||
|
||||
data = {
|
||||
'url': self.word.replace('www.', ''),
|
||||
'Submit1': 'Submit'
|
||||
}
|
||||
response = requests.post('https://suip.biz/', headers=headers, params=params, data=data)
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
hosts: list = str(soup.find('pre')).splitlines()
|
||||
self.clean_hosts(hosts)
|
||||
params = (
|
||||
('act', 'amass'),
|
||||
)
|
||||
# change act to amass now
|
||||
response = requests.post('https://suip.biz/', headers=headers, params=params, data=data)
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
hosts: list = str(soup.find('pre')).splitlines()
|
||||
self.clean_hosts(hosts)
|
||||
|
||||
def get_hostnames(self) -> set:
|
||||
return self.totalhosts
|
||||
|
||||
def process(self):
|
||||
self.do_search()
|
||||
print('\tSearching results.')
|
||||
|
||||
def clean_hosts(self, soup_hosts):
|
||||
for host in soup_hosts:
|
||||
host = str(host).strip()
|
||||
if len(host) > 1 and 'pre' not in host:
|
||||
if host[0] == '.':
|
||||
self.totalhosts.add(host[1:])
|
||||
else:
|
||||
self.totalhosts.add(host)
|
||||
|
|
@ -87,6 +87,7 @@ def get_supportedengines() -> Set[Union[str, Any]]:
|
|||
'netcraft',
|
||||
'otx',
|
||||
'securityTrails',
|
||||
'suip',
|
||||
'spyse',
|
||||
'threatcrowd',
|
||||
'trello',
|
||||
|
|
|
|||
|
|
@ -14,31 +14,35 @@ class Checker:
|
|||
|
||||
def __init__(self, hosts: list):
|
||||
self.hosts = hosts
|
||||
self.realhosts = []
|
||||
self.realhosts: list = []
|
||||
self.addresses: set = set()
|
||||
|
||||
@staticmethod
|
||||
async def query(host, resolver) -> [list, str]:
|
||||
try:
|
||||
result = await resolver.gethostbyname(host, socket.AF_INET)
|
||||
return result
|
||||
addresses = result.addresses
|
||||
if addresses == [] or addresses is None or result is None:
|
||||
return f"{host}:", tuple()
|
||||
else:
|
||||
return f"{host}:{', '.join(map(str, addresses))}", addresses
|
||||
except Exception:
|
||||
# print(f'An error occurred in query: {e}')
|
||||
return f"{host}:"
|
||||
return f"{host}:", tuple()
|
||||
|
||||
def check(self):
|
||||
async def query_all(self, resolver) -> list:
|
||||
results = await asyncio.gather(*[asyncio.create_task(self.query(host, resolver))
|
||||
for host in self.hosts])
|
||||
return results
|
||||
|
||||
async def check(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
resolver = aiodns.DNSResolver(loop=loop)
|
||||
for host in self.hosts:
|
||||
resp = self.query(host, resolver)
|
||||
result = loop.run_until_complete(resp)
|
||||
true_result = ''
|
||||
if isinstance(result, str):
|
||||
true_result = result
|
||||
elif result != '' and not isinstance(result, str) and result.addresses is not None \
|
||||
and result.addresses != []:
|
||||
result = result.addresses
|
||||
result.sort()
|
||||
true_result = f"{host}:{', '.join(map(str, result))}"
|
||||
self.realhosts.append(true_result)
|
||||
loop.close()
|
||||
return self.realhosts
|
||||
resolver = aiodns.DNSResolver(loop=loop, timeout=4)
|
||||
results = await self.query_all(resolver)
|
||||
for host, address in results:
|
||||
self.realhosts.append(host)
|
||||
self.addresses.update({addr for addr in address})
|
||||
# address may be a list of ips
|
||||
# and do a set comprehension to remove duplicates
|
||||
self.realhosts.sort()
|
||||
self.addresses = list(self.addresses)
|
||||
return self.realhosts, self.addresses
|
||||
|
|
|
|||
Loading…
Reference in a new issue