mirror of
https://github.com/laramies/theHarvester.git
synced 2024-09-22 00:06:30 +08:00
Matt
GitHub
commit
5f470fba56
4
.github/ISSUE_TEMPLATE/issue-template.md
vendored
4
.github/ISSUE_TEMPLATE/issue-template.md
vendored
|
|
@ -14,11 +14,11 @@ A clear and concise description of what the bug, feature request,
|
|||
or other request is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
Steps to reproduce the behaviour:
|
||||
1. Run tool like this: '...'
|
||||
2. See error
|
||||
|
||||
**Expected behavior**
|
||||
**Expected behaviour**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots**
|
||||
|
|
|
|||
12
.github/workflows/theHarvester.yml
vendored
12
.github/workflows/theHarvester.yml
vendored
|
|
@ -10,7 +10,6 @@ on:
|
|||
- '*'
|
||||
jobs:
|
||||
Python:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
max-parallel: 4
|
||||
|
|
@ -25,6 +24,7 @@ jobs:
|
|||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Run theHarvester module baidu
|
||||
|
|
@ -35,6 +35,10 @@ jobs:
|
|||
run: |
|
||||
python theHarvester.py -d metasploit.com -b bing
|
||||
|
||||
- name: Run theHarvester module certspotter
|
||||
run: |
|
||||
python theHarvester.py -d metasploit.com -b certspotter
|
||||
|
||||
- name: Run theHarvester module crtsh
|
||||
run: |
|
||||
python theHarvester.py -d metasploit.com -b crtsh
|
||||
|
|
@ -108,6 +112,6 @@ jobs:
|
|||
- name: Test with pytest
|
||||
run: |
|
||||
pytest
|
||||
# - name: Static type checking with mypy
|
||||
# run: |
|
||||
# mypy --pretty *.py
|
||||
- name: Static type checking with mypy
|
||||
run: |
|
||||
mypy --pretty theHarvester/*/*.py
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
|
|
@ -7,4 +7,7 @@
|
|||
debug_results.txt
|
||||
venv
|
||||
.mypy_cache
|
||||
.pytest_cache
|
||||
.pytest_cache
|
||||
build/
|
||||
dist/
|
||||
theHarvester.egg-info/
|
||||
23
.travis.yml
23
.travis.yml
|
|
@ -2,29 +2,30 @@ dist: bionic
|
|||
language: python
|
||||
cache: pip
|
||||
matrix:
|
||||
include:
|
||||
- python: '3.7'
|
||||
env: TEST_SUITE=suite_3_7
|
||||
- python: '3.8-dev'
|
||||
env: TEST_SUITE=suite_3_8_dev
|
||||
include:
|
||||
- python: '3.7'
|
||||
env: TEST_SUITE=suite_3_7
|
||||
- python: '3.8'
|
||||
env: TEST_SUITE=suite_3_8
|
||||
before_install:
|
||||
- pip install -r requirements.txt
|
||||
- pip install pipenv
|
||||
- pipenv install --dev
|
||||
install:
|
||||
- python setup.py test
|
||||
script:
|
||||
- python theHarvester.py -d metasploit.com -b baidu,bing,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,otx,intelx,threatcrowd,trello,twitter,virustotal,yahoo -l 200
|
||||
- python theHarvester.py -d metasploit.com -b baidu,bing,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,otx,intelx,threatcrowd,trello,twitter,virustotal,yahoo
|
||||
-l 200
|
||||
- pytest
|
||||
- flake8 . --count --show-source --statistics
|
||||
#- mypy --pretty *.py
|
||||
- mypy --pretty theHarvester/discovery/*.py
|
||||
notifications:
|
||||
email: false
|
||||
slack:
|
||||
rooms:
|
||||
secure: QDZfGOGT1lSxBFDFgzVjKQODljKeckj7c4mTg77WLQ5Nnb1rnNeFciIxu8eHONYry6s60ucB5dXJ8Ju4c8fyPWUNKJ9S3PfuikvaXgf/jLePJ4XVxRUUNMCTxO8Ja3wJh5wqj/SP8gi+RWdW95SyI3KU8kAqLOE+XTqFp9wOW4eGV4vj+pp+X+9tRyS8myvBqsS2xBTWzctl2qRhHXv7mqTQ3SSaj6niio023s0+ZSMDtgKeBQ6EucnYinJVjIHwSWUUMky1ogkK4UgcLxmaHQflmpZdlFB3B3c4uhaa5dq5cbi7CTV74SiAr3F630LBVhpN753NXPr3Vx1kOoAdBl7INv6aU/y99wNTZDvhTuYeIBdIKYalzfg09nCMXVjfw2nrdMel8VvBfVcKYcg4y3eoCYvSoL1YQY5ifyBlGHUi5ZV5TodmMHrRC4tVdIuoLmuiyuyCVKJRpQVyWfSVH5zmulIwXBwuMARzC8OrdeA0gPT6FWthu2lCiUlQn+3xOY+z0eHO5bFcVbA7Aa/BHCZwvduqD7nkKftZ/wt6UKRU0rUkRIh+fZ2llY4ZyxA6S1xr5ct/SD+wpPsSmFRj0/dpneOJxKvqbNYfw/BTOSicNYbGYwTvconpt78pkX1+vU0/1/C3E+lNSvr3QxDpojottxcBsH13yv7CuDPCPOA=
|
||||
secure: DQXvpVqLJ1NKa4zOVrrLuHjd5yCY8tdLm4QjSILc5g7NGN5QY3wVmC3m7KWq3RsqdepeiJbd3mgLYhfo6TA/tAaZxEYXKEdafWGF7ayJcEJS/fn0GuLqhOaS/PzRYSeBMQH5KodfvJQpVFzfHPj9AoIHOrHVH3x192RzIS3hRyR8kZgSCrTgxiDjTeWUzLvg/w7ikEVqVFMh73cQJegVA6A5mkHeUf20NmKzo+e0fGU7Sktk38YyNOdi3fbAiACR6ah1clSB7HaBg3VDiAmQCE8O2tftgcU6ihhnTi6d4i8Lf/traQznQ3mvSbFcw5Pedo8eXaLDhAuqwzMb3uWE9jr+zLlDa8s6+ADNVO/ISu+xV1zpnTdcjATKHaBfsNFntLij1hnyeuTEbhLRAB1T7wc+uAWVlJkkDZK08610a8NWalRtV17U8u8lJbcKWQ4IBnclG6DE+zpgsMHZpcswyeMF092mRZzUbgXG9+nbRcp1JqhgiLJUZdg5jXX7NoLdk7irbrZU4aTFqhbz3P3NexafFDXZEsp1Z1eY0uppRsd0vt8E8rX/HMw9OWHgkg7GDATZSqMu1kgJoSQQg1U3ApXacsl6WBAndLdYF+MyHJMLgzewdAJ4y4qvVMb/VkTJ8Q6PicjwlqyEP5PRLZk7fech4kuTVUqyuTibd5t8D5k=
|
||||
on_success: always
|
||||
on_failure: always
|
||||
template:
|
||||
- "Repo `%{repository_slug}` *%{result}* build (<%{build_url}|#%{build_number}>) for commit (<%{compare_url}|%{commit}>) on branch `%{branch}`."
|
||||
- "Execution time: *%{duration}*"
|
||||
- "Message: %{message}"
|
||||
- "By: %{author}"
|
||||
- "By: %{author}"
|
||||
17
Pipfile
17
Pipfile
|
|
@ -5,21 +5,20 @@ name = "pypi"
|
|||
|
||||
[packages]
|
||||
aiodns = "==2.0.0"
|
||||
beautifulsoup4 = "==4.8.0"
|
||||
aiohttp = "==3.5.4"
|
||||
beautifulsoup4 = "==4.8.1"
|
||||
dnspython = "==1.16.0"
|
||||
flake8 = "==3.7.8"
|
||||
grequests = "==0.4.0"
|
||||
mypy = "==0.730"
|
||||
netaddr = "==0.7.19"
|
||||
plotly = "==4.1.1"
|
||||
pytest = "==5.2.0"
|
||||
plotly = "==4.2.1"
|
||||
pyyaml = "==5.1.2"
|
||||
requests = "==2.22.0"
|
||||
retrying = "==1.3.3"
|
||||
shodan = "==1.19.0"
|
||||
texttable = "==1.6.2"
|
||||
retrying = "==1.3.3"
|
||||
|
||||
[dev-packages]
|
||||
|
||||
[requires]
|
||||
python_version = "3.7"
|
||||
flake8 = "==3.7.8"
|
||||
mypy = "==0.740"
|
||||
mypy-extensions = "==0.4.3"
|
||||
pytest = "==5.2.1"
|
||||
|
|
|
|||
382
Pipfile.lock
generated
382
Pipfile.lock
generated
|
|
@ -1,12 +1,10 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "af937a3b86c64e37abd6f44f3ff30f08f9c2660a270206637f8cc1cb87416fbb"
|
||||
"sha256": "7e45ee0725c108a6f56760e8a6540c9400948fe984b9bb1ba7b29139a753b6ae"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3.7"
|
||||
},
|
||||
"requires": {},
|
||||
"sources": [
|
||||
{
|
||||
"name": "pypi",
|
||||
|
|
@ -24,28 +22,14 @@
|
|||
"index": "pypi",
|
||||
"version": "==2.0.0"
|
||||
},
|
||||
"atomicwrites": {
|
||||
"hashes": [
|
||||
"sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
|
||||
"sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
|
||||
],
|
||||
"version": "==1.3.0"
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2",
|
||||
"sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396"
|
||||
],
|
||||
"version": "==19.2.0"
|
||||
},
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612",
|
||||
"sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b",
|
||||
"sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469"
|
||||
"sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
|
||||
"sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931",
|
||||
"sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.8.0"
|
||||
"version": "==4.8.1"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
|
|
@ -56,36 +40,40 @@
|
|||
},
|
||||
"cffi": {
|
||||
"hashes": [
|
||||
"sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774",
|
||||
"sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d",
|
||||
"sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90",
|
||||
"sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b",
|
||||
"sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63",
|
||||
"sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45",
|
||||
"sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25",
|
||||
"sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3",
|
||||
"sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b",
|
||||
"sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647",
|
||||
"sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016",
|
||||
"sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4",
|
||||
"sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb",
|
||||
"sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753",
|
||||
"sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7",
|
||||
"sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9",
|
||||
"sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f",
|
||||
"sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8",
|
||||
"sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f",
|
||||
"sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc",
|
||||
"sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42",
|
||||
"sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3",
|
||||
"sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909",
|
||||
"sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45",
|
||||
"sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d",
|
||||
"sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512",
|
||||
"sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff",
|
||||
"sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201"
|
||||
"sha256:08f99e8b38d5134d504aa7e486af8e4fde66a2f388bbecc270cdd1e00fa09ff8",
|
||||
"sha256:1112d2fc92a867a6103bce6740a549e74b1d320cf28875609f6e93857eee4f2d",
|
||||
"sha256:1b9ab50c74e075bd2ae489853c5f7f592160b379df53b7f72befcbe145475a36",
|
||||
"sha256:24eff2997436b6156c2f30bed215c782b1d8fd8c6a704206053c79af95962e45",
|
||||
"sha256:2eff642fbc9877a6449026ad66bf37c73bf4232505fb557168ba5c502f95999b",
|
||||
"sha256:362e896cea1249ed5c2a81cf6477fabd9e1a5088aa7ea08358a4c6b0998294d2",
|
||||
"sha256:40eddb3589f382cb950f2dcf1c39c9b8d7bd5af20665ce273815b0d24635008b",
|
||||
"sha256:5ed40760976f6b8613d4a0db5e423673ca162d4ed6c9ed92d1f4e58a47ee01b5",
|
||||
"sha256:632c6112c1e914c486f06cfe3f0cc507f44aa1e00ebf732cedb5719e6aa0466a",
|
||||
"sha256:64d84f0145e181f4e6cc942088603c8db3ae23485c37eeda71cb3900b5e67cb4",
|
||||
"sha256:6cb4edcf87d0e7f5bdc7e5c1a0756fbb37081b2181293c5fdf203347df1cd2a2",
|
||||
"sha256:6f19c9df4785305669335b934c852133faed913c0faa63056248168966f7a7d5",
|
||||
"sha256:719537b4c5cd5218f0f47826dd705fb7a21d83824920088c4214794457113f3f",
|
||||
"sha256:7b0e337a70e58f1a36fb483fd63880c9e74f1db5c532b4082bceac83df1523fa",
|
||||
"sha256:853376efeeb8a4ae49a737d5d30f5db8cdf01d9319695719c4af126488df5a6a",
|
||||
"sha256:85bbf77ffd12985d76a69d2feb449e35ecdcb4fc54a5f087d2bd54158ae5bb0c",
|
||||
"sha256:8978115c6f0b0ce5880bc21c967c65058be8a15f1b81aa5fdbdcbea0e03952d1",
|
||||
"sha256:8f7eec920bc83692231d7306b3e311586c2e340db2dc734c43c37fbf9c981d24",
|
||||
"sha256:8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226",
|
||||
"sha256:92068ebc494b5f9826b822cec6569f1f47b9a446a3fef477e1d11d7fac9ea895",
|
||||
"sha256:b57e1c8bcdd7340e9c9d09613b5e7fdd0c600be142f04e2cc1cc8cb7c0b43529",
|
||||
"sha256:ba956c9b44646bc1852db715b4a252e52a8f5a4009b57f1dac48ba3203a7bde1",
|
||||
"sha256:ca42034c11eb447497ea0e7b855d87ccc2aebc1e253c22e7d276b8599c112a27",
|
||||
"sha256:dc9b2003e9a62bbe0c84a04c61b0329e86fccd85134a78d7aca373bbbf788165",
|
||||
"sha256:dd308802beb4b2961af8f037becbdf01a1e85009fdfc14088614c1b3c383fae5",
|
||||
"sha256:e77cd105b19b8cd721d101687fcf665fd1553eb7b57556a1ef0d453b6fc42faa",
|
||||
"sha256:f56dff1bd81022f1c980754ec721fb8da56192b026f17f0f99b965da5ab4fbd2",
|
||||
"sha256:fa4cc13c03ea1d0d37ce8528e0ecc988d2365e8ac64d8d86cafab4038cb4ce89",
|
||||
"sha256:fa8cf1cb974a9f5911d2a0303f6adc40625c05578d8e7ff5d313e1e27850bd59",
|
||||
"sha256:fb003019f06d5fc0aa4738492ad8df1fa343b8a37cbcf634018ad78575d185df",
|
||||
"sha256:fd409b7778167c3bcc836484a8f49c0e0b93d3e745d975749f83aa5d18a5822f",
|
||||
"sha256:fe5d65a3ee38122003245a82303d11ac05ff36531a8f5ce4bc7d4bbc012797e1"
|
||||
],
|
||||
"version": "==1.12.3"
|
||||
"version": "==1.13.0"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
|
|
@ -123,21 +111,6 @@
|
|||
"index": "pypi",
|
||||
"version": "==1.16.0"
|
||||
},
|
||||
"entrypoints": {
|
||||
"hashes": [
|
||||
"sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19",
|
||||
"sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"
|
||||
],
|
||||
"version": "==0.3"
|
||||
},
|
||||
"flake8": {
|
||||
"hashes": [
|
||||
"sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548",
|
||||
"sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.7.8"
|
||||
},
|
||||
"gevent": {
|
||||
"hashes": [
|
||||
"sha256:0774babec518a24d9a7231d4e689931f31b332c4517a771e532002614e270a64",
|
||||
|
|
@ -206,51 +179,6 @@
|
|||
],
|
||||
"version": "==2.8"
|
||||
},
|
||||
"importlib-metadata": {
|
||||
"hashes": [
|
||||
"sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26",
|
||||
"sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af"
|
||||
],
|
||||
"markers": "python_version < '3.8'",
|
||||
"version": "==0.23"
|
||||
},
|
||||
"mccabe": {
|
||||
"hashes": [
|
||||
"sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
|
||||
"sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
|
||||
],
|
||||
"version": "==0.6.1"
|
||||
},
|
||||
"more-itertools": {
|
||||
"hashes": [
|
||||
"sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832",
|
||||
"sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"
|
||||
],
|
||||
"version": "==7.2.0"
|
||||
},
|
||||
"mypy": {
|
||||
"hashes": [
|
||||
"sha256:1d98fd818ad3128a5408148c9e4a5edce6ed6b58cc314283e631dd5d9216527b",
|
||||
"sha256:22ee018e8fc212fe601aba65d3699689dd29a26410ef0d2cc1943de7bec7e3ac",
|
||||
"sha256:3a24f80776edc706ec8d05329e854d5b9e464cd332e25cde10c8da2da0a0db6c",
|
||||
"sha256:42a78944e80770f21609f504ca6c8173f7768043205b5ac51c9144e057dcf879",
|
||||
"sha256:4b2b20106973548975f0c0b1112eceb4d77ed0cafe0a231a1318f3b3a22fc795",
|
||||
"sha256:591a9625b4d285f3ba69f541c84c0ad9e7bffa7794da3fa0585ef13cf95cb021",
|
||||
"sha256:5b4b70da3d8bae73b908a90bb2c387b977e59d484d22c604a2131f6f4397c1a3",
|
||||
"sha256:84edda1ffeda0941b2ab38ecf49302326df79947fa33d98cdcfbf8ca9cf0bb23",
|
||||
"sha256:b2b83d29babd61b876ae375786960a5374bba0e4aba3c293328ca6ca5dc448dd",
|
||||
"sha256:cc4502f84c37223a1a5ab700649b5ab1b5e4d2bf2d426907161f20672a21930b",
|
||||
"sha256:e29e24dd6e7f39f200a5bb55dcaa645d38a397dd5a6674f6042ef02df5795046"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.730"
|
||||
},
|
||||
"mypy-extensions": {
|
||||
"hashes": [
|
||||
"sha256:a161e3b917053de87dbe469987e173e49fb454eca10ef28b48b384538cc11458"
|
||||
],
|
||||
"version": "==0.4.2"
|
||||
},
|
||||
"netaddr": {
|
||||
"hashes": [
|
||||
"sha256:38aeec7cdd035081d3a4c306394b19d677623bf76fa0913f6695127c7753aefd",
|
||||
|
|
@ -259,34 +187,13 @@
|
|||
"index": "pypi",
|
||||
"version": "==0.7.19"
|
||||
},
|
||||
"packaging": {
|
||||
"hashes": [
|
||||
"sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47",
|
||||
"sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"
|
||||
],
|
||||
"version": "==19.2"
|
||||
},
|
||||
"plotly": {
|
||||
"hashes": [
|
||||
"sha256:0f0cd50a8bad5acadfc95af367820da8ed3e1569d83ef13bb218cc4259dcf219",
|
||||
"sha256:b1ff1f29dffca9085847dcb3b8a92d4c3750c433996cf620b0b3c9dcf8a7b268"
|
||||
"sha256:593418bbbd325ee020b7d0381a9452c603558981bde05a303b860455eb907574",
|
||||
"sha256:6650ddb4da3aa94dcaa32e0779e611c6b17f371b5250ffdbf5ece6d66ba4483b"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.1.1"
|
||||
},
|
||||
"pluggy": {
|
||||
"hashes": [
|
||||
"sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6",
|
||||
"sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34"
|
||||
],
|
||||
"version": "==0.13.0"
|
||||
},
|
||||
"py": {
|
||||
"hashes": [
|
||||
"sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
|
||||
"sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
|
||||
],
|
||||
"version": "==1.8.0"
|
||||
"version": "==4.2.1"
|
||||
},
|
||||
"pycares": {
|
||||
"hashes": [
|
||||
|
|
@ -306,41 +213,12 @@
|
|||
],
|
||||
"version": "==3.0.0"
|
||||
},
|
||||
"pycodestyle": {
|
||||
"hashes": [
|
||||
"sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56",
|
||||
"sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"
|
||||
],
|
||||
"version": "==2.5.0"
|
||||
},
|
||||
"pycparser": {
|
||||
"hashes": [
|
||||
"sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3"
|
||||
],
|
||||
"version": "==2.19"
|
||||
},
|
||||
"pyflakes": {
|
||||
"hashes": [
|
||||
"sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0",
|
||||
"sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"
|
||||
],
|
||||
"version": "==2.1.1"
|
||||
},
|
||||
"pyparsing": {
|
||||
"hashes": [
|
||||
"sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80",
|
||||
"sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4"
|
||||
],
|
||||
"version": "==2.4.2"
|
||||
},
|
||||
"pytest": {
|
||||
"hashes": [
|
||||
"sha256:13c1c9b22127a77fc684eee24791efafcef343335d855e3573791c68588fe1a5",
|
||||
"sha256:d8ba7be9466f55ef96ba203fc0f90d0cf212f2f927e69186e1353e30bc7f62e5"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==5.2.0"
|
||||
},
|
||||
"pyyaml": {
|
||||
"hashes": [
|
||||
"sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9",
|
||||
|
|
@ -403,22 +281,179 @@
|
|||
"index": "pypi",
|
||||
"version": "==1.6.2"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
|
||||
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
|
||||
],
|
||||
"version": "==1.25.6"
|
||||
},
|
||||
"xlsxwriter": {
|
||||
"hashes": [
|
||||
"sha256:00e9c337589ec67a69f1220f47409146ab1affd8eb1e8eaad23f35685bd23e47",
|
||||
"sha256:5a5e2195a4672d17db79839bbdf1006a521adb57eaceea1c335ae4b3d19f088f"
|
||||
],
|
||||
"version": "==1.2.2"
|
||||
}
|
||||
},
|
||||
"develop": {
|
||||
"atomicwrites": {
|
||||
"hashes": [
|
||||
"sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
|
||||
"sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
|
||||
],
|
||||
"version": "==1.3.0"
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
|
||||
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
|
||||
],
|
||||
"version": "==19.3.0"
|
||||
},
|
||||
"entrypoints": {
|
||||
"hashes": [
|
||||
"sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19",
|
||||
"sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"
|
||||
],
|
||||
"version": "==0.3"
|
||||
},
|
||||
"flake8": {
|
||||
"hashes": [
|
||||
"sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548",
|
||||
"sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.7.8"
|
||||
},
|
||||
"importlib-metadata": {
|
||||
"hashes": [
|
||||
"sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26",
|
||||
"sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af"
|
||||
],
|
||||
"markers": "python_version < '3.8'",
|
||||
"version": "==0.23"
|
||||
},
|
||||
"mccabe": {
|
||||
"hashes": [
|
||||
"sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
|
||||
"sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
|
||||
],
|
||||
"version": "==0.6.1"
|
||||
},
|
||||
"more-itertools": {
|
||||
"hashes": [
|
||||
"sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832",
|
||||
"sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"
|
||||
],
|
||||
"version": "==7.2.0"
|
||||
},
|
||||
"mypy": {
|
||||
"hashes": [
|
||||
"sha256:1521c186a3d200c399bd5573c828ea2db1362af7209b2adb1bb8532cea2fb36f",
|
||||
"sha256:31a046ab040a84a0fc38bc93694876398e62bc9f35eca8ccbf6418b7297f4c00",
|
||||
"sha256:3b1a411909c84b2ae9b8283b58b48541654b918e8513c20a400bb946aa9111ae",
|
||||
"sha256:48c8bc99380575deb39f5d3400ebb6a8a1cb5cc669bbba4d3bb30f904e0a0e7d",
|
||||
"sha256:540c9caa57a22d0d5d3c69047cc9dd0094d49782603eb03069821b41f9e970e9",
|
||||
"sha256:672e418425d957e276c291930a3921b4a6413204f53fe7c37cad7bc57b9a3391",
|
||||
"sha256:6ed3b9b3fdc7193ea7aca6f3c20549b377a56f28769783a8f27191903a54170f",
|
||||
"sha256:9371290aa2cad5ad133e4cdc43892778efd13293406f7340b9ffe99d5ec7c1d9",
|
||||
"sha256:ace6ac1d0f87d4072f05b5468a084a45b4eda970e4d26704f201e06d47ab2990",
|
||||
"sha256:b428f883d2b3fe1d052c630642cc6afddd07d5cd7873da948644508be3b9d4a7",
|
||||
"sha256:d5bf0e6ec8ba346a2cf35cb55bf4adfddbc6b6576fcc9e10863daa523e418dbb",
|
||||
"sha256:d7574e283f83c08501607586b3167728c58e8442947e027d2d4c7dcd6d82f453",
|
||||
"sha256:dc889c84241a857c263a2b1cd1121507db7d5b5f5e87e77147097230f374d10b",
|
||||
"sha256:f4748697b349f373002656bf32fede706a0e713d67bfdcf04edf39b1f61d46eb"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.740"
|
||||
},
|
||||
"mypy-extensions": {
|
||||
"hashes": [
|
||||
"sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
|
||||
"sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.4.3"
|
||||
},
|
||||
"packaging": {
|
||||
"hashes": [
|
||||
"sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47",
|
||||
"sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"
|
||||
],
|
||||
"version": "==19.2"
|
||||
},
|
||||
"pluggy": {
|
||||
"hashes": [
|
||||
"sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6",
|
||||
"sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34"
|
||||
],
|
||||
"version": "==0.13.0"
|
||||
},
|
||||
"py": {
|
||||
"hashes": [
|
||||
"sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
|
||||
"sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
|
||||
],
|
||||
"version": "==1.8.0"
|
||||
},
|
||||
"pycodestyle": {
|
||||
"hashes": [
|
||||
"sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56",
|
||||
"sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"
|
||||
],
|
||||
"version": "==2.5.0"
|
||||
},
|
||||
"pyflakes": {
|
||||
"hashes": [
|
||||
"sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0",
|
||||
"sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"
|
||||
],
|
||||
"version": "==2.1.1"
|
||||
},
|
||||
"pyparsing": {
|
||||
"hashes": [
|
||||
"sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80",
|
||||
"sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4"
|
||||
],
|
||||
"version": "==2.4.2"
|
||||
},
|
||||
"pytest": {
|
||||
"hashes": [
|
||||
"sha256:7e4800063ccfc306a53c461442526c5571e1462f61583506ce97e4da6a1d88c8",
|
||||
"sha256:ca563435f4941d0cb34767301c27bc65c510cb82e90b9ecf9cb52dc2c63caaa0"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==5.2.1"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
||||
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
||||
],
|
||||
"version": "==1.12.0"
|
||||
},
|
||||
"typed-ast": {
|
||||
"hashes": [
|
||||
"sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161",
|
||||
"sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e",
|
||||
"sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e",
|
||||
"sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0",
|
||||
"sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c",
|
||||
"sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47",
|
||||
"sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631",
|
||||
"sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4",
|
||||
"sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34",
|
||||
"sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b",
|
||||
"sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2",
|
||||
"sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e",
|
||||
"sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a",
|
||||
"sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233",
|
||||
"sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1",
|
||||
"sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36",
|
||||
"sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d",
|
||||
"sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a",
|
||||
"sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66",
|
||||
"sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"
|
||||
],
|
||||
"version": "==1.4.0"
|
||||
|
|
@ -431,13 +466,6 @@
|
|||
],
|
||||
"version": "==3.7.4"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
|
||||
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
|
||||
],
|
||||
"version": "==1.25.6"
|
||||
},
|
||||
"wcwidth": {
|
||||
"hashes": [
|
||||
"sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
|
||||
|
|
@ -445,13 +473,6 @@
|
|||
],
|
||||
"version": "==0.1.7"
|
||||
},
|
||||
"xlsxwriter": {
|
||||
"hashes": [
|
||||
"sha256:48c04e6cec449a562455d7cf40305de1190adfc821e834e023425461f345dfc2",
|
||||
"sha256:7f41b1dbd6b38dd1ce243156cac54d1dae50a41f4a45871596ca67b0abcfb3e8"
|
||||
],
|
||||
"version": "==1.2.1"
|
||||
},
|
||||
"zipp": {
|
||||
"hashes": [
|
||||
"sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
|
||||
|
|
@ -459,6 +480,5 @@
|
|||
],
|
||||
"version": "==0.6.0"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
26
README.md
26
README.md
|
|
@ -17,7 +17,9 @@ Passive:
|
|||
|
||||
* bing: Microsoft search engine - www.bing.com
|
||||
|
||||
* bingapi: Microsoft search engine, through the API (Requires API key, see below.)
|
||||
* bingapi: Microsoft search engine, through the API (Requires an API key, see below.)
|
||||
|
||||
* CertSpotter: Cert Spotter monitors Certificate Transparency logs - https://sslmate.com/certspotter/
|
||||
|
||||
* crtsh: Comodo Certificate search - www.crt.sh
|
||||
|
||||
|
|
@ -29,27 +31,29 @@ Passive:
|
|||
|
||||
* Exalead: a Meta search engine - https://www.exalead.com/search
|
||||
|
||||
* github-code: Github code search engine (Requires Github Personal Access Token, see below.) - www.github.com
|
||||
* github-code: Github code search engine (Requires a Github Personal Access Token, see below.) - www.github.com
|
||||
|
||||
* google: Google search engine (Optional Google dorking.) - www.google.com
|
||||
|
||||
* hunter: Hunter search engine (Requires API key, see below.) - www.hunter.io
|
||||
* hunter: Hunter search engine (Requires an API key, see below.) - www.hunter.io
|
||||
|
||||
* intelx: Intelx search engine (Requires API key, see below.) - www.intelx.io
|
||||
* intelx: Intelx search engine (Requires an API key, see below.) - www.intelx.io
|
||||
|
||||
* linkedin: Google search engine, specific search for Linkedin users - www.linkedin.com
|
||||
* linkedin: Google search engine, specific search for LinkedIn users - www.linkedin.com
|
||||
|
||||
* netcraft: Netcraft Data Mining - www.netcraft.com
|
||||
* netcraft: Internet Security and Data Mining - www.netcraft.com
|
||||
|
||||
* otx: AlienVault Open Threat Exchange - https://otx.alienvault.com
|
||||
|
||||
* securityTrails: Security Trails search engine, the world's largest repository<br>
|
||||
of historical DNS data (Requires API key, see below.) - www.securitytrails.com
|
||||
of historical DNS data (Requires an API key, see below.) - www.securitytrails.com
|
||||
|
||||
* shodan: Shodan search engine, will search for ports and banners from discovered<br>
|
||||
hosts - www.shodanhq.com
|
||||
|
||||
* Spyse: Web research tools for professionals(Requires an API key) - https://spyse.com/
|
||||
* Spyse: Web research tools for professionals (Requires an API key.) - https://spyse.com/
|
||||
|
||||
* Suip: Web research tools that can take over 10 minutes to run, but worth the wait. - https://suip.biz/
|
||||
|
||||
* threatcrowd: Open source threat intelligence - www.threatcrowd.org
|
||||
|
||||
|
|
@ -73,7 +77,7 @@ Modules that require an API key:
|
|||
--------------------------------
|
||||
Add your keys to api-keys.yaml
|
||||
|
||||
* bingapi
|
||||
* bing
|
||||
* github
|
||||
* hunter
|
||||
* intelx
|
||||
|
|
@ -84,8 +88,8 @@ Add your keys to api-keys.yaml
|
|||
Dependencies:
|
||||
-------------
|
||||
* Python 3.7+
|
||||
* python3 -m pip install -r requirements.txt
|
||||
* Recommend that you use a virtualenv when cloning from git
|
||||
* python3 -m pip install pipenv
|
||||
* pipenv install
|
||||
|
||||
Comments, bugs, or requests?
|
||||
----------------------------
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
aiodns==2.0.0
|
||||
aiohttp==3.5.4
|
||||
beautifulsoup4==4.8.0
|
||||
dnspython==1.16.0
|
||||
flake8==3.7.8
|
||||
gevent==1.4.0
|
||||
grequests==0.4.0
|
||||
mypy==0.730
|
||||
mypy==0.740
|
||||
netaddr==0.7.19
|
||||
plotly==4.1.1
|
||||
plotly==4.2.1
|
||||
pytest==5.2.0
|
||||
PyYaml==5.1.2
|
||||
requests==2.22.0
|
||||
shodan==1.19.0
|
||||
texttable==1.6.2
|
||||
retrying==1.3.3
|
||||
retrying==1.3.3
|
||||
|
|
|
|||
8
setup.py
Normal file → Executable file
8
setup.py
Normal file → Executable file
|
|
@ -1,7 +1,7 @@
|
|||
import setuptools
|
||||
from theHarvester.lib.core import Core
|
||||
|
||||
with open("README.md", "r") as fh:
|
||||
with open('README.md', 'r') as fh:
|
||||
long_description = fh.read()
|
||||
|
||||
setuptools.setup(
|
||||
|
|
@ -22,19 +22,19 @@
|
|||
|
||||
classifiers=[
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.6",
|
||||
"Programming Language :: Python :: 3.7",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
|
||||
"Operating System :: OS Independent",
|
||||
],
|
||||
data_files=[
|
||||
('share/dict/theHarvester', [
|
||||
('/etc/theHarvester', [
|
||||
'wordlists/general/common.txt',
|
||||
'wordlists/dns-big.txt',
|
||||
'wordlists/dns-names.txt',
|
||||
'wordlists/dorks.txt',
|
||||
'wordlists/names_small.txt'
|
||||
'wordlists/names_small.txt',
|
||||
'api-keys.yaml'
|
||||
]
|
||||
)
|
||||
],
|
||||
|
|
|
|||
32
tests/discovery/test_certspotter.py
Normal file
32
tests/discovery/test_certspotter.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
#!/usr/bin/env python3
|
||||
# coding=utf-8
|
||||
from theHarvester.lib.core import *
|
||||
from theHarvester.discovery import certspottersearch
|
||||
import requests
|
||||
import pytest
|
||||
|
||||
|
||||
class TestCertspotter(object):
|
||||
@staticmethod
|
||||
def domain() -> str:
|
||||
return 'metasploit.com'
|
||||
|
||||
def test_api(self):
|
||||
base_url = f'https://api.certspotter.com/v1/issuances?domain={TestCertspotter.domain()}&expand=dns_names'
|
||||
headers = {'User-Agent': Core.get_user_agent()}
|
||||
request = requests.get(base_url, headers=headers)
|
||||
assert request.status_code == 200
|
||||
|
||||
def test_search(self):
|
||||
search = certspottersearch.SearchCertspoter(TestCertspotter.domain())
|
||||
search.process()
|
||||
assert isinstance(search.get_hostnames(), set)
|
||||
|
||||
def test_search_no_results(self):
|
||||
search = certspottersearch.SearchCertspoter('radiant.eu')
|
||||
search.process()
|
||||
assert len(search.get_hostnames()) == 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main()
|
||||
195
tests/test_theHarvester.py
Normal file
195
tests/test_theHarvester.py
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
import os
|
||||
import sys
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
import theHarvester.__main__ as harvester
|
||||
|
||||
domain = 'metasploit.com'
|
||||
sys.argv = args = [os.path.curdir + 'theHarvester.py', '-d', domain, '-b', 'domain']
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.baidusearch.SearchBaidu')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_baidu(stash, search_engine):
|
||||
args[-1] = 'baidu'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 2
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.bingsearch.SearchBing')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_bing(stash, search_engine):
|
||||
args[-1] = 'bing'
|
||||
harvester.start()
|
||||
args[-1] = 'bingapi'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 4
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.certspottersearch.SearchCertspoter')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_certspotter(stash, search_engine):
|
||||
args[-1] = 'certspotter'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 1
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.crtsh.SearchCrtsh')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_crtsh(stash, search_engine):
|
||||
args[-1] = 'crtsh'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 1
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.dnsdumpster.SearchDnsDumpster')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_dnsdumpster(stash, search_engine):
|
||||
args[-1] = 'dnsdumpster'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 1
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.dogpilesearch.SearchDogpile')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_dogpile(stash, search_engine):
|
||||
args[-1] = 'dogpile'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 2
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.duckduckgosearch.SearchDuckDuckGo')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_duckduckgo(stash, search_engine):
|
||||
args[-1] = 'duckduckgo'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 2
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.githubcode.SearchGithubCode')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_github(stash, search_engine):
|
||||
args[-1] = 'github-code'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 2
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.exaleadsearch.SearchExalead')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_exalead(stash, search_engine):
|
||||
args[-1] = 'exalead'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 2
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.googlesearch.SearchGoogle')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_google(stash, search_engine):
|
||||
args[-1] = 'google'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 2
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.huntersearch.SearchHunter')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_hunter(stash, search_engine):
|
||||
args[-1] = 'hunter'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 2
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.intelxsearch.SearchIntelx')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_intelx(stash, search_engine):
|
||||
args[-1] = 'intelx'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 2
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.linkedinsearch.SearchLinkedin')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_linkedin(stash, search_engine):
|
||||
args[-1] = 'linkedin'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 1
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.linkedinsearch.SearchLinkedin')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_linkedin_links(stash, search_engine):
|
||||
args[-1] = 'linkedin_links'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 1
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.netcraft.SearchNetcraft')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_netcraft(stash, search_engine):
|
||||
args[-1] = 'netcraft'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 1
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.otxsearch.SearchOtx')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_otx(stash, search_engine):
|
||||
args[-1] = 'otx'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 2
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.securitytrailssearch.SearchSecuritytrail')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_security_trails(stash, search_engine):
|
||||
args[-1] = 'securityTrails'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 2
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.suip.SearchSuip')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_suip(stash, search_engine):
|
||||
args[-1] = 'suip'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 1
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.threatcrowd.SearchThreatcrowd')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_threatcrowd(stash, search_engine):
|
||||
args[-1] = 'threatcrowd'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 1
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.trello.SearchTrello')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_trello(stash, search_engine):
|
||||
search_engine().get_results = MagicMock(return_value=('user@trello.com', 'trello', 'trello.com'))
|
||||
args[-1] = 'trello'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 3
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.twittersearch.SearchTwitter')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_twitter(stash, search_engine):
|
||||
args[-1] = 'twitter'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 1
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.virustotal.SearchVirustotal')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_virustotal(stash, search_engine):
|
||||
args[-1] = 'virustotal'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 1
|
||||
|
||||
|
||||
@patch('theHarvester.discovery.yahoosearch.SearchYahoo')
|
||||
@patch('theHarvester.lib.stash.StashManager')
|
||||
def test_yahoo(stash, search_engine):
|
||||
args[-1] = 'yahoo'
|
||||
harvester.start()
|
||||
assert stash().store_all.call_count == 2
|
||||
|
|
@ -3,9 +3,11 @@
|
|||
# Note: This script runs theHarvester
|
||||
from platform import python_version
|
||||
import sys
|
||||
if python_version()[0:3] < '3.6':
|
||||
print('\033[93m[!] Make sure you have Python 3.6+ installed, quitting.\n\n \033[0m')
|
||||
sys.exit(1)
|
||||
import asyncio
|
||||
|
||||
if python_version()[0:3] < '3.7':
|
||||
print('\033[93m[!] Make sure you have Python 3.7+ installed, quitting.\n\n \033[0m')
|
||||
sys.exit(1)
|
||||
from theHarvester import __main__
|
||||
__main__.entry_point()
|
||||
|
||||
asyncio.run(__main__.entry_point())
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
Core.banner()
|
||||
|
||||
|
||||
def start():
|
||||
async def start():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='theHarvester is used to gather open source intelligence (OSINT) on a\n'
|
||||
'company or domain.')
|
||||
|
|
@ -40,43 +40,112 @@ def start():
|
|||
parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False,
|
||||
action='store_true')
|
||||
parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str)
|
||||
parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, crtsh, dnsdumpster,
|
||||
parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, certspotter, crtsh, dnsdumpster,
|
||||
dogpile, duckduckgo, github-code, google,
|
||||
hunter, intelx,
|
||||
linkedin, linkedin_links, netcraft, otx, securityTrails, spyse, threatcrowd,
|
||||
trello, twitter, vhost, virustotal, yahoo''')
|
||||
linkedin, linkedin_links, netcraft, otx, securityTrails, spyse(disabled for now), threatcrowd,
|
||||
trello, twitter, vhost, virustotal, yahoo, all''')
|
||||
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
db = stash.stash_manager()
|
||||
db = stash.StashManager()
|
||||
db.do_init()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
all_emails = []
|
||||
all_hosts = []
|
||||
all_ip = []
|
||||
all_emails: list = []
|
||||
all_hosts: list = []
|
||||
all_ip: list = []
|
||||
dnsbrute = args.dns_brute
|
||||
dnslookup = args.dns_lookup
|
||||
dnsserver = args.dns_server
|
||||
dnstld = args.dns_tld
|
||||
engines = []
|
||||
filename = args.filename # type: str
|
||||
full = []
|
||||
filename: str = args.filename
|
||||
full: list = []
|
||||
google_dorking = args.google_dork
|
||||
host_ip = []
|
||||
limit = args.limit # type: int
|
||||
host_ip: list = []
|
||||
limit: int = args.limit
|
||||
ports_scanning = args.port_scan
|
||||
shodan = args.shodan
|
||||
start = args.start # type: int
|
||||
start: int = args.start
|
||||
takeover_check = False
|
||||
trello_urls = []
|
||||
vhost = []
|
||||
all_urls: list = []
|
||||
vhost: list = []
|
||||
virtual = args.virtual_host
|
||||
word = args.domain # type: str
|
||||
word: str = args.domain
|
||||
|
||||
async def store(search_engine: Any, source: str, process_param: Any = None, store_host: bool = False,
|
||||
store_emails: bool = False, store_ip: bool = False, store_people: bool = False,
|
||||
store_data: bool = False, store_links: bool = False, store_results: bool = False) -> None:
|
||||
"""
|
||||
Persist details into the database.
|
||||
The details to be stored is controlled by the parameters passed to the method.
|
||||
|
||||
:param search_engine: search engine to fetch details from
|
||||
:param source: source against which the details (corresponding to the search engine) need to be persisted
|
||||
:param process_param: any parameters to be passed to the search engine
|
||||
eg: Google needs google_dorking
|
||||
:param store_host: whether to store hosts
|
||||
:param store_emails: whether to store emails
|
||||
:param store_ip: whether to store IP address
|
||||
:param store_people: whether to store user details
|
||||
:param store_data: whether to fetch host from method get_data() and persist
|
||||
:param store_links: whether to store links
|
||||
:param store_results: whether to fetch details from get_results() and persist
|
||||
"""
|
||||
await search_engine.process() if process_param is None else await search_engine.process(process_param)
|
||||
db_stash = stash.StashManager()
|
||||
|
||||
if store_host:
|
||||
host_names = filter(await search_engine.get_hostnames())
|
||||
all_hosts.extend(host_names)
|
||||
db_stash.store_all(word, all_hosts, 'host', source)
|
||||
if store_emails:
|
||||
email_list = filter(await search_engine.get_emails())
|
||||
db_stash.store_all(word, email_list, 'email', source)
|
||||
if store_ip:
|
||||
ips_list = search_engine.get_ips()
|
||||
all_ip.extend(ips_list)
|
||||
db_stash.store_all(word, all_ip, 'ip', source)
|
||||
if store_data:
|
||||
data = filter(search_engine.get_data())
|
||||
all_hosts.extend(data)
|
||||
db.store_all(word, all_hosts, 'host', source)
|
||||
if store_results:
|
||||
email_list, host_names, urls = search_engine.get_results()
|
||||
all_emails.extend(email_list)
|
||||
host_names = filter(host_names)
|
||||
all_urls.extend(filter(urls))
|
||||
all_hosts.extend(host_names)
|
||||
db.store_all(word, all_hosts, 'host', source)
|
||||
db.store_all(word, all_emails, 'email', source)
|
||||
if store_people:
|
||||
people_list = search_engine.get_people()
|
||||
db_stash.store_all(word, people_list, 'people', source)
|
||||
if len(people_list) == 0:
|
||||
print('\n[*] No users found.\n\n')
|
||||
else:
|
||||
print('\n[*] Users found: ' + str(len(people_list)))
|
||||
print('---------------------')
|
||||
for usr in sorted(list(set(people_list))):
|
||||
print(usr)
|
||||
if store_links:
|
||||
links = search_engine.get_links()
|
||||
db.store_all(word, links, 'name', engineitem)
|
||||
if len(links) == 0:
|
||||
print('\n[*] No links found.\n\n')
|
||||
else:
|
||||
print(f'\n[*] Links found: {len(links)}')
|
||||
print('---------------------')
|
||||
for link in sorted(list(set(links))):
|
||||
print(link)
|
||||
|
||||
if args.source is not None:
|
||||
engines = sorted(set(map(str.strip, args.source.split(','))))
|
||||
if args.source.lower() != 'all':
|
||||
engines = sorted(set(map(str.strip, args.source.split(','))))
|
||||
else:
|
||||
engines = Core.get_supportedengines()
|
||||
# Iterate through search engines in order
|
||||
if set(engines).issubset(Core.get_supportedengines()):
|
||||
print(f'\033[94m[*] Target: {word} \n \033[0m')
|
||||
|
|
@ -87,13 +156,7 @@ def start():
|
|||
from theHarvester.discovery import baidusearch
|
||||
try:
|
||||
baidu_search = baidusearch.SearchBaidu(word, limit)
|
||||
baidu_search.process()
|
||||
all_emails = filter(baidu_search.get_emails())
|
||||
hosts = filter(baidu_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'baidu')
|
||||
db.store_all(word, all_emails, 'email', 'baidu')
|
||||
await store(baidu_search, engineitem, store_host=True, store_emails=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
@ -107,43 +170,38 @@ def start():
|
|||
bingapi += 'yes'
|
||||
else:
|
||||
bingapi += 'no'
|
||||
bing_search.process(bingapi)
|
||||
all_emails = filter(bing_search.get_emails())
|
||||
hosts = filter(bing_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'email', 'bing')
|
||||
db.store_all(word, all_hosts, 'host', 'bing')
|
||||
await store(bing_search, 'bing', process_param=bingapi, store_host=True, store_emails=True)
|
||||
except Exception as e:
|
||||
if isinstance(e, MissingKey):
|
||||
print(e)
|
||||
else:
|
||||
pass
|
||||
|
||||
elif engineitem == 'certspotter':
|
||||
print('\033[94m[*] Searching CertSpotter. \033[0m')
|
||||
from theHarvester.discovery import certspottersearch
|
||||
try:
|
||||
certspotter_search = certspottersearch.SearchCertspoter(word)
|
||||
await store(certspotter_search, engineitem, None, store_host=True)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
elif engineitem == 'crtsh':
|
||||
try:
|
||||
print('\033[94m[*] Searching CRT.sh. \033[0m')
|
||||
from theHarvester.discovery import crtsh
|
||||
crtsh_search = crtsh.SearchCrtsh(word)
|
||||
crtsh_search.process()
|
||||
hosts = filter(crtsh_search.get_data())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'CRTsh')
|
||||
await store(crtsh_search, 'CRTsh', store_data=True)
|
||||
|
||||
except Exception:
|
||||
print(f'\033[93m[!] An timeout occurred with crtsh, cannot find {args.domain}\033[0m')
|
||||
print(f'\033[93m[!] A timeout occurred with crtsh, cannot find {args.domain}\033[0m')
|
||||
|
||||
elif engineitem == 'dnsdumpster':
|
||||
try:
|
||||
print('\033[94m[*] Searching DNSdumpster. \033[0m')
|
||||
from theHarvester.discovery import dnsdumpster
|
||||
dns_dumpster_search = dnsdumpster.SearchDnsDumpster(word)
|
||||
dns_dumpster_search.process()
|
||||
hosts = filter(dns_dumpster_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'dnsdumpster')
|
||||
await store(dns_dumpster_search, engineitem, store_host=True)
|
||||
except Exception as e:
|
||||
print(f'\033[93m[!] An error occurred with dnsdumpster: {e} \033[0m')
|
||||
|
||||
|
|
@ -152,14 +210,7 @@ def start():
|
|||
print('\033[94m[*] Searching Dogpile. \033[0m')
|
||||
from theHarvester.discovery import dogpilesearch
|
||||
dogpile_search = dogpilesearch.SearchDogpile(word, limit)
|
||||
dogpile_search.process()
|
||||
emails = filter(dogpile_search.get_emails())
|
||||
hosts = filter(dogpile_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
all_emails.extend(emails)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'email', 'dogpile')
|
||||
db.store_all(word, all_hosts, 'host', 'dogpile')
|
||||
await store(dogpile_search, engineitem, store_host=True, store_emails=True)
|
||||
except Exception as e:
|
||||
print(f'\033[93m[!] An error occurred with Dogpile: {e} \033[0m')
|
||||
|
||||
|
|
@ -167,28 +218,14 @@ def start():
|
|||
print('\033[94m[*] Searching DuckDuckGo. \033[0m')
|
||||
from theHarvester.discovery import duckduckgosearch
|
||||
duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(word, limit)
|
||||
duckduckgo_search.process()
|
||||
emails = filter(duckduckgo_search.get_emails())
|
||||
hosts = filter(duckduckgo_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
all_emails.extend(emails)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'email', 'duckduckgo')
|
||||
db.store_all(word, all_hosts, 'host', 'duckduckgo')
|
||||
await store(duckduckgo_search, engineitem, store_host=True, store_emails=True)
|
||||
|
||||
elif engineitem == 'github-code':
|
||||
print('\033[94m[*] Searching Github (code). \033[0m')
|
||||
try:
|
||||
from theHarvester.discovery import githubcode
|
||||
github_search = githubcode.SearchGithubCode(word, limit)
|
||||
github_search.process()
|
||||
emails = filter(github_search.get_emails())
|
||||
all_emails.extend(emails)
|
||||
hosts = filter(github_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'github-code')
|
||||
db.store_all(word, all_emails, 'email', 'github-code')
|
||||
await store(github_search, engineitem, store_host=True, store_emails=True)
|
||||
except MissingKey as ex:
|
||||
print(ex)
|
||||
else:
|
||||
|
|
@ -198,27 +235,14 @@ def start():
|
|||
print('\033[94m[*] Searching Exalead \033[0m')
|
||||
from theHarvester.discovery import exaleadsearch
|
||||
exalead_search = exaleadsearch.SearchExalead(word, limit, start)
|
||||
exalead_search.process()
|
||||
emails = filter(exalead_search.get_emails())
|
||||
all_emails.extend(emails)
|
||||
hosts = filter(exalead_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'exalead')
|
||||
db.store_all(word, all_emails, 'email', 'exalead')
|
||||
await store(exalead_search, engineitem, store_host=True, store_emails=True)
|
||||
|
||||
elif engineitem == 'google':
|
||||
print('\033[94m[*] Searching Google. \033[0m')
|
||||
from theHarvester.discovery import googlesearch
|
||||
google_search = googlesearch.SearchGoogle(word, limit, start)
|
||||
google_search.process(google_dorking)
|
||||
emails = filter(google_search.get_emails())
|
||||
all_emails.extend(emails)
|
||||
hosts = filter(google_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'google')
|
||||
db.store_all(word, all_emails, 'email', 'google')
|
||||
await store(google_search, engineitem, process_param=google_dorking, store_host=True,
|
||||
store_emails=True)
|
||||
|
||||
elif engineitem == 'hunter':
|
||||
print('\033[94m[*] Searching Hunter. \033[0m')
|
||||
|
|
@ -226,14 +250,7 @@ def start():
|
|||
# Import locally or won't work.
|
||||
try:
|
||||
hunter_search = huntersearch.SearchHunter(word, limit, start)
|
||||
hunter_search.process()
|
||||
emails = filter(hunter_search.get_emails())
|
||||
all_emails.extend(emails)
|
||||
hosts = filter(hunter_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'hunter')
|
||||
db.store_all(word, all_emails, 'email', 'hunter')
|
||||
await store(hunter_search, engineitem, store_host=True, store_emails=True)
|
||||
except Exception as e:
|
||||
if isinstance(e, MissingKey):
|
||||
print(e)
|
||||
|
|
@ -246,14 +263,7 @@ def start():
|
|||
# Import locally or won't work.
|
||||
try:
|
||||
intelx_search = intelxsearch.SearchIntelx(word, limit)
|
||||
intelx_search.process()
|
||||
emails = filter(intelx_search.get_emails())
|
||||
all_emails.extend(emails)
|
||||
hosts = filter(intelx_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'intelx')
|
||||
db.store_all(word, all_emails, 'email', 'intelx')
|
||||
await store(intelx_search, engineitem, store_host=True, store_emails=True)
|
||||
except Exception as e:
|
||||
if isinstance(e, MissingKey):
|
||||
print(e)
|
||||
|
|
@ -264,60 +274,26 @@ def start():
|
|||
print('\033[94m[*] Searching Linkedin. \033[0m')
|
||||
from theHarvester.discovery import linkedinsearch
|
||||
linkedin_search = linkedinsearch.SearchLinkedin(word, limit)
|
||||
linkedin_search.process()
|
||||
people = linkedin_search.get_people()
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, people, 'name', 'linkedin')
|
||||
|
||||
if len(people) == 0:
|
||||
print('\n[*] No users found Linkedin.\n\n')
|
||||
else:
|
||||
print(f'\n[*] Users found: {len(people)}')
|
||||
print('---------------------')
|
||||
for user in sorted(list(set(people))):
|
||||
print(user)
|
||||
await store(linkedin_search, engineitem, store_people=True)
|
||||
|
||||
elif engineitem == 'linkedin_links':
|
||||
print('\033[94m[*] Searching Linkedin. \033[0m')
|
||||
from theHarvester.discovery import linkedinsearch
|
||||
linkedin_links_search = linkedinsearch.SearchLinkedin(word, limit)
|
||||
linkedin_links_search.process()
|
||||
people = linkedin_links_search.get_links()
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, people, 'name', 'linkedin')
|
||||
|
||||
if len(people) == 0:
|
||||
print('\n[*] No links found Linkedin.\n\n')
|
||||
else:
|
||||
print(f'\n[*] Links found: {len(people)}')
|
||||
print('---------------------')
|
||||
for user in sorted(list(set(people))):
|
||||
print(user)
|
||||
await store(linkedin_links_search, 'linkedin', store_links=True)
|
||||
|
||||
elif engineitem == 'netcraft':
|
||||
print('\033[94m[*] Searching Netcraft. \033[0m')
|
||||
from theHarvester.discovery import netcraft
|
||||
netcraft_search = netcraft.SearchNetcraft(word)
|
||||
netcraft_search.process()
|
||||
hosts = filter(netcraft_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'netcraft')
|
||||
await store(netcraft_search, engineitem, store_host=True)
|
||||
|
||||
elif engineitem == 'otx':
|
||||
print('\033[94m[*] Searching AlienVault OTX. \033[0m')
|
||||
from theHarvester.discovery import otxsearch
|
||||
try:
|
||||
otxsearch_search = otxsearch.SearchOtx(word)
|
||||
otxsearch_search.process()
|
||||
hosts = filter(otxsearch_search.get_hostnames())
|
||||
all_hosts.extend(list(hosts))
|
||||
ips = filter(otxsearch_search.get_ips())
|
||||
all_ip.extend(list(ips))
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'otx')
|
||||
db.store_all(word, all_ip, 'ip', 'otx')
|
||||
await store(otxsearch_search, engineitem, store_host=True, store_ip=True)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
|
|
@ -326,15 +302,7 @@ def start():
|
|||
from theHarvester.discovery import securitytrailssearch
|
||||
try:
|
||||
securitytrails_search = securitytrailssearch.SearchSecuritytrail(word)
|
||||
securitytrails_search.process()
|
||||
hosts = filter(securitytrails_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, hosts, 'host', 'securityTrails')
|
||||
ips = securitytrails_search.get_ips()
|
||||
all_ip.extend(ips)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, ips, 'ip', 'securityTrails')
|
||||
await store(securitytrails_search, engineitem, store_host=True, store_ip=True)
|
||||
except Exception as e:
|
||||
if isinstance(e, MissingKey):
|
||||
print(e)
|
||||
|
|
@ -342,45 +310,37 @@ def start():
|
|||
pass
|
||||
|
||||
elif engineitem == 'suip':
|
||||
print('\033[94m[*] Searching suip. \033[0m')
|
||||
print('\033[94m[*] Searching Suip. This module can take 10+ mins to run but it is worth it.\033[0m')
|
||||
from theHarvester.discovery import suip
|
||||
try:
|
||||
suip_search = suip.SearchSuip(word)
|
||||
suip_search.process()
|
||||
hosts = filter(suip_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'suip')
|
||||
await store(suip_search, engineitem, store_host=True)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
elif engineitem == 'spyse':
|
||||
print('\033[94m[*] Searching Spyse. \033[0m')
|
||||
from theHarvester.discovery import spyse
|
||||
try:
|
||||
spysesearch_search = spyse.SearchSpyse(word)
|
||||
spysesearch_search.process()
|
||||
hosts = filter(spysesearch_search.get_hostnames())
|
||||
all_hosts.extend(list(hosts))
|
||||
# ips = filter(spysesearch_search.get_ips())
|
||||
# all_ip.extend(list(ips))
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'spyse')
|
||||
# db.store_all(word, all_ip, 'ip', 'spyse')
|
||||
except Exception as e:
|
||||
print(e)
|
||||
# elif engineitem == 'spyse':
|
||||
# print('\033[94m[*] Searching Spyse. \033[0m')
|
||||
# from theHarvester.discovery import spyse
|
||||
# try:
|
||||
# spysesearch_search = spyse.SearchSpyse(word)
|
||||
# spysesearch_search.process()
|
||||
# hosts = filter(spysesearch_search.get_hostnames())
|
||||
# all_hosts.extend(list(hosts))
|
||||
# # ips = filter(spysesearch_search.get_ips())
|
||||
# # all_ip.extend(list(ips))
|
||||
# all_hosts.extend(hosts)
|
||||
# db = stash.stash_manager()
|
||||
# db.store_all(word, all_hosts, 'host', 'spyse')
|
||||
# # db.store_all(word, all_ip, 'ip', 'spyse')
|
||||
# except Exception as e:
|
||||
# print(e)
|
||||
|
||||
elif engineitem == 'threatcrowd':
|
||||
print('\033[94m[*] Searching Threatcrowd. \033[0m')
|
||||
from theHarvester.discovery import threatcrowd
|
||||
try:
|
||||
threatcrowd_search = threatcrowd.SearchThreatcrowd(word)
|
||||
threatcrowd_search.process()
|
||||
hosts = filter(threatcrowd_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'threatcrowd')
|
||||
await store(threatcrowd_search, engineitem, store_host=True)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
|
|
@ -389,55 +349,25 @@ def start():
|
|||
from theHarvester.discovery import trello
|
||||
# Import locally or won't work.
|
||||
trello_search = trello.SearchTrello(word)
|
||||
trello_search.process()
|
||||
emails, hosts, urls = trello_search.get_results()
|
||||
all_emails.extend(emails)
|
||||
hosts = filter(hosts)
|
||||
trello_urls = filter(urls)
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, hosts, 'host', 'trello')
|
||||
db.store_all(word, emails, 'email', 'trello')
|
||||
await store(trello_search, engineitem, store_results=True)
|
||||
|
||||
elif engineitem == 'twitter':
|
||||
print('\033[94m[*] Searching Twitter usernames using Google. \033[0m')
|
||||
from theHarvester.discovery import twittersearch
|
||||
twitter_search = twittersearch.SearchTwitter(word, limit)
|
||||
twitter_search.process()
|
||||
people = twitter_search.get_people()
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, people, 'name', 'twitter')
|
||||
|
||||
if len(people) == 0:
|
||||
print('\n[*] No users found.\n\n')
|
||||
else:
|
||||
print('\n[*] Users found: ' + str(len(people)))
|
||||
print('---------------------')
|
||||
for user in sorted(list(set(people))):
|
||||
print(user)
|
||||
await store(twitter_search, engineitem, store_people=True)
|
||||
|
||||
elif engineitem == 'virustotal':
|
||||
print('\033[94m[*] Searching VirusTotal. \033[0m')
|
||||
from theHarvester.discovery import virustotal
|
||||
virustotal_search = virustotal.SearchVirustotal(word)
|
||||
virustotal_search.process()
|
||||
hosts = filter(virustotal_search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'virustotal')
|
||||
await store(virustotal_search, engineitem, store_host=True)
|
||||
|
||||
elif engineitem == 'yahoo':
|
||||
print('\033[94m[*] Searching Yahoo. \033[0m')
|
||||
from theHarvester.discovery import yahoosearch
|
||||
yahoo_search = yahoosearch.SearchYahoo(word, limit)
|
||||
yahoo_search.process()
|
||||
hosts = yahoo_search.get_hostnames()
|
||||
emails = yahoo_search.get_emails()
|
||||
all_hosts.extend(filter(hosts))
|
||||
all_emails.extend(filter(emails))
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'yahoo')
|
||||
db.store_all(word, all_emails, 'email', 'yahoo')
|
||||
await store(yahoo_search, engineitem, store_host=True, store_emails=True)
|
||||
else:
|
||||
print('\033[93m[!] Invalid source.\n\n \033[0m')
|
||||
sys.exit(1)
|
||||
|
|
@ -478,14 +408,15 @@ def start():
|
|||
print('---------------------')
|
||||
all_hosts = sorted(list(set(all_hosts)))
|
||||
full_host = hostchecker.Checker(all_hosts)
|
||||
full, ips = asyncio.run(full_host.check())
|
||||
db = stash.stash_manager()
|
||||
# full, ips = asyncio.run(full_host.check())
|
||||
full, ips = await full_host.check()
|
||||
db = stash.StashManager()
|
||||
for host in full:
|
||||
host = str(host)
|
||||
print(host)
|
||||
host_ip = [netaddr_ip.format() for netaddr_ip in sorted([netaddr.IPAddress(ip) for ip in ips])]
|
||||
db.store_all(word, host_ip, 'ip', 'DNS-resolver')
|
||||
length_urls = len(trello_urls)
|
||||
length_urls = len(all_urls)
|
||||
if length_urls == 0:
|
||||
if len(engines) >= 1 and 'trello' in engines:
|
||||
print('\n[*] No Trello URLs found.')
|
||||
|
|
@ -493,7 +424,7 @@ def start():
|
|||
total = length_urls
|
||||
print('\n[*] Trello URLs found: ' + str(total))
|
||||
print('--------------------')
|
||||
for url in sorted(trello_urls):
|
||||
for url in sorted(all_urls):
|
||||
print(url)
|
||||
|
||||
# DNS brute force
|
||||
|
|
@ -501,7 +432,7 @@ def start():
|
|||
if dnsbrute is True:
|
||||
print('\n[*] Starting DNS brute force.')
|
||||
a = dnssearch.DnsForce(word, dnsserver, verbose=True)
|
||||
res = a.process()
|
||||
a.process()
|
||||
# print('\n[*] Hosts found after DNS brute force:')
|
||||
# for y in res:
|
||||
# print('-------------------------------------')
|
||||
|
|
@ -640,7 +571,7 @@ def start():
|
|||
if filename != "":
|
||||
try:
|
||||
print('\n[*] Reporting started.')
|
||||
db = stash.stash_manager()
|
||||
db = stash.StashManager()
|
||||
scanboarddata = db.getscanboarddata()
|
||||
latestscanresults = db.getlatestscanresults(word)
|
||||
previousscanresults = db.getlatestscanresults(word, previousday=True)
|
||||
|
|
@ -702,10 +633,10 @@ def start():
|
|||
file.write('<banner><!--' + res[1] + '--></banner>')
|
||||
reg_server = re.compile('Server:.*')
|
||||
temp = reg_server.findall(res[1])
|
||||
if temp != []:
|
||||
if temp:
|
||||
shodanalysis.append(res[0] + ':' + temp[0])
|
||||
file.write('</shodan>')
|
||||
if shodanalysis != []:
|
||||
if shodanalysis:
|
||||
shodanalysis = sorted(set(shodanalysis))
|
||||
file.write('<servers>')
|
||||
for x in shodanalysis:
|
||||
|
|
@ -722,9 +653,9 @@ def start():
|
|||
sys.exit(0)
|
||||
|
||||
|
||||
def entry_point():
|
||||
async def entry_point():
|
||||
try:
|
||||
start()
|
||||
await start()
|
||||
except KeyboardInterrupt:
|
||||
print('\n\n\033[93m[!] ctrl+c detected from user, quitting.\n\n \033[0m')
|
||||
except Exception as error_entry_point:
|
||||
|
|
@ -733,4 +664,4 @@ def entry_point():
|
|||
|
||||
|
||||
if __name__ == '__main__':
|
||||
entry_point()
|
||||
asyncio.run(main=entry_point())
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
__all__ = ['baidusearch',
|
||||
'bingsearch',
|
||||
'crtsh',
|
||||
'certspottersearch',
|
||||
'dnssearch',
|
||||
'dogpilesearch',
|
||||
'duckduckgosearch',
|
||||
|
|
|
|||
|
|
@ -12,25 +12,28 @@ def __init__(self, word, limit):
|
|||
self.hostname = 'www.baidu.com'
|
||||
self.limit = limit
|
||||
|
||||
def do_search(self):
|
||||
async def do_search(self):
|
||||
headers = {
|
||||
'Host': self.hostname,
|
||||
'User-agent': Core.get_user_agent()
|
||||
}
|
||||
base_url = f'https://{self.server}/s?wd=%40{self.word}&pnxx&oq={self.word}'
|
||||
urls = [base_url.replace("xx", str(num)) for num in range(0, self.limit, 10) if num <= self.limit]
|
||||
req = (grequests.get(url, headers=headers, timeout=5) for url in urls)
|
||||
"""req = (grequests.get(url, headers=headers, timeout=5) for url in urls)
|
||||
responses = grequests.imap(req, size=5)
|
||||
for response in responses:
|
||||
self.total_results += response.content.decode('UTF-8')
|
||||
self.total_results += response.content.decode('UTF-8')"""
|
||||
responses = await async_fetcher.fetch_all(urls, headers=headers)
|
||||
for response in responses:
|
||||
self.total_results += response
|
||||
|
||||
def process(self):
|
||||
self.do_search()
|
||||
async def process(self):
|
||||
await self.do_search()
|
||||
|
||||
def get_emails(self):
|
||||
async def get_emails(self):
|
||||
rawres = myparser.Parser(self.total_results, self.word)
|
||||
return rawres.emails()
|
||||
|
||||
def get_hostnames(self):
|
||||
async def get_hostnames(self):
|
||||
rawres = myparser.Parser(self.total_results, self.word)
|
||||
return rawres.hostnames()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
from theHarvester.discovery.constants import *
|
||||
from theHarvester.lib.core import *
|
||||
from theHarvester.parsers import myparser
|
||||
import grequests
|
||||
# import grequests
|
||||
from theHarvester.lib.core import async_fetcher
|
||||
|
||||
|
||||
class SearchBing:
|
||||
|
|
@ -17,7 +18,8 @@ def __init__(self, word, limit, start):
|
|||
self.bingApi = Core.bing_key()
|
||||
self.counter = start
|
||||
|
||||
def do_search(self):
|
||||
async def do_search(self):
|
||||
print('hello from bing do search')
|
||||
headers = {
|
||||
'Host': self.hostname,
|
||||
'Cookie': 'SRCHHPGUSR=ADLT=DEMOTE&NRSLT=50',
|
||||
|
|
@ -26,12 +28,11 @@ def do_search(self):
|
|||
}
|
||||
base_url = f'https://{self.server}/search?q=%40"{self.word}"&count=50&first=xx'
|
||||
urls = [base_url.replace("xx", str(num)) for num in range(0, self.limit, 50) if num <= self.limit]
|
||||
req = (grequests.get(url, headers=headers, timeout=5) for url in urls)
|
||||
responses = grequests.imap(req, size=5)
|
||||
responses = await async_fetcher.fetch_all(urls, headers=headers)
|
||||
for response in responses:
|
||||
self.total_results += response.content.decode('UTF-8')
|
||||
self.total_results += response
|
||||
|
||||
def do_search_api(self):
|
||||
async def do_search_api(self):
|
||||
url = 'https://api.cognitive.microsoft.com/bing/v7.0/search?'
|
||||
params = {
|
||||
'q': self.word,
|
||||
|
|
@ -41,12 +42,10 @@ def do_search_api(self):
|
|||
'safesearch': 'Off'
|
||||
}
|
||||
headers = {'User-Agent': Core.get_user_agent(), 'Ocp-Apim-Subscription-Key': self.bingApi}
|
||||
grequests_resp = grequests.get(url=url, headers=headers, params=params)
|
||||
response = grequests.map([grequests_resp])
|
||||
self.results = response[0].content.decode('UTF-8')
|
||||
self.results = await async_fetcher.fetch_all([url], headers=headers, params=params)
|
||||
self.total_results += self.results
|
||||
|
||||
def do_search_vhost(self):
|
||||
async def do_search_vhost(self):
|
||||
headers = {
|
||||
'Host': self.hostname,
|
||||
'Cookie': 'mkt=en-US;ui=en-US;SRCHHPGUSR=NEWWND=0&ADLT=DEMOTE&NRSLT=50',
|
||||
|
|
@ -55,33 +54,32 @@ def do_search_vhost(self):
|
|||
}
|
||||
base_url = f'http://{self.server}/search?q=ip:{self.word}&go=&count=50&FORM=QBHL&qs=n&first=xx'
|
||||
urls = [base_url.replace("xx", str(num)) for num in range(0, self.limit, 50) if num <= self.limit]
|
||||
req = (grequests.get(url, headers=headers, timeout=5) for url in urls)
|
||||
responses = grequests.imap(req, size=5)
|
||||
responses = await async_fetcher.fetch_all(urls, headers=headers)
|
||||
for response in responses:
|
||||
self.total_results += response.content.decode('UTF-8')
|
||||
self.total_results += response
|
||||
|
||||
def get_emails(self):
|
||||
async def get_emails(self):
|
||||
rawres = myparser.Parser(self.total_results, self.word)
|
||||
return rawres.emails()
|
||||
|
||||
def get_hostnames(self):
|
||||
async def get_hostnames(self):
|
||||
rawres = myparser.Parser(self.total_results, self.word)
|
||||
return rawres.hostnames()
|
||||
|
||||
def get_allhostnames(self):
|
||||
async def get_allhostnames(self):
|
||||
rawres = myparser.Parser(self.total_results, self.word)
|
||||
return rawres.hostnames_all()
|
||||
|
||||
def process(self, api):
|
||||
async def process(self, api):
|
||||
if api == 'yes':
|
||||
if self.bingApi is None:
|
||||
raise MissingKey(True)
|
||||
else:
|
||||
if api == 'yes':
|
||||
self.do_search_api()
|
||||
await self.do_search_api()
|
||||
else:
|
||||
self.do_search()
|
||||
await self.do_search()
|
||||
print(f'\tSearching {self.counter} results.')
|
||||
|
||||
def process_vhost(self):
|
||||
self.do_search_vhost()
|
||||
async def process_vhost(self):
|
||||
await self.do_search_vhost()
|
||||
|
|
|
|||
29
theHarvester/discovery/certspottersearch.py
Normal file
29
theHarvester/discovery/certspottersearch.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
from theHarvester.lib.core import *
|
||||
import requests
|
||||
|
||||
|
||||
class SearchCertspoter:
|
||||
|
||||
def __init__(self, word):
|
||||
self.word = word
|
||||
self.totalhosts = set()
|
||||
|
||||
def do_search(self) -> None:
|
||||
base_url = f'https://api.certspotter.com/v1/issuances?domain={self.word}&expand=dns_names'
|
||||
headers = {'User-Agent': Core.get_user_agent()}
|
||||
try:
|
||||
request = requests.get(base_url, headers=headers)
|
||||
response = request.json()
|
||||
for dct in response:
|
||||
for key, value in dct.items():
|
||||
if key == 'dns_names':
|
||||
self.totalhosts.update({name for name in value if name})
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
def get_hostnames(self) -> set:
|
||||
return self.totalhosts
|
||||
|
||||
def process(self):
|
||||
self.do_search()
|
||||
print('\tSearching results.')
|
||||
|
|
@ -8,21 +8,20 @@ class DnsForce:
|
|||
|
||||
def __init__(self, domain, dnsserver, verbose=False):
|
||||
self.domain = domain
|
||||
self.file = 'wordlists/dns-names.txt'
|
||||
self.subdo = False
|
||||
self.verbose = verbose
|
||||
try:
|
||||
f = open(self.file, 'r')
|
||||
except Exception:
|
||||
print('Error opening DNS dictionary file.')
|
||||
sys.exit()
|
||||
self.list = f.readlines()
|
||||
with open('wordlists/dns-names.txt', 'r') as file:
|
||||
self.list = file.readlines()
|
||||
except FileNotFoundError:
|
||||
with open('/etc/theHarvester/dns-names.txt', 'r') as file:
|
||||
self.list = file.readlines()
|
||||
|
||||
def run(self, host):
|
||||
hostname = str(host.split('\n')[0]) + '.' + str(self.domain)
|
||||
if self.verbose:
|
||||
ESC = chr(27)
|
||||
sys.stdout.write(ESC + '[2K' + ESC + '[G')
|
||||
esc = chr(27)
|
||||
sys.stdout.write(esc + '[2K' + esc + '[G')
|
||||
sys.stdout.write('\r' + hostname + ' - ')
|
||||
sys.stdout.flush()
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ def __init__(self, word, limit):
|
|||
self.word = word
|
||||
self.total_results = ""
|
||||
self.server = 'api.github.com'
|
||||
self.hostname = 'api.github.com'
|
||||
self.limit = limit
|
||||
self.counter = 0
|
||||
self.page = 1
|
||||
|
|
@ -51,13 +50,13 @@ def fragments_from_response(response: Response) -> List[str]:
|
|||
return [fragment for fragment in fragments if fragment is not None]
|
||||
|
||||
@staticmethod
|
||||
def page_from_response(page: str, response: Response) -> Optional[int]:
|
||||
def page_from_response(page: str, response: Response) -> Optional[Any]:
|
||||
page_link = response.links.get(page)
|
||||
if page_link:
|
||||
parsed = urlparse.urlparse(page_link.get("url"))
|
||||
params = urlparse.parse_qs(parsed.query)
|
||||
page = params.get('page') or [None]
|
||||
page_number = page[0] and int(page[0])
|
||||
pages: List[Any] = params.get('page', [None])
|
||||
page_number = pages[0] and int(pages[0])
|
||||
return page_number
|
||||
else:
|
||||
return None
|
||||
|
|
@ -82,7 +81,7 @@ def do_search(self, page: Optional[int]) -> Response:
|
|||
else:
|
||||
url = f'https://{self.server}/search/code?q="{self.word}"&page={page}'
|
||||
headers = {
|
||||
'Host': self.hostname,
|
||||
'Host': self.server,
|
||||
'User-agent': Core.get_user_agent(),
|
||||
'Accept': "application/vnd.github.v3.text-match+json",
|
||||
'Authorization': 'token {}'.format(self.key)
|
||||
|
|
|
|||
|
|
@ -155,4 +155,4 @@ def send_dorks(self): # Helper function to minimize code reusability.
|
|||
time.sleep(getDelay())
|
||||
self.totalresults += self.results
|
||||
except Exception as e:
|
||||
print(f'\tException Occurred {e}')
|
||||
print(f'\tException Occurred {e}')
|
||||
|
|
@ -24,6 +24,10 @@ def port_scanner(self, host, ports):
|
|||
except Exception as e:
|
||||
print(e)
|
||||
self.lock.release()
|
||||
|
||||
if(len(self.ports)) == 0:
|
||||
print("No ports found on host: {0}".format(host))
|
||||
|
||||
return openports
|
||||
|
||||
def process(self):
|
||||
|
|
|
|||
|
|
@ -40,4 +40,4 @@ def search_ip(self, ip):
|
|||
except Exception as e:
|
||||
print(f'Error occurred in the Shodan IP search module: {e}')
|
||||
finally:
|
||||
return self.hostdatarow
|
||||
return self.hostdatarow
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
from theHarvester.discovery.constants import *
|
||||
from theHarvester.lib.core import *
|
||||
import requests
|
||||
from pprint import pprint
|
||||
|
||||
|
||||
class SearchSpyse:
|
||||
|
|
@ -20,7 +19,6 @@ def do_search(self):
|
|||
headers = {'User-Agent': Core.get_user_agent()}
|
||||
request = requests.get(base_url, headers=headers)
|
||||
self.results = request.json()
|
||||
pprint(self.results)
|
||||
# self.totalresults += self.results
|
||||
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
from theHarvester.lib.core import *
|
||||
from bs4 import BeautifulSoup
|
||||
import requests
|
||||
import aiohttp
|
||||
import asyncio
|
||||
|
||||
|
||||
class SearchSuip:
|
||||
|
|
@ -12,37 +14,54 @@ def __init__(self, word: str):
|
|||
self.totalhosts: set = set()
|
||||
self.totalips: set = set()
|
||||
|
||||
def do_search(self):
|
||||
async def request(self, url, params):
|
||||
headers = {'User-Agent': Core.get_user_agent()}
|
||||
params = (
|
||||
('act', 'subfinder'),
|
||||
)
|
||||
data = {'url': self.word.replace('www.', ''), 'Submit1': 'Submit'}
|
||||
timeout = aiohttp.ClientTimeout(total=300)
|
||||
# by default timeout is 5 minutes we will change that to 6 minutes
|
||||
# Depending on the domain and if it has a lot of subdomains you may want to tweak it
|
||||
# The results are well worth the wait :)
|
||||
try:
|
||||
async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
|
||||
async with session.post(url, params=params, data=data) as resp:
|
||||
await asyncio.sleep(3)
|
||||
return await resp.text()
|
||||
except Exception as e:
|
||||
print(f'An exception has occurred: {e}')
|
||||
return ''
|
||||
|
||||
data = {
|
||||
'url': self.word.replace('www.', ''),
|
||||
'Submit1': 'Submit'
|
||||
}
|
||||
response = requests.post('https://suip.biz/', headers=headers, params=params, data=data)
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
hosts: list = str(soup.find('pre')).splitlines()
|
||||
self.clean_hosts(hosts)
|
||||
params = (
|
||||
('act', 'amass'),
|
||||
)
|
||||
# change act to amass now
|
||||
response = requests.post('https://suip.biz/', headers=headers, params=params, data=data)
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
hosts: list = str(soup.find('pre')).splitlines()
|
||||
self.clean_hosts(hosts)
|
||||
async def handler(self, url):
|
||||
first_data = [url, (('act', 'subfinder'),), ]
|
||||
second_data = [url, (('act', 'amass'),), ]
|
||||
async_requests = [
|
||||
self.request(url=url, params=params)
|
||||
for url, params in [first_data, second_data]
|
||||
]
|
||||
results = await asyncio.gather(*async_requests)
|
||||
return results
|
||||
|
||||
def get_hostnames(self) -> set:
|
||||
async def do_search(self):
|
||||
try:
|
||||
results = await self.handler(url="https://suip.biz/")
|
||||
for result in results:
|
||||
# results has both responses in a list
|
||||
# iterate through them and parse out the urls
|
||||
soup = BeautifulSoup(str(result), 'html.parser')
|
||||
hosts: list = str(soup.find('pre')).splitlines()
|
||||
await self.clean_hosts(hosts)
|
||||
except Exception as e:
|
||||
print('An exception has occurred: ', e)
|
||||
import traceback as t
|
||||
t.print_exc()
|
||||
|
||||
async def get_hostnames(self) -> set:
|
||||
return self.totalhosts
|
||||
|
||||
def process(self):
|
||||
self.do_search()
|
||||
async def process(self):
|
||||
await self.do_search()
|
||||
print('\tSearching results.')
|
||||
|
||||
def clean_hosts(self, soup_hosts):
|
||||
async def clean_hosts(self, soup_hosts):
|
||||
for host in soup_hosts:
|
||||
host = str(host).strip()
|
||||
if len(host) > 1 and 'pre' not in host:
|
||||
|
|
|
|||
|
|
@ -3,54 +3,91 @@
|
|||
import random
|
||||
from typing import Set, Union, Any
|
||||
import yaml
|
||||
import asyncio
|
||||
import aiohttp
|
||||
|
||||
|
||||
class Core:
|
||||
@staticmethod
|
||||
def version() -> str:
|
||||
return '3.1.0.dev2'
|
||||
return '3.1.1dev3'
|
||||
|
||||
@staticmethod
|
||||
def bing_key() -> str:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['bing']['key']
|
||||
try:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
except FileNotFoundError:
|
||||
with open('/etc/theHarvester/api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['bing']['key']
|
||||
return keys['apikeys']['bing']['key']
|
||||
|
||||
@staticmethod
|
||||
def github_key() -> str:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['github']['key']
|
||||
try:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
except FileNotFoundError:
|
||||
with open('/etc/theHarvester/api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['github']['key']
|
||||
return keys['apikeys']['github']['key']
|
||||
|
||||
@staticmethod
|
||||
def hunter_key() -> str:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
try:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
except FileNotFoundError:
|
||||
with open('/etc/theHarvester/api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['hunter']['key']
|
||||
return keys['apikeys']['hunter']['key']
|
||||
|
||||
@staticmethod
|
||||
def intelx_key() -> str:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['intelx']['key']
|
||||
try:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
except FileNotFoundError:
|
||||
with open('/etc/theHarvester/api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['intelx']['key']
|
||||
return keys['apikeys']['intelx']['key']
|
||||
|
||||
@staticmethod
|
||||
def security_trails_key() -> str:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['securityTrails']['key']
|
||||
try:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
except FileNotFoundError:
|
||||
with open('/etc/theHarvester/api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['securityTrails']['key']
|
||||
return keys['apikeys']['securityTrails']['key']
|
||||
|
||||
@staticmethod
|
||||
def shodan_key() -> str:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['shodan']['key']
|
||||
try:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
except FileNotFoundError:
|
||||
with open('/etc/theHarvester/api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['shodan']['key']
|
||||
return keys['apikeys']['shodan']['key']
|
||||
|
||||
@staticmethod
|
||||
def spyse_key() -> str:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['spyse']['key']
|
||||
try:
|
||||
with open('api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
except FileNotFoundError:
|
||||
with open('/etc/theHarvester/api-keys.yaml', 'r') as api_keys:
|
||||
keys = yaml.safe_load(api_keys)
|
||||
return keys['apikeys']['spyse']['key']
|
||||
return keys['apikeys']['spyse']['key']
|
||||
|
||||
@staticmethod
|
||||
def banner() -> None:
|
||||
|
|
@ -61,18 +98,19 @@ def banner() -> None:
|
|||
print(r"* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *")
|
||||
print(r"* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *")
|
||||
print('* *')
|
||||
print(f'* theHarvester {Core.version()} *')
|
||||
print('* theHarvester {} *'.format(Core.version()))
|
||||
print('* Coded by Christian Martorella *')
|
||||
print('* Edge-Security Research *')
|
||||
print('* cmartorella@edge-security.com *')
|
||||
print('* *')
|
||||
print('******************************************************************* \n\n \033[0m')
|
||||
print('******************************************************************* \n\n\033[0m')
|
||||
|
||||
@staticmethod
|
||||
def get_supportedengines() -> Set[Union[str, Any]]:
|
||||
supportedengines = {'baidu',
|
||||
'bing',
|
||||
'bingapi',
|
||||
'certspotter',
|
||||
'crtsh',
|
||||
'dnsdumpster',
|
||||
'dogpile',
|
||||
|
|
@ -332,3 +370,34 @@ def get_user_agent() -> str:
|
|||
'Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00'
|
||||
]
|
||||
return random.choice(user_agents)
|
||||
|
||||
|
||||
class async_fetcher:
|
||||
|
||||
@staticmethod
|
||||
async def fetch(session, url, params='') -> str:
|
||||
# This fetch method solely focuses on get requests
|
||||
# TODO determine if method for post requests is necessary
|
||||
if len(params) == '':
|
||||
async with session.get(url, params=params) as response:
|
||||
await asyncio.sleep(3)
|
||||
return await response.text()
|
||||
else:
|
||||
async with session.get(url) as response:
|
||||
await asyncio.sleep(3)
|
||||
return await response.text()
|
||||
|
||||
@staticmethod
|
||||
async def fetch_all(urls, headers='', params='') -> list:
|
||||
timeout = aiohttp.ClientTimeout(total=10)
|
||||
if len(headers) == 0:
|
||||
headers = {'User-Agent': Core.get_user_agent()}
|
||||
if len(params) == 0:
|
||||
async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
|
||||
texts = await asyncio.gather(*[async_fetcher.fetch(session, url) for url in urls])
|
||||
return texts
|
||||
else:
|
||||
# Indicates the request has certain params
|
||||
async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
|
||||
texts = await asyncio.gather(*[async_fetcher.fetch(session, url, params) for url in urls])
|
||||
return texts
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
import aiodns
|
||||
import asyncio
|
||||
import socket
|
||||
from typing import Tuple, Any
|
||||
|
||||
|
||||
class Checker:
|
||||
|
|
@ -18,7 +19,7 @@ def __init__(self, hosts: list):
|
|||
self.addresses: set = set()
|
||||
|
||||
@staticmethod
|
||||
async def query(host, resolver) -> [list, str]:
|
||||
async def query(host, resolver) -> Tuple[str, Any]:
|
||||
try:
|
||||
result = await resolver.gethostbyname(host, socket.AF_INET)
|
||||
addresses = result.addresses
|
||||
|
|
@ -42,7 +43,7 @@ async def check(self):
|
|||
self.realhosts.append(host)
|
||||
self.addresses.update({addr for addr in address})
|
||||
# address may be a list of ips
|
||||
# and do a set comprehension to remove uniques
|
||||
# and do a set comprehension to remove duplicates
|
||||
self.realhosts.sort()
|
||||
self.addresses = list(self.addresses)
|
||||
return self.realhosts, self.addresses
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
import plotly.graph_objs as go
|
||||
|
||||
try:
|
||||
db = stash.stash_manager()
|
||||
db = stash.StashManager()
|
||||
db.do_init()
|
||||
except Exception as error:
|
||||
print(f'{error}')
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
import sqlite3
|
||||
|
||||
|
||||
class stash_manager:
|
||||
class StashManager:
|
||||
|
||||
def __init__(self):
|
||||
self.db = "stash.sqlite"
|
||||
|
|
@ -18,7 +18,7 @@ def __init__(self):
|
|||
def do_init(self):
|
||||
conn = sqlite3.connect(self.db)
|
||||
c = conn.cursor()
|
||||
c.execute('CREATE TABLE results (domain text, resource text, type text, find_date date, source text)')
|
||||
c.execute('CREATE TABLE IF NOT EXISTS results (domain text, resource text, type text, find_date date, source text)')
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return
|
||||
|
|
@ -131,7 +131,7 @@ def getlatestscanresults(self, domain, previousday=False):
|
|||
self.previousscanresults = results
|
||||
return self.previousscanresults
|
||||
except Exception as e:
|
||||
print('Error in getting the previous scan results from the database: ' + str(e))
|
||||
print(f'Error in getting the previous scan results from the database: {e}')
|
||||
else:
|
||||
try:
|
||||
c = conn.cursor()
|
||||
|
|
@ -148,9 +148,9 @@ def getlatestscanresults(self, domain, previousday=False):
|
|||
self.latestscanresults = results
|
||||
return self.latestscanresults
|
||||
except Exception as e:
|
||||
print('Error in getting the latest scan results from the database: ' + str(e))
|
||||
print(f'Error in getting the latest scan results from the database: {e}')
|
||||
except Exception as e:
|
||||
print('Error connecting to theHarvester database: ' + str(e))
|
||||
print(f'Error connecting to theHarvester database: {e}')
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ def generatepreviousscanresults(self, previousscanresults):
|
|||
'''
|
||||
return html
|
||||
except Exception as e:
|
||||
print('Error generating the previous scan results HTML code: ' + str(e))
|
||||
print(f'Error generating the previous scan results HTML code: {e}')
|
||||
|
||||
def generatelatestscanresults(self, latestscanresults):
|
||||
try:
|
||||
|
|
|
|||
Loading…
Reference in a new issue