Initial commit of restAPI

.github/workflows/theHarvester.yml

@@ -151,3 +151,4 @@ jobs:
     - name: Static type checking with mypy
       run: |
         mypy --pretty theHarvester/*/*.py
+        mypy --pretty theHarvester/*/*/*.py

requirements/base.txt

@@ -6,6 +6,7 @@ beautifulsoup4==4.9.3
 censys==1.1.1
 certifi==2020.12.5
 dnspython==2.1.0
+fastapi==0.63.0
 netaddr==0.8.0
 plotly==4.14.3
 pyppeteer==0.2.5
@@ -13,7 +14,11 @@ PyYAML==5.4.1
 requests==2.25.1
 retrying==1.3.3
 rocketreach==2.1.2
+setuptools==54.2.0
 shodan==1.25.0
+slowapi==0.1.4
+starlette==0.14.2
 texttable==1.6.3
 lxml==4.6.3
+uvicorn==0.13.4
 uvloop==0.15.2; platform_system != "Windows"

restfulHarvest.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+import uvicorn
+import theHarvester.lib.api.api as api
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-H', '--host', default='127.0.0.1', help='IP address to listen on default is 127.0.0.1')
+parser.add_argument('-p', '--port', default=5000, help='Port to bind the web server to, default is 5000')
+parser.add_argument('-l', '--log-level', default='info', help='Set logging level, default is info but [critical|error|warning|info|debug|trace] can be set')
+
+args = parser.parse_args()
+
+if __name__ == "__main__":
+    uvicorn.run(app=api.app, host=args.host, port=args.port, log_level=args.log_level)

theHarvester/lib/api/api_example.py

@@ -0,0 +1,54 @@
+"""
+Example script to query theHarvester rest API, obtain results, and write out to stdout as well as an html
+"""
+
+import asyncio
+import pprint
+
+import aiohttp
+
+
+async def fetch_json(session, url):
+    async with session.get(url) as response:
+        return await response.json()
+
+
+async def fetch(session, url):
+    async with session.get(url) as response:
+        return await response.text()
+
+
+async def main():
+    """
+    Just a simple example of how to interact with the rest api
+    you can easily use requests instead of aiohttp or whatever you best see fit
+    """
+    url = "http://127.0.0.1:5000"
+    domain = "netflix.com"
+    query_url = f'{url}/query?limit=300&filename=output&source=bing,baidu,duckduckgo,dogpile&domain={domain}'
+    async with aiohttp.ClientSession() as session:
+        fetched_json = await fetch_json(session, query_url)
+        emails = fetched_json["emails"]
+        ips = fetched_json["ips"]
+        urls = fetched_json["urls"]
+        html_filename = fetched_json["html_file"]
+
+    async with aiohttp.ClientSession() as session:
+        html_file = await fetch(session, f"{url}{html_filename}")
+
+    if len(html_file) > 0:
+        with open('results.html', 'w+') as fp:
+            fp.write(html_file)
+
+    print('Emails found: ')
+    pprint.pprint(emails, indent=4)
+    print('\n')
+    print('Ips found: ')
+    pprint.pprint(ips, indent=4)
+    print('\n')
+    print('Urls found: ')
+    pprint.pprint(urls, indent=4)
+
+
+if __name__ == '__main__':
+    asyncio.run(main())