From 16c792719e8720e0ffbe74a321256d025dc9e08b Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Tue, 31 Dec 2019 00:31:19 +0000 Subject: [PATCH] Fix flake8 errors and update pipenv.lock --- Pipfile.lock | 60 ++++++++++++--------------- theHarvester/discovery/threatcrowd.py | 9 ++-- theHarvester/lib/reportgraph.py | 4 +- theHarvester/lib/stash.py | 9 ++-- 4 files changed, 34 insertions(+), 48 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 62c3266b..a63f6437 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "26a684cf27c35ec099638a43ccfcec6f9b44c176680a1b70698971d291527bd6" + "sha256": "db30e9238d9e3525a4640e043831ac8c039b4af08d806b3c1c9a7510a8062334" }, "pipfile-spec": 6, "requires": {}, @@ -40,6 +40,13 @@ "index": "pypi", "version": "==3.6.2" }, + "aiosqlite": { + "hashes": [ + "sha256:4f02314a42db6722dc26f2a6119c64e3f05f141f57bbf2b1e1f9fd741b6d7fb8" + ], + "index": "pypi", + "version": "==0.11.0" + }, "async-timeout": { "hashes": [ "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", @@ -185,25 +192,25 @@ }, "multidict": { "hashes": [ - "sha256:20081b14c923d2c5122c13e060d0ee334e078e1802c36006b20c8d7a59ee6a52", - "sha256:335344a3c3b19845c73a7826f359c51c4a12a1ccd2392b5f572a63b452bfc771", - "sha256:49e80c53659c7ac50ec1c4b5fa50f045b67fffeb5b735dccb6205e4ff122e8b6", - "sha256:615a282acd530a1bc1b01f069a8c5874cb7c2780c287a2895ad5ab7407540e9d", - "sha256:63d9a3d93a514549760cb68c82864966bddb6ab53a3326931c8db9ad29414603", - "sha256:77264002c184538df5dcb4fc1de5df6803587fa30bbe12203a7a3436b8aafc0f", - "sha256:7dd6f6a51b64d0a6473bc30c53e1d73fcb461c437f43662b7d6d701bd90db253", - "sha256:7f4e591ec80619e74c50b7800f9db9b0e01d2099094767050dfe2e78e1c41839", - "sha256:824716bba5a4efd74ddd36a3830efb9e49860149514ef7c41aac0144757ebb5d", - "sha256:8f30ead697c2e37147d82ba8019952b5ea99bd3d1052f1f1ebff951eaa953209", - "sha256:a03efe8b7591c77d9ad4b9d81dcfb9c96e538ae25eb114385f35f4d7ffa3bac2", - "sha256:b86e8e33a0a24240b293e7fad233a7e886bae6e51ca6923d39f4e313dd1d5578", - "sha256:c1c64c93b8754a5cebd495d136f47a5ca93cbfceba532e306a768c27a0c1292b", - "sha256:d4dafdcfbf0ac80fc5f00603f0ce43e487c654ae34a656e4749f175d9832b1b5", - "sha256:daf6d89e47418e38af98e1f2beb3fe0c8aa34806f681d04df314c0f131dcf01d", - "sha256:e03b7addca96b9eb24d6eabbdc3041e8f71fd47b316e0f3c0fa993fc7b99002c", - "sha256:ff53a434890a16356bc45c0b90557efd89d0e5a820dbab37015d7ee630c6707a" + "sha256:0f04bf4c15d8417401a10a650c349ccc0285943681bfd87d3690587d7714a9b4", + "sha256:15a61c0df2d32487e06f6084eabb48fd9e8b848315e397781a70caf9670c9d78", + "sha256:3c5e2dcbe6b04cbb4303e47a896757a77b676c5e5db5528be7ff92f97ba7ab95", + "sha256:5d2b32b890d9e933d3ced417924261802a857abdee9507b68c75014482145c03", + "sha256:5e5fb8bfebf87f2e210306bf9dd8de2f1af6782b8b78e814060ae9254ab1f297", + "sha256:63ba2be08d82ea2aa8b0f7942a74af4908664d26cb4ff60c58eadb1e33e7da00", + "sha256:73740fcdb38f0adcec85e97db7557615b50ec4e5a3e73e35878720bcee963382", + "sha256:78bed18e7f1eb21f3d10ff3acde900b4d630098648fe1d65bb4abfb3e22c4900", + "sha256:a02fade7b5476c4f88efe9593ff2f3286698d8c6d715ba4f426954f73f382026", + "sha256:aacbde3a8875352a640efa2d1b96e5244a29b0f8df79cbf1ec6470e86fd84697", + "sha256:be813fb9e5ce41a5a99a29cdb857144a1bd6670883586f995b940a4878dc5238", + "sha256:bfcad6da0b8839f01a819602aaa5c5a5b4c85ecbfae9b261a31df3d9262fb31e", + "sha256:c2bfc0db3166e68515bc4a2b9164f4f75ae9c793e9635f8651f2c9ffc65c8dad", + "sha256:c66d11870ae066499a3541963e6ce18512ca827c2aaeaa2f4e37501cee39ac5d", + "sha256:cc7f2202b753f880c2e4123f9aacfdb94560ba893e692d24af271dac41f8b8d9", + "sha256:d1f45e5bb126662ba66ee579831ce8837b1fd978115c9657e32eb3c75b92973d", + "sha256:ed5f3378c102257df9e2dc9ce6468dabf68bee9ec34969cfdc472631aba00316" ], - "version": "==4.7.2" + "version": "==4.7.3" }, "netaddr": { "hashes": [ @@ -396,14 +403,6 @@ "index": "pypi", "version": "==3.7.9" }, - "importlib-metadata": { - "hashes": [ - "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45", - "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f" - ], - "markers": "python_version < '3.8'", - "version": "==1.3.0" - }, "mccabe": { "hashes": [ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", @@ -542,13 +541,6 @@ "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" ], "version": "==0.1.7" - }, - "zipp": { - "hashes": [ - "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", - "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" - ], - "version": "==0.6.0" } } } diff --git a/theHarvester/discovery/threatcrowd.py b/theHarvester/discovery/threatcrowd.py index 2d9bcfe5..3c893043 100644 --- a/theHarvester/discovery/threatcrowd.py +++ b/theHarvester/discovery/threatcrowd.py @@ -1,6 +1,5 @@ from theHarvester.lib.core import * from theHarvester.parsers import myparser -import grequests class SearchThreatcrowd: @@ -10,7 +9,7 @@ def __init__(self, word): self.results = "" self.totalresults = "" - def do_search(self): + async def do_search(self): base_url = f'https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={self.word}' headers = {'User-Agent': Core.get_user_agent()} try: @@ -21,10 +20,10 @@ def do_search(self): print(e) self.totalresults += self.results - def get_hostnames(self): + async def get_hostnames(self): return myparser.Parser(self.results, self.word).hostnames() def process(self): - self.do_search() - self.get_hostnames() + await self.do_search() + await self.get_hostnames() print('\tSearching results.') diff --git a/theHarvester/lib/reportgraph.py b/theHarvester/lib/reportgraph.py index 73922cc1..ccbdd5d8 100644 --- a/theHarvester/lib/reportgraph.py +++ b/theHarvester/lib/reportgraph.py @@ -4,8 +4,6 @@ import plotly.graph_objs as go - - class GraphGenerator: def __init__(self, domain): @@ -95,4 +93,4 @@ def drawscattergraphscanhistory(self, domain, scanhistorydomain): output_type='div') return scatterchartcode except Exception as e: - print(f'Error generating HTML for the historical graph for domain: {e}') \ No newline at end of file + print(f'Error generating HTML for the historical graph for domain: {e}') diff --git a/theHarvester/lib/stash.py b/theHarvester/lib/stash.py index 6c3db5f1..4c700113 100644 --- a/theHarvester/lib/stash.py +++ b/theHarvester/lib/stash.py @@ -252,19 +252,16 @@ async def latestscanchartdata(self, domain): cursor = await conn.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="host"''', (domain, latestdate,)) scandetailshost = await cursor.fetchall() self.latestscandomain["scandetailshost"] = scandetailshost - cursor = await conn.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="email"''', - (domain, latestdate,)) + cursor = await conn.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="email"''', (domain, latestdate,)) scandetailsemail = await cursor.fetchall() self.latestscandomain["scandetailsemail"] = scandetailsemail cursor = await conn.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="ip"''', (domain, latestdate,)) scandetailsip = await cursor.fetchall() self.latestscandomain["scandetailsip"] = scandetailsip - cursor = await conn.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="vhost"''', - (domain, latestdate,)) + cursor = await conn.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="vhost"''', (domain, latestdate,)) scandetailsvhost = await cursor.fetchall() self.latestscandomain["scandetailsvhost"] = scandetailsvhost - cursor = await conn.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="shodan"''', - (domain, latestdate,)) + cursor = await conn.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="shodan"''', (domain, latestdate,)) scandetailsshodan = await cursor.fetchall() self.latestscandomain["scandetailsshodan"] = scandetailsshodan return self.latestscandomain