From b0865ba05311387c76b336644d7822678a24d5bd Mon Sep 17 00:00:00 2001 From: David Mougeolle Date: Fri, 20 Mar 2020 00:46:16 +0100 Subject: [PATCH] Remove useless async generator utilities --- theHarvester/__main__.py | 1 - theHarvester/discovery/dnssearch.py | 3 +- theHarvester/lib/itertools.py | 48 ----------------------------- 3 files changed, 1 insertion(+), 51 deletions(-) delete mode 100644 theHarvester/lib/itertools.py diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 8a46a5fc..e491d19c 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -478,7 +478,6 @@ async def handler(lst): generate_postprocessing_callback, reverse_all_ips_in_range, serialize_ip_range) - from theHarvester.lib.itertools import merge_async_generators # reverse each iprange in a separate task __reverse_dns_tasks = {} diff --git a/theHarvester/discovery/dnssearch.py b/theHarvester/discovery/dnssearch.py index e6f42762..e6a1c98f 100644 --- a/theHarvester/discovery/dnssearch.py +++ b/theHarvester/discovery/dnssearch.py @@ -11,13 +11,12 @@ from __future__ import absolute_import, division, print_function import asyncio -import functools import re import sys from aiodns import DNSResolver from ipaddress import IPv4Address, IPv4Network -from typing import AsyncGenerator, Awaitable, Callable, Iterable, List +from typing import Awaitable, Callable, Iterable, List # TODO: need big focus on performance and results parsing, now does the basic. diff --git a/theHarvester/lib/itertools.py b/theHarvester/lib/itertools.py deleted file mode 100644 index f1d023d7..00000000 --- a/theHarvester/lib/itertools.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -========= -Itertools -========= - -Async iterator / generator tools. -""" - -from __future__ import absolute_import, division, print_function - -import asyncio - -from typing import Any, AsyncIterable - -# TODO: need big focus on performance and results parsing, now does the basic. - -##################################################################### -# DNS FORCE -##################################################################### - -def merge_async_generators( - *aiters: AsyncIterable[Any]) -> AsyncIterable[Any]: - """ - Merge several async generators into a single one. - The merged generator provides items in the order of availability. - - Parameters - ---------- - aiters: AsyncIterable. - A list of iterators / generators. - - Returns - ------- - out: AsyncIterable. - The merged iterator / generator. - """ - # merge async iterators, proof of concept - queue = asyncio.Queue(1) - async def drain(aiter): - async for item in aiter: - await queue.put(item) - async def merged(): - while not all(task.done() for task in tasks) or not queue.empty(): - yield await queue.get() - tasks = [asyncio.create_task(drain(aiter)) for aiter in aiters] - return merged()