Remove useless async generator utilities

This commit is contained in:
David Mougeolle 2020-03-20 00:46:16 +01:00
parent 295bda21ea
commit b0865ba053
3 changed files with 1 additions and 51 deletions

View file

@ -478,7 +478,6 @@ async def handler(lst):
generate_postprocessing_callback,
reverse_all_ips_in_range,
serialize_ip_range)
from theHarvester.lib.itertools import merge_async_generators
# reverse each iprange in a separate task
__reverse_dns_tasks = {}

View file

@ -11,13 +11,12 @@
from __future__ import absolute_import, division, print_function
import asyncio
import functools
import re
import sys
from aiodns import DNSResolver
from ipaddress import IPv4Address, IPv4Network
from typing import AsyncGenerator, Awaitable, Callable, Iterable, List
from typing import Awaitable, Callable, Iterable, List
# TODO: need big focus on performance and results parsing, now does the basic.

View file

@ -1,48 +0,0 @@
# -*- coding: utf-8 -*-
"""
=========
Itertools
=========
Async iterator / generator tools.
"""
from __future__ import absolute_import, division, print_function
import asyncio
from typing import Any, AsyncIterable
# TODO: need big focus on performance and results parsing, now does the basic.
#####################################################################
# DNS FORCE
#####################################################################
def merge_async_generators(
*aiters: AsyncIterable[Any]) -> AsyncIterable[Any]:
"""
Merge several async generators into a single one.
The merged generator provides items in the order of availability.
Parameters
----------
aiters: AsyncIterable.
A list of iterators / generators.
Returns
-------
out: AsyncIterable.
The merged iterator / generator.
"""
# merge async iterators, proof of concept
queue = asyncio.Queue(1)
async def drain(aiter):
async for item in aiter:
await queue.put(item)
async def merged():
while not all(task.done() for task in tasks) or not queue.empty():
yield await queue.get()
tasks = [asyncio.create_task(drain(aiter)) for aiter in aiters]
return merged()