From 518e88df9724f8328d9f6aea41ded3e65b230761 Mon Sep 17 00:00:00 2001 From: mpa Date: Wed, 22 Apr 2020 09:49:53 +0400 Subject: [PATCH] feat(proxy chain): Chained proxy support in aiohttp session Add ChainProxyConnector support, !pin pydantic to "1.4", add documentation on aiohttp connector. --- aiogram/api/client/bot.py | 8 +- aiogram/api/client/session/aiohttp.py | 99 ++++++---- aiogram/api/client/session/asyncio.py | 177 ------------------ aiogram/api/client/session/base.py | 10 +- docs/api/client/session/aiohttp.md | 76 ++++++++ mkdocs.yml | 3 + poetry.lock | 151 ++++++++++----- pyproject.toml | 4 +- .../test_session/test_aiohttp_session.py | 27 ++- .../test_session/test_asyncio_session.py | 3 - 10 files changed, 281 insertions(+), 277 deletions(-) delete mode 100644 aiogram/api/client/session/asyncio.py create mode 100644 docs/api/client/session/aiohttp.md delete mode 100644 tests/test_api/test_client/test_session/test_asyncio_session.py diff --git a/aiogram/api/client/bot.py b/aiogram/api/client/bot.py index 27293d83..47fde6fc 100644 --- a/aiogram/api/client/bot.py +++ b/aiogram/api/client/bot.py @@ -2,7 +2,7 @@ from __future__ import annotations import datetime from contextlib import asynccontextmanager -from typing import Any, AsyncIterator, List, Optional, TypeVar, Union +from typing import Any, AsyncIterator, List, Optional, TypeVar, Union, cast from async_lru import alru_cache @@ -110,7 +110,7 @@ from ..types import ( WebhookInfo, ) from .session.aiohttp import AiohttpSession -from .session.base import BaseSession +from .session.base import BaseSession, PT T = TypeVar("T") @@ -121,12 +121,12 @@ class Bot(ContextInstanceMixin["Bot"]): """ def __init__( - self, token: str, session: Optional[BaseSession] = None, parse_mode: Optional[str] = None + self, token: str, session: Optional[BaseSession[PT]] = None, parse_mode: Optional[str] = None ) -> None: validate_token(token) if session is None: - session = AiohttpSession() + session = cast(BaseSession[PT], AiohttpSession()) self.session = session self.parse_mode = parse_mode diff --git a/aiogram/api/client/session/aiohttp.py b/aiogram/api/client/session/aiohttp.py index de0e35b8..fdc59a7a 100644 --- a/aiogram/api/client/session/aiohttp.py +++ b/aiogram/api/client/session/aiohttp.py @@ -1,6 +1,19 @@ from __future__ import annotations -from typing import AsyncGenerator, Callable, Optional, TypeVar, Type, Tuple, Dict, Union, cast +from typing import ( + Any, + AsyncGenerator, + Callable, + Optional, + Iterable, + TypeVar, + Type, + Tuple, + Dict, + List, + Union, + cast, +) from aiohttp import ClientSession, ClientTimeout, FormData, BasicAuth, TCPConnector @@ -9,22 +22,63 @@ from aiogram.api.methods import Request, TelegramMethod from .base import PRODUCTION, BaseSession, TelegramAPIServer T = TypeVar("T") -_ProxyType = Union[str, Tuple[str, BasicAuth]] +_ProxyBasic = Union[str, Tuple[str, BasicAuth]] +_ProxyChain = Iterable[_ProxyBasic] +_ProxyType = Union[_ProxyChain, _ProxyBasic] + + +def _retrieve_basic(basic: _ProxyBasic) -> Dict[str, Any]: + from aiohttp_socks.utils import parse_proxy_url # type: ignore + + proxy_auth: Optional[BasicAuth] = None + + if isinstance(basic, str): + proxy_url = basic + else: + proxy_url, proxy_auth = basic + + proxy_type, host, port, username, password = parse_proxy_url(proxy_url) + if isinstance(proxy_auth, BasicAuth): + username = proxy_auth.login + password = proxy_auth.password + + return dict( + proxy_type=proxy_type, + host=host, + port=port, + username=username, + password=password, + rdns=True, + ) + + +def _prepare_connector(chain_or_plain: _ProxyType) -> Tuple[Type["TCPConnector"], Dict[str, Any]]: + from aiohttp_socks import ProxyInfo, ProxyConnector, ChainProxyConnector # type: ignore + + # since tuple is Iterable(compatible with _ProxyChain) object, we assume that + # user wants chained proxies if tuple is a pair of string(url) and BasicAuth + if isinstance(chain_or_plain, str) or ( + isinstance(chain_or_plain, tuple) and len(chain_or_plain) == 2 + ): + return ProxyConnector, _retrieve_basic(chain_or_plain) + + infos: List[ProxyInfo] = [] + for basic in chain_or_plain: + infos.append(ProxyInfo(**_retrieve_basic(basic))) + + return ChainProxyConnector, dict(proxy_infos=infos) class AiohttpSession(BaseSession[_ProxyType]): def __init__( self, api: TelegramAPIServer = PRODUCTION, - json_loads: Optional[Callable[..., str]] = None, + json_loads: Optional[Callable[[str, ...], Any]] = None, json_dumps: Optional[Callable[..., str]] = None, proxy: Optional[_ProxyType] = None, ): super(AiohttpSession, self).__init__( - api=api, - json_loads=json_loads, - json_dumps=json_dumps, - proxy=proxy + api=api, json_loads=json_loads, json_dumps=json_dumps, proxy=proxy ) self._session: Optional[ClientSession] = None self._connector_type: Type[TCPConnector] = TCPConnector @@ -32,47 +86,24 @@ class AiohttpSession(BaseSession[_ProxyType]): if self.proxy: try: - from aiohttp_socks import ProxyConnector - from aiohttp_socks.utils import parse_proxy_url + self._connector_type, self._connector_init = _prepare_connector(self.proxy) except ImportError as exc: # pragma: no cover raise UserWarning( "In order to use aiohttp client for proxy requests, install " "https://pypi.org/project/aiohttp-socks/" ) from exc - if isinstance(self.proxy, str): - proxy_url, proxy_auth = self.proxy, None - else: - proxy_url, proxy_auth = self.proxy - - self._connector_type = ProxyConnector - - proxy_type, host, port, username, password = parse_proxy_url(proxy_url) - if proxy_auth: - if not username: - username = proxy_auth.login - if not password: - password = proxy_auth.password - - self._connector_init.update( - dict( - proxy_type=proxy_type, host=host, port=port, - username=username, password=password, - rdns=True, - ) - ) - async def create_session(self) -> ClientSession: if self._session is None or self._session.closed: - self._session = ClientSession() + self._session = ClientSession(connector=self._connector_type(**self._connector_init)) return self._session - async def close(self): + async def close(self) -> None: if self._session is not None and not self._session.closed: await self._session.close() - def build_form_data(self, request: Request): + def build_form_data(self, request: Request) -> FormData: form = FormData(quote_fields=False) for key, value in request.data.items(): if value is None: diff --git a/aiogram/api/client/session/asyncio.py b/aiogram/api/client/session/asyncio.py deleted file mode 100644 index 0a26793e..00000000 --- a/aiogram/api/client/session/asyncio.py +++ /dev/null @@ -1,177 +0,0 @@ -from __future__ import annotations - -import io -import uuid -import asyncio -from urllib.parse import urlencode, urlparse, ParseResult -from collections import deque -from typing import AsyncGenerator, Callable, Optional, TypeVar, Set, Deque, Tuple, Dict, cast -from contextlib import asynccontextmanager - -from aiogram.api.methods import Request, TelegramMethod - -from aiogram.api.client.session.base import PRODUCTION, BaseSession, TelegramAPIServer - - -T = TypeVar("T") -StreamType = Tuple[asyncio.StreamReader, asyncio.StreamWriter] - - -def _get_boundary() -> bytes: - return b"%032x" % uuid.uuid4().int - - -async def _get_headers(reader: asyncio.StreamReader) -> Optional[bytes]: - headers = await reader.readuntil(b"\r\n\r\n") - if headers[-4:] != b"\r\n\r\n": - return None - return headers - - -class AsyncioSession(BaseSession): - def __init__( - self, - api: TelegramAPIServer = PRODUCTION, - json_loads: Optional[Callable] = None, - json_dumps: Optional[Callable] = None, - ) -> None: - super().__init__( - api=api, json_loads=json_loads, json_dumps=json_dumps, - ) - - self._closed = False - # we use stream req-time semaphore - self._semaphore = asyncio.Semaphore() - - # keep some connections' underlying socket open with the help of the following dss - self._connections_deque: Deque[Optional[StreamType]] = deque() - self._busy_connections: Set[Optional[StreamType]] = set() - - async def _encode_multipart_formdata(self, request: Request) -> Tuple[bytes, bytes]: - boundary = _get_boundary() - body = io.BytesIO() - - for key, val in request.data.items(): - if val is None: - continue - - part = ( - b"--%b\r\n" - b'content-disposition: form-data; name="%b"\r\n\r\n' - b"%b" - b"\r\n" % (boundary, key.encode(), str(self.prepare_value(val)).encode()) - ) - - body.write(part) - - for key, file in request.files.items(): # type: ignore - headers = ( - b"--%b\r\n" - b"content-disposition:" - b" form-data;" - b' name="%b";' - b' filename="%b"' - b"\r\n\r\n" % (boundary, key.encode(), (file.filename or key).encode(),) - ) - - body.write(headers) - - async for chunk in file.read(file.chunk_size): - body.write(chunk) - - body.write(b"\r\n") - - body.write(b"--%b--\r\n\r\n" % boundary) - return b"multipart/form-data; boundary=%b" % boundary, body.getvalue() - - def _encode_formdata(self, request: Request) -> Tuple[bytes, bytes]: - raw_data: Dict[str, str] = {} - for key, val in request.data.items(): - if val is None: - continue - raw_data[key] = str(self.prepare_value(val)) - data = urlencode(raw_data) - return b"application/x-www-form-urlencoded", data.encode() - - async def form_request(self, parsed: ParseResult, request: Request): - plain_http = b"POST %b HTTP/1.1\r\n" b"host: %b\r\n" % ( - str(parsed.path).encode(), - str(parsed.hostname).encode(), - ) - - if request.files: - content_type, data = await self._encode_multipart_formdata(request) - else: - content_type, data = self._encode_formdata(request) - - plain_http += ( - b"content-length: %i\r\n" - b"content-type: %b\r\n" - b"\r\n" - b"%b" % (len(data or ""), content_type, data) - ) - - return plain_http - - @asynccontextmanager - async def _get_stream(self, host: str, port: int) -> AsyncGenerator[StreamType, None]: # type: ignore - await self._semaphore.acquire() - rw = self._connections_deque.popleft() if self._connections_deque else None - self._busy_connections.add(rw) - try: - if rw is None: - rw = await asyncio.open_connection(host=host, port=port, ssl=True) - yield rw - finally: - self._busy_connections.discard(rw) - self._connections_deque.append(rw) - self._semaphore.release() - - async def make_request(self, token: str, method: TelegramMethod[T]) -> T: - request = method.build_request() - parsed = urlparse(self.api.api_url(token=token, method=request.method)) - plain_http = await self.form_request(parsed, request) - - async with self._get_stream( - parsed.hostname, parsed.port or 443 - ) as stream: # type: StreamType - r, w = stream - w.write(plain_http) - await w.drain() - - headers = await _get_headers(r) - if not headers: - raise asyncio.CancelledError("Could not properly read headers") - - headers = headers.lower() - - index = headers.index(b"content-length:") + 16 - json_data = self.json_loads( - await r.readexactly(int(headers[index : headers.index(b"\r", index)])) - ) - - response = method.build_response(json_data) - self.raise_for_status(response) - return cast(T, response.result) - - async def stream_content( - self, url: str, timeout: int, chunk_size: int - ) -> AsyncGenerator[bytes, None]: - yield b"" # todo - - async def close(self): - if self._closed: - return - - self._closed = True - - async def _close(_: asyncio.StreamReader, w: asyncio.StreamWriter): - w.close() - await w.wait_closed() - - await asyncio.gather( - *(_close(*rw) for rw in (*self._connections_deque, *self._busy_connections) if rw) - ) - - self._connections_deque = deque([None] * len(self._connections_deque)) - self._busy_connections.clear() diff --git a/aiogram/api/client/session/base.py b/aiogram/api/client/session/base.py index eb50de1d..8b417a10 100644 --- a/aiogram/api/client/session/base.py +++ b/aiogram/api/client/session/base.py @@ -12,16 +12,16 @@ from ...methods import Response, TelegramMethod from ..telegram import PRODUCTION, TelegramAPIServer T = TypeVar("T") -_ProxyType = TypeVar("_ProxyType") +PT = TypeVar("PT") -class BaseSession(abc.ABC, Generic[_ProxyType]): +class BaseSession(abc.ABC, Generic[PT]): def __init__( self, api: Optional[TelegramAPIServer] = None, - json_loads: Optional[Callable[..., str]] = None, + json_loads: Optional[Callable[[str, ...], Any]] = None, json_dumps: Optional[Callable[..., str]] = None, - proxy: Optional[_ProxyType] = None, + proxy: Optional[PT] = None, ) -> None: if api is None: api = PRODUCTION @@ -74,7 +74,7 @@ class BaseSession(abc.ABC, Generic[_ProxyType]): return {k: self.clean_json(v) for k, v in value.items() if v is not None} return value - async def __aenter__(self) -> BaseSession: + async def __aenter__(self) -> BaseSession[PT]: return self async def __aexit__( diff --git a/docs/api/client/session/aiohttp.md b/docs/api/client/session/aiohttp.md new file mode 100644 index 00000000..9eab5ede --- /dev/null +++ b/docs/api/client/session/aiohttp.md @@ -0,0 +1,76 @@ +# Aiohttp session + +AiohttpSession represents a wrapper-class around `ClientSession` from [aiohttp]('https://pypi.org/project/aiohttp/') + +Currently `AiohttpSession` is a default session used in `aiogram.Bot` + +## Usage example + +```python +from aiogram import Bot +from aiogram.api.client.session.aiohttp import AiohttpSession + +session = AiohttpSession() +Bot('token', session=session) +``` + + +## Proxy requests in AiohttpSession + +In order to use AiohttpSession with proxy connector you have to install [aiohttp-socks]('https://pypi.org/project/aiohttp-socks/') + +Binding session to bot: +```python +from aiogram import Bot +from aiogram.api.client.session.aiohttp import AiohttpSession + +session = AiohttpSession(proxy="protocol://host:port/") +Bot(token="bot token", session=session) +``` + +!!! note "Protocols" + Only following protocols are supported: http(tunneling), socks4(a), socks5 as aiohttp_socks documentation claims. + + +### Authorization + +Proxy authorization credentials can be specified in proxy URL or come as an instance of `aiohttp.BasicAuth` containing +login and password. + +Consider examples: +```python +from aiohttp import BasicAuth +from aiogram.api.client.session.aiohttp import AiohttpSession + +auth = BasicAuth(login="user", password="password") +session = AiohttpSession(proxy=("protocol://host:port", auth)) +# or simply include your basic auth credential in URL +session = AiohttpSession(proxy="protocol://user:password@host:port") +``` + +!!! note "Credential priorities" + Aiogram prefers `BasicAuth` over username and password in URL, so + if proxy URL contains login and password and `BasicAuth` object is passed at the same time + aiogram will use login and password from `BasicAuth` instance. + + +### Proxy chains + +Since [aiohttp-socks]('https://pypi.org/project/aiohttp-socks/') supports proxy chains, you're able to use them in aiogram + +Example of chain proxies: +```python +from aiohttp import BasicAuth +from aiogram.api.client.session.aiohttp import AiohttpSession + +auth = BasicAuth(login="user", password="password") +session = AiohttpSession( + proxy={"protocol0://host0:port0", + "protocol1://user:password@host1:port1", + ("protocol2://host2:port2", auth),} # can be any iterable if not set +) +``` + +## Location + +- `from aiogram.api.client.session.aiohttp import AiohttpSession` diff --git a/mkdocs.yml b/mkdocs.yml index 747b5f4e..0d77e640 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -41,6 +41,9 @@ nav: - install.md - Bot API: - api/index.md + - Client: + - Session: + - aiohttp: api/client/session/aiohttp.md - Methods: - Available methods: api/methods/index.md - Getting updates: diff --git a/poetry.lock b/poetry.lock index e74f26aa..1a98fcae 100644 --- a/poetry.lock +++ b/poetry.lock @@ -24,6 +24,18 @@ yarl = ">=1.0,<2.0" [package.extras] speedups = ["aiodns", "brotlipy", "cchardet"] +[[package]] +category = "main" +description = "Proxy connector for aiohttp" +name = "aiohttp-socks" +optional = false +python-versions = "*" +version = "0.3.8" + +[package.dependencies] +aiohttp = ">=2.3.2" +attrs = ">=19.2.0" + [[package]] category = "dev" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." @@ -171,7 +183,7 @@ description = "Code coverage measurement for Python" name = "coverage" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" -version = "5.0.4" +version = "5.1" [package.extras] toml = ["toml"] @@ -310,15 +322,15 @@ category = "dev" description = "An autocompletion tool for Python that can be used for text editors." name = "jedi" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "0.16.0" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.17.0" [package.dependencies] -parso = ">=0.5.2" +parso = ">=0.7.0" [package.extras] qa = ["flake8 (3.7.9)"] -testing = ["colorama (0.4.1)", "docopt", "pytest (>=3.9.0,<5.0.0)"] +testing = ["colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"] [[package]] category = "dev" @@ -326,7 +338,7 @@ description = "A very fast and expressive template engine." name = "jinja2" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.11.1" +version = "2.11.2" [package.dependencies] MarkupSafe = ">=0.23" @@ -334,6 +346,14 @@ MarkupSafe = ">=0.23" [package.extras] i18n = ["Babel (>=0.8)"] +[[package]] +category = "dev" +description = "Lightweight pipelining: using Python functions as pipeline jobs." +name = "joblib" +optional = false +python-versions = "*" +version = "0.14.1" + [[package]] category = "dev" description = "Python LiveReload is an awesome tool for web developers" @@ -508,13 +528,16 @@ description = "Natural Language Toolkit" name = "nltk" optional = false python-versions = "*" -version = "3.4.5" +version = "3.5" [package.dependencies] -six = "*" +click = "*" +joblib = "*" +regex = "*" +tqdm = "*" [package.extras] -all = ["pyparsing", "scikit-learn", "python-crfsuite", "matplotlib", "scipy", "gensim", "requests", "twython", "numpy"] +all = ["requests", "numpy", "python-crfsuite", "scikit-learn", "twython", "pyparsing", "scipy", "matplotlib", "gensim"] corenlp = ["requests"] machine_learning = ["gensim", "numpy", "python-crfsuite", "scikit-learn", "scipy"] plot = ["matplotlib"] @@ -539,7 +562,7 @@ description = "A Python Parser" name = "parso" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "0.6.2" +version = "0.7.0" [package.extras] testing = ["docopt", "pytest (>=3.0.7)"] @@ -833,6 +856,17 @@ optional = false python-versions = ">= 3.5" version = "6.0.4" +[[package]] +category = "dev" +description = "Fast, Extensible Progress Meter" +name = "tqdm" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*" +version = "4.45.0" + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "argopt", "pydoc-markdown"] + [[package]] category = "dev" description = "Traitlets Python config system" @@ -909,9 +943,10 @@ testing = ["jaraco.itertools", "func-timeout"] [extras] fast = ["uvloop"] +proxy = ["aiohttp-socks"] [metadata] -content-hash = "bce1bc7bf9eb949283094490a084d484a3d2f7b0d992ea3a4ea1e75401f6e2da" +content-hash = "de211bfe6e7caebbe7b3f7107e30587e0c137d99ae9ac6e9d2c3dd833d731049" python-versions = "^3.7" [metadata.files] @@ -933,6 +968,10 @@ aiohttp = [ {file = "aiohttp-3.6.2-py3-none-any.whl", hash = "sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4"}, {file = "aiohttp-3.6.2.tar.gz", hash = "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326"}, ] +aiohttp-socks = [ + {file = "aiohttp_socks-0.3.8-py3-none-any.whl", hash = "sha256:e13dbb8913ccf3e236a33dd6247d0dcd7f7a908373a0a36053f98a8f18525364"}, + {file = "aiohttp_socks-0.3.8.tar.gz", hash = "sha256:143dff61f5c9e75a5f8482d1c01f98153869139c39f5c600935b2b4b52089bf7"}, +] appdirs = [ {file = "appdirs-1.4.3-py2.py3-none-any.whl", hash = "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e"}, {file = "appdirs-1.4.3.tar.gz", hash = "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92"}, @@ -989,37 +1028,37 @@ colorama = [ {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, ] coverage = [ - {file = "coverage-5.0.4-cp27-cp27m-macosx_10_12_x86_64.whl", hash = "sha256:8a620767b8209f3446197c0e29ba895d75a1e272a36af0786ec70fe7834e4307"}, - {file = "coverage-5.0.4-cp27-cp27m-macosx_10_13_intel.whl", hash = "sha256:73aa6e86034dad9f00f4bbf5a666a889d17d79db73bc5af04abd6c20a014d9c8"}, - {file = "coverage-5.0.4-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:408ce64078398b2ee2ec08199ea3fcf382828d2f8a19c5a5ba2946fe5ddc6c31"}, - {file = "coverage-5.0.4-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:cda33311cb9fb9323958a69499a667bd728a39a7aa4718d7622597a44c4f1441"}, - {file = "coverage-5.0.4-cp27-cp27m-win32.whl", hash = "sha256:5f587dfd83cb669933186661a351ad6fc7166273bc3e3a1531ec5c783d997aac"}, - {file = "coverage-5.0.4-cp27-cp27m-win_amd64.whl", hash = "sha256:9fad78c13e71546a76c2f8789623eec8e499f8d2d799f4b4547162ce0a4df435"}, - {file = "coverage-5.0.4-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:2e08c32cbede4a29e2a701822291ae2bc9b5220a971bba9d1e7615312efd3037"}, - {file = "coverage-5.0.4-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:922fb9ef2c67c3ab20e22948dcfd783397e4c043a5c5fa5ff5e9df5529074b0a"}, - {file = "coverage-5.0.4-cp35-cp35m-macosx_10_12_x86_64.whl", hash = "sha256:c3fc325ce4cbf902d05a80daa47b645d07e796a80682c1c5800d6ac5045193e5"}, - {file = "coverage-5.0.4-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:046a1a742e66d065d16fb564a26c2a15867f17695e7f3d358d7b1ad8a61bca30"}, - {file = "coverage-5.0.4-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6ad6ca45e9e92c05295f638e78cd42bfaaf8ee07878c9ed73e93190b26c125f7"}, - {file = "coverage-5.0.4-cp35-cp35m-win32.whl", hash = "sha256:eda55e6e9ea258f5e4add23bcf33dc53b2c319e70806e180aecbff8d90ea24de"}, - {file = "coverage-5.0.4-cp35-cp35m-win_amd64.whl", hash = "sha256:4a8a259bf990044351baf69d3b23e575699dd60b18460c71e81dc565f5819ac1"}, - {file = "coverage-5.0.4-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:f372cdbb240e09ee855735b9d85e7f50730dcfb6296b74b95a3e5dea0615c4c1"}, - {file = "coverage-5.0.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a37c6233b28e5bc340054cf6170e7090a4e85069513320275a4dc929144dccf0"}, - {file = "coverage-5.0.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:443be7602c790960b9514567917af538cac7807a7c0c0727c4d2bbd4014920fd"}, - {file = "coverage-5.0.4-cp36-cp36m-win32.whl", hash = "sha256:165a48268bfb5a77e2d9dbb80de7ea917332a79c7adb747bd005b3a07ff8caf0"}, - {file = "coverage-5.0.4-cp36-cp36m-win_amd64.whl", hash = "sha256:0a907199566269e1cfa304325cc3b45c72ae341fbb3253ddde19fa820ded7a8b"}, - {file = "coverage-5.0.4-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:513e6526e0082c59a984448f4104c9bf346c2da9961779ede1fc458e8e8a1f78"}, - {file = "coverage-5.0.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:3844c3dab800ca8536f75ae89f3cf566848a3eb2af4d9f7b1103b4f4f7a5dad6"}, - {file = "coverage-5.0.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:641e329e7f2c01531c45c687efcec8aeca2a78a4ff26d49184dce3d53fc35014"}, - {file = "coverage-5.0.4-cp37-cp37m-win32.whl", hash = "sha256:db1d4e38c9b15be1521722e946ee24f6db95b189d1447fa9ff18dd16ba89f732"}, - {file = "coverage-5.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:62061e87071497951155cbccee487980524d7abea647a1b2a6eb6b9647df9006"}, - {file = "coverage-5.0.4-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:65a7e00c00472cd0f59ae09d2fb8a8aaae7f4a0cf54b2b74f3138d9f9ceb9cb2"}, - {file = "coverage-5.0.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1f66cf263ec77af5b8fe14ef14c5e46e2eb4a795ac495ad7c03adc72ae43fafe"}, - {file = "coverage-5.0.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:85596aa5d9aac1bf39fe39d9fa1051b0f00823982a1de5766e35d495b4a36ca9"}, - {file = "coverage-5.0.4-cp38-cp38-win32.whl", hash = "sha256:86a0ea78fd851b313b2e712266f663e13b6bc78c2fb260b079e8b67d970474b1"}, - {file = "coverage-5.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:03f630aba2b9b0d69871c2e8d23a69b7fe94a1e2f5f10df5049c0df99db639a0"}, - {file = "coverage-5.0.4-cp39-cp39-win32.whl", hash = "sha256:7c9762f80a25d8d0e4ab3cb1af5d9dffbddb3ee5d21c43e3474c84bf5ff941f7"}, - {file = "coverage-5.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:4482f69e0701139d0f2c44f3c395d1d1d37abd81bfafbf9b6efbe2542679d892"}, - {file = "coverage-5.0.4.tar.gz", hash = "sha256:1b60a95fc995649464e0cd48cecc8288bac5f4198f21d04b8229dc4097d76823"}, + {file = "coverage-5.1-cp27-cp27m-macosx_10_12_x86_64.whl", hash = "sha256:0cb4be7e784dcdc050fc58ef05b71aa8e89b7e6636b99967fadbdba694cf2b65"}, + {file = "coverage-5.1-cp27-cp27m-macosx_10_13_intel.whl", hash = "sha256:c317eaf5ff46a34305b202e73404f55f7389ef834b8dbf4da09b9b9b37f76dd2"}, + {file = "coverage-5.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:b83835506dfc185a319031cf853fa4bb1b3974b1f913f5bb1a0f3d98bdcded04"}, + {file = "coverage-5.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5f2294dbf7875b991c381e3d5af2bcc3494d836affa52b809c91697449d0eda6"}, + {file = "coverage-5.1-cp27-cp27m-win32.whl", hash = "sha256:de807ae933cfb7f0c7d9d981a053772452217df2bf38e7e6267c9cbf9545a796"}, + {file = "coverage-5.1-cp27-cp27m-win_amd64.whl", hash = "sha256:bf9cb9a9fd8891e7efd2d44deb24b86d647394b9705b744ff6f8261e6f29a730"}, + {file = "coverage-5.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:acf3763ed01af8410fc36afea23707d4ea58ba7e86a8ee915dfb9ceff9ef69d0"}, + {file = "coverage-5.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:dec5202bfe6f672d4511086e125db035a52b00f1648d6407cc8e526912c0353a"}, + {file = "coverage-5.1-cp35-cp35m-macosx_10_12_x86_64.whl", hash = "sha256:7a5bdad4edec57b5fb8dae7d3ee58622d626fd3a0be0dfceda162a7035885ecf"}, + {file = "coverage-5.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1601e480b9b99697a570cea7ef749e88123c04b92d84cedaa01e117436b4a0a9"}, + {file = "coverage-5.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:dbe8c6ae7534b5b024296464f387d57c13caa942f6d8e6e0346f27e509f0f768"}, + {file = "coverage-5.1-cp35-cp35m-win32.whl", hash = "sha256:a027ef0492ede1e03a8054e3c37b8def89a1e3c471482e9f046906ba4f2aafd2"}, + {file = "coverage-5.1-cp35-cp35m-win_amd64.whl", hash = "sha256:0e61d9803d5851849c24f78227939c701ced6704f337cad0a91e0972c51c1ee7"}, + {file = "coverage-5.1-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:2d27a3f742c98e5c6b461ee6ef7287400a1956c11421eb574d843d9ec1f772f0"}, + {file = "coverage-5.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:66460ab1599d3cf894bb6baee8c684788819b71a5dc1e8fa2ecc152e5d752019"}, + {file = "coverage-5.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:5c542d1e62eece33c306d66fe0a5c4f7f7b3c08fecc46ead86d7916684b36d6c"}, + {file = "coverage-5.1-cp36-cp36m-win32.whl", hash = "sha256:2742c7515b9eb368718cd091bad1a1b44135cc72468c731302b3d641895b83d1"}, + {file = "coverage-5.1-cp36-cp36m-win_amd64.whl", hash = "sha256:dead2ddede4c7ba6cb3a721870f5141c97dc7d85a079edb4bd8d88c3ad5b20c7"}, + {file = "coverage-5.1-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:01333e1bd22c59713ba8a79f088b3955946e293114479bbfc2e37d522be03355"}, + {file = "coverage-5.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:e1ea316102ea1e1770724db01998d1603ed921c54a86a2efcb03428d5417e489"}, + {file = "coverage-5.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:adeb4c5b608574a3d647011af36f7586811a2c1197c861aedb548dd2453b41cd"}, + {file = "coverage-5.1-cp37-cp37m-win32.whl", hash = "sha256:782caea581a6e9ff75eccda79287daefd1d2631cc09d642b6ee2d6da21fc0a4e"}, + {file = "coverage-5.1-cp37-cp37m-win_amd64.whl", hash = "sha256:00f1d23f4336efc3b311ed0d807feb45098fc86dee1ca13b3d6768cdab187c8a"}, + {file = "coverage-5.1-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:402e1744733df483b93abbf209283898e9f0d67470707e3c7516d84f48524f55"}, + {file = "coverage-5.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:a3f3654d5734a3ece152636aad89f58afc9213c6520062db3978239db122f03c"}, + {file = "coverage-5.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:6402bd2fdedabbdb63a316308142597534ea8e1895f4e7d8bf7476c5e8751fef"}, + {file = "coverage-5.1-cp38-cp38-win32.whl", hash = "sha256:8fa0cbc7ecad630e5b0f4f35b0f6ad419246b02bc750de7ac66db92667996d24"}, + {file = "coverage-5.1-cp38-cp38-win_amd64.whl", hash = "sha256:79a3cfd6346ce6c13145731d39db47b7a7b859c0272f02cdb89a3bdcbae233a0"}, + {file = "coverage-5.1-cp39-cp39-win32.whl", hash = "sha256:a82b92b04a23d3c8a581fc049228bafde988abacba397d57ce95fe95e0338ab4"}, + {file = "coverage-5.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb28a7245de68bf29f6fb199545d072d1036a1917dca17a1e75bbb919e14ee8e"}, + {file = "coverage-5.1.tar.gz", hash = "sha256:f90bfc4ad18450c80b024036eaf91e4a246ae287701aaa88eaebebf150868052"}, ] decorator = [ {file = "decorator-4.4.2-py2.py3-none-any.whl", hash = "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760"}, @@ -1061,12 +1100,16 @@ isort = [ {file = "isort-4.3.21.tar.gz", hash = "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1"}, ] jedi = [ - {file = "jedi-0.16.0-py2.py3-none-any.whl", hash = "sha256:b4f4052551025c6b0b0b193b29a6ff7bdb74c52450631206c262aef9f7159ad2"}, - {file = "jedi-0.16.0.tar.gz", hash = "sha256:d5c871cb9360b414f981e7072c52c33258d598305280fef91c6cae34739d65d5"}, + {file = "jedi-0.17.0-py2.py3-none-any.whl", hash = "sha256:cd60c93b71944d628ccac47df9a60fec53150de53d42dc10a7fc4b5ba6aae798"}, + {file = "jedi-0.17.0.tar.gz", hash = "sha256:df40c97641cb943661d2db4c33c2e1ff75d491189423249e989bcea4464f3030"}, ] jinja2 = [ - {file = "Jinja2-2.11.1-py2.py3-none-any.whl", hash = "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49"}, - {file = "Jinja2-2.11.1.tar.gz", hash = "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250"}, + {file = "Jinja2-2.11.2-py2.py3-none-any.whl", hash = "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035"}, + {file = "Jinja2-2.11.2.tar.gz", hash = "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0"}, +] +joblib = [ + {file = "joblib-0.14.1-py2.py3-none-any.whl", hash = "sha256:bdb4fd9b72915ffb49fde2229ce482dd7ae79d842ed8c2b4c932441495af1403"}, + {file = "joblib-0.14.1.tar.gz", hash = "sha256:0630eea4f5664c463f23fbf5dcfc54a2bc6168902719fa8e19daf033022786c8"}, ] livereload = [ {file = "livereload-2.6.1-py2.py3-none-any.whl", hash = "sha256:78d55f2c268a8823ba499305dcac64e28ddeb9a92571e12d543cd304faf5817b"}, @@ -1140,6 +1183,11 @@ markupsafe = [ {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6"}, {file = "MarkupSafe-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2"}, {file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-win32.whl", hash = "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"}, {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"}, ] mccabe = [ @@ -1201,16 +1249,15 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] nltk = [ - {file = "nltk-3.4.5.win32.exe", hash = "sha256:a08bdb4b8a1c13de16743068d9eb61c8c71c2e5d642e8e08205c528035843f82"}, - {file = "nltk-3.4.5.zip", hash = "sha256:bed45551259aa2101381bbdd5df37d44ca2669c5c3dad72439fa459b29137d94"}, + {file = "nltk-3.5.zip", hash = "sha256:845365449cd8c5f9731f7cb9f8bd6fd0767553b9d53af9eb1b3abf7700936b35"}, ] packaging = [ {file = "packaging-20.3-py2.py3-none-any.whl", hash = "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752"}, {file = "packaging-20.3.tar.gz", hash = "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3"}, ] parso = [ - {file = "parso-0.6.2-py2.py3-none-any.whl", hash = "sha256:8515fc12cfca6ee3aa59138741fc5624d62340c97e401c74875769948d4f2995"}, - {file = "parso-0.6.2.tar.gz", hash = "sha256:0c5659e0c6eba20636f99a04f469798dca8da279645ce5c387315b2c23912157"}, + {file = "parso-0.7.0-py2.py3-none-any.whl", hash = "sha256:158c140fc04112dc45bca311633ae5033c2c2a7b732fa33d0955bad8152a8dd0"}, + {file = "parso-0.7.0.tar.gz", hash = "sha256:908e9fae2144a076d72ae4e25539143d40b8e3eafbaeae03c1bfe226f4cdf12c"}, ] pathspec = [ {file = "pathspec-0.8.0-py2.py3-none-any.whl", hash = "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0"}, @@ -1364,6 +1411,10 @@ tornado = [ {file = "tornado-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:c58d56003daf1b616336781b26d184023ea4af13ae143d9dda65e31e534940b9"}, {file = "tornado-6.0.4.tar.gz", hash = "sha256:0fe2d45ba43b00a41cd73f8be321a44936dc1aba233dee979f17a042b83eb6dc"}, ] +tqdm = [ + {file = "tqdm-4.45.0-py2.py3-none-any.whl", hash = "sha256:ea9e3fd6bd9a37e8783d75bfc4c1faf3c6813da6bd1c3e776488b41ec683af94"}, + {file = "tqdm-4.45.0.tar.gz", hash = "sha256:00339634a22c10a7a22476ee946bbde2dbe48d042ded784e4d88e0236eca5d81"}, +] traitlets = [ {file = "traitlets-4.3.3-py2.py3-none-any.whl", hash = "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44"}, {file = "traitlets-4.3.3.tar.gz", hash = "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7"}, diff --git a/pyproject.toml b/pyproject.toml index 4434365c..2506fd52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,12 +34,12 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.7" aiohttp = "^3.6" -pydantic = "^1.2" +pydantic = "1.4" Babel = "^2.7" aiofiles = "^0.4.0" uvloop = {version = "^0.14.0", markers = "sys_platform == 'darwin' or sys_platform == 'linux'", optional = true} async_lru = "^1.0" -aiohttp-socks = {version = "^0.3.4", optional = true} +aiohttp-socks = {version = "^0.3.8", optional = true} [tool.poetry.dev-dependencies] uvloop = {version = "^0.14.0", markers = "sys_platform == 'darwin' or sys_platform == 'linux'"} diff --git a/tests/test_api/test_client/test_session/test_aiohttp_session.py b/tests/test_api/test_client/test_session/test_aiohttp_session.py index 62608a81..2658407f 100644 --- a/tests/test_api/test_client/test_session/test_aiohttp_session.py +++ b/tests/test_api/test_client/test_session/test_aiohttp_session.py @@ -56,6 +56,29 @@ class TestAiohttpSession: aiohttp_session = await session.create_session() assert isinstance(aiohttp_session.connector, aiohttp_socks.ProxyConnector) + @pytest.mark.asyncio + async def test_create_proxy_session_chained_proxies(self): + session = AiohttpSession( + proxy=[ + "socks4://proxy.url/", + 'socks5://proxy.url/', + 'http://user:password@127.0.0.1:3128' + ] + ) + + assert isinstance(session.proxy, list) + + assert isinstance(session._connector_init, dict) + assert isinstance(session._connector_init["proxy_infos"], list) + assert isinstance(session._connector_init["proxy_infos"][0], aiohttp_socks.ProxyInfo) + + assert session._connector_init["proxy_infos"][0].proxy_type is aiohttp_socks.ProxyType.SOCKS4 + assert session._connector_init["proxy_infos"][1].proxy_type is aiohttp_socks.ProxyType.SOCKS5 + assert session._connector_init["proxy_infos"][2].proxy_type is aiohttp_socks.ProxyType.HTTP + + aiohttp_session = await session.create_session() + assert isinstance(aiohttp_session.connector, aiohttp_socks.ChainProxyConnector) + @pytest.mark.asyncio async def test_close_session(self): session = AiohttpSession() @@ -167,5 +190,5 @@ class TestAiohttpSession: ) as mocked_close: async with session as ctx: assert session == ctx - mocked_close.awaited_once() - mocked_create_session.awaited_once() + await mocked_close.awaited_once() + await mocked_create_session.awaited_once() diff --git a/tests/test_api/test_client/test_session/test_asyncio_session.py b/tests/test_api/test_client/test_session/test_asyncio_session.py deleted file mode 100644 index 5b9c3b39..00000000 --- a/tests/test_api/test_client/test_session/test_asyncio_session.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -todo -"""