Add httpx client and set it as default

Closes #279
This commit is contained in:
Boger 2020-03-12 20:35:46 +03:00
parent 0bd7fc2c7e
commit 086806e202
6 changed files with 413 additions and 13 deletions

View file

@ -6,8 +6,8 @@ from typing import Any, Optional, TypeVar
from ...utils.mixins import ContextInstanceMixin, DataMixin
from ...utils.token import extract_bot_id, validate_token
from ..methods import TelegramMethod
from .session.aiohttp import AiohttpSession
from .session.base import BaseSession
from .session.httpx import HttpxSession
T = TypeVar("T")
@ -23,7 +23,7 @@ class BaseBot(ContextInstanceMixin, DataMixin):
validate_token(token)
if session is None:
session = AiohttpSession()
session = HttpxSession()
self.session = session
self.parse_mode = parse_mode

View file

@ -0,0 +1,82 @@
from __future__ import annotations
import warnings
from typing import Any, AsyncGenerator, Callable, Dict, Optional, Tuple, TypeVar, Union, cast
from httpx import AsyncClient
from aiogram.api.client.session.base import BaseSession
from aiogram.api.client.telegram import PRODUCTION, TelegramAPIServer
from aiogram.api.methods import Request, TelegramMethod
from aiogram.api.types import InputFile
from aiogram.utils.warnings import CodeHasNoEffect
T = TypeVar("T")
class HttpxSession(BaseSession):
def __init__(
self,
api: TelegramAPIServer = PRODUCTION,
json_loads: Optional[Callable] = None,
json_dumps: Optional[Callable] = None,
):
super(HttpxSession, self).__init__(
api=api, json_loads=json_loads, json_dumps=json_dumps,
)
self._client: Optional[AsyncClient] = None
async def create_session(self) -> AsyncClient:
if self._client is None:
self._client = AsyncClient()
return self._client
async def close(self):
if self._client is not None:
await self._client.aclose()
def build_form_data(self, request: Request):
form_data: Dict[str, Union[str, int, bool]] = {}
files: Dict[str, Tuple[InputFile, str]] = {}
for key, value in request.data.items():
if value is None:
continue
form_data[key] = self.prepare_value(value)
if request.files:
for key, input_file in request.files.items():
filename = input_file.filename or key
files[key] = (input_file, filename)
return form_data, files
async def make_request(self, token: str, call: TelegramMethod[T]) -> T:
session = await self.create_session()
request = call.build_request()
url = self.api.api_url(token=token, method=request.method)
form_data, files = self.build_form_data(request)
resp = await session.post(url=url, data=form_data, files=files)
raw_result = resp.json()
# we need cast because JSON can return list, but not in our Telegram API case
raw_result = cast(Dict[str, Any], raw_result)
response = call.build_response(raw_result)
self.raise_for_status(response)
return cast(T, response.result)
async def stream_content(
self, url: str, timeout: int, chunk_size: int
) -> AsyncGenerator[bytes, None]:
warnings.warn("httpx doesn't support `chunk_size` yet", CodeHasNoEffect)
session = await self.create_session()
async with session.stream(method="GET", url=url, timeout=timeout) as resp:
async for chunk in resp.aiter_bytes():
yield chunk
async def __aenter__(self) -> HttpxSession:
await self.create_session()
return self

166
poetry.lock generated
View file

@ -72,7 +72,6 @@ version = "3.0.1"
[[package]]
category = "dev"
description = "Enhance the standard unittest package with features for testing asyncio libraries"
marker = "python_version < \"3.8\""
name = "asynctest"
optional = false
python-versions = ">=3.5"
@ -140,6 +139,14 @@ typed-ast = ">=1.4.0"
[package.extras]
d = ["aiohttp (>=3.3.2)", "aiohttp-cors"]
[[package]]
category = "main"
description = "Python package for providing Mozilla's CA Bundle."
name = "certifi"
optional = false
python-versions = "*"
version = "2019.11.28"
[[package]]
category = "main"
description = "Universal encoding detector for Python 2 and 3"
@ -219,6 +226,69 @@ flake8 = ">=3.3.0"
jinja2 = ">=2.9.0"
pygments = ">=2.2.0"
[[package]]
category = "main"
description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
name = "h11"
optional = false
python-versions = "*"
version = "0.9.0"
[[package]]
category = "main"
description = "HTTP/2 State-Machine based protocol implementation"
name = "h2"
optional = false
python-versions = "*"
version = "3.2.0"
[package.dependencies]
hpack = ">=3.0,<4"
hyperframe = ">=5.2.0,<6"
[[package]]
category = "main"
description = "Pure-Python HPACK header compression"
name = "hpack"
optional = false
python-versions = "*"
version = "3.0.0"
[[package]]
category = "main"
description = "Chromium HSTS Preload list as a Python package and updated daily"
name = "hstspreload"
optional = false
python-versions = ">=3.6"
version = "2020.3.12"
[[package]]
category = "main"
description = "The next generation HTTP client."
name = "httpx"
optional = false
python-versions = ">=3.6"
version = "0.12.0"
[package.dependencies]
certifi = "*"
chardet = ">=3.0.0,<4.0.0"
h11 = ">=0.8,<0.10"
h2 = ">=3.0.0,<4.0.0"
hstspreload = "*"
idna = ">=2.0.0,<3.0.0"
rfc3986 = ">=1.3,<2"
sniffio = ">=1.0.0,<2.0.0"
urllib3 = ">=1.0.0,<2.0.0"
[[package]]
category = "main"
description = "HTTP/2 framing layer for Python"
name = "hyperframe"
optional = false
python-versions = "*"
version = "5.2.0"
[[package]]
category = "main"
description = "Internationalized Domain Names in Applications (IDNA)"
@ -759,6 +829,29 @@ optional = false
python-versions = "*"
version = "2020.2.20"
[[package]]
category = "dev"
description = "A utility for mocking out the Python HTTPX library."
name = "respx"
optional = false
python-versions = ">=3.6"
version = "0.10.1"
[package.dependencies]
asynctest = "*"
httpx = ">=0.12,<0.13"
[[package]]
category = "main"
description = "Validating URI References per RFC 3986"
name = "rfc3986"
optional = false
python-versions = "*"
version = "1.3.2"
[package.extras]
idna2008 = ["idna"]
[[package]]
category = "dev"
description = "Python 2 and 3 compatibility utilities"
@ -767,6 +860,14 @@ optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
version = "1.14.0"
[[package]]
category = "main"
description = "Sniff out which async library your code is running under"
name = "sniffio"
optional = false
python-versions = ">=3.5"
version = "1.1.0"
[[package]]
category = "dev"
description = "Python Library for Tom's Obvious, Minimal Language"
@ -815,6 +916,19 @@ optional = false
python-versions = "*"
version = "3.7.4.1"
[[package]]
category = "main"
description = "HTTP library with thread-safe connection pooling, file post, and more."
name = "urllib3"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
version = "1.25.8"
[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"]
[[package]]
category = "main"
description = "Fast implementation of asyncio event loop on top of libuv"
@ -861,7 +975,7 @@ testing = ["jaraco.itertools", "func-timeout"]
fast = ["uvloop"]
[metadata]
content-hash = "2eb50b5b57d0fac4780f1eb3f92ff129d891fd346e0c00856c1a56c58feffb03"
content-hash = "d71d0606da0aeda76ffb026cc7bab6445522cef5e5b60207005e3447d6297684"
python-versions = "^3.7"
[metadata.files]
@ -926,6 +1040,10 @@ black = [
{file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"},
{file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"},
]
certifi = [
{file = "certifi-2019.11.28-py2.py3-none-any.whl", hash = "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3"},
{file = "certifi-2019.11.28.tar.gz", hash = "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"},
]
chardet = [
{file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
{file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"},
@ -987,6 +1105,29 @@ flake8-html = [
{file = "flake8-html-0.4.0.tar.gz", hash = "sha256:44bec37f142e97c4a5b2cf10efe24ed253617a9736878851a594d4763011e742"},
{file = "flake8_html-0.4.0-py2.py3-none-any.whl", hash = "sha256:f372cd599ba9a374943eaa75a9cce30408cf4c0cc2251bc5194e6b0d3fc2bc3a"},
]
h11 = [
{file = "h11-0.9.0-py2.py3-none-any.whl", hash = "sha256:4bc6d6a1238b7615b266ada57e0618568066f57dd6fa967d1290ec9309b2f2f1"},
{file = "h11-0.9.0.tar.gz", hash = "sha256:33d4bca7be0fa039f4e84d50ab00531047e53d6ee8ffbc83501ea602c169cae1"},
]
h2 = [
{file = "h2-3.2.0-py2.py3-none-any.whl", hash = "sha256:61e0f6601fa709f35cdb730863b4e5ec7ad449792add80d1410d4174ed139af5"},
{file = "h2-3.2.0.tar.gz", hash = "sha256:875f41ebd6f2c44781259005b157faed1a5031df3ae5aa7bcb4628a6c0782f14"},
]
hpack = [
{file = "hpack-3.0.0-py2.py3-none-any.whl", hash = "sha256:0edd79eda27a53ba5be2dfabf3b15780928a0dff6eb0c60a3d6767720e970c89"},
{file = "hpack-3.0.0.tar.gz", hash = "sha256:8eec9c1f4bfae3408a3f30500261f7e6a65912dc138526ea054f9ad98892e9d2"},
]
hstspreload = [
{file = "hstspreload-2020.3.12.tar.gz", hash = "sha256:0f02fd8f4fd40eb8dd406742c2a2531685dbdb6f7e96f3eb88c80b856de658fe"},
]
httpx = [
{file = "httpx-0.12.0-py3-none-any.whl", hash = "sha256:add141cad7602f58289287fd8e8b7adb610550e2c183712b31860ac7e113c150"},
{file = "httpx-0.12.0.tar.gz", hash = "sha256:a3e82b1fec1e672e500c650b5d54a7353f7d20497f1fbfc6faae5f66aecd91d1"},
]
hyperframe = [
{file = "hyperframe-5.2.0-py2.py3-none-any.whl", hash = "sha256:5187962cb16dcc078f23cb5a4b110098d546c3f41ff2d4038a9896893bbd0b40"},
{file = "hyperframe-5.2.0.tar.gz", hash = "sha256:a9f5c17f2cc3c719b917c4f33ed1c61bd1f8dfac4b1bd23b7c80b3400971b41f"},
]
idna = [
{file = "idna-2.9-py2.py3-none-any.whl", hash = "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"},
{file = "idna-2.9.tar.gz", hash = "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb"},
@ -1083,6 +1224,11 @@ markupsafe = [
{file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6"},
{file = "MarkupSafe-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2"},
{file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"},
{file = "MarkupSafe-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15"},
{file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2"},
{file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42"},
{file = "MarkupSafe-1.1.1-cp38-cp38-win32.whl", hash = "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b"},
{file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"},
{file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"},
]
mccabe = [
@ -1283,10 +1429,22 @@ regex = [
{file = "regex-2020.2.20-cp38-cp38-win_amd64.whl", hash = "sha256:7f78f963e62a61e294adb6ff5db901b629ef78cb2a1cfce3cf4eeba80c1c67aa"},
{file = "regex-2020.2.20.tar.gz", hash = "sha256:9e9624440d754733eddbcd4614378c18713d2d9d0dc647cf9c72f64e39671be5"},
]
respx = [
{file = "respx-0.10.1-py2.py3-none-any.whl", hash = "sha256:43aca802e0fd0c964865b07f101943e7b5902ea070ec94cf8e84a39db8729b06"},
{file = "respx-0.10.1.tar.gz", hash = "sha256:190d1fb5bddaf6fcc1319a3cdfbd682c77d7167017b3283cbe79b8fb74927135"},
]
rfc3986 = [
{file = "rfc3986-1.3.2-py2.py3-none-any.whl", hash = "sha256:df4eba676077cefb86450c8f60121b9ae04b94f65f85b69f3f731af0516b7b18"},
{file = "rfc3986-1.3.2.tar.gz", hash = "sha256:0344d0bd428126ce554e7ca2b61787b6a28d2bbd19fc70ed2dd85efe31176405"},
]
six = [
{file = "six-1.14.0-py2.py3-none-any.whl", hash = "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"},
{file = "six-1.14.0.tar.gz", hash = "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a"},
]
sniffio = [
{file = "sniffio-1.1.0-py3-none-any.whl", hash = "sha256:20ed6d5b46f8ae136d00b9dcb807615d83ed82ceea6b2058cecb696765246da5"},
{file = "sniffio-1.1.0.tar.gz", hash = "sha256:8e3810100f69fe0edd463d02ad407112542a11ffdc29f67db2bf3771afb87a21"},
]
toml = [
{file = "toml-0.10.0-py2.7.egg", hash = "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3"},
{file = "toml-0.10.0-py2.py3-none-any.whl", hash = "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"},
@ -1333,6 +1491,10 @@ typing-extensions = [
{file = "typing_extensions-3.7.4.1-py3-none-any.whl", hash = "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"},
{file = "typing_extensions-3.7.4.1.tar.gz", hash = "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2"},
]
urllib3 = [
{file = "urllib3-1.25.8-py2.py3-none-any.whl", hash = "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc"},
{file = "urllib3-1.25.8.tar.gz", hash = "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"},
]
uvloop = [
{file = "uvloop-0.14.0-cp35-cp35m-macosx_10_11_x86_64.whl", hash = "sha256:08b109f0213af392150e2fe6f81d33261bb5ce968a288eb698aad4f46eb711bd"},
{file = "uvloop-0.14.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:4544dcf77d74f3a84f03dd6278174575c44c67d7165d4c42c71db3fdc3860726"},

View file

@ -38,6 +38,7 @@ Babel = "^2.7"
aiofiles = "^0.4.0"
uvloop = {version = "^0.14.0", markers = "sys_platform == 'darwin' or sys_platform == 'linux'", optional = true}
async_lru = "^1.0"
httpx = "^0.12.0"
[tool.poetry.dev-dependencies]
uvloop = {version = "^0.14.0", markers = "sys_platform == 'darwin' or sys_platform == 'linux'"}
@ -62,6 +63,7 @@ pymdown-extensions = "^6.1"
lxml = "^4.4"
ipython = "^7.10"
markdown-include = "^0.5.1"
RESPX = "^0.10.1"
[tool.poetry.extras]
fast = ["uvloop"]

View file

@ -1,7 +1,7 @@
import pytest
from aiogram.api.client.base import BaseBot
from aiogram.api.client.session.aiohttp import AiohttpSession
from aiogram.api.client.session.httpx import HttpxSession
from aiogram.api.methods import GetMe
try:
@ -13,7 +13,7 @@ except ImportError:
class TestBaseBot:
def test_init(self):
base_bot = BaseBot("42:TEST")
assert isinstance(base_bot.session, AiohttpSession)
assert isinstance(base_bot.session, HttpxSession)
assert base_bot.id == 42
def test_hashable(self):
@ -32,7 +32,7 @@ class TestBaseBot:
method = GetMe()
with patch(
"aiogram.api.client.session.aiohttp.AiohttpSession.make_request",
"aiogram.api.client.session.httpx.HttpxSession.make_request",
new_callable=CoroutineMock,
) as mocked_make_request:
await base_bot(method)
@ -40,11 +40,11 @@ class TestBaseBot:
@pytest.mark.asyncio
async def test_close(self):
base_bot = BaseBot("42:TEST", session=AiohttpSession())
base_bot = BaseBot("42:TEST", session=HttpxSession())
await base_bot.session.create_session()
with patch(
"aiogram.api.client.session.aiohttp.AiohttpSession.close", new_callable=CoroutineMock
"aiogram.api.client.session.httpx.HttpxSession.close", new_callable=CoroutineMock
) as mocked_close:
await base_bot.close()
mocked_close.assert_awaited()
@ -53,11 +53,9 @@ class TestBaseBot:
@pytest.mark.parametrize("close", [True, False])
async def test_context_manager(self, close: bool):
with patch(
"aiogram.api.client.session.aiohttp.AiohttpSession.close", new_callable=CoroutineMock
"aiogram.api.client.session.httpx.HttpxSession.close", new_callable=CoroutineMock
) as mocked_close:
async with BaseBot("42:TEST", session=AiohttpSession()).context(
auto_close=close
) as bot:
async with BaseBot("42:TEST", session=HttpxSession()).context(auto_close=close) as bot:
assert isinstance(bot, BaseBot)
if close:
mocked_close.assert_awaited()

View file

@ -0,0 +1,156 @@
import re
from typing import AsyncContextManager, AsyncGenerator
import aiohttp
import httpx
import pytest
import respx
from aresponses import ResponsesMockServer
from pip._vendor.packaging.version import Version
from respx import HTTPXMock
from aiogram.api.client.session.httpx import HttpxSession
from aiogram.api.client.telegram import PRODUCTION
from aiogram.api.methods import Request, TelegramMethod
from aiogram.api.types import InputFile
try:
from asynctest import CoroutineMock, patch
except ImportError:
from unittest.mock import AsyncMock as CoroutineMock, patch # type: ignore
@pytest.fixture
def httpx_mock():
with respx.mock() as httpx_mock:
yield httpx_mock
class BareInputFile(InputFile):
async def read(self, chunk_size: int):
yield b""
class TestHttpxSession:
@pytest.mark.asyncio
async def test_create_session(self):
session = HttpxSession()
assert session._client is None
httpx_session = await session.create_session()
assert session._client is not None
assert isinstance(httpx_session, httpx.AsyncClient)
@pytest.mark.asyncio
async def test_close_session(self):
session = HttpxSession()
await session.create_session()
with patch("httpx.AsyncClient.aclose", new=CoroutineMock()) as mocked_close:
await session.close()
mocked_close.assert_called_once()
def test_build_form_data_with_data_only(self):
request = Request(
method="method",
data={
"str": "value",
"int": 42,
"bool": True,
"null": None,
"list": ["foo"],
"dict": {"bar": "baz"},
},
)
session = HttpxSession()
form, files = session.build_form_data(request)
fields = list(form.keys()) + list(files.keys())
assert len(fields) == 5
assert all(isinstance(field, str) for field in fields)
assert "null" not in fields
def test_build_form_data_with_files(self):
request = Request(
method="method",
data={"key": "value"},
files={"document": BareInputFile(filename="file.txt")},
)
session = HttpxSession()
form, files = session.build_form_data(request)
assert len(form) + len(files) == 2
assert "document" in files
assert files["document"][1] == "file.txt"
assert isinstance(files["document"][0], BareInputFile)
@pytest.mark.asyncio
async def test_make_request(self, httpx_mock: HTTPXMock):
httpx_mock.post(
url=re.compile(r".*/bot42:TEST/method"),
status_code=200,
content='{"ok": true, "result": 42}',
content_type="application/json",
)
session = HttpxSession()
class TestMethod(TelegramMethod[int]):
__returning__ = int
def build_request(self) -> Request:
return Request(method="method", data={})
call = TestMethod()
with patch(
"aiogram.api.client.session.base.BaseSession.raise_for_status"
) as patched_raise_for_status:
result = await session.make_request("42:TEST", call)
assert isinstance(result, int)
assert result == 42
assert patched_raise_for_status.called_once()
# Update right Version if httpx still didn't implement it
@pytest.mark.skipif(
Version(httpx.__version__) <= Version("0.12.0"),
reason="old httpx doesn't support chunk_size",
)
@pytest.mark.asyncio
async def test_stream_content(self, httpx_mock: HTTPXMock):
httpx_mock.get(
url=re.compile(".*"), status_code=200, content=b"\f" * 10,
)
session = HttpxSession()
stream = session.stream_content(
"https://www.python.org/static/img/python-logo.png", timeout=5, chunk_size=1
)
assert isinstance(stream, AsyncGenerator)
size = 0
async for chunk in stream:
assert isinstance(chunk, bytes)
chunk_size = len(chunk)
assert chunk_size == 1
size += chunk_size
assert size == 10
@pytest.mark.asyncio
async def test_context_manager(self):
session = HttpxSession()
assert isinstance(session, AsyncContextManager)
with patch(
"aiogram.api.client.session.httpx.HttpxSession.create_session",
new_callable=CoroutineMock,
) as mocked_create_session, patch(
"aiogram.api.client.session.httpx.HttpxSession.close", new_callable=CoroutineMock
) as mocked_close:
async with session as ctx:
assert session == ctx
mocked_close.awaited_once()
mocked_create_session.awaited_once()