Reject oversized media before fully downloading it (#51896)
Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Bourtoule
parent
585bb07c70
commit
2ec6f0d71e
@@ -4,16 +4,25 @@
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
from collections.abc import Generator
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.connections import HTTPConnection
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.run_batch import (
|
||||
BatchRequestOutput,
|
||||
BatchTranscriptionRequest,
|
||||
download_bytes_from_url,
|
||||
make_transcription_wrapper,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.utils.mem_constants import MiB_bytes
|
||||
|
||||
CHAT_MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
EMBEDDING_MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
@@ -281,6 +290,74 @@ INPUT_REASONING_BATCH = "\n".join(
|
||||
)
|
||||
|
||||
MINIMAL_WAV_BASE64 = "UklGRigAAABXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YQQAAAAAAP9/"
|
||||
_EXACT_LIMIT_AUDIO = b"a" * MiB_bytes
|
||||
_OVERSIZED_BASE64_AUDIO = "A" * (4 * ((MiB_bytes + 2) // 3))
|
||||
_OVERSIZED_HTTP_AUDIO = b"b" * (MiB_bytes + 1)
|
||||
|
||||
|
||||
class _BatchHTTPServer(ThreadingHTTPServer):
|
||||
request_paths: list[str]
|
||||
|
||||
|
||||
class _BatchHTTPHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
server = self.server
|
||||
assert isinstance(server, _BatchHTTPServer)
|
||||
server.request_paths.append(self.path)
|
||||
|
||||
if self.path == "/exact-limit":
|
||||
self._send_body(_EXACT_LIMIT_AUDIO)
|
||||
return
|
||||
|
||||
if self.path == "/oversized-chunked":
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self._write_body(_OVERSIZED_HTTP_AUDIO)
|
||||
return
|
||||
|
||||
self.send_error(404, "Unknown batch HTTP test path")
|
||||
|
||||
def _send_body(self, body: bytes) -> None:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self._write_body(body)
|
||||
|
||||
def _write_body(self, body: bytes) -> None:
|
||||
try:
|
||||
for offset in range(0, len(body), 64 * 1024):
|
||||
self.wfile.write(body[offset : offset + 64 * 1024])
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
self.close_connection = True
|
||||
|
||||
def log_message(self, fmt: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def batch_http_server() -> Generator[_BatchHTTPServer, None, None]:
|
||||
server = _BatchHTTPServer(("127.0.0.1", 0), _BatchHTTPHandler)
|
||||
server.request_paths = []
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield server
|
||||
finally:
|
||||
server.shutdown()
|
||||
thread.join()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def _batch_http_url(server: _BatchHTTPServer, path: str) -> str:
|
||||
return f"http://localhost:{server.server_port}{path}"
|
||||
|
||||
|
||||
async def _close_async_connection(connection: HTTPConnection) -> None:
|
||||
if connection._async_client is not None:
|
||||
await connection._async_client.close()
|
||||
|
||||
|
||||
INPUT_TRANSCRIPTION_BATCH = (
|
||||
json.dumps(
|
||||
{
|
||||
@@ -761,8 +838,15 @@ def test_tool_calling():
|
||||
|
||||
def _make_aiohttp_mocks(response_data: bytes = b"fake-data", status: int = 200):
|
||||
"""Create mock objects that simulate aiohttp.ClientSession context managers."""
|
||||
|
||||
async def iter_chunked(chunk_size: int):
|
||||
del chunk_size
|
||||
yield response_data
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = status
|
||||
mock_resp.content_length = len(response_data)
|
||||
mock_resp.content.iter_chunked = iter_chunked
|
||||
mock_resp.read = AsyncMock(return_value=response_data)
|
||||
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
||||
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
||||
@@ -892,3 +976,79 @@ async def test_download_bytes_backslash_bypass():
|
||||
await download_bytes_from_url(
|
||||
bypass_url, allowed_media_domains=["evil.internal"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_wrapper_rejects_oversized_data_url_before_decode(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||
handler = AsyncMock()
|
||||
wrapped_handler = make_transcription_wrapper(is_translation=False)(handler)
|
||||
request = BatchTranscriptionRequest.model_validate(
|
||||
{
|
||||
"model": SPEECH_LARGE_MODEL_NAME,
|
||||
"file_url": f"data:audio/wav;base64,{_OVERSIZED_BASE64_AUDIO}",
|
||||
"response_format": "json",
|
||||
}
|
||||
)
|
||||
|
||||
with patch("vllm.entrypoints.openai.run_batch.base64.b64decode") as decode:
|
||||
response = await wrapped_handler(request)
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert "Maximum file size exceeded" in response.error.message
|
||||
decode.assert_not_called()
|
||||
handler.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_bytes_allows_http_body_at_audio_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
batch_http_server: _BatchHTTPServer,
|
||||
):
|
||||
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||
connection = HTTPConnection()
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.openai.run_batch.global_http_connection",
|
||||
connection,
|
||||
)
|
||||
|
||||
try:
|
||||
result = await download_bytes_from_url(
|
||||
_batch_http_url(batch_http_server, "/exact-limit")
|
||||
)
|
||||
assert result == _EXACT_LIMIT_AUDIO
|
||||
finally:
|
||||
await _close_async_connection(connection)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_wrapper_rejects_oversized_http_before_handler(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
batch_http_server: _BatchHTTPServer,
|
||||
):
|
||||
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||
connection = HTTPConnection()
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.openai.run_batch.global_http_connection",
|
||||
connection,
|
||||
)
|
||||
handler = AsyncMock()
|
||||
wrapped_handler = make_transcription_wrapper(is_translation=False)(handler)
|
||||
request = BatchTranscriptionRequest.model_validate(
|
||||
{
|
||||
"model": SPEECH_LARGE_MODEL_NAME,
|
||||
"file_url": _batch_http_url(batch_http_server, "/oversized-chunked"),
|
||||
"response_format": "json",
|
||||
}
|
||||
)
|
||||
|
||||
try:
|
||||
response = await wrapped_handler(request)
|
||||
finally:
|
||||
await _close_async_connection(connection)
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert "Maximum file size exceeded" in response.error.message
|
||||
handler.assert_not_awaited()
|
||||
|
||||
@@ -41,7 +41,6 @@ async def test_chunk_offsets_are_cumulative_not_nominal():
|
||||
overlap_chunk_second=1,
|
||||
min_energy_split_window_size=1600,
|
||||
)
|
||||
serving.max_audio_filesize_mb = 100.0
|
||||
serving.model_cls = MagicMock()
|
||||
serving.model_cls.validate_language.side_effect = lambda lang: lang
|
||||
serving.model_cls.supports_explicit_language_detection = False
|
||||
|
||||
@@ -13,6 +13,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import aiohttp
|
||||
import pytest
|
||||
|
||||
from vllm.connections import MediaDownloadSizeExceededError
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.exceptions import VLLMClientError, VLLMUnprocessableEntityError
|
||||
from vllm.multimodal.media import MediaConnector
|
||||
@@ -134,6 +135,19 @@ class TestMediaConnectorErrorHandling:
|
||||
|
||||
assert isinstance(exc_info.value, aiohttp.ClientConnectionError)
|
||||
|
||||
def test_fetch_image_download_size_limit_error(self):
|
||||
connector = MediaConnector()
|
||||
|
||||
with patch.object(
|
||||
connector.connection, "get_bytes", new_callable=MagicMock
|
||||
) as mock_get:
|
||||
mock_get.side_effect = MediaDownloadSizeExceededError(256 * 1024 * 1024)
|
||||
|
||||
with pytest.raises(VLLMUnprocessableEntityError) as exc_info:
|
||||
connector.fetch_image("https://example.com/image.jpg")
|
||||
|
||||
assert "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB=256" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestErrorResponse:
|
||||
"""Tests for error response creation."""
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.connections import HTTPConnection
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal.media import AudioMediaIO, MediaConnector
|
||||
from vllm.utils.mem_constants import KiB_bytes, MiB_bytes
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
|
||||
class _SyncResponse:
|
||||
def __init__(self, chunks: list[bytes], content_length: int | None = None):
|
||||
self.headers = (
|
||||
{} if content_length is None else {"content-length": str(content_length)}
|
||||
)
|
||||
self._chunks = chunks
|
||||
self.iterated = 0
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def iter_content(self, chunk_size: int) -> Iterator[bytes]:
|
||||
assert chunk_size == 64 * KiB_bytes
|
||||
for chunk in self._chunks:
|
||||
self.iterated += 1
|
||||
yield chunk
|
||||
|
||||
|
||||
class _AsyncContent:
|
||||
def __init__(self, chunks: list[bytes]):
|
||||
self._chunks = chunks
|
||||
self.iterated = 0
|
||||
|
||||
async def iter_chunked(self, chunk_size: int) -> AsyncIterator[bytes]:
|
||||
assert chunk_size == 64 * KiB_bytes
|
||||
for chunk in self._chunks:
|
||||
self.iterated += 1
|
||||
yield chunk
|
||||
|
||||
|
||||
class _AsyncResponse:
|
||||
def __init__(self, chunks: list[bytes], content_length: int | None = None):
|
||||
self.content_length = content_length
|
||||
self.content = _AsyncContent(chunks)
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_args):
|
||||
return None
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def test_audio_base64_rejects_before_decode(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||
max_encoded_chars = 4 * ((MiB_bytes + 2) // 3)
|
||||
|
||||
with (
|
||||
patch("vllm.multimodal.media.audio.pybase64.b64decode") as decode,
|
||||
pytest.raises(VLLMValidationError, match="Maximum file size exceeded"),
|
||||
):
|
||||
AudioMediaIO().load_base64("audio/wav", "A" * (max_encoded_chars + 1))
|
||||
|
||||
decode.assert_not_called()
|
||||
|
||||
|
||||
def test_sync_http_reader_rejects_from_content_length_before_body_read():
|
||||
response = _SyncResponse([b"A" * 10], content_length=MiB_bytes + 1)
|
||||
connection = HTTPConnection()
|
||||
|
||||
with (
|
||||
patch.object(connection, "get_response", return_value=response) as get_response,
|
||||
pytest.raises(VLLMValidationError, match="Maximum file size exceeded"),
|
||||
):
|
||||
connection.get_bytes("https://example.test/audio", max_bytes=MiB_bytes)
|
||||
|
||||
assert get_response.call_args.kwargs["stream"] is True
|
||||
assert response.iterated == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_http_reader_stops_after_first_over_limit_chunk():
|
||||
response = _AsyncResponse([b"A" * (64 * KiB_bytes), b"B", b"C" * 10])
|
||||
connection = HTTPConnection()
|
||||
|
||||
async def get_async_response(*_args, **_kwargs):
|
||||
return response
|
||||
|
||||
with (
|
||||
patch.object(connection, "get_async_response", side_effect=get_async_response),
|
||||
pytest.raises(VLLMValidationError, match="Maximum file size exceeded"),
|
||||
):
|
||||
await connection.async_get_bytes(
|
||||
"https://example.test/audio",
|
||||
max_bytes=64 * KiB_bytes,
|
||||
)
|
||||
|
||||
assert response.content.iterated == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_connector_threads_byte_limit_to_http_reader(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||
|
||||
class _Connection:
|
||||
async def async_get_bytes(self, _url: str, **kwargs):
|
||||
assert kwargs["max_bytes"] == MiB_bytes
|
||||
raise VLLMValidationError("Maximum file size exceeded")
|
||||
|
||||
connector = MediaConnector(connection=_Connection()) # type: ignore[arg-type]
|
||||
with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"):
|
||||
await connector.fetch_audio_async("https://example.test/audio")
|
||||
@@ -0,0 +1,192 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
from collections.abc import Generator
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.connections import HTTPConnection, HTTPResponseSizeExceededError
|
||||
|
||||
_ONE_MIB = 1024 * 1024
|
||||
_SMALL_BODY = b"a" * (_ONE_MIB // 2)
|
||||
_LARGE_BODY = b"b" * (2 * _ONE_MIB)
|
||||
_CHUNK_SIZE = 64 * 1024
|
||||
|
||||
|
||||
class _MediaHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
body = _LARGE_BODY if "large" in self.path else _SMALL_BODY
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
if self.path != "/chunked-large":
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
|
||||
try:
|
||||
for offset in range(0, len(body), _CHUNK_SIZE):
|
||||
self.wfile.write(body[offset : offset + _CHUNK_SIZE])
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
self.close_connection = True
|
||||
|
||||
def log_message(self, fmt: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_media_server() -> Generator[str, None, None]:
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), _MediaHandler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield f"http://127.0.0.1:{server.server_port}"
|
||||
finally:
|
||||
server.shutdown()
|
||||
thread.join()
|
||||
server.server_close()
|
||||
|
||||
|
||||
class _SyncResponse:
|
||||
def __init__(self, chunks: list[bytes]) -> None:
|
||||
self.headers: dict[str, str] = {}
|
||||
self._chunks = chunks
|
||||
self.content_accessed = False
|
||||
self.iterated = 0
|
||||
|
||||
@property
|
||||
def content(self) -> bytes:
|
||||
self.content_accessed = True
|
||||
raise AssertionError("bounded reads must not access response.content")
|
||||
|
||||
def iter_content(self, chunk_size: int):
|
||||
del chunk_size
|
||||
for chunk in self._chunks:
|
||||
self.iterated += 1
|
||||
yield chunk
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
return False
|
||||
|
||||
|
||||
class _AsyncContent:
|
||||
def __init__(self, chunks: list[bytes]) -> None:
|
||||
self._chunks = chunks
|
||||
self.iterated = 0
|
||||
|
||||
async def iter_chunked(self, chunk_size: int):
|
||||
del chunk_size
|
||||
for chunk in self._chunks:
|
||||
self.iterated += 1
|
||||
yield chunk
|
||||
|
||||
|
||||
class _AsyncResponse:
|
||||
def __init__(self, chunks: list[bytes]) -> None:
|
||||
self.content = _AsyncContent(chunks)
|
||||
self.content_length = None
|
||||
self.read = AsyncMock(
|
||||
side_effect=AssertionError("bounded reads must not call response.read()")
|
||||
)
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, traceback):
|
||||
return False
|
||||
|
||||
|
||||
def test_get_bytes_with_limit_streams_instead_of_materializing() -> None:
|
||||
response = _SyncResponse([b"abcd", b"efgh", b"ijkl"])
|
||||
connection = HTTPConnection()
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
connection,
|
||||
"get_response",
|
||||
return_value=response,
|
||||
) as get_response,
|
||||
pytest.raises(HTTPResponseSizeExceededError, match="maximum size"),
|
||||
):
|
||||
connection.get_bytes("http://example.com/media", max_bytes=4)
|
||||
|
||||
get_response.assert_called_once_with(
|
||||
"http://example.com/media",
|
||||
stream=True,
|
||||
timeout=None,
|
||||
allow_redirects=True,
|
||||
)
|
||||
assert response.content_accessed is False
|
||||
assert response.iterated == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_bytes_with_limit_streams_instead_of_materializing() -> None:
|
||||
response = _AsyncResponse([b"abcd", b"efgh", b"ijkl"])
|
||||
connection = HTTPConnection()
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
connection,
|
||||
"get_async_response",
|
||||
new=AsyncMock(return_value=response),
|
||||
),
|
||||
pytest.raises(HTTPResponseSizeExceededError, match="maximum size"),
|
||||
):
|
||||
await connection.async_get_bytes("http://example.com/media", max_bytes=4)
|
||||
|
||||
response.read.assert_not_awaited()
|
||||
assert response.content.iterated == 2
|
||||
|
||||
|
||||
def test_get_bytes_rejects_content_length_over_limit(
|
||||
monkeypatch: pytest.MonkeyPatch, local_media_server: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", 1, raising=False)
|
||||
connection = HTTPConnection(reuse_client=False)
|
||||
|
||||
with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB"):
|
||||
connection.get_bytes(f"{local_media_server}/large")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_bytes_rejects_stream_over_limit_without_content_length(
|
||||
monkeypatch: pytest.MonkeyPatch, local_media_server: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", 1, raising=False)
|
||||
connection = HTTPConnection(reuse_client=False)
|
||||
|
||||
try:
|
||||
with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB"):
|
||||
await connection.async_get_bytes(f"{local_media_server}/chunked-large")
|
||||
finally:
|
||||
if connection._async_client is not None:
|
||||
await connection._async_client.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_bytes_allows_body_within_limit(
|
||||
monkeypatch: pytest.MonkeyPatch, local_media_server: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", 1, raising=False)
|
||||
connection = HTTPConnection(reuse_client=False)
|
||||
|
||||
assert connection.get_bytes(f"{local_media_server}/small") == _SMALL_BODY
|
||||
|
||||
try:
|
||||
actual = await connection.async_get_bytes(f"{local_media_server}/small")
|
||||
assert actual == _SMALL_BODY
|
||||
finally:
|
||||
if connection._async_client is not None:
|
||||
await connection._async_client.close()
|
||||
Reference in New Issue
Block a user