[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -42,7 +42,7 @@ from vllm.entrypoints.openai.models.serving import (
|
||||
)
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.exceptions import QueueOverflowError, VLLMValidationError
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
@@ -848,6 +848,9 @@ class MockEngine:
|
||||
renderer: MagicMock = field(default_factory=MagicMock)
|
||||
errored: bool = False
|
||||
|
||||
def check_admission(self, n: int = 1, request_id: str | None = None) -> None:
|
||||
pass
|
||||
|
||||
|
||||
async def _async_serving_chat_init():
|
||||
engine = MockEngine()
|
||||
@@ -944,6 +947,35 @@ async def test_serving_chat_returns_correct_model_name():
|
||||
assert await serving_chat.create_chat_completion(req) == MODEL_NAME
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [True, False])
|
||||
async def test_admission_rejection_escapes_before_response_starts(stream):
|
||||
"""Overload rejections must propagate out of create_chat_completion.
|
||||
|
||||
For streaming this is the only chance to return a real HTTP status: once
|
||||
StreamingResponse is constructed the 200 has already been sent and the
|
||||
rejection would degrade into an in-band SSE error chunk.
|
||||
"""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
mock_engine.check_admission.side_effect = QueueOverflowError()
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
req = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "what is 1+1?"}],
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
with pytest.raises(QueueOverflowError):
|
||||
await serving_chat.create_chat_completion(req)
|
||||
|
||||
mock_engine.generate.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_chat_should_set_correct_max_tokens():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
|
||||
@@ -38,6 +38,7 @@ async def test_non_streaming_cancel_aborts_engine_requests(
|
||||
):
|
||||
engine_client = SimpleNamespace(
|
||||
errored=False,
|
||||
check_admission=Mock(),
|
||||
generate=Mock(side_effect=lambda *_args, **_kwargs: _never_finishes()),
|
||||
abort=AsyncMock(),
|
||||
is_tracing_enabled=AsyncMock(return_value=False),
|
||||
@@ -104,6 +105,7 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
|
||||
started_request_ids: list[str] = []
|
||||
engine_client = SimpleNamespace(
|
||||
errored=False,
|
||||
check_admission=Mock(),
|
||||
generate=Mock(
|
||||
side_effect=lambda *_args, **_kwargs: _records_start_then_never_finishes(
|
||||
started_request_ids, _args[2]
|
||||
|
||||
Reference in New Issue
Block a user