[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-31 14:26:49 +00:00
committed by GitHub
parent 76ff0cdff2
commit dafbef15a1
18 changed files with 644 additions and 35 deletions
@@ -42,7 +42,7 @@ from vllm.entrypoints.openai.models.serving import (
)
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.exceptions import VLLMValidationError
from vllm.exceptions import QueueOverflowError, VLLMValidationError
from vllm.inputs import TokensPrompt
from vllm.multimodal.inputs import PlaceholderRange
from vllm.outputs import CompletionOutput, RequestOutput
@@ -848,6 +848,9 @@ class MockEngine:
renderer: MagicMock = field(default_factory=MagicMock)
errored: bool = False
def check_admission(self, n: int = 1, request_id: str | None = None) -> None:
pass
async def _async_serving_chat_init():
engine = MockEngine()
@@ -944,6 +947,35 @@ async def test_serving_chat_returns_correct_model_name():
assert await serving_chat.create_chat_completion(req) == MODEL_NAME
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [True, False])
async def test_admission_rejection_escapes_before_response_starts(stream):
"""Overload rejections must propagate out of create_chat_completion.
For streaming this is the only chance to return a real HTTP status: once
StreamingResponse is constructed the 200 has already been sent and the
rejection would degrade into an in-band SSE error chunk.
"""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
mock_engine.check_admission.side_effect = QueueOverflowError()
serving_chat = _build_serving_chat(mock_engine)
req = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "what is 1+1?"}],
stream=stream,
)
with pytest.raises(QueueOverflowError):
await serving_chat.create_chat_completion(req)
mock_engine.generate.assert_not_called()
@pytest.mark.asyncio
async def test_serving_chat_should_set_correct_max_tokens():
mock_engine = MagicMock(spec=AsyncLLM)
@@ -38,6 +38,7 @@ async def test_non_streaming_cancel_aborts_engine_requests(
):
engine_client = SimpleNamespace(
errored=False,
check_admission=Mock(),
generate=Mock(side_effect=lambda *_args, **_kwargs: _never_finishes()),
abort=AsyncMock(),
is_tracing_enabled=AsyncMock(return_value=False),
@@ -104,6 +105,7 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
started_request_ids: list[str] = []
engine_client = SimpleNamespace(
errored=False,
check_admission=Mock(),
generate=Mock(
side_effect=lambda *_args, **_kwargs: _records_start_then_never_finishes(
started_request_ids, _args[2]