[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-31 14:26:49 +00:00
committed by GitHub
parent 76ff0cdff2
commit dafbef15a1
18 changed files with 644 additions and 35 deletions
@@ -38,6 +38,7 @@ async def test_non_streaming_cancel_aborts_engine_requests(
):
engine_client = SimpleNamespace(
errored=False,
check_admission=Mock(),
generate=Mock(side_effect=lambda *_args, **_kwargs: _never_finishes()),
abort=AsyncMock(),
is_tracing_enabled=AsyncMock(return_value=False),
@@ -104,6 +105,7 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
started_request_ids: list[str] = []
engine_client = SimpleNamespace(
errored=False,
check_admission=Mock(),
generate=Mock(
side_effect=lambda *_args, **_kwargs: _records_start_then_never_finishes(
started_request_ids, _args[2]