[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -38,6 +38,7 @@ async def test_non_streaming_cancel_aborts_engine_requests(
|
||||
):
|
||||
engine_client = SimpleNamespace(
|
||||
errored=False,
|
||||
check_admission=Mock(),
|
||||
generate=Mock(side_effect=lambda *_args, **_kwargs: _never_finishes()),
|
||||
abort=AsyncMock(),
|
||||
is_tracing_enabled=AsyncMock(return_value=False),
|
||||
@@ -104,6 +105,7 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
|
||||
started_request_ids: list[str] = []
|
||||
engine_client = SimpleNamespace(
|
||||
errored=False,
|
||||
check_admission=Mock(),
|
||||
generate=Mock(
|
||||
side_effect=lambda *_args, **_kwargs: _records_start_then_never_finishes(
|
||||
started_request_ids, _args[2]
|
||||
|
||||
Reference in New Issue
Block a user