[Core] Move max_concurrent_batches to VllmConfig (#44274)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-02 08:57:25 -07:00
committed by GitHub
parent 774e552397
commit cab5c9a2a9
11 changed files with 25 additions and 37 deletions
+12 -5
View File
@@ -5,6 +5,7 @@ import copy
import time
import uuid
from concurrent.futures import Future, ThreadPoolExecutor
from unittest.mock import PropertyMock, patch
import pytest
from transformers import AutoTokenizer
@@ -293,10 +294,6 @@ def test_engine_core_concurrent_batches():
# Use the thread pool instead of creating a new thread
return self.thread_pool.submit(_execute)
@property
def max_concurrent_batches(self) -> int:
return 2
def shutdown(self):
if hasattr(self, "thread_pool"):
self.thread_pool.shutdown(wait=False)
@@ -314,7 +311,17 @@ def test_engine_core_concurrent_batches():
async_scheduling=False,
)
vllm_config = engine_args.create_engine_config()
with set_default_torch_num_threads(1):
# Force two concurrent batches to exercise the batch queue independently
# of async scheduling (which is disabled above).
with (
set_default_torch_num_threads(1),
patch.object(
VllmConfig,
"max_concurrent_batches",
new_callable=PropertyMock,
return_value=2,
),
):
engine_core = EngineCore(
vllm_config=vllm_config, log_stats=False, executor_class=DummyExecutor
)
@@ -1212,7 +1212,6 @@ def test_engine_core_proc_instantiation_cuda_empty(monkeypatch: pytest.MonkeyPat
mock_executor.get_kv_cache_specs.return_value = [{"default": mock_spec}]
mock_executor.determine_available_memory.return_value = [1024 * 1024 * 1024]
mock_executor.initialize_from_config.return_value = None
mock_executor.max_concurrent_batches = 1
return mock_executor