[Core] Move max_concurrent_batches to VllmConfig (#44274)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -284,7 +284,7 @@ def test_multiproc_executor_pipeline_parallel():
|
||||
assert output_rank == 2, "Output rank should be 2 (first rank of last PP stage)"
|
||||
|
||||
# Verify max_concurrent_batches for pipeline parallel
|
||||
assert executor.max_concurrent_batches == 2, (
|
||||
assert vllm_config.max_concurrent_batches == 2, (
|
||||
"Max concurrent batches should equal PP size"
|
||||
)
|
||||
|
||||
|
||||
@@ -83,7 +83,7 @@ def assert_executor(executor, tp_size, pp_size):
|
||||
assert executor._get_output_rank() == expected_output_rank
|
||||
|
||||
if pp_size > 1:
|
||||
assert executor.max_concurrent_batches == pp_size
|
||||
assert executor.vllm_config.max_concurrent_batches == pp_size
|
||||
|
||||
executor.check_health()
|
||||
assert not executor.is_failed
|
||||
|
||||
@@ -87,10 +87,6 @@ class DummyExecutor(UniProcExecutor):
|
||||
self.collective_rpc("init_worker", args=([kwargs],))
|
||||
self.collective_rpc("init_device")
|
||||
|
||||
@property
|
||||
def max_concurrent_batches(self) -> int:
|
||||
return 2
|
||||
|
||||
def shutdown(self):
|
||||
if hasattr(self, "thread_pool"):
|
||||
self.thread_pool.shutdown(wait=False)
|
||||
|
||||
@@ -5,6 +5,7 @@ import copy
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from unittest.mock import PropertyMock, patch
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
@@ -293,10 +294,6 @@ def test_engine_core_concurrent_batches():
|
||||
# Use the thread pool instead of creating a new thread
|
||||
return self.thread_pool.submit(_execute)
|
||||
|
||||
@property
|
||||
def max_concurrent_batches(self) -> int:
|
||||
return 2
|
||||
|
||||
def shutdown(self):
|
||||
if hasattr(self, "thread_pool"):
|
||||
self.thread_pool.shutdown(wait=False)
|
||||
@@ -314,7 +311,17 @@ def test_engine_core_concurrent_batches():
|
||||
async_scheduling=False,
|
||||
)
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
with set_default_torch_num_threads(1):
|
||||
# Force two concurrent batches to exercise the batch queue independently
|
||||
# of async scheduling (which is disabled above).
|
||||
with (
|
||||
set_default_torch_num_threads(1),
|
||||
patch.object(
|
||||
VllmConfig,
|
||||
"max_concurrent_batches",
|
||||
new_callable=PropertyMock,
|
||||
return_value=2,
|
||||
),
|
||||
):
|
||||
engine_core = EngineCore(
|
||||
vllm_config=vllm_config, log_stats=False, executor_class=DummyExecutor
|
||||
)
|
||||
|
||||
@@ -1212,7 +1212,6 @@ def test_engine_core_proc_instantiation_cuda_empty(monkeypatch: pytest.MonkeyPat
|
||||
mock_executor.get_kv_cache_specs.return_value = [{"default": mock_spec}]
|
||||
mock_executor.determine_available_memory.return_value = [1024 * 1024 * 1024]
|
||||
mock_executor.initialize_from_config.return_value = None
|
||||
mock_executor.max_concurrent_batches = 1
|
||||
|
||||
return mock_executor
|
||||
|
||||
|
||||
Reference in New Issue
Block a user