[Core] Move max_concurrent_batches to VllmConfig (#44274)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-02 08:57:25 -07:00
committed by GitHub
parent 774e552397
commit cab5c9a2a9
11 changed files with 25 additions and 37 deletions
+1 -1
View File
@@ -284,7 +284,7 @@ def test_multiproc_executor_pipeline_parallel():
assert output_rank == 2, "Output rank should be 2 (first rank of last PP stage)" assert output_rank == 2, "Output rank should be 2 (first rank of last PP stage)"
# Verify max_concurrent_batches for pipeline parallel # Verify max_concurrent_batches for pipeline parallel
assert executor.max_concurrent_batches == 2, ( assert vllm_config.max_concurrent_batches == 2, (
"Max concurrent batches should equal PP size" "Max concurrent batches should equal PP size"
) )
+1 -1
View File
@@ -83,7 +83,7 @@ def assert_executor(executor, tp_size, pp_size):
assert executor._get_output_rank() == expected_output_rank assert executor._get_output_rank() == expected_output_rank
if pp_size > 1: if pp_size > 1:
assert executor.max_concurrent_batches == pp_size assert executor.vllm_config.max_concurrent_batches == pp_size
executor.check_health() executor.check_health()
assert not executor.is_failed assert not executor.is_failed
@@ -87,10 +87,6 @@ class DummyExecutor(UniProcExecutor):
self.collective_rpc("init_worker", args=([kwargs],)) self.collective_rpc("init_worker", args=([kwargs],))
self.collective_rpc("init_device") self.collective_rpc("init_device")
@property
def max_concurrent_batches(self) -> int:
return 2
def shutdown(self): def shutdown(self):
if hasattr(self, "thread_pool"): if hasattr(self, "thread_pool"):
self.thread_pool.shutdown(wait=False) self.thread_pool.shutdown(wait=False)
+12 -5
View File
@@ -5,6 +5,7 @@ import copy
import time import time
import uuid import uuid
from concurrent.futures import Future, ThreadPoolExecutor from concurrent.futures import Future, ThreadPoolExecutor
from unittest.mock import PropertyMock, patch
import pytest import pytest
from transformers import AutoTokenizer from transformers import AutoTokenizer
@@ -293,10 +294,6 @@ def test_engine_core_concurrent_batches():
# Use the thread pool instead of creating a new thread # Use the thread pool instead of creating a new thread
return self.thread_pool.submit(_execute) return self.thread_pool.submit(_execute)
@property
def max_concurrent_batches(self) -> int:
return 2
def shutdown(self): def shutdown(self):
if hasattr(self, "thread_pool"): if hasattr(self, "thread_pool"):
self.thread_pool.shutdown(wait=False) self.thread_pool.shutdown(wait=False)
@@ -314,7 +311,17 @@ def test_engine_core_concurrent_batches():
async_scheduling=False, async_scheduling=False,
) )
vllm_config = engine_args.create_engine_config() vllm_config = engine_args.create_engine_config()
with set_default_torch_num_threads(1): # Force two concurrent batches to exercise the batch queue independently
# of async scheduling (which is disabled above).
with (
set_default_torch_num_threads(1),
patch.object(
VllmConfig,
"max_concurrent_batches",
new_callable=PropertyMock,
return_value=2,
),
):
engine_core = EngineCore( engine_core = EngineCore(
vllm_config=vllm_config, log_stats=False, executor_class=DummyExecutor vllm_config=vllm_config, log_stats=False, executor_class=DummyExecutor
) )
@@ -1212,7 +1212,6 @@ def test_engine_core_proc_instantiation_cuda_empty(monkeypatch: pytest.MonkeyPat
mock_executor.get_kv_cache_specs.return_value = [{"default": mock_spec}] mock_executor.get_kv_cache_specs.return_value = [{"default": mock_spec}]
mock_executor.determine_available_memory.return_value = [1024 * 1024 * 1024] mock_executor.determine_available_memory.return_value = [1024 * 1024 * 1024]
mock_executor.initialize_from_config.return_value = None mock_executor.initialize_from_config.return_value = None
mock_executor.max_concurrent_batches = 1
return mock_executor return mock_executor
+9
View File
@@ -487,6 +487,15 @@ class VllmConfig:
] ]
return hash_str return hash_str
@property
def max_concurrent_batches(self) -> int:
# PP requires PP-size concurrent batches to fill the pipeline.
# Async scheduling requires 2 concurrent batches to overlap.
pp_size = self.parallel_config.pipeline_parallel_size
if pp_size > 1:
return pp_size
return 2 if self.scheduler_config.async_scheduling else 1
@property @property
def num_speculative_tokens(self) -> int: def num_speculative_tokens(self) -> int:
if ( if (
+1 -1
View File
@@ -188,7 +188,7 @@ class EngineCore:
# Batch queue for scheduled batches. This enables us to asynchronously # Batch queue for scheduled batches. This enables us to asynchronously
# schedule and execute batches, and is required by pipeline parallelism # schedule and execute batches, and is required by pipeline parallelism
# to eliminate pipeline bubbles. # to eliminate pipeline bubbles.
self.batch_queue_size = self.model_executor.max_concurrent_batches self.batch_queue_size = vllm_config.max_concurrent_batches
self.batch_queue: ( self.batch_queue: (
deque[tuple[Future[ModelRunnerOutput], SchedulerOutput, Future[Any]]] | None deque[tuple[Future[ModelRunnerOutput], SchedulerOutput, Future[Any]]] | None
) = None ) = None
-4
View File
@@ -253,10 +253,6 @@ class Executor(ABC):
output: list[DraftTokenIds] = self.collective_rpc("take_draft_token_ids") output: list[DraftTokenIds] = self.collective_rpc("take_draft_token_ids")
return output[0] return output[0]
@property
def max_concurrent_batches(self) -> int:
return 1
def profile(self, is_start: bool = True, profile_prefix: str | None = None): def profile(self, is_start: bool = True, profile_prefix: str | None = None):
self.collective_rpc("profile", args=(is_start, profile_prefix)) self.collective_rpc("profile", args=(is_start, profile_prefix))
+1 -7
View File
@@ -15,7 +15,7 @@ from concurrent.futures import Future, InvalidStateError
from contextlib import suppress from contextlib import suppress
from dataclasses import dataclass from dataclasses import dataclass
from enum import Enum, auto from enum import Enum, auto
from functools import cached_property, partial from functools import partial
from multiprocessing.connection import Connection from multiprocessing.connection import Connection
from multiprocessing.process import BaseProcess from multiprocessing.process import BaseProcess
from multiprocessing.synchronize import Lock as LockType from multiprocessing.synchronize import Lock as LockType
@@ -472,12 +472,6 @@ class MultiprocExecutor(Executor):
self.collective_rpc("check_health", timeout=10) self.collective_rpc("check_health", timeout=10)
return return
@cached_property
def max_concurrent_batches(self) -> int:
# PP requires PP-size concurrent batches to fill the pipeline.
pp_size = self.parallel_config.pipeline_parallel_size
return 2 if pp_size <= 1 and self.scheduler_config.async_scheduling else pp_size
def _get_output_rank(self) -> int: def _get_output_rank(self) -> int:
# Only returns ModelRunnerOutput from TP rank=0 and PP rank=-1 # Only returns ModelRunnerOutput from TP rank=0 and PP rank=-1
# (the first TP worker of the last PP stage). # (the first TP worker of the last PP stage).
-8
View File
@@ -96,14 +96,6 @@ class RayDistributedExecutor(Executor):
self.scheduler_output: SchedulerOutput | None = None self.scheduler_output: SchedulerOutput | None = None
@property
def max_concurrent_batches(self) -> int:
"""Ray distributed executor supports pipeline parallelism,
meaning that it allows PP size batches to be executed concurrently.
"""
pp_size = self.parallel_config.pipeline_parallel_size
return 2 if pp_size <= 1 and self.scheduler_config.async_scheduling else pp_size
def shutdown(self) -> None: def shutdown(self) -> None:
if logger: if logger:
# Somehow logger can be None here. # Somehow logger can be None here.
-5
View File
@@ -3,7 +3,6 @@
import os import os
from collections.abc import Callable from collections.abc import Callable
from concurrent.futures import Future from concurrent.futures import Future
from functools import cached_property
from multiprocessing import Lock from multiprocessing import Lock
from typing import Any from typing import Any
@@ -77,10 +76,6 @@ class UniProcExecutor(Executor):
local_rank = int(device_info[1]) if len(device_info) > 1 else 0 local_rank = int(device_info[1]) if len(device_info) > 1 else 0
return distributed_init_method, 0, local_rank return distributed_init_method, 0, local_rank
@cached_property
def max_concurrent_batches(self) -> int:
return 2 if self.scheduler_config.async_scheduling else 1
def collective_rpc( # type: ignore[override] def collective_rpc( # type: ignore[override]
self, self,
method: str | Callable, method: str | Callable,