[Core] Move max_concurrent_batches to VllmConfig (#44274)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -253,10 +253,6 @@ class Executor(ABC):
|
||||
output: list[DraftTokenIds] = self.collective_rpc("take_draft_token_ids")
|
||||
return output[0]
|
||||
|
||||
@property
|
||||
def max_concurrent_batches(self) -> int:
|
||||
return 1
|
||||
|
||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
|
||||
self.collective_rpc("profile", args=(is_start, profile_prefix))
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from concurrent.futures import Future, InvalidStateError
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from functools import cached_property, partial
|
||||
from functools import partial
|
||||
from multiprocessing.connection import Connection
|
||||
from multiprocessing.process import BaseProcess
|
||||
from multiprocessing.synchronize import Lock as LockType
|
||||
@@ -472,12 +472,6 @@ class MultiprocExecutor(Executor):
|
||||
self.collective_rpc("check_health", timeout=10)
|
||||
return
|
||||
|
||||
@cached_property
|
||||
def max_concurrent_batches(self) -> int:
|
||||
# PP requires PP-size concurrent batches to fill the pipeline.
|
||||
pp_size = self.parallel_config.pipeline_parallel_size
|
||||
return 2 if pp_size <= 1 and self.scheduler_config.async_scheduling else pp_size
|
||||
|
||||
def _get_output_rank(self) -> int:
|
||||
# Only returns ModelRunnerOutput from TP rank=0 and PP rank=-1
|
||||
# (the first TP worker of the last PP stage).
|
||||
|
||||
@@ -96,14 +96,6 @@ class RayDistributedExecutor(Executor):
|
||||
|
||||
self.scheduler_output: SchedulerOutput | None = None
|
||||
|
||||
@property
|
||||
def max_concurrent_batches(self) -> int:
|
||||
"""Ray distributed executor supports pipeline parallelism,
|
||||
meaning that it allows PP size batches to be executed concurrently.
|
||||
"""
|
||||
pp_size = self.parallel_config.pipeline_parallel_size
|
||||
return 2 if pp_size <= 1 and self.scheduler_config.async_scheduling else pp_size
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if logger:
|
||||
# Somehow logger can be None here.
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Future
|
||||
from functools import cached_property
|
||||
from multiprocessing import Lock
|
||||
from typing import Any
|
||||
|
||||
@@ -77,10 +76,6 @@ class UniProcExecutor(Executor):
|
||||
local_rank = int(device_info[1]) if len(device_info) > 1 else 0
|
||||
return distributed_init_method, 0, local_rank
|
||||
|
||||
@cached_property
|
||||
def max_concurrent_batches(self) -> int:
|
||||
return 2 if self.scheduler_config.async_scheduling else 1
|
||||
|
||||
def collective_rpc( # type: ignore[override]
|
||||
self,
|
||||
method: str | Callable,
|
||||
|
||||
Reference in New Issue
Block a user