[Core] Move max_concurrent_batches to VllmConfig (#44274)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -3,7 +3,6 @@
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Future
|
||||
from functools import cached_property
|
||||
from multiprocessing import Lock
|
||||
from typing import Any
|
||||
|
||||
@@ -77,10 +76,6 @@ class UniProcExecutor(Executor):
|
||||
local_rank = int(device_info[1]) if len(device_info) > 1 else 0
|
||||
return distributed_init_method, 0, local_rank
|
||||
|
||||
@cached_property
|
||||
def max_concurrent_batches(self) -> int:
|
||||
return 2 if self.scheduler_config.async_scheduling else 1
|
||||
|
||||
def collective_rpc( # type: ignore[override]
|
||||
self,
|
||||
method: str | Callable,
|
||||
|
||||
Reference in New Issue
Block a user