[Feature] Add fault tolerance framework (simplified) for DP+EP external LB deployments (#44428)
Signed-off-by: fangyuchu <fangyuchu@qq.com> Signed-off-by: a798347923 <2645302020@qq.com> Signed-off-by: TianZhuo <2770730562@qq.com> Signed-off-by: a798347923 <39047817+a798347923@users.noreply.github.com> Signed-off-by: 205150940 <112750056+205150940@users.noreply.github.com> Signed-off-by: w00689259 <wangzhuo66@huawei.com> Signed-off-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com> Signed-off-by: zWaNg3 <389750525@qq.com> Signed-off-by: yzchang-plus <1078477584@qq.com> Signed-off-by: Jade Zheng <zheng.shoujian@outlook.com> Co-authored-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com> Co-authored-by: a798347923 <2645302020@qq.com> Co-authored-by: TianZhuo <2770730562@qq.com> Co-authored-by: 205150940 <112750056+205150940@users.noreply.github.com> Co-authored-by: a798347923 <39047817+a798347923@users.noreply.github.com> Co-authored-by: w00689259 <wangzhuo66@huawei.com> Co-authored-by: zWaNg3 <389750525@qq.com> Co-authored-by: yzchang-plus <1078477584@qq.com> Co-authored-by: Jade Zheng <zheng.shoujian@outlook.com>
This commit is contained in:
co-authored by
zWaNg3
a798347923
TianZhuo
205150940
a798347923
w00689259
zWaNg3
yzchang-plus
Jade Zheng
parent
33ef67e9fb
commit
0b0bd2b5f6
@@ -42,6 +42,7 @@ from vllm.config import (
|
||||
DiffusionConfig,
|
||||
ECTransferConfig,
|
||||
EPLBConfig,
|
||||
FaultToleranceConfig,
|
||||
KernelConfig,
|
||||
KVEventsConfig,
|
||||
KVTransferConfig,
|
||||
@@ -724,6 +725,11 @@ class EngineArgs:
|
||||
optimization_level: OptimizationLevel = VllmConfig.optimization_level
|
||||
performance_mode: PerformanceMode = VllmConfig.performance_mode
|
||||
|
||||
fault_tolerance_config: FaultToleranceConfig = get_field(
|
||||
ParallelConfig, "fault_tolerance_config"
|
||||
)
|
||||
enable_fault_tolerance: bool = ParallelConfig.enable_fault_tolerance
|
||||
|
||||
kv_offloading_size: float | None = CacheConfig.kv_offloading_size
|
||||
kv_offloading_backend: KVOffloadingBackend = CacheConfig.kv_offloading_backend
|
||||
tokens_only: bool = False
|
||||
@@ -756,6 +762,16 @@ class EngineArgs:
|
||||
self.weight_transfer_config = WeightTransferConfig(
|
||||
**self.weight_transfer_config
|
||||
)
|
||||
if isinstance(self.fault_tolerance_config, dict):
|
||||
if not self.enable_fault_tolerance:
|
||||
logger.warning(
|
||||
"--fault-tolerance-config was passed. Fault tolerance is being "
|
||||
"automatically enabled."
|
||||
)
|
||||
self.enable_fault_tolerance = True
|
||||
self.fault_tolerance_config = FaultToleranceConfig(
|
||||
**self.fault_tolerance_config
|
||||
)
|
||||
if isinstance(self.ir_op_priority, dict):
|
||||
self.ir_op_priority = IrOpPriorityConfig(**self.ir_op_priority)
|
||||
|
||||
@@ -1153,6 +1169,12 @@ class EngineArgs:
|
||||
parallel_group.add_argument(
|
||||
"--worker-extension-cls", **parallel_kwargs["worker_extension_cls"]
|
||||
)
|
||||
parallel_group.add_argument(
|
||||
"--enable-fault-tolerance", **parallel_kwargs["enable_fault_tolerance"]
|
||||
)
|
||||
parallel_group.add_argument(
|
||||
"--fault-tolerance-config", **parallel_kwargs["fault_tolerance_config"]
|
||||
)
|
||||
|
||||
# KV cache arguments
|
||||
cache_kwargs = get_kwargs(CacheConfig)
|
||||
@@ -1619,6 +1641,7 @@ class EngineArgs:
|
||||
def from_cli_args(cls, args: argparse.Namespace):
|
||||
# Get the list of attributes of this dataclass.
|
||||
attrs = [attr.name for attr in dataclasses.fields(cls)]
|
||||
|
||||
# Set the attributes from the parsed arguments.
|
||||
engine_args = cls(
|
||||
**{attr: getattr(args, attr) for attr in attrs if hasattr(args, attr)}
|
||||
@@ -2024,6 +2047,12 @@ class EngineArgs:
|
||||
data_parallel_external_lb = (
|
||||
self.data_parallel_external_lb or self.data_parallel_rank is not None
|
||||
)
|
||||
if self.enable_fault_tolerance and not data_parallel_external_lb:
|
||||
raise ValueError(
|
||||
"Fault tolerance requires external load balancer mode "
|
||||
"(--data-parallel-external-lb or --data-parallel-rank). "
|
||||
"Internal LB mode is not supported."
|
||||
)
|
||||
if (
|
||||
self.data_parallel_size > 1
|
||||
and data_parallel_external_lb
|
||||
@@ -2181,6 +2210,8 @@ class EngineArgs:
|
||||
_api_process_count=self._api_process_count,
|
||||
_api_process_rank=self._api_process_rank,
|
||||
assigned_physical_gpu_ids=self._resolve_device_ids(),
|
||||
enable_fault_tolerance=self.enable_fault_tolerance,
|
||||
fault_tolerance_config=self.fault_tolerance_config,
|
||||
numa_bind=self.numa_bind,
|
||||
numa_bind_nodes=self.numa_bind_nodes,
|
||||
numa_bind_cpus=self.numa_bind_cpus,
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
from vllm.v1.fault_tolerance.utils import FaultToleranceRequest, FaultToleranceResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.engine import PauseMode
|
||||
@@ -234,6 +235,16 @@ class EngineClient(ABC):
|
||||
"""Perform a collective RPC call to the given path."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def handle_fault(
|
||||
self, fault_tolerance_request: FaultToleranceRequest
|
||||
) -> FaultToleranceResult:
|
||||
"""send fault tolerance instruction to the engine"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_status(self):
|
||||
"""Get fault tolerance status of all engines."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||
"""Get supported tasks"""
|
||||
raise NotImplementedError
|
||||
|
||||
Reference in New Issue
Block a user