[Feature] Add fault tolerance framework (simplified) for DP+EP external LB deployments (#44428)

Signed-off-by: fangyuchu <fangyuchu@qq.com>
Signed-off-by: a798347923 <2645302020@qq.com>
Signed-off-by: TianZhuo <2770730562@qq.com>
Signed-off-by: a798347923 <39047817+a798347923@users.noreply.github.com>
Signed-off-by: 205150940 <112750056+205150940@users.noreply.github.com>
Signed-off-by: w00689259 <wangzhuo66@huawei.com>
Signed-off-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com>
Signed-off-by: zWaNg3 <389750525@qq.com>
Signed-off-by: yzchang-plus <1078477584@qq.com>
Signed-off-by: Jade Zheng <zheng.shoujian@outlook.com>
Co-authored-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com>
Co-authored-by: a798347923 <2645302020@qq.com>
Co-authored-by: TianZhuo <2770730562@qq.com>
Co-authored-by: 205150940 <112750056+205150940@users.noreply.github.com>
Co-authored-by: a798347923 <39047817+a798347923@users.noreply.github.com>
Co-authored-by: w00689259 <wangzhuo66@huawei.com>
Co-authored-by: zWaNg3 <389750525@qq.com>
Co-authored-by: yzchang-plus <1078477584@qq.com>
Co-authored-by: Jade Zheng <zheng.shoujian@outlook.com>
This commit is contained in:
fangyuchu
2026-07-25 11:49:09 -04:00
committed by GitHub
co-authored by zWaNg3 a798347923 TianZhuo 205150940 a798347923 w00689259 zWaNg3 yzchang-plus Jade Zheng
parent 33ef67e9fb
commit 0b0bd2b5f6
27 changed files with 1088 additions and 7 deletions
+31
View File
@@ -42,6 +42,7 @@ from vllm.config import (
DiffusionConfig,
ECTransferConfig,
EPLBConfig,
FaultToleranceConfig,
KernelConfig,
KVEventsConfig,
KVTransferConfig,
@@ -724,6 +725,11 @@ class EngineArgs:
optimization_level: OptimizationLevel = VllmConfig.optimization_level
performance_mode: PerformanceMode = VllmConfig.performance_mode
fault_tolerance_config: FaultToleranceConfig = get_field(
ParallelConfig, "fault_tolerance_config"
)
enable_fault_tolerance: bool = ParallelConfig.enable_fault_tolerance
kv_offloading_size: float | None = CacheConfig.kv_offloading_size
kv_offloading_backend: KVOffloadingBackend = CacheConfig.kv_offloading_backend
tokens_only: bool = False
@@ -756,6 +762,16 @@ class EngineArgs:
self.weight_transfer_config = WeightTransferConfig(
**self.weight_transfer_config
)
if isinstance(self.fault_tolerance_config, dict):
if not self.enable_fault_tolerance:
logger.warning(
"--fault-tolerance-config was passed. Fault tolerance is being "
"automatically enabled."
)
self.enable_fault_tolerance = True
self.fault_tolerance_config = FaultToleranceConfig(
**self.fault_tolerance_config
)
if isinstance(self.ir_op_priority, dict):
self.ir_op_priority = IrOpPriorityConfig(**self.ir_op_priority)
@@ -1153,6 +1169,12 @@ class EngineArgs:
parallel_group.add_argument(
"--worker-extension-cls", **parallel_kwargs["worker_extension_cls"]
)
parallel_group.add_argument(
"--enable-fault-tolerance", **parallel_kwargs["enable_fault_tolerance"]
)
parallel_group.add_argument(
"--fault-tolerance-config", **parallel_kwargs["fault_tolerance_config"]
)
# KV cache arguments
cache_kwargs = get_kwargs(CacheConfig)
@@ -1619,6 +1641,7 @@ class EngineArgs:
def from_cli_args(cls, args: argparse.Namespace):
# Get the list of attributes of this dataclass.
attrs = [attr.name for attr in dataclasses.fields(cls)]
# Set the attributes from the parsed arguments.
engine_args = cls(
**{attr: getattr(args, attr) for attr in attrs if hasattr(args, attr)}
@@ -2024,6 +2047,12 @@ class EngineArgs:
data_parallel_external_lb = (
self.data_parallel_external_lb or self.data_parallel_rank is not None
)
if self.enable_fault_tolerance and not data_parallel_external_lb:
raise ValueError(
"Fault tolerance requires external load balancer mode "
"(--data-parallel-external-lb or --data-parallel-rank). "
"Internal LB mode is not supported."
)
if (
self.data_parallel_size > 1
and data_parallel_external_lb
@@ -2181,6 +2210,8 @@ class EngineArgs:
_api_process_count=self._api_process_count,
_api_process_rank=self._api_process_rank,
assigned_physical_gpu_ids=self._resolve_device_ids(),
enable_fault_tolerance=self.enable_fault_tolerance,
fault_tolerance_config=self.fault_tolerance_config,
numa_bind=self.numa_bind,
numa_bind_nodes=self.numa_bind_nodes,
numa_bind_cpus=self.numa_bind_cpus,
+11
View File
@@ -20,6 +20,7 @@ from vllm.sampling_params import SamplingParams
from vllm.tasks import SupportedTask
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.input_processor import InputProcessor
from vllm.v1.fault_tolerance.utils import FaultToleranceRequest, FaultToleranceResult
if TYPE_CHECKING:
from vllm.v1.engine import PauseMode
@@ -234,6 +235,16 @@ class EngineClient(ABC):
"""Perform a collective RPC call to the given path."""
raise NotImplementedError
async def handle_fault(
self, fault_tolerance_request: FaultToleranceRequest
) -> FaultToleranceResult:
"""send fault tolerance instruction to the engine"""
raise NotImplementedError
async def get_status(self):
"""Get fault tolerance status of all engines."""
raise NotImplementedError
async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
"""Get supported tasks"""
raise NotImplementedError