[Feature] Add fault tolerance framework (simplified) for DP+EP external LB deployments (#44428)

Signed-off-by: fangyuchu <fangyuchu@qq.com>
Signed-off-by: a798347923 <2645302020@qq.com>
Signed-off-by: TianZhuo <2770730562@qq.com>
Signed-off-by: a798347923 <39047817+a798347923@users.noreply.github.com>
Signed-off-by: 205150940 <112750056+205150940@users.noreply.github.com>
Signed-off-by: w00689259 <wangzhuo66@huawei.com>
Signed-off-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com>
Signed-off-by: zWaNg3 <389750525@qq.com>
Signed-off-by: yzchang-plus <1078477584@qq.com>
Signed-off-by: Jade Zheng <zheng.shoujian@outlook.com>
Co-authored-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com>
Co-authored-by: a798347923 <2645302020@qq.com>
Co-authored-by: TianZhuo <2770730562@qq.com>
Co-authored-by: 205150940 <112750056+205150940@users.noreply.github.com>
Co-authored-by: a798347923 <39047817+a798347923@users.noreply.github.com>
Co-authored-by: w00689259 <wangzhuo66@huawei.com>
Co-authored-by: zWaNg3 <389750525@qq.com>
Co-authored-by: yzchang-plus <1078477584@qq.com>
Co-authored-by: Jade Zheng <zheng.shoujian@outlook.com>
This commit is contained in:
fangyuchu
2026-07-25 11:49:09 -04:00
committed by GitHub
co-authored by zWaNg3 a798347923 TianZhuo 205150940 a798347923 w00689259 zWaNg3 yzchang-plus Jade Zheng
parent 33ef67e9fb
commit 0b0bd2b5f6
27 changed files with 1088 additions and 7 deletions
+15
View File
@@ -5,6 +5,7 @@ import contextlib
import numpy as np
import torch
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
from vllm.v1.outputs import AsyncModelRunnerOutput, LogprobsTensors, ModelRunnerOutput
from vllm.v1.worker.gpu.sample.output import SamplerOutput
@@ -17,6 +18,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
num_sampled_tokens: torch.Tensor,
main_stream: torch.cuda.Stream,
copy_stream: torch.cuda.Stream,
check_ep_fault: bool = False,
):
# NOTE(woosuk): We must retain references to the GPU tensors,
# as the copy operations are performed on a different CUDA stream than
@@ -26,6 +28,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
self.num_sampled_tokens = num_sampled_tokens
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
self.copy_event = torch.cuda.Event(blocking=True)
self._has_fault: torch.Tensor | None = None
with stream(copy_stream, main_stream):
copy_stream.wait_stream(main_stream)
@@ -44,6 +47,9 @@ class AsyncOutput(AsyncModelRunnerOutput):
k: v.to_cpu_nonblocking() if v is not None else None
for k, v in self.model_runner_output.prompt_logprobs_dict.items()
}
if check_ep_fault:
has_fault = get_ep_all2all_manager().query_fault()
self._has_fault = has_fault.to("cpu", non_blocking=True)
self.copy_event.record(copy_stream)
def get_output(self) -> ModelRunnerOutput:
@@ -67,6 +73,15 @@ class AsyncOutput(AsyncModelRunnerOutput):
if self.logprobs_tensors is not None:
self.model_runner_output.logprobs = self.logprobs_tensors.tolists()
self.model_runner_output.prompt_logprobs_dict = self.prompt_logprobs_dict
if self._has_fault is not None and self._has_fault.item():
mask = get_ep_all2all_manager().query_active_mask()
raise RuntimeError(
"Fault detected in EP all2all communication: "
"one or more ranks timed out during dispatch/combine. "
f"Mask: {mask.cpu().tolist()}"
)
return self.model_runner_output