[Feature] Detect all2all peer fault with fault tolerance backend and prevent corrupted output (#43637)

Signed-off-by: fangyuchu <fangyuchu@qq.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
fangyuchu
2026-06-30 09:00:25 -07:00
committed by GitHub
co-authored by Tyler Michael Smith mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent e840f0d3f5
commit 245888ff77
5 changed files with 82 additions and 4 deletions
+19
View File
@@ -58,6 +58,7 @@ from vllm.logger import init_logger
from vllm.lora.layers import LoRAMapping, LoRAMappingType
from vllm.model_executor.layers.attention import Attention, MLAAttention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
)
@@ -249,6 +250,7 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
async_output_copy_stream: torch.cuda.Stream,
vocab_size: int,
routed_experts: RoutedExpertsTensors | None = None,
check_ep_fault: bool = False,
):
self._model_runner_output = model_runner_output
self._invalid_req_indices = invalid_req_indices
@@ -262,6 +264,7 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
self.vocab_size = vocab_size
self._logprobs_tensors = logprobs_tensors
self._routed_experts = routed_experts
self._has_fault: torch.Tensor | None = None
# Initiate the copy on a separate stream, but do not synchronize it.
default_stream = torch.cuda.current_stream()
@@ -280,6 +283,9 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
if self._routed_experts is not None
else None
)
if check_ep_fault:
has_fault = get_ep_all2all_manager().query_fault()
self._has_fault = has_fault.to("cpu", non_blocking=True)
self.async_copy_ready_event.record()
def get_output(self) -> ModelRunnerOutput:
@@ -316,6 +322,14 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
output.routed_experts = self._routed_experts_cpu.tolists()
del self._routed_experts
if self._has_fault is not None and self._has_fault.item():
mask = get_ep_all2all_manager().query_active_mask()
raise RuntimeError(
"Fault detected in EP all2all communication: "
"one or more ranks timed out during dispatch/combine. "
f"Mask: {mask.cpu().tolist()}"
)
return output
@@ -445,6 +459,10 @@ class GPUModelRunner(
self.device = device
self.dtype = self.model_config.dtype
self.check_ep_fault = False
if parallel_config.data_parallel_size > 1 and self.model_config.is_moe:
self.check_ep_fault = get_ep_all2all_manager().support_fault_tolerance
self.kv_cache_dtype = kv_cache_dtype_str_to_dtype(
cache_config.cache_dtype, self.model_config
)
@@ -4695,6 +4713,7 @@ class GPUModelRunner(
async_output_copy_stream=self._get_or_create_async_output_copy_stream(),
vocab_size=self.input_batch.vocab_size,
routed_experts=routed_experts_snapshot,
check_ep_fault=self.check_ep_fault,
)
with record_function_or_nullcontext(
"gpu_model_runner: set_async_sampled_token_ids"