[Feature] Detect all2all peer fault with fault tolerance backend and prevent corrupted output (#43637)
Signed-off-by: fangyuchu <fangyuchu@qq.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
e840f0d3f5
commit
245888ff77
@@ -58,6 +58,7 @@ from vllm.logger import init_logger
|
||||
from vllm.lora.layers import LoRAMapping, LoRAMappingType
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
)
|
||||
@@ -249,6 +250,7 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
async_output_copy_stream: torch.cuda.Stream,
|
||||
vocab_size: int,
|
||||
routed_experts: RoutedExpertsTensors | None = None,
|
||||
check_ep_fault: bool = False,
|
||||
):
|
||||
self._model_runner_output = model_runner_output
|
||||
self._invalid_req_indices = invalid_req_indices
|
||||
@@ -262,6 +264,7 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
self.vocab_size = vocab_size
|
||||
self._logprobs_tensors = logprobs_tensors
|
||||
self._routed_experts = routed_experts
|
||||
self._has_fault: torch.Tensor | None = None
|
||||
|
||||
# Initiate the copy on a separate stream, but do not synchronize it.
|
||||
default_stream = torch.cuda.current_stream()
|
||||
@@ -280,6 +283,9 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
if self._routed_experts is not None
|
||||
else None
|
||||
)
|
||||
if check_ep_fault:
|
||||
has_fault = get_ep_all2all_manager().query_fault()
|
||||
self._has_fault = has_fault.to("cpu", non_blocking=True)
|
||||
self.async_copy_ready_event.record()
|
||||
|
||||
def get_output(self) -> ModelRunnerOutput:
|
||||
@@ -316,6 +322,14 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
output.routed_experts = self._routed_experts_cpu.tolists()
|
||||
del self._routed_experts
|
||||
|
||||
if self._has_fault is not None and self._has_fault.item():
|
||||
mask = get_ep_all2all_manager().query_active_mask()
|
||||
raise RuntimeError(
|
||||
"Fault detected in EP all2all communication: "
|
||||
"one or more ranks timed out during dispatch/combine. "
|
||||
f"Mask: {mask.cpu().tolist()}"
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
@@ -445,6 +459,10 @@ class GPUModelRunner(
|
||||
self.device = device
|
||||
self.dtype = self.model_config.dtype
|
||||
|
||||
self.check_ep_fault = False
|
||||
if parallel_config.data_parallel_size > 1 and self.model_config.is_moe:
|
||||
self.check_ep_fault = get_ep_all2all_manager().support_fault_tolerance
|
||||
|
||||
self.kv_cache_dtype = kv_cache_dtype_str_to_dtype(
|
||||
cache_config.cache_dtype, self.model_config
|
||||
)
|
||||
@@ -4695,6 +4713,7 @@ class GPUModelRunner(
|
||||
async_output_copy_stream=self._get_or_create_async_output_copy_stream(),
|
||||
vocab_size=self.input_batch.vocab_size,
|
||||
routed_experts=routed_experts_snapshot,
|
||||
check_ep_fault=self.check_ep_fault,
|
||||
)
|
||||
with record_function_or_nullcontext(
|
||||
"gpu_model_runner: set_async_sampled_token_ids"
|
||||
|
||||
Reference in New Issue
Block a user