[MRV2] Enable routed-experts capture (#50721)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
aoshen02
2026-08-03 21:00:13 +00:00
committed by GitHub
co-authored by Nick Hill
parent e279f71583
commit 42ab184ea7
10 changed files with 335 additions and 227 deletions
+8
View File
@@ -11,6 +11,7 @@ from vllm.v1.outputs import (
LogprobsTensors,
ModelRunnerOutput,
PoolerOutput,
RoutedExpertsTensors,
)
from vllm.v1.worker.gpu.sample.output import SamplerOutput
@@ -24,6 +25,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
main_stream: torch.cuda.Stream,
copy_stream: torch.cuda.Stream,
check_ep_fault: bool = False,
routed_experts: RoutedExpertsTensors | None = None,
):
# NOTE(woosuk): We must retain references to the GPU tensors,
# as the copy operations are performed on a different CUDA stream than
@@ -31,6 +33,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
self.model_runner_output = model_runner_output
self.sampler_output = sampler_output
self.num_sampled_tokens = num_sampled_tokens
self.routed_experts = routed_experts
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
self.copy_event = torch.cuda.Event(blocking=True)
self._has_fault: torch.Tensor | None = None
@@ -48,6 +51,9 @@ class AsyncOutput(AsyncModelRunnerOutput):
if sampler_output.num_nans is not None:
self.num_nans = async_copy_to_np(sampler_output.num_nans)
self.num_sampled_tokens_np = async_copy_to_np(num_sampled_tokens)
self.routed_experts_cpu: RoutedExpertsTensors | None = None
if routed_experts is not None:
self.routed_experts_cpu = routed_experts.to_cpu_nonblocking()
self.prompt_logprobs_dict = {
k: v.to_cpu_nonblocking() if v is not None else None
for k, v in self.model_runner_output.prompt_logprobs_dict.items()
@@ -78,6 +84,8 @@ class AsyncOutput(AsyncModelRunnerOutput):
if self.logprobs_tensors is not None:
self.model_runner_output.logprobs = self.logprobs_tensors.tolists()
self.model_runner_output.prompt_logprobs_dict = self.prompt_logprobs_dict
if self.routed_experts_cpu is not None:
self.model_runner_output.routed_experts = self.routed_experts_cpu.tolists()
if self._has_fault is not None and self._has_fault.item():
mask = get_ep_all2all_manager().query_active_mask()