[MRV2] Enable routed-experts capture (#50721)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -11,6 +11,7 @@ from vllm.v1.outputs import (
|
||||
LogprobsTensors,
|
||||
ModelRunnerOutput,
|
||||
PoolerOutput,
|
||||
RoutedExpertsTensors,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
|
||||
@@ -24,6 +25,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
main_stream: torch.cuda.Stream,
|
||||
copy_stream: torch.cuda.Stream,
|
||||
check_ep_fault: bool = False,
|
||||
routed_experts: RoutedExpertsTensors | None = None,
|
||||
):
|
||||
# NOTE(woosuk): We must retain references to the GPU tensors,
|
||||
# as the copy operations are performed on a different CUDA stream than
|
||||
@@ -31,6 +33,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
self.model_runner_output = model_runner_output
|
||||
self.sampler_output = sampler_output
|
||||
self.num_sampled_tokens = num_sampled_tokens
|
||||
self.routed_experts = routed_experts
|
||||
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
|
||||
self.copy_event = torch.cuda.Event(blocking=True)
|
||||
self._has_fault: torch.Tensor | None = None
|
||||
@@ -48,6 +51,9 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
if sampler_output.num_nans is not None:
|
||||
self.num_nans = async_copy_to_np(sampler_output.num_nans)
|
||||
self.num_sampled_tokens_np = async_copy_to_np(num_sampled_tokens)
|
||||
self.routed_experts_cpu: RoutedExpertsTensors | None = None
|
||||
if routed_experts is not None:
|
||||
self.routed_experts_cpu = routed_experts.to_cpu_nonblocking()
|
||||
self.prompt_logprobs_dict = {
|
||||
k: v.to_cpu_nonblocking() if v is not None else None
|
||||
for k, v in self.model_runner_output.prompt_logprobs_dict.items()
|
||||
@@ -78,6 +84,8 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
if self.logprobs_tensors is not None:
|
||||
self.model_runner_output.logprobs = self.logprobs_tensors.tolists()
|
||||
self.model_runner_output.prompt_logprobs_dict = self.prompt_logprobs_dict
|
||||
if self.routed_experts_cpu is not None:
|
||||
self.model_runner_output.routed_experts = self.routed_experts_cpu.tolists()
|
||||
|
||||
if self._has_fault is not None and self._has_fault.item():
|
||||
mask = get_ep_all2all_manager().query_active_mask()
|
||||
|
||||
Reference in New Issue
Block a user