[Bugfix][MoE] Restore TP-sharded routed-expert capture for DP=1 (#61388)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: Ao Shen <281812371+aoshen02@users.noreply.github.com> Signed-off-by: Ao Shen <aoshen@inferact.ai> Co-authored-by: Ao Shen <281812371+aoshen02@users.noreply.github.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Ao Shen <aoshen@inferact.ai>
This commit is contained in:
co-authored by
Ao Shen
OpenAI Codex
Ao Shen
parent
ef38e8d006
commit
a8be1500ba
@@ -4,7 +4,7 @@
|
||||
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -16,6 +16,9 @@ from vllm.forward_context import (
|
||||
ForwardContext,
|
||||
override_forward_context,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
)
|
||||
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
|
||||
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
@@ -173,6 +176,31 @@ def test_replay_batch_keeps_each_request_window(state):
|
||||
assert context.dp_metadata is None and context.is_padding is None
|
||||
|
||||
|
||||
def test_replay_context_restores_tp_routing_rows(state):
|
||||
"""Capture uses the compact replay batch's size, not the outer batch's."""
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
|
||||
replay, _ = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
context = replay.forward_context
|
||||
assert context.is_padding is None and context.dp_metadata is None
|
||||
count = len(REPLAY_ROWS)
|
||||
assert context.batch_descriptor.num_tokens == count
|
||||
capturer = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer)
|
||||
capturer.dp_rank, capturer.tp_size = 0, 2
|
||||
capturer.device_buffer = torch.full((401, 1, 2), -1, device=DEVICE)
|
||||
expected = torch.tensor(REPLAY_ROWS, device=DEVICE)[:, None].expand(-1, 2)
|
||||
gathered = torch.cat((expected, torch.full((1, 2), -1, device=DEVICE)))
|
||||
with (
|
||||
override_forward_context(context),
|
||||
patch(
|
||||
"vllm.model_executor.layers.fused_moe.routed_experts_capturer.get_tp_group"
|
||||
) as group,
|
||||
):
|
||||
group.return_value.all_gather.return_value = gathered
|
||||
capturer.capture(0, gathered[: (count + 1) // 2])
|
||||
torch.testing.assert_close(capturer.device_buffer[:count, 0], expected)
|
||||
assert (capturer.device_buffer[count:] == -1).all()
|
||||
|
||||
|
||||
def test_replay_batch_keeps_device_decode_boundaries(state):
|
||||
"""Adaptive verification resizes decodes on the GPU (CPU [2, 2], device
|
||||
[1, 3]): their rows stay and the boundaries shift only by trimmed rows."""
|
||||
|
||||
Reference in New Issue
Block a user