[MRV2] Enable routed-experts capture (#50721)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
aoshen02
2026-08-03 21:00:13 +00:00
committed by GitHub
co-authored by Nick Hill
parent e279f71583
commit 42ab184ea7
10 changed files with 335 additions and 227 deletions
@@ -2,15 +2,19 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import types
from types import SimpleNamespace
from unittest.mock import patch
from unittest.mock import Mock, patch
import pytest
import torch
from vllm.config import VllmConfig
from vllm.config.compilation import CompilationMode
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
bind_routed_experts_capturer,
get_routed_experts_attn_gid,
)
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
@@ -118,97 +122,12 @@ def test_base_router_capture_with_eplb_enabled():
assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
def test_gpu_model_runner_binds_router_capture(monkeypatch):
from vllm.v1.worker import gpu_model_runner as gmr
class _DummyRouter:
_routing_replay_out: torch.Tensor | None = None
class DummyFusedMoE:
def __init__(self):
self.layer_id = 7
self.router = _make_router()
self.routed_experts = _make_modular_routed_experts()
self._quant_method = self.routed_experts.quant_method
class DummyCapturer:
def __init__(self):
self.calls = []
def capture(self, layer_id, topk_ids):
self.calls.append((layer_id, topk_ids))
dummy_module = DummyFusedMoE()
# Patch the runtime import inside _bind_routed_experts_capturer.
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
dummy_self = types.SimpleNamespace(
model=types.SimpleNamespace(modules=lambda: [dummy_module])
)
capturer = DummyCapturer()
gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, capturer)
assert dummy_module.router.capture_fn is not None
dummy_module.router.capture_fn(torch.tensor([[5, 6]]))
assert len(capturer.calls) == 1
layer_id, topk_ids = capturer.calls[0]
assert layer_id == 7
assert torch.equal(topk_ids, torch.tensor([[5, 6]]))
def test_gpu_model_runner_binding_stage(monkeypatch):
from vllm.v1.worker import gpu_model_runner as gmr
class DummyFusedMoE:
def __init__(self):
self.layer_id = 11
self.router = _make_router()
self.routed_experts = _make_modular_routed_experts()
self._quant_method = self.routed_experts.quant_method
class DummyCapturer:
def __init__(self):
self.calls = []
def capture(self, layer_id, topk_ids):
self.calls.append((layer_id, topk_ids))
dummy_module = DummyFusedMoE()
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
dummy_self = types.SimpleNamespace(
model=types.SimpleNamespace(modules=lambda: [dummy_module])
)
# Before binding, no capture hook.
assert dummy_module.router.capture_fn is None
capturer = DummyCapturer()
gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, capturer)
# After binding, hook should exist and be callable.
assert callable(dummy_module.router.capture_fn)
dummy_module.router.capture_fn(torch.tensor([[9, 10]]))
assert len(capturer.calls) == 1
def test_gpu_model_runner_does_not_bind_draft_router_capture(monkeypatch):
from vllm.v1.worker import gpu_model_runner as gmr
def test_public_binding_only_visits_target_model(monkeypatch):
class DummyFusedMoE:
def __init__(self, layer_id):
self.layer_id = layer_id
self.router = _make_router()
self.routed_experts = _make_modular_routed_experts()
self._quant_method = self.routed_experts.quant_method
self._quant_method = _make_modular_routed_experts().quant_method
target_module = DummyFusedMoE(layer_id=7)
draft_module = DummyFusedMoE(layer_id=0)
@@ -216,27 +135,21 @@ def test_gpu_model_runner_does_not_bind_draft_router_capture(monkeypatch):
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
calls = []
capturer = types.SimpleNamespace(capture=lambda *args: calls.append(args))
dummy_self = types.SimpleNamespace(
model=types.SimpleNamespace(modules=lambda: [target_module]),
compilation_config=types.SimpleNamespace(
static_forward_context={
"model.layers.7.mlp.experts": target_module,
"mtp.layers.0.mlp.experts": draft_module,
}
),
bind_routed_experts_capturer(
types.SimpleNamespace(modules=lambda: [target_module]), capturer
)
capturer = types.SimpleNamespace(capture=lambda *_: None)
gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, capturer)
assert target_module.router.capture_fn is not None
assert draft_module.router.capture_fn is None
topk_ids = torch.tensor([[5, 6]])
target_module.router.capture_fn(topk_ids)
assert calls == [(7, topk_ids)]
def test_gpu_model_runner_rejects_monolithic_without_replay_support(monkeypatch):
from vllm.v1.worker import gpu_model_runner as gmr
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
class DummyFusedMoE:
def __init__(self):
self.layer_id = 3
@@ -269,12 +182,10 @@ def test_gpu_model_runner_rejects_monolithic_without_replay_support(monkeypatch)
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
dummy_self = types.SimpleNamespace(
model=types.SimpleNamespace(modules=lambda: [dummy_module])
)
with pytest.raises(ValueError, match="monolithic MoE kernel"):
gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, DummyCapturer())
bind_routed_experts_capturer(
types.SimpleNamespace(modules=lambda: [dummy_module]), DummyCapturer()
)
def test_routed_experts_capturer_single_dp_no_metadata():
@@ -333,3 +244,111 @@ def test_routed_experts_capturer_dp_unexpected_batch_raises():
):
capturer.capture(layer_id=0, topk_ids=topk)
assert capturer.device_buffer[0, 0, 0].item() == -1
def test_routed_experts_attention_group_is_shared_and_fail_closed(monkeypatch):
class FullAttentionSpec:
pass
monkeypatch.setattr(f"{_REC_MODULE}.FullAttentionSpec", FullAttentionSpec)
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(kv_cache_spec=object()),
SimpleNamespace(kv_cache_spec=FullAttentionSpec()),
]
)
assert get_routed_experts_attn_gid(config) == 1
with pytest.raises(ValueError, match="requires a full-attention KV cache group"):
get_routed_experts_attn_gid(SimpleNamespace(kv_cache_groups=[]))
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
def test_mrv2_async_output_returns_existing_routed_experts_field():
from vllm.v1.outputs import ModelRunnerOutput, RoutedExpertsTensors
from vllm.v1.worker.gpu.async_utils import AsyncOutput
from vllm.v1.worker.gpu.sample.output import SamplerOutput
routed_experts = RoutedExpertsTensors(
routing_data=torch.arange(6, dtype=torch.int32, device="cuda").reshape(3, 1, 2),
slot_mapping=torch.tensor([11, 12, 13], device="cuda"),
)
num_sampled = torch.tensor([1], dtype=torch.int32, device="cuda")
sampler_output = SamplerOutput(
sampled_token_ids=torch.tensor([[1]], device="cuda"),
logprobs_tensors=None,
num_nans=None,
num_sampled=num_sampled,
num_rejected=torch.tensor([0], dtype=torch.int32, device="cuda"),
)
output = AsyncOutput(
model_runner_output=ModelRunnerOutput(req_ids=["req"], req_id_to_index={}),
sampler_output=sampler_output,
num_sampled_tokens=num_sampled,
main_stream=torch.cuda.current_stream(),
copy_stream=torch.cuda.Stream(),
routed_experts=routed_experts,
).get_output()
assert output.routed_experts is not None
assert output.routed_experts.routing_data[:, 0, 0].tolist() == [0, 2, 4]
assert output.routed_experts.slot_mapping.tolist() == [11, 12, 13]
@pytest.mark.parametrize("rank", [0, 1])
def test_all_tp_ranks_initialize_capture(monkeypatch, rank):
pytest.importorskip("vllm.vllm_flash_attn", exc_type=ImportError)
import vllm.v1.worker.gpu.model_runner as model_runner
capturer = Mock()
constructor = Mock(return_value=capturer)
bind = Mock()
monkeypatch.setattr(model_runner, "RoutedExpertsCapturer", constructor)
monkeypatch.setattr(model_runner, "bind_routed_experts_capturer", bind)
runner = model_runner.GPUModelRunner.__new__(model_runner.GPUModelRunner)
runner.max_num_tokens = 32
runner.vllm_config = SimpleNamespace(parallel_config=SimpleNamespace(rank=rank))
runner.kv_cache_config = SimpleNamespace()
runner.model = Mock()
runner.init_routed_experts_capturer()
constructor.assert_called_once_with(
max_num_batched_tokens=32,
vllm_config=runner.vllm_config,
kv_cache_config=runner.kv_cache_config,
)
bind.assert_called_once_with(runner.model, capturer)
assert runner.routed_experts_capturer is capturer
def test_v2_model_runner_accepts_routed_experts(monkeypatch):
monkeypatch.setattr("importlib.metadata.entry_points", lambda **_: ())
config = SimpleNamespace(
model_config=SimpleNamespace(
enable_return_routed_experts=True,
use_mla=False,
logits_processors=None,
enable_prompt_embeds=False,
),
speculative_config=None,
parallel_config=SimpleNamespace(
prefill_context_parallel_size=1,
tensor_parallel_size=1,
distributed_executor_backend=None,
pipeline_parallel_size=1,
enable_dbo=False,
enable_elastic_ep=False,
),
compilation_config=SimpleNamespace(
mode=CompilationMode.NONE,
pass_config=SimpleNamespace(enable_sp=False),
),
cache_config=SimpleNamespace(kv_sharing_fast_prefill=False),
ec_transfer_config=None,
)
unsupported = VllmConfig._get_v2_model_runner_unsupported_features(config)
assert "routed experts capture" not in unsupported