[MRV2] Enable routed-experts capture (#50721)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -2,15 +2,19 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import types
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
bind_routed_experts_capturer,
|
||||
get_routed_experts_attn_gid,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
|
||||
|
||||
@@ -118,97 +122,12 @@ def test_base_router_capture_with_eplb_enabled():
|
||||
assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
|
||||
|
||||
|
||||
def test_gpu_model_runner_binds_router_capture(monkeypatch):
|
||||
from vllm.v1.worker import gpu_model_runner as gmr
|
||||
|
||||
class _DummyRouter:
|
||||
_routing_replay_out: torch.Tensor | None = None
|
||||
|
||||
class DummyFusedMoE:
|
||||
def __init__(self):
|
||||
self.layer_id = 7
|
||||
self.router = _make_router()
|
||||
self.routed_experts = _make_modular_routed_experts()
|
||||
self._quant_method = self.routed_experts.quant_method
|
||||
|
||||
class DummyCapturer:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def capture(self, layer_id, topk_ids):
|
||||
self.calls.append((layer_id, topk_ids))
|
||||
|
||||
dummy_module = DummyFusedMoE()
|
||||
|
||||
# Patch the runtime import inside _bind_routed_experts_capturer.
|
||||
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
|
||||
|
||||
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
|
||||
|
||||
dummy_self = types.SimpleNamespace(
|
||||
model=types.SimpleNamespace(modules=lambda: [dummy_module])
|
||||
)
|
||||
|
||||
capturer = DummyCapturer()
|
||||
gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, capturer)
|
||||
|
||||
assert dummy_module.router.capture_fn is not None
|
||||
dummy_module.router.capture_fn(torch.tensor([[5, 6]]))
|
||||
|
||||
assert len(capturer.calls) == 1
|
||||
layer_id, topk_ids = capturer.calls[0]
|
||||
assert layer_id == 7
|
||||
assert torch.equal(topk_ids, torch.tensor([[5, 6]]))
|
||||
|
||||
|
||||
def test_gpu_model_runner_binding_stage(monkeypatch):
|
||||
from vllm.v1.worker import gpu_model_runner as gmr
|
||||
|
||||
class DummyFusedMoE:
|
||||
def __init__(self):
|
||||
self.layer_id = 11
|
||||
self.router = _make_router()
|
||||
self.routed_experts = _make_modular_routed_experts()
|
||||
self._quant_method = self.routed_experts.quant_method
|
||||
|
||||
class DummyCapturer:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def capture(self, layer_id, topk_ids):
|
||||
self.calls.append((layer_id, topk_ids))
|
||||
|
||||
dummy_module = DummyFusedMoE()
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
|
||||
|
||||
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
|
||||
|
||||
dummy_self = types.SimpleNamespace(
|
||||
model=types.SimpleNamespace(modules=lambda: [dummy_module])
|
||||
)
|
||||
|
||||
# Before binding, no capture hook.
|
||||
assert dummy_module.router.capture_fn is None
|
||||
|
||||
capturer = DummyCapturer()
|
||||
gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, capturer)
|
||||
|
||||
# After binding, hook should exist and be callable.
|
||||
assert callable(dummy_module.router.capture_fn)
|
||||
dummy_module.router.capture_fn(torch.tensor([[9, 10]]))
|
||||
assert len(capturer.calls) == 1
|
||||
|
||||
|
||||
def test_gpu_model_runner_does_not_bind_draft_router_capture(monkeypatch):
|
||||
from vllm.v1.worker import gpu_model_runner as gmr
|
||||
|
||||
def test_public_binding_only_visits_target_model(monkeypatch):
|
||||
class DummyFusedMoE:
|
||||
def __init__(self, layer_id):
|
||||
self.layer_id = layer_id
|
||||
self.router = _make_router()
|
||||
self.routed_experts = _make_modular_routed_experts()
|
||||
self._quant_method = self.routed_experts.quant_method
|
||||
self._quant_method = _make_modular_routed_experts().quant_method
|
||||
|
||||
target_module = DummyFusedMoE(layer_id=7)
|
||||
draft_module = DummyFusedMoE(layer_id=0)
|
||||
@@ -216,27 +135,21 @@ def test_gpu_model_runner_does_not_bind_draft_router_capture(monkeypatch):
|
||||
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
|
||||
|
||||
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
|
||||
calls = []
|
||||
capturer = types.SimpleNamespace(capture=lambda *args: calls.append(args))
|
||||
|
||||
dummy_self = types.SimpleNamespace(
|
||||
model=types.SimpleNamespace(modules=lambda: [target_module]),
|
||||
compilation_config=types.SimpleNamespace(
|
||||
static_forward_context={
|
||||
"model.layers.7.mlp.experts": target_module,
|
||||
"mtp.layers.0.mlp.experts": draft_module,
|
||||
}
|
||||
),
|
||||
bind_routed_experts_capturer(
|
||||
types.SimpleNamespace(modules=lambda: [target_module]), capturer
|
||||
)
|
||||
|
||||
capturer = types.SimpleNamespace(capture=lambda *_: None)
|
||||
gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, capturer)
|
||||
|
||||
assert target_module.router.capture_fn is not None
|
||||
assert draft_module.router.capture_fn is None
|
||||
topk_ids = torch.tensor([[5, 6]])
|
||||
target_module.router.capture_fn(topk_ids)
|
||||
assert calls == [(7, topk_ids)]
|
||||
|
||||
|
||||
def test_gpu_model_runner_rejects_monolithic_without_replay_support(monkeypatch):
|
||||
from vllm.v1.worker import gpu_model_runner as gmr
|
||||
|
||||
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
|
||||
class DummyFusedMoE:
|
||||
def __init__(self):
|
||||
self.layer_id = 3
|
||||
@@ -269,12 +182,10 @@ def test_gpu_model_runner_rejects_monolithic_without_replay_support(monkeypatch)
|
||||
|
||||
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
|
||||
|
||||
dummy_self = types.SimpleNamespace(
|
||||
model=types.SimpleNamespace(modules=lambda: [dummy_module])
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="monolithic MoE kernel"):
|
||||
gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, DummyCapturer())
|
||||
bind_routed_experts_capturer(
|
||||
types.SimpleNamespace(modules=lambda: [dummy_module]), DummyCapturer()
|
||||
)
|
||||
|
||||
|
||||
def test_routed_experts_capturer_single_dp_no_metadata():
|
||||
@@ -333,3 +244,111 @@ def test_routed_experts_capturer_dp_unexpected_batch_raises():
|
||||
):
|
||||
capturer.capture(layer_id=0, topk_ids=topk)
|
||||
assert capturer.device_buffer[0, 0, 0].item() == -1
|
||||
|
||||
|
||||
def test_routed_experts_attention_group_is_shared_and_fail_closed(monkeypatch):
|
||||
class FullAttentionSpec:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(f"{_REC_MODULE}.FullAttentionSpec", FullAttentionSpec)
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(kv_cache_spec=object()),
|
||||
SimpleNamespace(kv_cache_spec=FullAttentionSpec()),
|
||||
]
|
||||
)
|
||||
assert get_routed_experts_attn_gid(config) == 1
|
||||
|
||||
with pytest.raises(ValueError, match="requires a full-attention KV cache group"):
|
||||
get_routed_experts_attn_gid(SimpleNamespace(kv_cache_groups=[]))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_mrv2_async_output_returns_existing_routed_experts_field():
|
||||
from vllm.v1.outputs import ModelRunnerOutput, RoutedExpertsTensors
|
||||
from vllm.v1.worker.gpu.async_utils import AsyncOutput
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
|
||||
routed_experts = RoutedExpertsTensors(
|
||||
routing_data=torch.arange(6, dtype=torch.int32, device="cuda").reshape(3, 1, 2),
|
||||
slot_mapping=torch.tensor([11, 12, 13], device="cuda"),
|
||||
)
|
||||
num_sampled = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
sampler_output = SamplerOutput(
|
||||
sampled_token_ids=torch.tensor([[1]], device="cuda"),
|
||||
logprobs_tensors=None,
|
||||
num_nans=None,
|
||||
num_sampled=num_sampled,
|
||||
num_rejected=torch.tensor([0], dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
output = AsyncOutput(
|
||||
model_runner_output=ModelRunnerOutput(req_ids=["req"], req_id_to_index={}),
|
||||
sampler_output=sampler_output,
|
||||
num_sampled_tokens=num_sampled,
|
||||
main_stream=torch.cuda.current_stream(),
|
||||
copy_stream=torch.cuda.Stream(),
|
||||
routed_experts=routed_experts,
|
||||
).get_output()
|
||||
|
||||
assert output.routed_experts is not None
|
||||
assert output.routed_experts.routing_data[:, 0, 0].tolist() == [0, 2, 4]
|
||||
assert output.routed_experts.slot_mapping.tolist() == [11, 12, 13]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rank", [0, 1])
|
||||
def test_all_tp_ranks_initialize_capture(monkeypatch, rank):
|
||||
pytest.importorskip("vllm.vllm_flash_attn", exc_type=ImportError)
|
||||
import vllm.v1.worker.gpu.model_runner as model_runner
|
||||
|
||||
capturer = Mock()
|
||||
constructor = Mock(return_value=capturer)
|
||||
bind = Mock()
|
||||
monkeypatch.setattr(model_runner, "RoutedExpertsCapturer", constructor)
|
||||
monkeypatch.setattr(model_runner, "bind_routed_experts_capturer", bind)
|
||||
|
||||
runner = model_runner.GPUModelRunner.__new__(model_runner.GPUModelRunner)
|
||||
runner.max_num_tokens = 32
|
||||
runner.vllm_config = SimpleNamespace(parallel_config=SimpleNamespace(rank=rank))
|
||||
runner.kv_cache_config = SimpleNamespace()
|
||||
runner.model = Mock()
|
||||
|
||||
runner.init_routed_experts_capturer()
|
||||
|
||||
constructor.assert_called_once_with(
|
||||
max_num_batched_tokens=32,
|
||||
vllm_config=runner.vllm_config,
|
||||
kv_cache_config=runner.kv_cache_config,
|
||||
)
|
||||
bind.assert_called_once_with(runner.model, capturer)
|
||||
assert runner.routed_experts_capturer is capturer
|
||||
|
||||
|
||||
def test_v2_model_runner_accepts_routed_experts(monkeypatch):
|
||||
monkeypatch.setattr("importlib.metadata.entry_points", lambda **_: ())
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
enable_return_routed_experts=True,
|
||||
use_mla=False,
|
||||
logits_processors=None,
|
||||
enable_prompt_embeds=False,
|
||||
),
|
||||
speculative_config=None,
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=1,
|
||||
tensor_parallel_size=1,
|
||||
distributed_executor_backend=None,
|
||||
pipeline_parallel_size=1,
|
||||
enable_dbo=False,
|
||||
enable_elastic_ep=False,
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
mode=CompilationMode.NONE,
|
||||
pass_config=SimpleNamespace(enable_sp=False),
|
||||
),
|
||||
cache_config=SimpleNamespace(kv_sharing_fast_prefill=False),
|
||||
ec_transfer_config=None,
|
||||
)
|
||||
|
||||
unsupported = VllmConfig._get_v2_model_runner_unsupported_features(config)
|
||||
|
||||
assert "routed experts capture" not in unsupported
|
||||
|
||||
Reference in New Issue
Block a user