|
|
|
@@ -2,15 +2,19 @@
|
|
|
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
|
import types
|
|
|
|
|
from types import SimpleNamespace
|
|
|
|
|
from unittest.mock import patch
|
|
|
|
|
from unittest.mock import Mock, patch
|
|
|
|
|
|
|
|
|
|
import pytest
|
|
|
|
|
import torch
|
|
|
|
|
|
|
|
|
|
from vllm.config import VllmConfig
|
|
|
|
|
from vllm.config.compilation import CompilationMode
|
|
|
|
|
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
|
|
|
|
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
|
|
|
|
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
|
|
|
|
RoutedExpertsCapturer,
|
|
|
|
|
bind_routed_experts_capturer,
|
|
|
|
|
get_routed_experts_attn_gid,
|
|
|
|
|
)
|
|
|
|
|
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
|
|
|
|
|
|
|
|
|
@@ -118,97 +122,12 @@ def test_base_router_capture_with_eplb_enabled():
|
|
|
|
|
assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_gpu_model_runner_binds_router_capture(monkeypatch):
|
|
|
|
|
from vllm.v1.worker import gpu_model_runner as gmr
|
|
|
|
|
|
|
|
|
|
class _DummyRouter:
|
|
|
|
|
_routing_replay_out: torch.Tensor | None = None
|
|
|
|
|
|
|
|
|
|
class DummyFusedMoE:
|
|
|
|
|
def __init__(self):
|
|
|
|
|
self.layer_id = 7
|
|
|
|
|
self.router = _make_router()
|
|
|
|
|
self.routed_experts = _make_modular_routed_experts()
|
|
|
|
|
self._quant_method = self.routed_experts.quant_method
|
|
|
|
|
|
|
|
|
|
class DummyCapturer:
|
|
|
|
|
def __init__(self):
|
|
|
|
|
self.calls = []
|
|
|
|
|
|
|
|
|
|
def capture(self, layer_id, topk_ids):
|
|
|
|
|
self.calls.append((layer_id, topk_ids))
|
|
|
|
|
|
|
|
|
|
dummy_module = DummyFusedMoE()
|
|
|
|
|
|
|
|
|
|
# Patch the runtime import inside _bind_routed_experts_capturer.
|
|
|
|
|
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
|
|
|
|
|
|
|
|
|
|
dummy_self = types.SimpleNamespace(
|
|
|
|
|
model=types.SimpleNamespace(modules=lambda: [dummy_module])
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
capturer = DummyCapturer()
|
|
|
|
|
gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, capturer)
|
|
|
|
|
|
|
|
|
|
assert dummy_module.router.capture_fn is not None
|
|
|
|
|
dummy_module.router.capture_fn(torch.tensor([[5, 6]]))
|
|
|
|
|
|
|
|
|
|
assert len(capturer.calls) == 1
|
|
|
|
|
layer_id, topk_ids = capturer.calls[0]
|
|
|
|
|
assert layer_id == 7
|
|
|
|
|
assert torch.equal(topk_ids, torch.tensor([[5, 6]]))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_gpu_model_runner_binding_stage(monkeypatch):
|
|
|
|
|
from vllm.v1.worker import gpu_model_runner as gmr
|
|
|
|
|
|
|
|
|
|
class DummyFusedMoE:
|
|
|
|
|
def __init__(self):
|
|
|
|
|
self.layer_id = 11
|
|
|
|
|
self.router = _make_router()
|
|
|
|
|
self.routed_experts = _make_modular_routed_experts()
|
|
|
|
|
self._quant_method = self.routed_experts.quant_method
|
|
|
|
|
|
|
|
|
|
class DummyCapturer:
|
|
|
|
|
def __init__(self):
|
|
|
|
|
self.calls = []
|
|
|
|
|
|
|
|
|
|
def capture(self, layer_id, topk_ids):
|
|
|
|
|
self.calls.append((layer_id, topk_ids))
|
|
|
|
|
|
|
|
|
|
dummy_module = DummyFusedMoE()
|
|
|
|
|
|
|
|
|
|
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
|
|
|
|
|
|
|
|
|
|
dummy_self = types.SimpleNamespace(
|
|
|
|
|
model=types.SimpleNamespace(modules=lambda: [dummy_module])
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Before binding, no capture hook.
|
|
|
|
|
assert dummy_module.router.capture_fn is None
|
|
|
|
|
|
|
|
|
|
capturer = DummyCapturer()
|
|
|
|
|
gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, capturer)
|
|
|
|
|
|
|
|
|
|
# After binding, hook should exist and be callable.
|
|
|
|
|
assert callable(dummy_module.router.capture_fn)
|
|
|
|
|
dummy_module.router.capture_fn(torch.tensor([[9, 10]]))
|
|
|
|
|
assert len(capturer.calls) == 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_gpu_model_runner_does_not_bind_draft_router_capture(monkeypatch):
|
|
|
|
|
from vllm.v1.worker import gpu_model_runner as gmr
|
|
|
|
|
|
|
|
|
|
def test_public_binding_only_visits_target_model(monkeypatch):
|
|
|
|
|
class DummyFusedMoE:
|
|
|
|
|
def __init__(self, layer_id):
|
|
|
|
|
self.layer_id = layer_id
|
|
|
|
|
self.router = _make_router()
|
|
|
|
|
self.routed_experts = _make_modular_routed_experts()
|
|
|
|
|
self._quant_method = self.routed_experts.quant_method
|
|
|
|
|
self._quant_method = _make_modular_routed_experts().quant_method
|
|
|
|
|
|
|
|
|
|
target_module = DummyFusedMoE(layer_id=7)
|
|
|
|
|
draft_module = DummyFusedMoE(layer_id=0)
|
|
|
|
@@ -216,27 +135,21 @@ def test_gpu_model_runner_does_not_bind_draft_router_capture(monkeypatch):
|
|
|
|
|
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
|
|
|
|
|
calls = []
|
|
|
|
|
capturer = types.SimpleNamespace(capture=lambda *args: calls.append(args))
|
|
|
|
|
|
|
|
|
|
dummy_self = types.SimpleNamespace(
|
|
|
|
|
model=types.SimpleNamespace(modules=lambda: [target_module]),
|
|
|
|
|
compilation_config=types.SimpleNamespace(
|
|
|
|
|
static_forward_context={
|
|
|
|
|
"model.layers.7.mlp.experts": target_module,
|
|
|
|
|
"mtp.layers.0.mlp.experts": draft_module,
|
|
|
|
|
}
|
|
|
|
|
),
|
|
|
|
|
bind_routed_experts_capturer(
|
|
|
|
|
types.SimpleNamespace(modules=lambda: [target_module]), capturer
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
capturer = types.SimpleNamespace(capture=lambda *_: None)
|
|
|
|
|
gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, capturer)
|
|
|
|
|
|
|
|
|
|
assert target_module.router.capture_fn is not None
|
|
|
|
|
assert draft_module.router.capture_fn is None
|
|
|
|
|
topk_ids = torch.tensor([[5, 6]])
|
|
|
|
|
target_module.router.capture_fn(topk_ids)
|
|
|
|
|
assert calls == [(7, topk_ids)]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_gpu_model_runner_rejects_monolithic_without_replay_support(monkeypatch):
|
|
|
|
|
from vllm.v1.worker import gpu_model_runner as gmr
|
|
|
|
|
|
|
|
|
|
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
|
|
|
|
|
class DummyFusedMoE:
|
|
|
|
|
def __init__(self):
|
|
|
|
|
self.layer_id = 3
|
|
|
|
@@ -269,12 +182,10 @@ def test_gpu_model_runner_rejects_monolithic_without_replay_support(monkeypatch)
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
|
|
|
|
|
|
|
|
|
|
dummy_self = types.SimpleNamespace(
|
|
|
|
|
model=types.SimpleNamespace(modules=lambda: [dummy_module])
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
with pytest.raises(ValueError, match="monolithic MoE kernel"):
|
|
|
|
|
gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, DummyCapturer())
|
|
|
|
|
bind_routed_experts_capturer(
|
|
|
|
|
types.SimpleNamespace(modules=lambda: [dummy_module]), DummyCapturer()
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_routed_experts_capturer_single_dp_no_metadata():
|
|
|
|
@@ -333,3 +244,111 @@ def test_routed_experts_capturer_dp_unexpected_batch_raises():
|
|
|
|
|
):
|
|
|
|
|
capturer.capture(layer_id=0, topk_ids=topk)
|
|
|
|
|
assert capturer.device_buffer[0, 0, 0].item() == -1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_routed_experts_attention_group_is_shared_and_fail_closed(monkeypatch):
|
|
|
|
|
class FullAttentionSpec:
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(f"{_REC_MODULE}.FullAttentionSpec", FullAttentionSpec)
|
|
|
|
|
config = SimpleNamespace(
|
|
|
|
|
kv_cache_groups=[
|
|
|
|
|
SimpleNamespace(kv_cache_spec=object()),
|
|
|
|
|
SimpleNamespace(kv_cache_spec=FullAttentionSpec()),
|
|
|
|
|
]
|
|
|
|
|
)
|
|
|
|
|
assert get_routed_experts_attn_gid(config) == 1
|
|
|
|
|
|
|
|
|
|
with pytest.raises(ValueError, match="requires a full-attention KV cache group"):
|
|
|
|
|
get_routed_experts_attn_gid(SimpleNamespace(kv_cache_groups=[]))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
|
|
|
|
def test_mrv2_async_output_returns_existing_routed_experts_field():
|
|
|
|
|
from vllm.v1.outputs import ModelRunnerOutput, RoutedExpertsTensors
|
|
|
|
|
from vllm.v1.worker.gpu.async_utils import AsyncOutput
|
|
|
|
|
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
|
|
|
|
|
|
|
|
|
routed_experts = RoutedExpertsTensors(
|
|
|
|
|
routing_data=torch.arange(6, dtype=torch.int32, device="cuda").reshape(3, 1, 2),
|
|
|
|
|
slot_mapping=torch.tensor([11, 12, 13], device="cuda"),
|
|
|
|
|
)
|
|
|
|
|
num_sampled = torch.tensor([1], dtype=torch.int32, device="cuda")
|
|
|
|
|
sampler_output = SamplerOutput(
|
|
|
|
|
sampled_token_ids=torch.tensor([[1]], device="cuda"),
|
|
|
|
|
logprobs_tensors=None,
|
|
|
|
|
num_nans=None,
|
|
|
|
|
num_sampled=num_sampled,
|
|
|
|
|
num_rejected=torch.tensor([0], dtype=torch.int32, device="cuda"),
|
|
|
|
|
)
|
|
|
|
|
output = AsyncOutput(
|
|
|
|
|
model_runner_output=ModelRunnerOutput(req_ids=["req"], req_id_to_index={}),
|
|
|
|
|
sampler_output=sampler_output,
|
|
|
|
|
num_sampled_tokens=num_sampled,
|
|
|
|
|
main_stream=torch.cuda.current_stream(),
|
|
|
|
|
copy_stream=torch.cuda.Stream(),
|
|
|
|
|
routed_experts=routed_experts,
|
|
|
|
|
).get_output()
|
|
|
|
|
|
|
|
|
|
assert output.routed_experts is not None
|
|
|
|
|
assert output.routed_experts.routing_data[:, 0, 0].tolist() == [0, 2, 4]
|
|
|
|
|
assert output.routed_experts.slot_mapping.tolist() == [11, 12, 13]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("rank", [0, 1])
|
|
|
|
|
def test_all_tp_ranks_initialize_capture(monkeypatch, rank):
|
|
|
|
|
pytest.importorskip("vllm.vllm_flash_attn", exc_type=ImportError)
|
|
|
|
|
import vllm.v1.worker.gpu.model_runner as model_runner
|
|
|
|
|
|
|
|
|
|
capturer = Mock()
|
|
|
|
|
constructor = Mock(return_value=capturer)
|
|
|
|
|
bind = Mock()
|
|
|
|
|
monkeypatch.setattr(model_runner, "RoutedExpertsCapturer", constructor)
|
|
|
|
|
monkeypatch.setattr(model_runner, "bind_routed_experts_capturer", bind)
|
|
|
|
|
|
|
|
|
|
runner = model_runner.GPUModelRunner.__new__(model_runner.GPUModelRunner)
|
|
|
|
|
runner.max_num_tokens = 32
|
|
|
|
|
runner.vllm_config = SimpleNamespace(parallel_config=SimpleNamespace(rank=rank))
|
|
|
|
|
runner.kv_cache_config = SimpleNamespace()
|
|
|
|
|
runner.model = Mock()
|
|
|
|
|
|
|
|
|
|
runner.init_routed_experts_capturer()
|
|
|
|
|
|
|
|
|
|
constructor.assert_called_once_with(
|
|
|
|
|
max_num_batched_tokens=32,
|
|
|
|
|
vllm_config=runner.vllm_config,
|
|
|
|
|
kv_cache_config=runner.kv_cache_config,
|
|
|
|
|
)
|
|
|
|
|
bind.assert_called_once_with(runner.model, capturer)
|
|
|
|
|
assert runner.routed_experts_capturer is capturer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_v2_model_runner_accepts_routed_experts(monkeypatch):
|
|
|
|
|
monkeypatch.setattr("importlib.metadata.entry_points", lambda **_: ())
|
|
|
|
|
config = SimpleNamespace(
|
|
|
|
|
model_config=SimpleNamespace(
|
|
|
|
|
enable_return_routed_experts=True,
|
|
|
|
|
use_mla=False,
|
|
|
|
|
logits_processors=None,
|
|
|
|
|
enable_prompt_embeds=False,
|
|
|
|
|
),
|
|
|
|
|
speculative_config=None,
|
|
|
|
|
parallel_config=SimpleNamespace(
|
|
|
|
|
prefill_context_parallel_size=1,
|
|
|
|
|
tensor_parallel_size=1,
|
|
|
|
|
distributed_executor_backend=None,
|
|
|
|
|
pipeline_parallel_size=1,
|
|
|
|
|
enable_dbo=False,
|
|
|
|
|
enable_elastic_ep=False,
|
|
|
|
|
),
|
|
|
|
|
compilation_config=SimpleNamespace(
|
|
|
|
|
mode=CompilationMode.NONE,
|
|
|
|
|
pass_config=SimpleNamespace(enable_sp=False),
|
|
|
|
|
),
|
|
|
|
|
cache_config=SimpleNamespace(kv_sharing_fast_prefill=False),
|
|
|
|
|
ec_transfer_config=None,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
unsupported = VllmConfig._get_v2_model_runner_unsupported_features(config)
|
|
|
|
|
|
|
|
|
|
assert "routed experts capture" not in unsupported
|
|
|
|
|