Add MoNe experts pruning support (#50415)
Signed-off-by: Ilya Boytsov <ieboytsov@nebius.com> Signed-off-by: Misha Goin <mgoin64@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Ilya Boytsov <ieboytsov@nebius.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Ilya Boytsov
Codex
Misha Goin
Claude Opus 5.5
parent
1484266938
commit
189bbfb9f4
@@ -0,0 +1,422 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.modular_kernel_tools.parallel_utils import (
|
||||
ProcessGroupInfo,
|
||||
parallel_launch_with_config,
|
||||
)
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.expert_substitution import (
|
||||
ConstantExpertSubstitution,
|
||||
SubstitutedRoutedExperts,
|
||||
get_expert_substitution_spec,
|
||||
make_expert_substitution,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
)
|
||||
from vllm.model_executor.models.mixtral import MixtralMoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
MOE_PREFIX = "model.layers.1.mlp.experts"
|
||||
|
||||
|
||||
def _model_config(approximate_experts, **kwargs) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
hf_config=SimpleNamespace(approximate_experts=approximate_experts),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _expert_substitution(targets: dict[str, dict[int, str]]) -> dict:
|
||||
"""The ``compression_config.transform_config.expert_substitution`` format."""
|
||||
return {
|
||||
"version": 1,
|
||||
"router_semantics": {
|
||||
"preserve_logical_expert_ids": True,
|
||||
"preserve_router_weights": True,
|
||||
"renormalize_after_substitution": False,
|
||||
},
|
||||
"targets": {
|
||||
path: {
|
||||
"num_logical_experts": 5,
|
||||
"weight_layout": "compact_retained_experts",
|
||||
"replacements": {
|
||||
str(expert_id): {
|
||||
"format": "constant-v1",
|
||||
"tensors": {"value": name},
|
||||
}
|
||||
for expert_id, name in replacements.items()
|
||||
},
|
||||
}
|
||||
for path, replacements in targets.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _compressed_model_config(expert_substitution: dict, **kwargs) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
compression_config={
|
||||
"transform_config": {"expert_substitution": expert_substitution}
|
||||
}
|
||||
),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _substitution(hidden_size: int, dtype=torch.float32):
|
||||
# Experts 1 and 3 of 5 are substituted; 0, 2, 4 map to physical 0, 1, 2.
|
||||
return ConstantExpertSubstitution(1, 5, (1, 3), hidden_size, dtype)
|
||||
|
||||
|
||||
def test_parse_approximate_experts():
|
||||
assert get_expert_substitution_spec(None) is None
|
||||
assert get_expert_substitution_spec(_model_config(None)) is None
|
||||
spec = get_expert_substitution_spec(_model_config({"1": [3, 1], "2": [], 5: (0,)}))
|
||||
assert spec.experts == {1: (1, 3), 5: (0,)}
|
||||
assert spec.value_names == {}
|
||||
text_config = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(approximate_experts={"0": [2]}),
|
||||
hf_config=SimpleNamespace(),
|
||||
)
|
||||
assert get_expert_substitution_spec(text_config).experts == {0: (2,)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"approximate_experts",
|
||||
[[1, 2], {"layer": [1]}, {"1": [1, 1]}, {"1": [-1]}, {"1": ["x"]}],
|
||||
)
|
||||
def test_parse_approximate_experts_rejects_invalid_entries(approximate_experts):
|
||||
with pytest.raises(ValueError, match="approximate_experts"):
|
||||
get_expert_substitution_spec(_model_config(approximate_experts))
|
||||
|
||||
|
||||
def test_parse_expert_substitution_with_explicit_value_names():
|
||||
spec = get_expert_substitution_spec(
|
||||
_compressed_model_config(
|
||||
_expert_substitution(
|
||||
{
|
||||
MOE_PREFIX: {3: "shared", 1: "layer_1.expert_1"},
|
||||
"model.layers.2.mlp.experts": {0: "shared"},
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
assert spec.experts == {1: (1, 3), 2: (0,)}
|
||||
assert spec.value_names == {
|
||||
"shared": ((1, 3), (2, 0)),
|
||||
"layer_1.expert_1": ((1, 1),),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutate",
|
||||
[
|
||||
lambda raw: raw.update(version=2),
|
||||
lambda raw: raw["router_semantics"].update(renormalize_after_substitution=True),
|
||||
lambda raw: raw["targets"].update({"mlp.experts": raw["targets"][MOE_PREFIX]}),
|
||||
lambda raw: raw["targets"][MOE_PREFIX].update(weight_layout="dense"),
|
||||
lambda raw: raw["targets"][MOE_PREFIX]["replacements"]["1"].update(
|
||||
format="linear-v1"
|
||||
),
|
||||
lambda raw: raw["targets"][MOE_PREFIX]["replacements"]["1"].pop("tensors"),
|
||||
],
|
||||
)
|
||||
def test_parse_expert_substitution_rejects_unsupported_metadata(mutate):
|
||||
raw = _expert_substitution({MOE_PREFIX: {1: "value"}})
|
||||
mutate(raw)
|
||||
with pytest.raises(ValueError, match="invalid expert_substitution metadata"):
|
||||
get_expert_substitution_spec(_compressed_model_config(raw))
|
||||
|
||||
|
||||
def test_parse_rejects_both_formats():
|
||||
config = _compressed_model_config(_expert_substitution({MOE_PREFIX: {1: "v"}}))
|
||||
config.hf_config.approximate_experts = {"1": [1]}
|
||||
with pytest.raises(ValueError, match="both"):
|
||||
get_expert_substitution_spec(config)
|
||||
|
||||
|
||||
def test_make_expert_substitution_matches_layer_index():
|
||||
config = _model_config({"1": [3, 1]})
|
||||
substitution = make_expert_substitution(config, MOE_PREFIX, 5, 16, torch.bfloat16)
|
||||
|
||||
assert substitution is not None
|
||||
assert substitution.layer_idx == 1
|
||||
assert substitution.substituted_expert_ids == (1, 3)
|
||||
assert substitution.num_compute_experts == 3
|
||||
assert substitution.values.dtype == torch.bfloat16
|
||||
assert [substitution.physical_expert_id(i) for i in range(5)] == [0, -1, 1, -1, 2]
|
||||
assert (
|
||||
make_expert_substitution(config, "model.layers.2.mlp.experts", 5, 16, None)
|
||||
is None
|
||||
)
|
||||
assert make_expert_substitution(None, MOE_PREFIX, 5, 16, None) is None
|
||||
explicit = _compressed_model_config(_expert_substitution({MOE_PREFIX: {1: "v"}}))
|
||||
substitution = make_expert_substitution(explicit, MOE_PREFIX, 5, 16, None)
|
||||
assert substitution is not None
|
||||
assert substitution.substituted_expert_ids == (1,)
|
||||
with pytest.raises(ValueError, match="exactly one layer index"):
|
||||
make_expert_substitution(config, "mlp.experts", 5, 16, None)
|
||||
with pytest.raises(ValueError, match="within"):
|
||||
make_expert_substitution(config, MOE_PREFIX, 3, 16, None)
|
||||
with pytest.raises(ValueError, match="retain at least one expert"):
|
||||
make_expert_substitution(_model_config({"1": [0, 1]}), MOE_PREFIX, 2, 16, None)
|
||||
|
||||
|
||||
def test_constant_substitution_loads_values():
|
||||
substitution = _substitution(2)
|
||||
substitution.weight_loader(substitution.values, torch.tensor([1.0, 2.0]), 3)
|
||||
with pytest.raises(ValueError, match="missing approx_value .* \\[1\\]"):
|
||||
substitution.validate_loaded_values("layer")
|
||||
substitution.weight_loader(substitution.values, torch.tensor([3.0, 4.0]), 1)
|
||||
substitution.validate_loaded_values("layer")
|
||||
torch.testing.assert_close(
|
||||
substitution.values, torch.tensor([[3.0, 4.0], [1.0, 2.0]])
|
||||
)
|
||||
with pytest.raises(ValueError, match="not listed"):
|
||||
substitution.weight_loader(substitution.values, torch.ones(2), 2)
|
||||
with pytest.raises(ValueError, match="has shape"):
|
||||
substitution.weight_loader(substitution.values, torch.ones(3), 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [0, 1, 3, 7])
|
||||
@pytest.mark.parametrize("hidden_size", [2, 4])
|
||||
def test_constant_substitution_transforms_routes_and_computes_side_output(
|
||||
num_tokens, hidden_size
|
||||
):
|
||||
substitution = _substitution(2)
|
||||
substitution.values.data.copy_(torch.tensor([[10.0, 20.0], [30.0, 40.0]]))
|
||||
|
||||
rows = torch.arange(num_tokens) % 3
|
||||
topk_ids = torch.tensor([[1, 2], [4, 3], [99, -1]])[rows]
|
||||
topk_weights = torch.tensor([[0.25, 0.75], [0.6, 0.4], [0.5, 0.5]])[rows]
|
||||
constant_output = substitution.transform_routes(
|
||||
torch.zeros(num_tokens, hidden_size), topk_weights, topk_ids
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
topk_weights, torch.tensor([[0.0, 0.75], [0.6, 0.0], [0.0, 0.0]])[rows]
|
||||
)
|
||||
torch.testing.assert_close(topk_ids, torch.tensor([[0, 1], [2, 0], [0, 0]])[rows])
|
||||
expected = torch.zeros(num_tokens, hidden_size)
|
||||
expected[:, :2] = torch.tensor([[2.5, 5.0], [12.0, 16.0], [0.0, 0.0]])[rows]
|
||||
torch.testing.assert_close(constant_output, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"device",
|
||||
[
|
||||
"cpu",
|
||||
pytest.param(
|
||||
"cuda",
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="requires a CUDA-like platform",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_constant_substitution_preserves_router_weight_precision(dtype, device):
|
||||
substitution = _substitution(1, dtype).to(device)
|
||||
substitution.values.data.copy_(
|
||||
torch.tensor([[60000.0], [-60000.0]], dtype=dtype, device=device)
|
||||
)
|
||||
topk_weights = torch.tensor([[0.5001, 0.4999]], dtype=torch.float32, device=device)
|
||||
topk_ids = torch.tensor([[1, 3]], device=device)
|
||||
# Nearly cancelling constants must retain the difference between FP32 weights.
|
||||
expected = (topk_weights.double() @ substitution.values.double()).to(dtype)
|
||||
|
||||
actual = substitution.transform_routes(
|
||||
torch.zeros(1, 1, dtype=dtype, device=device), topk_weights, topk_ids
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
def _bare_substituted_routed_experts(**attrs) -> SubstitutedRoutedExperts:
|
||||
layer = SubstitutedRoutedExperts.__new__(SubstitutedRoutedExperts)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.expert_substitution = _substitution(2)
|
||||
for name, value in attrs.items():
|
||||
setattr(layer, name, value)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
@pytest.mark.parametrize("padded_input", [False, True])
|
||||
def test_substituted_routed_experts_adds_constant_on_one_tp_rank(tp_rank, padded_input):
|
||||
def apply(*, layer, x, topk_weights, topk_ids, **kwargs):
|
||||
torch.testing.assert_close(topk_weights, torch.tensor([[0.0, 0.75]]))
|
||||
torch.testing.assert_close(topk_ids, torch.tensor([[0, 1]]))
|
||||
return torch.ones(x.shape[0], 2)
|
||||
|
||||
layer = _bare_substituted_routed_experts(
|
||||
moe_config=SimpleNamespace(tp_rank=tp_rank),
|
||||
quant_method=SimpleNamespace(is_monolithic=False, apply=apply),
|
||||
)
|
||||
layer.expert_substitution.values.data.copy_(
|
||||
torch.tensor([[10.0, 20.0], [30.0, 40.0]])
|
||||
)
|
||||
actual = layer.forward_modular(
|
||||
torch.zeros(1, 4 if padded_input else 2),
|
||||
torch.tensor([[0.25, 0.75]]),
|
||||
torch.tensor([[1, 2]]),
|
||||
)
|
||||
expected = torch.ones(1, 2)
|
||||
if tp_rank == 0:
|
||||
expected += torch.tensor([[2.5, 5.0]])
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like platform"
|
||||
)
|
||||
def test_standard_fused_moe_model_discovers_substitution(dist_init):
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = _model_config(
|
||||
{"0": [1]}, dtype=torch.bfloat16, is_moe=True
|
||||
)
|
||||
vllm_config.kernel_config.moe_backend = "triton"
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
moe = MixtralMoE(
|
||||
num_experts=4,
|
||||
top_k=2,
|
||||
hidden_size=256,
|
||||
intermediate_size=512,
|
||||
params_dtype=torch.bfloat16,
|
||||
prefix="model.layers.0.block_sparse_moe",
|
||||
)
|
||||
|
||||
routed_experts = moe.experts.routed_experts
|
||||
assert isinstance(routed_experts, SubstitutedRoutedExperts)
|
||||
assert routed_experts.expert_substitution.substituted_expert_ids == (1,)
|
||||
assert routed_experts.w13_weight.shape[0] == 3
|
||||
assert moe.experts.moe_config.require_decomposed_backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like platform"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"moe_backend",
|
||||
[
|
||||
pytest.param("triton", id="triton"),
|
||||
pytest.param(
|
||||
"flashinfer_cutlass",
|
||||
id="flashinfer-cutlass",
|
||||
marks=pytest.mark.skipif(
|
||||
not FlashInferExperts._supports_current_device(),
|
||||
reason="FlashInfer CUTLASS is unavailable on this platform",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_fused_moe_combines_compact_experts_and_substitution(dist_init, moe_backend):
|
||||
hidden_size = 256
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = _model_config(
|
||||
{"0": [1, 3]}, dtype=torch.bfloat16, is_moe=True
|
||||
)
|
||||
vllm_config.compilation_config.static_forward_context = {}
|
||||
vllm_config.kernel_config.moe_backend = moe_backend
|
||||
|
||||
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
|
||||
init_workspace_manager(torch.accelerator.current_device_index())
|
||||
layer = FusedMoEFactory(
|
||||
num_experts=4,
|
||||
top_k=2,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=512,
|
||||
params_dtype=torch.bfloat16,
|
||||
prefix="model.layers.0.mlp.experts",
|
||||
renormalize=False,
|
||||
).cuda()
|
||||
substitution = layer.routed_experts.expert_substitution
|
||||
assert layer.routed_experts.w13_weight.shape[0] == 2
|
||||
with torch.no_grad():
|
||||
layer.routed_experts.w13_weight.normal_(0, 0.01)
|
||||
layer.routed_experts.w2_weight.normal_(0, 0.01)
|
||||
substitution.values.normal_(0, 0.01)
|
||||
layer._quant_method.process_weights_after_loading(layer.routed_experts)
|
||||
|
||||
hidden_states = torch.randn(8, hidden_size, dtype=torch.bfloat16, device="cuda")
|
||||
router_logits = torch.randn(8, 4, dtype=torch.float32, device="cuda")
|
||||
topk_weights, topk_ids = layer.router.select_experts(
|
||||
hidden_states.clone(), router_logits
|
||||
)
|
||||
expected = substitution.transform_routes(hidden_states, topk_weights, topk_ids)
|
||||
expected += layer._quant_method.apply(
|
||||
layer=layer.routed_experts,
|
||||
x=hidden_states.clone(),
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
shared_experts=None,
|
||||
shared_experts_input=None,
|
||||
)
|
||||
|
||||
get_forward_context().all_moe_layers = None
|
||||
actual = layer(hidden_states.clone(), router_logits)
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
def _tp_substitution_worker(
|
||||
process_group: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
_cpu_group,
|
||||
) -> None:
|
||||
with set_forward_context(None, vllm_config):
|
||||
init_workspace_manager(process_group.local_rank)
|
||||
layer = FusedMoEFactory(
|
||||
num_experts=4,
|
||||
top_k=1,
|
||||
hidden_size=256,
|
||||
intermediate_size=512,
|
||||
params_dtype=torch.bfloat16,
|
||||
prefix="model.layers.0.mlp.experts",
|
||||
renormalize=True,
|
||||
)
|
||||
with torch.no_grad():
|
||||
layer.routed_experts.w13_weight.zero_()
|
||||
layer.routed_experts.w2_weight.zero_()
|
||||
layer.routed_experts.expert_substitution.values.fill_(0.25)
|
||||
layer._quant_method.process_weights_after_loading(layer.routed_experts)
|
||||
|
||||
hidden_states = torch.zeros(2, 256, dtype=torch.bfloat16)
|
||||
router_logits = torch.tensor(
|
||||
[[-100.0, 100.0, -100.0, -100.0]] * 2,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
get_forward_context().all_moe_layers = None
|
||||
actual = layer(hidden_states, router_logits)
|
||||
torch.testing.assert_close(
|
||||
actual, torch.full_like(actual, 0.25), atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike() or current_platform.device_count() < 2,
|
||||
reason="requires two CUDA-like devices",
|
||||
)
|
||||
def test_constant_substitution_is_added_once_with_tp2():
|
||||
vllm_config = VllmConfig(parallel_config=ParallelConfig(tensor_parallel_size=2))
|
||||
vllm_config.model_config = _model_config(
|
||||
{"0": [1]},
|
||||
dtype=torch.bfloat16,
|
||||
is_moe=True,
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
)
|
||||
vllm_config.compilation_config.static_forward_context = {}
|
||||
vllm_config.kernel_config.moe_backend = "triton"
|
||||
|
||||
parallel_launch_with_config(2, _tp_substitution_worker, vllm_config, None)
|
||||
@@ -136,7 +136,13 @@ def test_arm_cpu_unquantized_kernel_selection(
|
||||
],
|
||||
)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.has_flashinfer",
|
||||
"vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe."
|
||||
"has_flashinfer_cutlass_fused_moe",
|
||||
return_value=False,
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe."
|
||||
"has_flashinfer_trtllm_fused_moe",
|
||||
return_value=False,
|
||||
)
|
||||
@patch(
|
||||
@@ -145,13 +151,18 @@ def test_arm_cpu_unquantized_kernel_selection(
|
||||
)
|
||||
def test_select_default_backend_by_platform(
|
||||
mock_aiter_enabled,
|
||||
mock_has_flashinfer,
|
||||
mock_has_flashinfer_trtllm,
|
||||
mock_has_flashinfer_cutlass,
|
||||
monkeypatch,
|
||||
platform_method,
|
||||
expected_backend,
|
||||
):
|
||||
"""Test default backend selection per platform with all optional
|
||||
accelerators (FlashInfer, AITER) disabled."""
|
||||
accelerators (FlashInfer, AITER) disabled.
|
||||
|
||||
Patch the feature probes directly because their cached results can outlive
|
||||
changes to has_flashinfer().
|
||||
"""
|
||||
with patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
|
||||
) as mock_platform:
|
||||
@@ -407,6 +418,41 @@ def test_select_cuda_flashinfer_trtllm_modular_for_standard_all2all(
|
||||
assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular"
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
|
||||
return_value=True,
|
||||
)
|
||||
@pytest.mark.parametrize("require_decomposed_backend", [False, True])
|
||||
def test_require_decomposed_backend_rejects_monolithic_kernels(
|
||||
mock_supports_current_device,
|
||||
require_decomposed_backend,
|
||||
):
|
||||
"""Expert substitution adjusts routes, so the kernel must not route itself."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
|
||||
TrtLlmBf16ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEActivationFormat,
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.routing_method = RoutingMethodType.Renormalize
|
||||
moe_config.require_decomposed_backend = require_decomposed_backend
|
||||
|
||||
supported, reason = TrtLlmBf16ExpertsMonolithic.is_supported_config(
|
||||
TrtLlmBf16ExpertsMonolithic,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
|
||||
assert supported != require_decomposed_backend
|
||||
assert (reason is not None and "routes adjusted" in reason) == (
|
||||
require_decomposed_backend
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
|
||||
return_value=True,
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
from transformers import DeepseekV2Config, DeepseekV2ForCausalLM
|
||||
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
DeviceConfig,
|
||||
KernelConfig,
|
||||
LoadConfig,
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.expert_substitution import (
|
||||
ConstantExpertSubstitution,
|
||||
)
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.models.expert_substitution import as_expert_substitution_model
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader, PPMissingLayer
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
MODEL_CONFIG = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(approximate_experts={"0": [1, 2], "1": [1, 2]})
|
||||
)
|
||||
|
||||
|
||||
def _compression_config(value_names: dict[int, dict[int, str]]) -> dict:
|
||||
"""Explicitly named constants given as ``{layer: {expert: tensor name}}``."""
|
||||
targets = {
|
||||
f"model.layers.{layer}.mlp.experts": {
|
||||
"num_logical_experts": 3,
|
||||
"weight_layout": "compact_retained_experts",
|
||||
"replacements": {
|
||||
str(expert): {"format": "constant-v1", "tensors": {"value": name}}
|
||||
for expert, name in names.items()
|
||||
},
|
||||
}
|
||||
for layer, names in value_names.items()
|
||||
}
|
||||
router_semantics = {
|
||||
"preserve_logical_expert_ids": True,
|
||||
"preserve_router_weights": True,
|
||||
"renormalize_after_substitution": False,
|
||||
}
|
||||
return {
|
||||
"transform_config": {
|
||||
"expert_substitution": {
|
||||
"version": 1,
|
||||
"router_semantics": router_semantics,
|
||||
"targets": targets,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# Expert 1 of both layers shares one explicitly named constant.
|
||||
EXPLICIT_MODEL_CONFIG = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
compression_config=_compression_config(
|
||||
{
|
||||
layer: {1: "constants.shared", 2: f"constants.layer_{layer}"}
|
||||
for layer in (0, 1)
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _approx_value(layer: int, expert: int) -> str:
|
||||
return f"model.layers.{layer}.mlp.experts.{expert}.approx_value"
|
||||
|
||||
|
||||
def _cpu_model(local_layers=(0, 1), tracked=True, model_config=MODEL_CONFIG):
|
||||
"""Two decoder layers; the others are placeholders as on another PP stage."""
|
||||
|
||||
class Model(torch.nn.Module):
|
||||
def __init__(self, prefix=""):
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.zeros(2), requires_grad=False)
|
||||
self.model = torch.nn.Module()
|
||||
self.model.layers = torch.nn.ModuleList()
|
||||
for index in range(2):
|
||||
if index not in local_layers:
|
||||
self.model.layers.append(PPMissingLayer())
|
||||
continue
|
||||
layer = torch.nn.Module()
|
||||
layer.mlp = torch.nn.Module()
|
||||
layer.mlp.experts = ConstantExpertSubstitution(
|
||||
index, 3, (1, 2), 2, torch.float32
|
||||
)
|
||||
self.model.layers.append(layer)
|
||||
|
||||
def load_weights(self, weights):
|
||||
loaded = AutoWeightsLoader(self).load_weights(weights)
|
||||
return loaded if tracked else None
|
||||
|
||||
return as_expert_substitution_model(Model, model_config)()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("local_layers", [(0, 1), (0,), (1,), ()])
|
||||
def test_loads_local_approx_values_and_passes_through_others(local_layers):
|
||||
"""Other PP stages and MTP drafts own none or some of the configured layers."""
|
||||
model = _cpu_model(local_layers)
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
("weight", torch.tensor([1.0, 2.0])),
|
||||
*(
|
||||
(_approx_value(layer, expert), torch.full((2,), 10.0 * layer + expert))
|
||||
for layer in range(2)
|
||||
for expert in (1, 2)
|
||||
),
|
||||
]
|
||||
)
|
||||
model.process_weights_after_loading()
|
||||
|
||||
assert loaded == {"weight"} | {
|
||||
f"model.layers.{layer}.mlp.experts.values" for layer in local_layers
|
||||
}
|
||||
for layer in local_layers:
|
||||
torch.testing.assert_close(
|
||||
model.model.layers[layer].mlp.experts.values,
|
||||
torch.tensor([[1.0, 1.0], [2.0, 2.0]]) + 10.0 * layer,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("local_layers", [(0, 1), (1,), ()])
|
||||
def test_loads_explicitly_named_values(local_layers):
|
||||
"""Named values are consumed even when their layer is built elsewhere."""
|
||||
model = _cpu_model(local_layers, model_config=EXPLICIT_MODEL_CONFIG)
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
("weight", torch.tensor([1.0, 2.0])),
|
||||
("constants.shared", torch.full((2,), 1.0)),
|
||||
("constants.layer_0", torch.full((2,), 2.0)),
|
||||
("constants.layer_1", torch.full((2,), 3.0)),
|
||||
]
|
||||
)
|
||||
model.process_weights_after_loading()
|
||||
|
||||
assert loaded == {"weight"} | {
|
||||
f"model.layers.{layer}.mlp.experts.values" for layer in local_layers
|
||||
}
|
||||
for layer in local_layers:
|
||||
torch.testing.assert_close(
|
||||
model.model.layers[layer].mlp.experts.values,
|
||||
torch.tensor([[1.0, 1.0], [2.0 + layer, 2.0 + layer]]),
|
||||
)
|
||||
|
||||
|
||||
def test_direct_reload_updates_values_incrementally():
|
||||
model = _cpu_model(local_layers=(0,), tracked=False)
|
||||
values = model.model.layers[0].mlp.experts.values
|
||||
model.load_weights([(_approx_value(0, 2), torch.ones(2))])
|
||||
with pytest.raises(ValueError, match="missing approx_value .* \\[1\\]"):
|
||||
model.process_weights_after_loading()
|
||||
|
||||
assert model.load_weights([(_approx_value(0, 1), torch.zeros(2))]) is None
|
||||
model.process_weights_after_loading()
|
||||
model.load_weights([(_approx_value(0, 2), torch.full((2,), 3.0))])
|
||||
torch.testing.assert_close(values, torch.tensor([[0.0, 0.0], [3.0, 3.0]]))
|
||||
|
||||
with pytest.raises(ValueError, match="not listed"):
|
||||
model.load_weights([(_approx_value(0, 0), torch.ones(2))])
|
||||
|
||||
|
||||
def test_layerwise_reload_is_rejected_and_keeps_weights():
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
record_metadata_for_reloading,
|
||||
)
|
||||
|
||||
model = _cpu_model(local_layers=(0,))
|
||||
record_metadata_for_reloading(model)
|
||||
loaded = [
|
||||
("weight", torch.tensor([3.0, 4.0])),
|
||||
(_approx_value(0, 1), torch.zeros(2)),
|
||||
(_approx_value(0, 2), torch.ones(2)),
|
||||
]
|
||||
model.load_weights(loaded)
|
||||
model.process_weights_after_loading()
|
||||
substitution = model.model.layers[0].mlp.experts
|
||||
values = substitution.values
|
||||
|
||||
initialize_layerwise_reload(model)
|
||||
with pytest.raises(NotImplementedError, match="layerwise weight reload"):
|
||||
model.load_weights([(name, weight + 1) for name, weight in loaded])
|
||||
|
||||
assert substitution.values is values
|
||||
torch.testing.assert_close(values, torch.tensor([[0.0, 0.0], [1.0, 1.0]]))
|
||||
torch.testing.assert_close(model.weight, torch.tensor([3.0, 4.0]))
|
||||
assert substitution.expert_substitution_routes.tolist() == [0, -1, -2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing_value", [False, True])
|
||||
def test_tensorizer_finalizes_local_substitution_values(monkeypatch, missing_value):
|
||||
"""Raw Tensorizer loading must reject an unloaded local constant row."""
|
||||
from vllm.model_executor.model_loader import tensorizer_loader
|
||||
|
||||
model = _cpu_model(local_layers=(0,))
|
||||
weights = [(_approx_value(0, 1), torch.zeros(2))]
|
||||
if not missing_value:
|
||||
weights.append((_approx_value(0, 2), torch.ones(2)))
|
||||
finalize = Mock(wraps=model.process_weights_after_loading)
|
||||
monkeypatch.setattr(model, "process_weights_after_loading", finalize)
|
||||
monkeypatch.setattr(tensorizer_loader, "initialize_model", lambda **kwargs: model)
|
||||
loader = object.__new__(tensorizer_loader.TensorizerLoader)
|
||||
monkeypatch.setattr(loader, "_get_weights_iterator", lambda: iter(weights))
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(dtype=torch.float32),
|
||||
device_config=SimpleNamespace(device="cpu"),
|
||||
)
|
||||
|
||||
if missing_value:
|
||||
with pytest.raises(ValueError, match=r"missing approx_value .* \[2\]"):
|
||||
loader._load_model_serialized_cpu(config)
|
||||
else:
|
||||
assert loader._load_model_serialized_cpu(config) is model
|
||||
torch.testing.assert_close(
|
||||
model.model.layers[0].mlp.experts.values,
|
||||
torch.tensor([[0.0, 0.0], [1.0, 1.0]]),
|
||||
)
|
||||
finalize.assert_called_once_with()
|
||||
|
||||
|
||||
def _write_tiny_deepseek_mone_checkpoint(
|
||||
model_dir: Path, explicit_names: bool
|
||||
) -> tuple[dict[str, torch.Tensor], str]:
|
||||
config = DeepseekV2Config(
|
||||
architectures=["DeepseekV2ForCausalLM"],
|
||||
vocab_size=32,
|
||||
hidden_size=16,
|
||||
intermediate_size=32,
|
||||
moe_intermediate_size=8,
|
||||
num_hidden_layers=1,
|
||||
num_attention_heads=2,
|
||||
num_key_value_heads=2,
|
||||
max_position_embeddings=32,
|
||||
first_k_dense_replace=0,
|
||||
n_routed_experts=3,
|
||||
n_shared_experts=1,
|
||||
num_experts_per_tok=1,
|
||||
n_group=1,
|
||||
topk_group=1,
|
||||
norm_topk_prob=False,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=4,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
tie_word_embeddings=False,
|
||||
dtype="float16",
|
||||
)
|
||||
experts = "model.layers.0.mlp.experts"
|
||||
if explicit_names:
|
||||
value_name = "model.layers.0.mlp.expert_replacements.1.value"
|
||||
config.compression_config = _compression_config({0: {1: value_name}})
|
||||
else:
|
||||
value_name = f"{experts}.1.approx_value"
|
||||
config.approximate_experts = {"0": [1]}
|
||||
|
||||
hf_weights = DeepseekV2ForCausalLM(config).to(torch.float16).state_dict()
|
||||
gate_up = hf_weights.pop(f"{experts}.gate_up_proj")
|
||||
down = hf_weights.pop(f"{experts}.down_proj")
|
||||
weights = {name: value.contiguous() for name, value in hf_weights.items()}
|
||||
for expert in (0, 2):
|
||||
gate, up = gate_up[expert].chunk(2, dim=0)
|
||||
weights[f"{experts}.{expert}.gate_proj.weight"] = gate.contiguous()
|
||||
weights[f"{experts}.{expert}.up_proj.weight"] = up.contiguous()
|
||||
weights[f"{experts}.{expert}.down_proj.weight"] = down[expert].contiguous()
|
||||
weights[value_name] = torch.arange(16, dtype=torch.float16)
|
||||
config.save_pretrained(model_dir)
|
||||
save_file(weights, model_dir / "model.safetensors")
|
||||
return weights, value_name
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like device"
|
||||
)
|
||||
@pytest.mark.usefixtures("dist_init", "workspace_init")
|
||||
@pytest.mark.parametrize(
|
||||
"explicit_names", [False, True], ids=["approximate_experts", "expert_substitution"]
|
||||
)
|
||||
def test_loads_mone_checkpoint_into_compact_experts(
|
||||
tmp_path: Path, explicit_names: bool
|
||||
):
|
||||
weights, value_name = _write_tiny_deepseek_mone_checkpoint(tmp_path, explicit_names)
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
tokenizer=str(tmp_path),
|
||||
skip_tokenizer_init=True,
|
||||
dtype="float16",
|
||||
max_model_len=32,
|
||||
enforce_eager=True,
|
||||
)
|
||||
load_config = LoadConfig(load_format="safetensors", use_tqdm_on_load=False)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
device_config=DeviceConfig(device="cuda"),
|
||||
load_config=load_config,
|
||||
attention_config=AttentionConfig(backend=AttentionBackendEnum.TRITON_MLA),
|
||||
kernel_config=KernelConfig(moe_backend="triton"),
|
||||
)
|
||||
model = get_model_loader(load_config).load_model(vllm_config, model_config)
|
||||
|
||||
experts = "model.layers.0.mlp.experts"
|
||||
routed_experts = model.model.layers[0].mlp.experts.routed_experts
|
||||
# Logical expert 2 is stored in physical row 1.
|
||||
torch.testing.assert_close(
|
||||
routed_experts.w13_weight[1].cpu(),
|
||||
torch.cat(
|
||||
[
|
||||
weights[f"{experts}.2.gate_proj.weight"],
|
||||
weights[f"{experts}.2.up_proj.weight"],
|
||||
]
|
||||
),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
routed_experts.w2_weight[1].cpu(), weights[f"{experts}.2.down_proj.weight"]
|
||||
)
|
||||
substitution = routed_experts.expert_substitution
|
||||
torch.testing.assert_close(
|
||||
substitution.values.cpu(), weights[value_name].unsqueeze(0)
|
||||
)
|
||||
|
||||
updated = weights[value_name] + 1
|
||||
assert model.load_weights([(value_name, updated)]) == {
|
||||
f"{experts}.routed_experts.expert_substitution.values"
|
||||
}
|
||||
torch.testing.assert_close(substitution.values.cpu(), updated.unsqueeze(0))
|
||||
@@ -142,6 +142,7 @@ def _megakernel_moe(moe_backend: str, **overrides) -> SimpleNamespace:
|
||||
is_act_and_mul=True,
|
||||
activation=MoEActivation.SILU,
|
||||
has_hash_routing=False,
|
||||
require_decomposed_backend=False,
|
||||
routing_method=RoutingMethodType.DeepseekV4,
|
||||
router_logits_dtype=torch.float32,
|
||||
hidden_dim=256,
|
||||
|
||||
Reference in New Issue
Block a user