[Model] Support DeepSeek-V4.1-Flash (#56214)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-11 02:11:19 -07:00
committed by GitHub
co-authored by Jee Jee Li andyluo7 OpenAI Codex zjy0516 Claude Opus 5
parent e6821ceac9
commit e77daef89e
49 changed files with 3996 additions and 138 deletions
+211 -4
View File
@@ -18,7 +18,13 @@ from vllm.models.deepseek_v4.nvidia.model import (
)
from vllm.models.deepseek_v4.nvidia.mtp import DeepSeekV4MTP
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.models.deepseek_v4_1.common.mm_preprocess import (
IMAGE_PAD_ID,
IMAGE_SENTINEL_BASE_ID,
)
from vllm.models.deepseek_v4_1.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(),
@@ -26,6 +32,133 @@ pytestmark = pytest.mark.skipif(
)
@pytest.fixture
def v41_moe_config(dist_init):
return SimpleNamespace(
model_config=SimpleNamespace(
hf_config=DeepseekV41Config(
text_config=dict(
hidden_size=128,
num_hidden_layers=2,
n_routed_experts=8,
num_experts_per_tok=2,
dspark_n_routed_experts=4,
dspark_num_experts_per_tok=3,
n_shared_experts=1,
moe_intermediate_size=128,
hidden_act="silu",
swiglu_limit=10.0,
norm_topk_prob=True,
topk_method="noaux_tc",
routed_scaling_factor=1.5,
),
),
),
quant_config=None,
kernel_config=SimpleNamespace(moe_backend="deep_gemm_mega_moe"),
parallel_config=SimpleNamespace(
enable_expert_parallel=True,
enable_eplb=False,
eplb_config=SimpleNamespace(num_redundant_experts=0),
),
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
)
@pytest.mark.parametrize("vision", [False, True])
@pytest.mark.parametrize("layer_id,num_experts,top_k", [(0, 8, 2), (2, 4, 3)])
def test_deepseek_v41_moe_routes_without_hash_table(
v41_moe_config, monkeypatch, vision, layer_id, num_experts, top_k
):
"""Main and draft layers select experts by score and preserve image routing."""
config = v41_moe_config.model_config.hf_config
config.vision_n_layers = int(vision)
with torch.device("cuda"):
moe = DeepseekV41MoE(v41_moe_config, prefix=f"model.layers.{layer_id}.ffn")
hidden_states = torch.randn(4, config.hidden_size)
input_ids = (
torch.tensor([42, IMAGE_SENTINEL_BASE_ID, IMAGE_PAD_ID, 129257])
if vision
else None
)
assert moe.gate.tid2eid is None
assert moe.gate.weight.shape == (num_experts, config.hidden_size)
assert isinstance(moe.experts, DeepseekV4MegaMoEExperts)
assert moe.experts.w13_weight.shape == (num_experts, 256, 64)
assert moe.experts.top_k == top_k
assert (config.n_routed_experts, config.num_experts_per_tok) == (8, 2)
with torch.no_grad():
moe.gate.weight.normal_(std=0.01)
moe.gate.e_score_correction_bias.copy_(torch.arange(num_experts, device="cuda"))
if vision:
moe.gate.bias_vl.copy_(-moe.gate.e_score_correction_bias)
scores = torch.nn.functional.softplus(
torch.nn.functional.linear(hidden_states, moe.gate.weight)
).sqrt()
bias = moe.gate.e_score_correction_bias
if vision:
image_mask = (input_ids == IMAGE_SENTINEL_BASE_ID) | (input_ids == IMAGE_PAD_ID)
bias = torch.where(image_mask[:, None], moe.gate.bias_vl, bias)
expected_ids = (scores + bias).topk(top_k, dim=-1).indices
expected_weights = scores.gather(1, expected_ids)
expected_weights *= config.routed_scaling_factor / expected_weights.sum(
dim=-1, keepdim=True
)
def check_routing(x, weights, ids, *, activation_clamp):
torch.testing.assert_close(ids, expected_ids)
torch.testing.assert_close(weights, expected_weights)
assert activation_clamp == config.swiglu_limit
return x.clone()
monkeypatch.setattr(moe.experts, "forward", check_routing)
monkeypatch.setattr(moe.shared_experts, "forward", lambda x: 2 * x)
torch.testing.assert_close(moe(hidden_states, input_ids), 3 * hidden_states)
@pytest.mark.parametrize(
"draft_experts,draft_top_k,expected", [(0, 0, (8, 2)), (4, 3, (4, 3))]
)
def test_deepseek_v41_fused_moe_uses_draft_counts_or_main_defaults(
v41_moe_config, monkeypatch, draft_experts, draft_top_k, expected
):
config = v41_moe_config.model_config.hf_config
config.dspark_n_routed_experts = draft_experts
config.dspark_num_experts_per_tok = draft_top_k
v41_moe_config.kernel_config.moe_backend = "auto"
captured = {}
def make_experts(**kwargs):
captured.update(kwargs)
return torch.nn.Identity()
monkeypatch.setattr(
"vllm.models.deepseek_v4.nvidia.model.FusedMoEFactory", make_experts
)
moe = DeepseekV41MoE(v41_moe_config, prefix="model.layers.2.ffn")
assert moe.gate.weight.shape == (expected[0], 128)
assert (captured["num_experts"], captured["top_k"]) == expected
assert captured["hash_indices_table"] is None
def test_deepseek_v4_moe_preserves_configured_hash_layers(v41_moe_config):
config = v41_moe_config.model_config.hf_config
config.num_hash_layers = 1
config.vocab_size = 32
moe = DeepseekV4MoE(
v41_moe_config,
prefix="model.layers.0.ffn",
num_hash_layers=config.num_hash_layers,
)
assert moe.gate.tid2eid.shape == (32, 2)
assert moe.gate.e_score_correction_bias is None
with pytest.raises(ValueError, match="hash MoE routing requires input_ids"):
moe(torch.zeros(1, 128))
def test_deepseek_v4_mega_moe_expert_mapping():
mapping = make_deepseek_v4_expert_params_mapping(2)
@@ -169,15 +302,79 @@ def test_deepseek_v4_mega_moe_weight_loader_uses_ep_expert_ownership():
assert torch.count_nonzero(experts.w13_weight[1]) == 0
@pytest.mark.parametrize("intermediate_size", [512, 2304])
def test_deepseek_v4_mega_moe_padding_preserves_weights(monkeypatch, intermediate_size):
"""Pad gate/up separately and zero every added down-projection column."""
experts = DeepseekV4MegaMoEExperts(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=2,
num_local_experts=2,
experts_start_idx=0,
top_k=1,
hidden_size=128,
intermediate_size=intermediate_size,
)
originals = []
for param in (
experts.w13_weight,
experts.w13_weight_scale,
experts.w2_weight,
experts.w2_weight_scale,
):
param.data.random_(1, 128)
originals.append(param.data.clone())
monkeypatch.setattr(experts, "_check_runtime_supported", lambda: None)
monkeypatch.setattr(
"vllm.utils.deep_gemm._import_deep_gemm",
lambda: SimpleNamespace(
transform_sf_into_required_layout=lambda sf, *args: sf,
transform_weights_for_mega_moe=lambda l1, l2: (l1, l2),
),
)
experts.finalize_weights()
padded_size = experts.intermediate_size
assert padded_size == (intermediate_size + 511) // 512 * 512
for actual, original in zip(experts._transformed_l1_weights, originals[:2]):
expected = (
original
if actual.dtype == torch.int8
else (experts._ue8m0_uint8_to_float(original))
)
actual = actual.view(expected.dtype).unflatten(1, (2, padded_size))
assert torch.equal(
actual[:, :, :intermediate_size],
expected.unflatten(1, (2, intermediate_size)),
)
assert torch.count_nonzero(actual[:, :, intermediate_size:]) == 0
for actual, original in zip(experts._transformed_l2_weights, originals[2:]):
expected = (
original
if actual.dtype == torch.int8
else (experts._ue8m0_uint8_to_float(original))
)
actual = actual.view(expected.dtype)
assert torch.equal(actual[..., : original.shape[-1]], expected)
assert torch.count_nonzero(actual[..., original.shape[-1] :]) == 0
transformed_l1 = experts._transformed_l1_weights
experts.finalize_weights()
assert experts._transformed_l1_weights is transformed_l1
@pytest.mark.parametrize(
"hidden_size,intermediate_size,block_size,fused",
"hidden_size,intermediate_size,block_size,fused,mxfp8",
[
pytest.param(128, 512, 128, True, id="aligned-block128"),
pytest.param(5120, 2304, 32, False, id="deepseek-v41-flash"),
pytest.param(128, 512, 128, True, False, id="aligned-block128"),
pytest.param(128, 512, 128, True, True, id="aligned-block128-mxfp8"),
pytest.param(5120, 2304, 32, False, False, id="deepseek-v41-flash"),
],
)
def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
monkeypatch, hidden_size, intermediate_size, block_size, fused
monkeypatch, hidden_size, intermediate_size, block_size, fused, mxfp8
):
"""V4.1-Flash's routed padding must preserve the serial shared-expert fallback."""
@@ -261,6 +458,16 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
),
),
)
if mxfp8:
for linear in (shared_experts.gate_up_proj, shared_experts.down_proj):
linear.weight_block_size = (1, 32)
linear.weight_scale = torch.nn.Parameter(
linear.weight_scale_inv.view(torch.uint8)
.repeat_interleave(128, dim=0)
.repeat_interleave(4, dim=1),
requires_grad=False,
)
del linear.weight_scale_inv
monkeypatch.setattr("vllm.utils.deep_gemm._import_deep_gemm", lambda: FakeDeepGemm)
original_gate_up_ptr = shared_experts.gate_up_proj.weight.data_ptr()