[Model] Support DeepSeek-V4.1-Flash (#56214)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-11 02:11:19 -07:00
committed by GitHub
co-authored by Jee Jee Li andyluo7 OpenAI Codex zjy0516 Claude Opus 5
parent e6821ceac9
commit e77daef89e
49 changed files with 3996 additions and 138 deletions
@@ -616,6 +616,8 @@ def test_dummy_maverick(
) -> None:
# Disable multiprocessing allows us to access model executor from LLM engine
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
# The forked test process may inherit CUDA state from test collection.
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
model_path = create_reduced_maverick_model(
original_model_name=original_model_name,
+9
View File
@@ -858,6 +858,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"DeepseekV4ForConditionalGeneration": _HfExamplesInfo(
"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
),
"DeepseekV41ForCausalLM": _HfExamplesInfo(
"deepseek-ai/DeepSeek-V4.1-Flash",
),
"Dots3NoteForCausalLM": _HfExamplesInfo(
"dots-studio/dots3-note-prev",
is_available_online=False,
@@ -1537,6 +1540,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
is_available_online=False,
use_original_num_layers=True, # DSpark has >1 draft block
),
"DSparkV41DraftModel": _HfExamplesInfo(
"deepseek-ai/DeepSeek-V4.1-Flash",
speculative_model="deepseek-ai/DeepSeek-V4.1-Flash", # draft in mtp.*
is_available_online=False,
use_original_num_layers=True, # DSpark has >1 draft block
),
"K3DSparkModel": _HfExamplesInfo(
"moonshotai/Kimi-K3",
speculative_model="Inferact/Kimi-K3-DSpark",
+211 -4
View File
@@ -18,7 +18,13 @@ from vllm.models.deepseek_v4.nvidia.model import (
)
from vllm.models.deepseek_v4.nvidia.mtp import DeepSeekV4MTP
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.models.deepseek_v4_1.common.mm_preprocess import (
IMAGE_PAD_ID,
IMAGE_SENTINEL_BASE_ID,
)
from vllm.models.deepseek_v4_1.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(),
@@ -26,6 +32,133 @@ pytestmark = pytest.mark.skipif(
)
@pytest.fixture
def v41_moe_config(dist_init):
return SimpleNamespace(
model_config=SimpleNamespace(
hf_config=DeepseekV41Config(
text_config=dict(
hidden_size=128,
num_hidden_layers=2,
n_routed_experts=8,
num_experts_per_tok=2,
dspark_n_routed_experts=4,
dspark_num_experts_per_tok=3,
n_shared_experts=1,
moe_intermediate_size=128,
hidden_act="silu",
swiglu_limit=10.0,
norm_topk_prob=True,
topk_method="noaux_tc",
routed_scaling_factor=1.5,
),
),
),
quant_config=None,
kernel_config=SimpleNamespace(moe_backend="deep_gemm_mega_moe"),
parallel_config=SimpleNamespace(
enable_expert_parallel=True,
enable_eplb=False,
eplb_config=SimpleNamespace(num_redundant_experts=0),
),
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
)
@pytest.mark.parametrize("vision", [False, True])
@pytest.mark.parametrize("layer_id,num_experts,top_k", [(0, 8, 2), (2, 4, 3)])
def test_deepseek_v41_moe_routes_without_hash_table(
v41_moe_config, monkeypatch, vision, layer_id, num_experts, top_k
):
"""Main and draft layers select experts by score and preserve image routing."""
config = v41_moe_config.model_config.hf_config
config.vision_n_layers = int(vision)
with torch.device("cuda"):
moe = DeepseekV41MoE(v41_moe_config, prefix=f"model.layers.{layer_id}.ffn")
hidden_states = torch.randn(4, config.hidden_size)
input_ids = (
torch.tensor([42, IMAGE_SENTINEL_BASE_ID, IMAGE_PAD_ID, 129257])
if vision
else None
)
assert moe.gate.tid2eid is None
assert moe.gate.weight.shape == (num_experts, config.hidden_size)
assert isinstance(moe.experts, DeepseekV4MegaMoEExperts)
assert moe.experts.w13_weight.shape == (num_experts, 256, 64)
assert moe.experts.top_k == top_k
assert (config.n_routed_experts, config.num_experts_per_tok) == (8, 2)
with torch.no_grad():
moe.gate.weight.normal_(std=0.01)
moe.gate.e_score_correction_bias.copy_(torch.arange(num_experts, device="cuda"))
if vision:
moe.gate.bias_vl.copy_(-moe.gate.e_score_correction_bias)
scores = torch.nn.functional.softplus(
torch.nn.functional.linear(hidden_states, moe.gate.weight)
).sqrt()
bias = moe.gate.e_score_correction_bias
if vision:
image_mask = (input_ids == IMAGE_SENTINEL_BASE_ID) | (input_ids == IMAGE_PAD_ID)
bias = torch.where(image_mask[:, None], moe.gate.bias_vl, bias)
expected_ids = (scores + bias).topk(top_k, dim=-1).indices
expected_weights = scores.gather(1, expected_ids)
expected_weights *= config.routed_scaling_factor / expected_weights.sum(
dim=-1, keepdim=True
)
def check_routing(x, weights, ids, *, activation_clamp):
torch.testing.assert_close(ids, expected_ids)
torch.testing.assert_close(weights, expected_weights)
assert activation_clamp == config.swiglu_limit
return x.clone()
monkeypatch.setattr(moe.experts, "forward", check_routing)
monkeypatch.setattr(moe.shared_experts, "forward", lambda x: 2 * x)
torch.testing.assert_close(moe(hidden_states, input_ids), 3 * hidden_states)
@pytest.mark.parametrize(
"draft_experts,draft_top_k,expected", [(0, 0, (8, 2)), (4, 3, (4, 3))]
)
def test_deepseek_v41_fused_moe_uses_draft_counts_or_main_defaults(
v41_moe_config, monkeypatch, draft_experts, draft_top_k, expected
):
config = v41_moe_config.model_config.hf_config
config.dspark_n_routed_experts = draft_experts
config.dspark_num_experts_per_tok = draft_top_k
v41_moe_config.kernel_config.moe_backend = "auto"
captured = {}
def make_experts(**kwargs):
captured.update(kwargs)
return torch.nn.Identity()
monkeypatch.setattr(
"vllm.models.deepseek_v4.nvidia.model.FusedMoEFactory", make_experts
)
moe = DeepseekV41MoE(v41_moe_config, prefix="model.layers.2.ffn")
assert moe.gate.weight.shape == (expected[0], 128)
assert (captured["num_experts"], captured["top_k"]) == expected
assert captured["hash_indices_table"] is None
def test_deepseek_v4_moe_preserves_configured_hash_layers(v41_moe_config):
config = v41_moe_config.model_config.hf_config
config.num_hash_layers = 1
config.vocab_size = 32
moe = DeepseekV4MoE(
v41_moe_config,
prefix="model.layers.0.ffn",
num_hash_layers=config.num_hash_layers,
)
assert moe.gate.tid2eid.shape == (32, 2)
assert moe.gate.e_score_correction_bias is None
with pytest.raises(ValueError, match="hash MoE routing requires input_ids"):
moe(torch.zeros(1, 128))
def test_deepseek_v4_mega_moe_expert_mapping():
mapping = make_deepseek_v4_expert_params_mapping(2)
@@ -169,15 +302,79 @@ def test_deepseek_v4_mega_moe_weight_loader_uses_ep_expert_ownership():
assert torch.count_nonzero(experts.w13_weight[1]) == 0
@pytest.mark.parametrize("intermediate_size", [512, 2304])
def test_deepseek_v4_mega_moe_padding_preserves_weights(monkeypatch, intermediate_size):
"""Pad gate/up separately and zero every added down-projection column."""
experts = DeepseekV4MegaMoEExperts(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=2,
num_local_experts=2,
experts_start_idx=0,
top_k=1,
hidden_size=128,
intermediate_size=intermediate_size,
)
originals = []
for param in (
experts.w13_weight,
experts.w13_weight_scale,
experts.w2_weight,
experts.w2_weight_scale,
):
param.data.random_(1, 128)
originals.append(param.data.clone())
monkeypatch.setattr(experts, "_check_runtime_supported", lambda: None)
monkeypatch.setattr(
"vllm.utils.deep_gemm._import_deep_gemm",
lambda: SimpleNamespace(
transform_sf_into_required_layout=lambda sf, *args: sf,
transform_weights_for_mega_moe=lambda l1, l2: (l1, l2),
),
)
experts.finalize_weights()
padded_size = experts.intermediate_size
assert padded_size == (intermediate_size + 511) // 512 * 512
for actual, original in zip(experts._transformed_l1_weights, originals[:2]):
expected = (
original
if actual.dtype == torch.int8
else (experts._ue8m0_uint8_to_float(original))
)
actual = actual.view(expected.dtype).unflatten(1, (2, padded_size))
assert torch.equal(
actual[:, :, :intermediate_size],
expected.unflatten(1, (2, intermediate_size)),
)
assert torch.count_nonzero(actual[:, :, intermediate_size:]) == 0
for actual, original in zip(experts._transformed_l2_weights, originals[2:]):
expected = (
original
if actual.dtype == torch.int8
else (experts._ue8m0_uint8_to_float(original))
)
actual = actual.view(expected.dtype)
assert torch.equal(actual[..., : original.shape[-1]], expected)
assert torch.count_nonzero(actual[..., original.shape[-1] :]) == 0
transformed_l1 = experts._transformed_l1_weights
experts.finalize_weights()
assert experts._transformed_l1_weights is transformed_l1
@pytest.mark.parametrize(
"hidden_size,intermediate_size,block_size,fused",
"hidden_size,intermediate_size,block_size,fused,mxfp8",
[
pytest.param(128, 512, 128, True, id="aligned-block128"),
pytest.param(5120, 2304, 32, False, id="deepseek-v41-flash"),
pytest.param(128, 512, 128, True, False, id="aligned-block128"),
pytest.param(128, 512, 128, True, True, id="aligned-block128-mxfp8"),
pytest.param(5120, 2304, 32, False, False, id="deepseek-v41-flash"),
],
)
def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
monkeypatch, hidden_size, intermediate_size, block_size, fused
monkeypatch, hidden_size, intermediate_size, block_size, fused, mxfp8
):
"""V4.1-Flash's routed padding must preserve the serial shared-expert fallback."""
@@ -261,6 +458,16 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
),
),
)
if mxfp8:
for linear in (shared_experts.gate_up_proj, shared_experts.down_proj):
linear.weight_block_size = (1, 32)
linear.weight_scale = torch.nn.Parameter(
linear.weight_scale_inv.view(torch.uint8)
.repeat_interleave(128, dim=0)
.repeat_interleave(4, dim=1),
requires_grad=False,
)
del linear.weight_scale_inv
monkeypatch.setattr("vllm.utils.deep_gemm._import_deep_gemm", lambda: FakeDeepGemm)
original_gate_up_ptr = shared_experts.gate_up_proj.weight.data_ptr()
+76
View File
@@ -186,3 +186,79 @@ def test_context_kv_weights_are_loaded_as_merged_linear_shards():
assert [weight.shard_id for _, weight in mapped] == [1, 0, 1, 1]
assert mapped[0][1].data_ptr() == mapped[1][1].data_ptr()
assert mapped[2][1].data_ptr() == mapped[3][1].data_ptr()
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"scale_dtype", [torch.uint8, torch.float8_e8m0fnu, torch.float32]
)
@pytest.mark.parametrize(
("checkpoint_name", "runtime_module", "shard_id"),
[
("mtp.0.attn.wq_a.scale", "model.layers.0.attn.fused_wqa_wkv", 0),
("mtp.0.attn.wkv.scale", "model.layers.0.attn.fused_wqa_wkv", 1),
("mtp.0.main_proj.scale", "model.main_proj", None),
(
"mtp.0.ffn.shared_experts.w1.scale",
"model.layers.0.ffn.shared_experts.gate_up_proj",
0,
),
(
"mtp.0.ffn.shared_experts.w2.scale",
"model.layers.0.ffn.shared_experts.down_proj",
None,
),
],
)
def test_v41_dspark_loads_linear_scales(
monkeypatch, scale_dtype, checkpoint_name, runtime_module, shard_id
):
"""Checkpoint ``.scale`` maps to the quant method's scale parameter and
loads untouched. MXFP8 block-scale expansion lives in the KMxfp8Static
loader (see tests/quantization/test_modelopt.py), not in load_weights."""
from vllm.models.deepseek_v4_1.nvidia import dspark
mxfp8 = scale_dtype != torch.float32
scale_name = "weight_scale" if mxfp8 else "weight_scale_inv"
runtime_name = f"{runtime_module}.{scale_name}"
raw = torch.tensor([[120, 127], [128, 130]], dtype=torch.uint8)
checkpoint_scale = raw.view(scale_dtype) if mxfp8 else raw.float()
param = nn.Parameter(torch.empty_like(checkpoint_scale), requires_grad=False)
shards = []
def load_scale(param, weight, *args):
shards.append(args)
assert weight.dtype == checkpoint_scale.dtype
param.copy_(weight)
param.weight_loader = load_scale
draft = SimpleNamespace(
config=SimpleNamespace(num_attention_heads=4, n_routed_experts=1),
quant_config=SimpleNamespace(
weight_block_size=[32, 32] if mxfp8 else [128, 128]
),
linear_scale_name=scale_name,
pad_shared_expert=False,
model=SimpleNamespace(
layers=[SimpleNamespace(ffn=SimpleNamespace(use_mega_moe=False))],
confidence_head=None,
),
named_parameters=lambda: [(runtime_name, param)],
process_weights_after_loading=lambda: None,
)
draft._remap_dspark_name = lambda name: (
dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name)
)
monkeypatch.setattr(dspark, "get_tensor_model_parallel_world_size", lambda: 4)
monkeypatch.setattr(dspark, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
dspark, "fused_moe_make_expert_params_mapping", lambda *a, **kw: []
)
loaded = dspark.DSparkDeepseekV4ForCausalLM.load_weights(
draft, [(checkpoint_name, checkpoint_scale)]
)
assert loaded == {runtime_name}
assert shards == [() if shard_id is None else (shard_id,)]
torch.testing.assert_close(param, checkpoint_scale)
+1
View File
@@ -93,6 +93,7 @@ def test_registry_imports(model_arch):
[
("LlamaForCausalLM", False, False, "bi-encoder"),
("LlavaForConditionalGeneration", True, True, "bi-encoder"),
("DeepseekV41ForCausalLM", True, False, "bi-encoder"),
("BertForSequenceClassification", False, False, "cross-encoder"),
("RobertaForSequenceClassification", False, False, "cross-encoder"),
("XLMRobertaForSequenceClassification", False, False, "cross-encoder"),