[Model] Support DeepSeek-V4.1-Flash (#56214)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jee Jee Li
andyluo7
OpenAI Codex
zjy0516
Claude Opus 5
parent
e6821ceac9
commit
e77daef89e
@@ -616,6 +616,8 @@ def test_dummy_maverick(
|
||||
) -> None:
|
||||
# Disable multiprocessing allows us to access model executor from LLM engine
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
# The forked test process may inherit CUDA state from test collection.
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
model_path = create_reduced_maverick_model(
|
||||
original_model_name=original_model_name,
|
||||
|
||||
@@ -858,6 +858,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"DeepseekV4ForConditionalGeneration": _HfExamplesInfo(
|
||||
"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
|
||||
),
|
||||
"DeepseekV41ForCausalLM": _HfExamplesInfo(
|
||||
"deepseek-ai/DeepSeek-V4.1-Flash",
|
||||
),
|
||||
"Dots3NoteForCausalLM": _HfExamplesInfo(
|
||||
"dots-studio/dots3-note-prev",
|
||||
is_available_online=False,
|
||||
@@ -1537,6 +1540,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
is_available_online=False,
|
||||
use_original_num_layers=True, # DSpark has >1 draft block
|
||||
),
|
||||
"DSparkV41DraftModel": _HfExamplesInfo(
|
||||
"deepseek-ai/DeepSeek-V4.1-Flash",
|
||||
speculative_model="deepseek-ai/DeepSeek-V4.1-Flash", # draft in mtp.*
|
||||
is_available_online=False,
|
||||
use_original_num_layers=True, # DSpark has >1 draft block
|
||||
),
|
||||
"K3DSparkModel": _HfExamplesInfo(
|
||||
"moonshotai/Kimi-K3",
|
||||
speculative_model="Inferact/Kimi-K3-DSpark",
|
||||
|
||||
@@ -18,7 +18,13 @@ from vllm.models.deepseek_v4.nvidia.model import (
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.mtp import DeepSeekV4MTP
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.models.deepseek_v4_1.common.mm_preprocess import (
|
||||
IMAGE_PAD_ID,
|
||||
IMAGE_SENTINEL_BASE_ID,
|
||||
)
|
||||
from vllm.models.deepseek_v4_1.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
@@ -26,6 +32,133 @@ pytestmark = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def v41_moe_config(dist_init):
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=DeepseekV41Config(
|
||||
text_config=dict(
|
||||
hidden_size=128,
|
||||
num_hidden_layers=2,
|
||||
n_routed_experts=8,
|
||||
num_experts_per_tok=2,
|
||||
dspark_n_routed_experts=4,
|
||||
dspark_num_experts_per_tok=3,
|
||||
n_shared_experts=1,
|
||||
moe_intermediate_size=128,
|
||||
hidden_act="silu",
|
||||
swiglu_limit=10.0,
|
||||
norm_topk_prob=True,
|
||||
topk_method="noaux_tc",
|
||||
routed_scaling_factor=1.5,
|
||||
),
|
||||
),
|
||||
),
|
||||
quant_config=None,
|
||||
kernel_config=SimpleNamespace(moe_backend="deep_gemm_mega_moe"),
|
||||
parallel_config=SimpleNamespace(
|
||||
enable_expert_parallel=True,
|
||||
enable_eplb=False,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=0),
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("vision", [False, True])
|
||||
@pytest.mark.parametrize("layer_id,num_experts,top_k", [(0, 8, 2), (2, 4, 3)])
|
||||
def test_deepseek_v41_moe_routes_without_hash_table(
|
||||
v41_moe_config, monkeypatch, vision, layer_id, num_experts, top_k
|
||||
):
|
||||
"""Main and draft layers select experts by score and preserve image routing."""
|
||||
config = v41_moe_config.model_config.hf_config
|
||||
config.vision_n_layers = int(vision)
|
||||
with torch.device("cuda"):
|
||||
moe = DeepseekV41MoE(v41_moe_config, prefix=f"model.layers.{layer_id}.ffn")
|
||||
hidden_states = torch.randn(4, config.hidden_size)
|
||||
input_ids = (
|
||||
torch.tensor([42, IMAGE_SENTINEL_BASE_ID, IMAGE_PAD_ID, 129257])
|
||||
if vision
|
||||
else None
|
||||
)
|
||||
assert moe.gate.tid2eid is None
|
||||
assert moe.gate.weight.shape == (num_experts, config.hidden_size)
|
||||
assert isinstance(moe.experts, DeepseekV4MegaMoEExperts)
|
||||
assert moe.experts.w13_weight.shape == (num_experts, 256, 64)
|
||||
assert moe.experts.top_k == top_k
|
||||
assert (config.n_routed_experts, config.num_experts_per_tok) == (8, 2)
|
||||
|
||||
with torch.no_grad():
|
||||
moe.gate.weight.normal_(std=0.01)
|
||||
moe.gate.e_score_correction_bias.copy_(torch.arange(num_experts, device="cuda"))
|
||||
if vision:
|
||||
moe.gate.bias_vl.copy_(-moe.gate.e_score_correction_bias)
|
||||
|
||||
scores = torch.nn.functional.softplus(
|
||||
torch.nn.functional.linear(hidden_states, moe.gate.weight)
|
||||
).sqrt()
|
||||
bias = moe.gate.e_score_correction_bias
|
||||
if vision:
|
||||
image_mask = (input_ids == IMAGE_SENTINEL_BASE_ID) | (input_ids == IMAGE_PAD_ID)
|
||||
bias = torch.where(image_mask[:, None], moe.gate.bias_vl, bias)
|
||||
expected_ids = (scores + bias).topk(top_k, dim=-1).indices
|
||||
expected_weights = scores.gather(1, expected_ids)
|
||||
expected_weights *= config.routed_scaling_factor / expected_weights.sum(
|
||||
dim=-1, keepdim=True
|
||||
)
|
||||
|
||||
def check_routing(x, weights, ids, *, activation_clamp):
|
||||
torch.testing.assert_close(ids, expected_ids)
|
||||
torch.testing.assert_close(weights, expected_weights)
|
||||
assert activation_clamp == config.swiglu_limit
|
||||
return x.clone()
|
||||
|
||||
monkeypatch.setattr(moe.experts, "forward", check_routing)
|
||||
monkeypatch.setattr(moe.shared_experts, "forward", lambda x: 2 * x)
|
||||
torch.testing.assert_close(moe(hidden_states, input_ids), 3 * hidden_states)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"draft_experts,draft_top_k,expected", [(0, 0, (8, 2)), (4, 3, (4, 3))]
|
||||
)
|
||||
def test_deepseek_v41_fused_moe_uses_draft_counts_or_main_defaults(
|
||||
v41_moe_config, monkeypatch, draft_experts, draft_top_k, expected
|
||||
):
|
||||
config = v41_moe_config.model_config.hf_config
|
||||
config.dspark_n_routed_experts = draft_experts
|
||||
config.dspark_num_experts_per_tok = draft_top_k
|
||||
v41_moe_config.kernel_config.moe_backend = "auto"
|
||||
captured = {}
|
||||
|
||||
def make_experts(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return torch.nn.Identity()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v4.nvidia.model.FusedMoEFactory", make_experts
|
||||
)
|
||||
moe = DeepseekV41MoE(v41_moe_config, prefix="model.layers.2.ffn")
|
||||
assert moe.gate.weight.shape == (expected[0], 128)
|
||||
assert (captured["num_experts"], captured["top_k"]) == expected
|
||||
assert captured["hash_indices_table"] is None
|
||||
|
||||
|
||||
def test_deepseek_v4_moe_preserves_configured_hash_layers(v41_moe_config):
|
||||
config = v41_moe_config.model_config.hf_config
|
||||
config.num_hash_layers = 1
|
||||
config.vocab_size = 32
|
||||
moe = DeepseekV4MoE(
|
||||
v41_moe_config,
|
||||
prefix="model.layers.0.ffn",
|
||||
num_hash_layers=config.num_hash_layers,
|
||||
)
|
||||
assert moe.gate.tid2eid.shape == (32, 2)
|
||||
assert moe.gate.e_score_correction_bias is None
|
||||
with pytest.raises(ValueError, match="hash MoE routing requires input_ids"):
|
||||
moe(torch.zeros(1, 128))
|
||||
|
||||
|
||||
def test_deepseek_v4_mega_moe_expert_mapping():
|
||||
mapping = make_deepseek_v4_expert_params_mapping(2)
|
||||
|
||||
@@ -169,15 +302,79 @@ def test_deepseek_v4_mega_moe_weight_loader_uses_ep_expert_ownership():
|
||||
assert torch.count_nonzero(experts.w13_weight[1]) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("intermediate_size", [512, 2304])
|
||||
def test_deepseek_v4_mega_moe_padding_preserves_weights(monkeypatch, intermediate_size):
|
||||
"""Pad gate/up separately and zero every added down-projection column."""
|
||||
experts = DeepseekV4MegaMoEExperts(
|
||||
SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
),
|
||||
num_experts=2,
|
||||
num_local_experts=2,
|
||||
experts_start_idx=0,
|
||||
top_k=1,
|
||||
hidden_size=128,
|
||||
intermediate_size=intermediate_size,
|
||||
)
|
||||
originals = []
|
||||
for param in (
|
||||
experts.w13_weight,
|
||||
experts.w13_weight_scale,
|
||||
experts.w2_weight,
|
||||
experts.w2_weight_scale,
|
||||
):
|
||||
param.data.random_(1, 128)
|
||||
originals.append(param.data.clone())
|
||||
monkeypatch.setattr(experts, "_check_runtime_supported", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
"vllm.utils.deep_gemm._import_deep_gemm",
|
||||
lambda: SimpleNamespace(
|
||||
transform_sf_into_required_layout=lambda sf, *args: sf,
|
||||
transform_weights_for_mega_moe=lambda l1, l2: (l1, l2),
|
||||
),
|
||||
)
|
||||
|
||||
experts.finalize_weights()
|
||||
padded_size = experts.intermediate_size
|
||||
assert padded_size == (intermediate_size + 511) // 512 * 512
|
||||
for actual, original in zip(experts._transformed_l1_weights, originals[:2]):
|
||||
expected = (
|
||||
original
|
||||
if actual.dtype == torch.int8
|
||||
else (experts._ue8m0_uint8_to_float(original))
|
||||
)
|
||||
actual = actual.view(expected.dtype).unflatten(1, (2, padded_size))
|
||||
assert torch.equal(
|
||||
actual[:, :, :intermediate_size],
|
||||
expected.unflatten(1, (2, intermediate_size)),
|
||||
)
|
||||
assert torch.count_nonzero(actual[:, :, intermediate_size:]) == 0
|
||||
for actual, original in zip(experts._transformed_l2_weights, originals[2:]):
|
||||
expected = (
|
||||
original
|
||||
if actual.dtype == torch.int8
|
||||
else (experts._ue8m0_uint8_to_float(original))
|
||||
)
|
||||
actual = actual.view(expected.dtype)
|
||||
assert torch.equal(actual[..., : original.shape[-1]], expected)
|
||||
assert torch.count_nonzero(actual[..., original.shape[-1] :]) == 0
|
||||
|
||||
transformed_l1 = experts._transformed_l1_weights
|
||||
experts.finalize_weights()
|
||||
assert experts._transformed_l1_weights is transformed_l1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_size,intermediate_size,block_size,fused",
|
||||
"hidden_size,intermediate_size,block_size,fused,mxfp8",
|
||||
[
|
||||
pytest.param(128, 512, 128, True, id="aligned-block128"),
|
||||
pytest.param(5120, 2304, 32, False, id="deepseek-v41-flash"),
|
||||
pytest.param(128, 512, 128, True, False, id="aligned-block128"),
|
||||
pytest.param(128, 512, 128, True, True, id="aligned-block128-mxfp8"),
|
||||
pytest.param(5120, 2304, 32, False, False, id="deepseek-v41-flash"),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
|
||||
monkeypatch, hidden_size, intermediate_size, block_size, fused
|
||||
monkeypatch, hidden_size, intermediate_size, block_size, fused, mxfp8
|
||||
):
|
||||
"""V4.1-Flash's routed padding must preserve the serial shared-expert fallback."""
|
||||
|
||||
@@ -261,6 +458,16 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
|
||||
),
|
||||
),
|
||||
)
|
||||
if mxfp8:
|
||||
for linear in (shared_experts.gate_up_proj, shared_experts.down_proj):
|
||||
linear.weight_block_size = (1, 32)
|
||||
linear.weight_scale = torch.nn.Parameter(
|
||||
linear.weight_scale_inv.view(torch.uint8)
|
||||
.repeat_interleave(128, dim=0)
|
||||
.repeat_interleave(4, dim=1),
|
||||
requires_grad=False,
|
||||
)
|
||||
del linear.weight_scale_inv
|
||||
monkeypatch.setattr("vllm.utils.deep_gemm._import_deep_gemm", lambda: FakeDeepGemm)
|
||||
|
||||
original_gate_up_ptr = shared_experts.gate_up_proj.weight.data_ptr()
|
||||
|
||||
@@ -186,3 +186,79 @@ def test_context_kv_weights_are_loaded_as_merged_linear_shards():
|
||||
assert [weight.shard_id for _, weight in mapped] == [1, 0, 1, 1]
|
||||
assert mapped[0][1].data_ptr() == mapped[1][1].data_ptr()
|
||||
assert mapped[2][1].data_ptr() == mapped[3][1].data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"scale_dtype", [torch.uint8, torch.float8_e8m0fnu, torch.float32]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("checkpoint_name", "runtime_module", "shard_id"),
|
||||
[
|
||||
("mtp.0.attn.wq_a.scale", "model.layers.0.attn.fused_wqa_wkv", 0),
|
||||
("mtp.0.attn.wkv.scale", "model.layers.0.attn.fused_wqa_wkv", 1),
|
||||
("mtp.0.main_proj.scale", "model.main_proj", None),
|
||||
(
|
||||
"mtp.0.ffn.shared_experts.w1.scale",
|
||||
"model.layers.0.ffn.shared_experts.gate_up_proj",
|
||||
0,
|
||||
),
|
||||
(
|
||||
"mtp.0.ffn.shared_experts.w2.scale",
|
||||
"model.layers.0.ffn.shared_experts.down_proj",
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_v41_dspark_loads_linear_scales(
|
||||
monkeypatch, scale_dtype, checkpoint_name, runtime_module, shard_id
|
||||
):
|
||||
"""Checkpoint ``.scale`` maps to the quant method's scale parameter and
|
||||
loads untouched. MXFP8 block-scale expansion lives in the KMxfp8Static
|
||||
loader (see tests/quantization/test_modelopt.py), not in load_weights."""
|
||||
from vllm.models.deepseek_v4_1.nvidia import dspark
|
||||
|
||||
mxfp8 = scale_dtype != torch.float32
|
||||
scale_name = "weight_scale" if mxfp8 else "weight_scale_inv"
|
||||
runtime_name = f"{runtime_module}.{scale_name}"
|
||||
raw = torch.tensor([[120, 127], [128, 130]], dtype=torch.uint8)
|
||||
checkpoint_scale = raw.view(scale_dtype) if mxfp8 else raw.float()
|
||||
param = nn.Parameter(torch.empty_like(checkpoint_scale), requires_grad=False)
|
||||
shards = []
|
||||
|
||||
def load_scale(param, weight, *args):
|
||||
shards.append(args)
|
||||
assert weight.dtype == checkpoint_scale.dtype
|
||||
param.copy_(weight)
|
||||
|
||||
param.weight_loader = load_scale
|
||||
draft = SimpleNamespace(
|
||||
config=SimpleNamespace(num_attention_heads=4, n_routed_experts=1),
|
||||
quant_config=SimpleNamespace(
|
||||
weight_block_size=[32, 32] if mxfp8 else [128, 128]
|
||||
),
|
||||
linear_scale_name=scale_name,
|
||||
pad_shared_expert=False,
|
||||
model=SimpleNamespace(
|
||||
layers=[SimpleNamespace(ffn=SimpleNamespace(use_mega_moe=False))],
|
||||
confidence_head=None,
|
||||
),
|
||||
named_parameters=lambda: [(runtime_name, param)],
|
||||
process_weights_after_loading=lambda: None,
|
||||
)
|
||||
draft._remap_dspark_name = lambda name: (
|
||||
dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name)
|
||||
)
|
||||
monkeypatch.setattr(dspark, "get_tensor_model_parallel_world_size", lambda: 4)
|
||||
monkeypatch.setattr(dspark, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
dspark, "fused_moe_make_expert_params_mapping", lambda *a, **kw: []
|
||||
)
|
||||
|
||||
loaded = dspark.DSparkDeepseekV4ForCausalLM.load_weights(
|
||||
draft, [(checkpoint_name, checkpoint_scale)]
|
||||
)
|
||||
|
||||
assert loaded == {runtime_name}
|
||||
assert shards == [() if shard_id is None else (shard_id,)]
|
||||
torch.testing.assert_close(param, checkpoint_scale)
|
||||
|
||||
@@ -93,6 +93,7 @@ def test_registry_imports(model_arch):
|
||||
[
|
||||
("LlamaForCausalLM", False, False, "bi-encoder"),
|
||||
("LlavaForConditionalGeneration", True, True, "bi-encoder"),
|
||||
("DeepseekV41ForCausalLM", True, False, "bi-encoder"),
|
||||
("BertForSequenceClassification", False, False, "cross-encoder"),
|
||||
("RobertaForSequenceClassification", False, False, "cross-encoder"),
|
||||
("XLMRobertaForSequenceClassification", False, False, "cross-encoder"),
|
||||
|
||||
Reference in New Issue
Block a user