[Model] Support DeepSeek-V4.1-Flash (#56214)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jee Jee Li
andyluo7
OpenAI Codex
zjy0516
Claude Opus 5
parent
e6821ceac9
commit
e77daef89e
+19
-9
@@ -15,6 +15,14 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Architecture -> the hf_text_config field naming its n-gram layers. A model is
|
||||
# only configurable here if it actually has such layers to store.
|
||||
_NGRAM_LAYER_FIELDS = {
|
||||
"DeepseekV41ForCausalLM": "engram_layer_ids",
|
||||
"Qwen4ExpForCausalLM": "ple_layer_ids",
|
||||
"Qwen4ExpForConditionalGeneration": "ple_layer_ids",
|
||||
}
|
||||
|
||||
|
||||
def _default_cpu_offload() -> bool:
|
||||
"""Honor the legacy environment variable only when the field is omitted."""
|
||||
@@ -40,23 +48,25 @@ class EngramConfig:
|
||||
Otherwise, each DP rank has a separate TP-sharded embedding replica."""
|
||||
|
||||
def verify_model_config(self, model_config: "ModelConfig | None") -> None:
|
||||
"""Reject Engram configuration for models without supported embeddings."""
|
||||
"""Reject Engram configuration for models without n-gram embeddings."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
supported_architectures = {
|
||||
"Qwen4ExpForCausalLM",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
}
|
||||
field = (
|
||||
_NGRAM_LAYER_FIELDS.get(model_config.architecture)
|
||||
if model_config is not None
|
||||
else None
|
||||
)
|
||||
if (
|
||||
model_config is None
|
||||
or model_config.architecture not in supported_architectures
|
||||
or field is None
|
||||
or not current_platform.is_cuda()
|
||||
or not getattr(model_config.hf_text_config, "ple_layer_ids", None)
|
||||
or not getattr(model_config.hf_text_config, field, None)
|
||||
):
|
||||
raise ValueError(
|
||||
"EngramConfig requires a model with supported Engram "
|
||||
"embeddings. Currently only the CUDA Qwen4Exp implementation "
|
||||
"with non-empty ple_layer_ids is supported."
|
||||
"embeddings. Currently only the CUDA Qwen4Exp and DeepSeek "
|
||||
"V4.1 implementations with non-empty n-gram layer ids are "
|
||||
"supported."
|
||||
)
|
||||
|
||||
def verify_parallel_config(self, parallel_config: "ParallelConfig") -> None:
|
||||
|
||||
@@ -165,6 +165,7 @@ FLASHINFER_MOE_EP_ARCHITECTURES = frozenset(
|
||||
{
|
||||
"DeepseekV4ForCausalLM",
|
||||
"DeepSeekV4MTPModel",
|
||||
"DeepseekV41ForCausalLM",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -679,12 +679,18 @@ class SpeculativeConfig:
|
||||
],
|
||||
}
|
||||
)
|
||||
if hf_config.model_type == "deepseek_v4":
|
||||
if hf_config.model_type in ("deepseek_v4", "deepseek_v41"):
|
||||
# V4.1 has no classic-MTP draft: its checkpoints ship DSpark stages
|
||||
# under ``mtp.*``, so only V4 gets an MTP architecture here. The
|
||||
# DSpark path rewrites ``architectures`` itself and needs only
|
||||
# ``n_predict``; ``method="mtp"`` on V4.1 is rejected below.
|
||||
is_v41 = hf_config.model_type == "deepseek_v41"
|
||||
hf_config.model_type = "deepseek_mtp"
|
||||
n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
|
||||
hf_config.update(
|
||||
{"n_predict": n_predict, "architectures": ["DeepSeekV4MTPModel"]}
|
||||
)
|
||||
overrides = {"n_predict": n_predict}
|
||||
if not is_v41:
|
||||
overrides["architectures"] = ["DeepSeekV4MTPModel"]
|
||||
hf_config.update(overrides)
|
||||
if hf_config.model_type in ("pangu_ultra_moe"):
|
||||
hf_config.model_type = "pangu_ultra_moe_mtp"
|
||||
if hf_config.model_type == "pangu_ultra_moe_mtp":
|
||||
@@ -1335,6 +1341,18 @@ class SpeculativeConfig:
|
||||
MTPModelTypes
|
||||
):
|
||||
self.method = "mtp"
|
||||
if (
|
||||
self.target_model_config is not None
|
||||
and self.target_model_config.hf_config.model_type
|
||||
== "deepseek_v41"
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek V4.1 has no classic-MTP draft: its "
|
||||
"checkpoints ship DSpark stages under mtp.* "
|
||||
"(main_proj/markov_head/confidence_head) and carry "
|
||||
"no e_proj/h_proj/enorm/hnorm/hc_head weights. Use "
|
||||
"speculative method 'dspark' instead of 'mtp'."
|
||||
)
|
||||
if (
|
||||
self.num_speculative_tokens > 1
|
||||
and self.draft_model_config.hf_config.model_type
|
||||
@@ -1398,12 +1416,26 @@ class SpeculativeConfig:
|
||||
and "Gemma4DSparkModel" not in self.draft_model_config.architectures
|
||||
and "K3DSparkModel" not in self.draft_model_config.architectures
|
||||
):
|
||||
# DeepSeek-V4 DSpark reuses the full DeepSeek-V4 config
|
||||
# DeepSeek-V4(.1) DSpark reuses the full target config
|
||||
# and its weights ship in the target checkpoint.
|
||||
self.draft_model_config.hf_config.model_type = "deepseek_v4"
|
||||
self.draft_model_config.hf_config.architectures = [
|
||||
"DSparkDraftModel"
|
||||
is_v41 = (
|
||||
self.target_model_config.hf_config.model_type == "deepseek_v41"
|
||||
)
|
||||
draft_hf_config = self.draft_model_config.hf_config
|
||||
draft_hf_config.model_type = (
|
||||
"deepseek_v41" if is_v41 else "deepseek_v4"
|
||||
)
|
||||
draft_hf_config.architectures = [
|
||||
"DSparkV41DraftModel" if is_v41 else "DSparkDraftModel"
|
||||
]
|
||||
if is_v41:
|
||||
# hf_config_override set n_predict to the number of
|
||||
# MTP stages (3), but one DSpark round drafts
|
||||
# dspark_block_size tokens; num_speculative_tokens
|
||||
# divisibility is checked against n_predict below.
|
||||
draft_hf_config.n_predict = getattr(
|
||||
draft_hf_config, "dspark_block_size", None
|
||||
) or getattr(draft_hf_config, "n_predict", None)
|
||||
self.draft_model_config.quantization = (
|
||||
self.target_model_config.quantization
|
||||
)
|
||||
|
||||
@@ -81,6 +81,7 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
|
||||
"DeepseekV4ForCausalLM",
|
||||
"DeepseekV4ForConditionalGeneration",
|
||||
"DeepSeekV4MTPModel",
|
||||
"DeepseekV41ForCausalLM",
|
||||
"Dots3NoteForCausalLM",
|
||||
"Dots3NoteMTPModel",
|
||||
"Glm5NextForCausalLM",
|
||||
|
||||
Reference in New Issue
Block a user