[Model] Support DeepSeek-V4.1-Flash (#56214)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-11 02:11:19 -07:00
committed by GitHub
co-authored by Jee Jee Li andyluo7 OpenAI Codex zjy0516 Claude Opus 5
parent e6821ceac9
commit e77daef89e
49 changed files with 3996 additions and 138 deletions
+23 -14
View File
@@ -22,6 +22,7 @@ from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.kv_cache_interface import (
AttentionSpec,
ChunkedLocalAttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
HiddenStateCacheSpec,
KpoolTailSpec,
@@ -2013,15 +2014,22 @@ def _get_packed_kv_cache_groups(
cdiv(len(names), n) * page for page, names in page_size_layers.items()
)
# Bytes a block must hold however the mamba buckets end up split: a mamba
# bucket can go down to one state per group, every other bucket's split is
# already fixed by the repeat pattern.
# Bytes a block must hold however the state buckets end up split: mamba,
# circular-buffer and unsplit sliding-window buckets can go down to one
# state per group, every other bucket's split is fixed by the repeat pattern.
def is_state_bucket(spec: UniformTypeKVCacheSpecs) -> bool:
if isinstance(spec.first_spec, (MambaSpec, CircularBufferSpec)):
return True
return repeats_per_group is None and isinstance(
spec.first_spec, SlidingWindowSpec
)
anchor_bytes = max(
(
widest_group_bytes(
page_size_layers,
len(spec.kv_cache_specs)
if isinstance(spec.first_spec, MambaSpec)
if is_state_bucket(spec)
else num_groups_for(spec, balanced),
)
for spec, page_size_layers, balanced in bucketed
@@ -2032,10 +2040,9 @@ def _get_packed_kv_cache_groups(
groups = []
for spec, page_size_layers, balanced in bucketed:
num_groups = num_groups_for(spec, balanced)
# `_align_hybrid_block_size` pads a mamba state up to one attention
# page, so cap a mamba group at the states a block already fits rather
# than let it widen the block.
if anchor_bytes and isinstance(spec.first_spec, MambaSpec):
# Cap a state group at the states a block already fits rather than let
# it widen the block.
if anchor_bytes and is_state_bucket(spec):
states_per_block = max(anchor_bytes // spec.first_spec.page_size_bytes, 1)
num_groups = max(
num_groups, cdiv(len(spec.kv_cache_specs), states_per_block)
@@ -2071,8 +2078,9 @@ def _is_deepseek_v4_eagle(vllm_config: VllmConfig) -> bool:
if spec_config is None or not spec_config.use_eagle():
return False
model_config = vllm_config.model_config
return (
model_config is not None and model_config.hf_config.model_type == "deepseek_v4"
return model_config is not None and model_config.hf_config.model_type in (
"deepseek_v4",
"deepseek_v41",
)
@@ -2093,9 +2101,9 @@ def _annotate_eagle_groups(
spec merging, wherever grouping happens to land. It is sufficient but
not necessary: a drafter whose spec is indistinguishable from the
target's cannot be found this way.
2. Model-scoped positional fallback for DeepseekV4, whose MTP block reuses
the target's own decoder layer and so carries no spec marker. Its draft
attention layer is always the last registered layer, so flag whichever
2. Model-scoped positional fallback for DeepseekV4/V4.1, whose MTP block
reuses the target's own decoder layer and so carries no spec marker. Its
draft attention layer is always the last registered layer, so flag whichever
group holds it. This rule is only valid where the groups partition
exactly the layers of ``kv_cache_spec``, which is true on the packed
grouping path and not in general; other callers must leave
@@ -2108,7 +2116,8 @@ def _annotate_eagle_groups(
kv_cache_spec: The kv cache spec of each attention layer, in layer
registration order. Only read by rule 2.
kv_cache_groups: Groups to annotate in place.
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4 packed group.
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4/V4.1 packed
group.
"""
spec_config = vllm_config.speculative_config
if spec_config is None or not spec_config.use_eagle_block_drop():