[Chore] Use upstream Transformers DeepseekV4Config (#61392)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Harry Mellor
2026-10-15 01:08:08 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Kevin H. Luu
parent 1c4c319792
commit b965ee564d
31 changed files with 155 additions and 120 deletions
+4 -1
View File
@@ -608,7 +608,10 @@ class DeepseekV4MoE(nn.Module):
self.image_sentinel_lo = (
IMAGE_SENTINEL_BASE_ID if getattr(config, "vision_n_layers", 0) > 0 else 0
)
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
num_hash_layers = getattr(config, "num_hash_layers", None) # Set by V4.1
if num_hash_layers is None:
num_hash_layers = config.mlp_layer_types.count("hash_moe")
is_hash_moe = extract_layer_index(prefix) < num_hash_layers
self.hash_indices_dtype = torch.int32
if is_hash_moe:
# hash MoE doesn't use e_score_correction_bias
+2 -1
View File
@@ -231,7 +231,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# we do this for because MTP layer is not included
# in the compress ratio list
if layer_id < config.num_hidden_layers:
self.compress_ratio = max(1, config.compress_ratios[layer_id])
layer_type = config.layer_types[layer_id]
self.compress_ratio = max(1, config.compress_rates.get(layer_type, 0))
else:
self.compress_ratio = 1
self.eps = config.rms_norm_eps
@@ -27,7 +27,7 @@ from typing import Any, assert_never
import numpy as np
import torch
from PIL import Image, ImageOps
from transformers import BatchFeature
from transformers import BatchFeature, DeepseekV4Config
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
@@ -48,7 +48,6 @@ from vllm.multimodal.processing.processor import (
UpdateMode,
_plan_prompt_updates,
)
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
IMAGE_START, IMAGE_PAD, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(5)
COMPRESS_PAD_TO = 4
@@ -34,6 +34,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
triton_scalar_specialization_rep,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.math_utils import next_power_of_2
@@ -393,7 +394,9 @@ class DequantizeAndGatherKCacheKernel(
compress_ratios = frozenset(
max(1, int(compress_ratio))
for compress_ratio in vllm_config.model_config.hf_config.compress_ratios
for compress_ratio in get_compress_ratios(
vllm_config.model_config.hf_config
)
)
return self._trace_dispatch(self.dispatch)(
zip_inputs(
@@ -658,7 +661,7 @@ class ComputeGlobalTopkIndicesAndLensKernel(
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
index_topk = vllm_config.model_config.hf_config.index_topk
compress_ratios = vllm_config.model_config.hf_config.compress_ratios
compress_ratios = get_compress_ratios(vllm_config.model_config.hf_config)
cache_block_size = vllm_config.cache_config.block_size
if index_topk <= 0 or cache_block_size <= 0:
return []
@@ -964,7 +967,7 @@ class CombineTopkSwaIndicesKernel(
hf_config = vllm_config.model_config.hf_config
index_topk = int(getattr(hf_config, "index_topk", 0) or 0)
compress_ratios = tuple(
sorted({max(1, int(ratio)) for ratio in hf_config.compress_ratios})
sorted({max(1, int(ratio)) for ratio in get_compress_ratios(hf_config)})
)
max_c128a_topk = (
((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128
@@ -1592,7 +1595,7 @@ class BuildFlashinferMixedSparseIndicesKernel(
image_widths = (0, max_image_tokens) if max_image_tokens > 0 else 0
index_topk = int(getattr(hf_config, "index_topk", 0) or 0)
compress_ratios = tuple(
sorted({max(1, int(ratio)) for ratio in hf_config.compress_ratios})
sorted({max(1, int(ratio)) for ratio in get_compress_ratios(hf_config)})
)
max_c128a_topk = (
((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128
@@ -40,6 +40,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalKwargsItem
from .mm_preprocess import (
COMPRESS_PAD_TO,
IMAGE_PLACEHOLDER,
IMAGE_SENTINEL_BASE_ID,
DeepseekV4VLDummyInputsBuilder,
@@ -105,6 +106,10 @@ class DeepseekV4ForConditionalGeneration(
# (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl.
requires_raw_input_tokens = True
supports_tower_connector_lora = True
# Image spans widen the sparse-SWA window in-kernel and the placeholder
# starts with a compressor-alignment pad
mm_prefix_clamp_sliding_window = True
mm_prefix_span_leading_pad_modulus = COMPRESS_PAD_TO
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
+3 -1
View File
@@ -145,7 +145,9 @@ class DeepseekV4MoE(nn.Module):
self.gate.e_score_correction_bias = None
self.gate.tid2eid = None
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count(
"hash_moe"
)
self.hash_indices_dtype = torch.int32
if is_hash_moe:
# hash MoE doesn't use e_score_correction_bias
+2 -2
View File
@@ -944,7 +944,7 @@ class DeepseekV4DecoderLayer(nn.Module):
vllm_config,
prefix=f"{prefix}.ffn",
use_sequence_parallel=self.use_sequence_parallel,
num_hash_layers=config.num_hash_layers,
num_hash_layers=config.mlp_layer_types.count("hash_moe"),
)
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
@@ -1207,7 +1207,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
prefix=f"{prefix}.layers",
)
self.has_local_hash_moe = self.start_layer < min(
self.end_layer, config.num_hash_layers
self.end_layer, config.mlp_layer_types.count("hash_moe")
)
if get_pp_group().is_last_rank:
@@ -18,6 +18,7 @@ from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
CuTeDSLLaunchSpec,
VllmCuTeDSLJitKernel,
)
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
class DequantGatherKCacheKernel(
@@ -329,7 +330,7 @@ class DequantGatherKCacheKernel(
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
block_size = vllm_config.cache_config.block_size
compress_ratios = vllm_config.model_config.hf_config.compress_ratios
compress_ratios = get_compress_ratios(vllm_config.model_config.hf_config)
return self._trace_dispatch(self.dispatch)(
zip_inputs(
dict(
@@ -436,7 +436,7 @@ class SparseAttnCompressNormRopeStoreC4Kernel(
rope_head_dim=hf_config.qk_rope_head_dim,
enabled=(
cache_config.block_size > 0
and 4 in hf_config.compress_ratios
and "compressed_sparse_attention" in hf_config.layer_types
and cache_config.cache_dtype == "fp8_ds_mla"
),
_when=lambda *, enabled: enabled,
@@ -934,7 +934,7 @@ class SparseAttnCompressNormRopeStoreFullC4Kernel(
rope_head_dim=hf_config.qk_rope_head_dim,
enabled=(
cache_config.block_size > 0
and 4 in hf_config.compress_ratios
and "compressed_sparse_attention" in hf_config.layer_types
and cache_config.cache_dtype != "fp8_ds_mla"
),
_when=lambda *, enabled: enabled,
@@ -1366,7 +1366,7 @@ class SparseAttnCompressC128Block8Kernel(
return self._trace_dispatch(self.dispatch)(
head_size=hf_config.head_dim,
state_width=hf_config.head_dim,
enabled=128 in hf_config.compress_ratios,
enabled="heavily_compressed_attention" in hf_config.layer_types,
_when=lambda *, enabled: enabled,
)
@@ -2040,7 +2040,7 @@ class SparseAttnNormRopeStoreFullKernel(
rope_head_dim=hf_config.qk_rope_head_dim,
enabled=(
cache_config.block_size > 0
and 128 in hf_config.compress_ratios
and "heavily_compressed_attention" in hf_config.layer_types
and cache_config.cache_dtype != "fp8_ds_mla"
),
_when=lambda *, enabled: enabled,
+3 -1
View File
@@ -638,7 +638,9 @@ class DeepseekV4MoE(nn.Module):
self.gate.e_score_correction_bias = None
self.gate.tid2eid = None
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count(
"hash_moe"
)
self.hash_indices_dtype = torch.int64 if self.use_mega_moe else torch.int32
if is_hash_moe:
# hash MoE doesn't use e_score_correction_bias
@@ -31,6 +31,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
TritonWarmupTensor,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.math_utils import cdiv, next_power_of_2
@@ -1012,7 +1013,7 @@ def _combine_topk_swa_warmup_inputs(vllm_config: Any) -> _WarmupInputRows:
"""
model_config = getattr(vllm_config, "model_config", None)
hf_config = getattr(model_config, "hf_config", None)
compress_ratios = getattr(hf_config, "compress_ratios", None) or [0]
compress_ratios = get_compress_ratios(hf_config) or [0]
topk = _hf_config_int(vllm_config, "index_topk", 0)
return zip_inputs(
*(