From b965ee564d1897fdaeddaeceecc316aa15a45312 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:08:08 +0100 Subject: [PATCH] [Chore] Use upstream Transformers DeepseekV4Config (#61392) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Kevin H. Luu --- .agents/skills/transformers-upgrade/SKILL.md | 23 ++++++-- tests/config/test_model_arch_config.py | 3 +- tests/distributed/test_eplb_spec_decode.py | 4 +- .../layers/test_fused_shared_expert.py | 3 +- .../processing/test_deepseek_v4_vl.py | 20 +++++-- tests/models/test_deepseek_v4_mega_moe.py | 4 +- tests/test_config.py | 6 +-- .../test_deepseek_v4_utils.py | 19 +++++++ .../attention/test_deepseek_v4_swa_visible.py | 27 +++++----- .../layers/attention/mla_attention.py | 3 +- vllm/models/deepseek_v4/amd/model.py | 5 +- vllm/models/deepseek_v4/attention.py | 3 +- .../deepseek_v4/common/mm_preprocess.py | 3 +- .../deepseek_v4/common/ops/cache_utils.py | 11 ++-- vllm/models/deepseek_v4/common/vl_model.py | 5 ++ vllm/models/deepseek_v4/cpu/model.py | 4 +- vllm/models/deepseek_v4/nvidia/model.py | 4 +- .../nvidia/ops/dequant_gather_k_cutedsl.py | 3 +- .../ops/sparse_attn_compress_cutedsl.py | 8 +-- vllm/models/deepseek_v4/xpu/model.py | 4 +- .../deepseek_v41/common/ops/cache_utils.py | 3 +- vllm/transformers_utils/config.py | 6 ++- vllm/transformers_utils/configs/__init__.py | 2 - .../transformers_utils/configs/deepseek_v4.py | 52 ------------------- vllm/transformers_utils/deepseek_v4_utils.py | 18 +++++++ .../model_arch_config_convertor.py | 5 +- .../backends/mla/compressor_utils.py | 3 +- vllm/v1/attention/backends/mla/indexer.py | 3 +- vllm/v1/attention/backends/mla/sparse_swa.py | 14 ++--- vllm/v1/spec_decode/llm_base_proposer.py | 5 +- vllm/v1/worker/gpu_model_runner.py | 2 +- 31 files changed, 155 insertions(+), 120 deletions(-) create mode 100644 tests/transformers_utils/test_deepseek_v4_utils.py delete mode 100755 vllm/transformers_utils/configs/deepseek_v4.py create mode 100644 vllm/transformers_utils/deepseek_v4_utils.py diff --git a/.agents/skills/transformers-upgrade/SKILL.md b/.agents/skills/transformers-upgrade/SKILL.md index de9065b6fa..847b292cc5 100644 --- a/.agents/skills/transformers-upgrade/SKILL.md +++ b/.agents/skills/transformers-upgrade/SKILL.md @@ -121,6 +121,9 @@ For each deleted config: Deleted processors follow the same steps in `vllm/transformers_utils/processors/__init__.py`. +Before merging, rebase and grep again for the old field names: PRs that land +in between still read them. + Done when every line of the inventory output is deleted, or kept with a reason that meets the bar in step 2. The PR also carries the config diff results and evals for any changed values. @@ -128,13 +131,15 @@ results and evals for any changed values. ## Upstreaming hazards Upstream configs are `@strict` dataclasses. Checkpoint keys upstream does not -declare survive as plain attributes. Defaults that only vLLM sets, and values -vLLM derives, vanish. +declare survive as plain attributes, but only when the checkpoint sets them. +Defaults that only vLLM sets, and values vLLM derives, vanish. | Hazard | Signal | Fix | | --- | --- | --- | | Upstream consumes legacy keys | `__post_init__` pops keys the model reads (`deepseek_v4` `compress_ratios`) | Read the upstream replacement (`layer_types`, `mlp_layer_types`, per-layer `rope_parameters`) | | Renamed or dropped fields | Diff shows `` for a field the model reads | Read the upstream name; derive dropped flags from what implies them (`enable_moe_block` from `num_experts`) | +| Undeclared field without a default | `config.x` raises `AttributeError` for configs that omit `x`, e.g. minimal test configs (A.X-K1 `scoring_func`). Dumps of real checkpoints miss it | Upstream ignoring `x` usually means vLLM can stop reading it; if vLLM truly needs it, read `getattr(config, "x", )` | +| Top-level config stops forwarding | The vendored top-level config forwarded reads to `text_config`; upstream does not. `getattr(config, "x", default)` silently takes the default, e.g. on MTP draft configs from `SpeculativeConfig.hf_config_override`, which the dumps do not build | Read from `get_text_config()`; grep `getattr` reads of every field, including speculative-decoding paths | | Type check on the config | `ctx.get_hf_config(SomeConfig)` relies on the vendored base class | Accept the upstream class too, e.g. a `Cosmos3ProcessingInfo` overriding `get_hf_config` | | Remote code takes over | Checkpoint `config.json` has `auto_map`; with `--trust-remote-code`, the hub class replaces the native one once vLLM stops registering its own | Check the remote class loads; if not, register the upstream class for that `model_type` | | Read-only upstream fields | Properties or derived fields break subclasses and `hf_overrides` (`nemotron_h` `hybrid_override_pattern`) | Override the source field (`layers_block_type`) instead | @@ -159,10 +164,10 @@ Version gates from the inventory need a decision: - **Patches citing a Transformers issue or PR:** - Delete the patch when its fix is in the floor release. - For an issue, find the PR that closed it and check that PR. A closed - issue is not a fixed one: #43329 was closed while the bug was still in - 5.16.1. + issue is not a fixed one: issues get closed while the bug is still in + the release. - Re-run the patched behaviour before deleting. A fix in the release does - not always cover vLLM's path: #47924 made Emu3's processor add BOS, but + not always cover vLLM's path: Emu3's processor gained BOS upstream, but vLLM tokenizes without the processor. - Links that only explain a design choice (e.g. a float32 RoPE note) are not patches; leave them. @@ -176,3 +181,11 @@ Version gates from the inventory need a decision: Add `tests/models/test_initialization.py -k ""` where a GPU is available, and a model eval for any model whose resolved config changed. + +Models in `vllm/models/` are high priority: whenever their config or model +code changes, run GSM8k (`tests/evals/gsm8k/`) and GPQA Diamond +(`tests/evals/gpt_oss/`) on `main` and the branch, even when the dumps show +no diff, and put both scores in the PR. FlashInfer autotuning picks kernel +tactics afresh on every server start, which moved DeepSeek-V4-Flash GSM8k by +about 1 point; pass `--kernel-config '{"enable_flashinfer_autotune": false}'` +to both servers so greedy outputs are comparable. diff --git a/tests/config/test_model_arch_config.py b/tests/config/test_model_arch_config.py index 65b4f440d8..a9409be09c 100644 --- a/tests/config/test_model_arch_config.py +++ b/tests/config/test_model_arch_config.py @@ -519,7 +519,8 @@ def test_deepseek_v4_convertor_splits_vision_architecture(): text-only DeepSeek-V4; the convertor routes it to the VL wrapper class by rewriting hf_config.architectures (model-class resolution reads the raw attribute).""" - from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config + from transformers import DeepseekV4Config + from vllm.transformers_utils.model_arch_config_convertor import ( DeepseekV4ModelArchConfigConvertor, ) diff --git a/tests/distributed/test_eplb_spec_decode.py b/tests/distributed/test_eplb_spec_decode.py index b49d06cd76..6f4e52e88d 100644 --- a/tests/distributed/test_eplb_spec_decode.py +++ b/tests/distributed/test_eplb_spec_decode.py @@ -7,12 +7,12 @@ from types import SimpleNamespace import lm_eval import pytest import torch +from transformers import DeepseekV4Config from tests.utils import large_gpu_mark from vllm.distributed.eplb.eplb_state import EplbState from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb from vllm.platforms import current_platform -from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config from vllm.v1.worker.gpu.eplb_utils import ( EPLBController, draft_model_supports_eplb, @@ -105,7 +105,7 @@ def _make_dsv4_dspark_hf_config() -> DeepseekV4Config: vocab_size=256, n_mtp_layers=2, enable_confidence_head=False, - compress_ratios=[1, 1], + layer_types=["sliding_attention"] * 2, ) diff --git a/tests/model_executor/layers/test_fused_shared_expert.py b/tests/model_executor/layers/test_fused_shared_expert.py index 0fcc2540e2..b279576e9a 100644 --- a/tests/model_executor/layers/test_fused_shared_expert.py +++ b/tests/model_executor/layers/test_fused_shared_expert.py @@ -182,7 +182,8 @@ def get_fse_test_model_config( head_dim=128, max_position_embeddings=128, sliding_window=None, - compress_ratios=[1], + layer_types=["sliding_attention"], + compress_rates={}, rope_theta=10000.0, compress_rope_theta=10000.0, rope_parameters={"rope_type": "default"}, diff --git a/tests/models/multimodal/processing/test_deepseek_v4_vl.py b/tests/models/multimodal/processing/test_deepseek_v4_vl.py index 5358fae970..0a28e71ea3 100644 --- a/tests/models/multimodal/processing/test_deepseek_v4_vl.py +++ b/tests/models/multimodal/processing/test_deepseek_v4_vl.py @@ -3,13 +3,27 @@ """DeepSeek-V4 VL processing helpers.""" from PIL import Image +from transformers import DeepseekV4Config from vllm.models.deepseek_v4.common.mm_preprocess import ( DeepseekV4VLProcessingInfo, DeepseekV4VLProcessor, build_image_block_pad_free, ) -from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config + +# Vision fields of deepseek-ai/DeepSeek-V4-Flash-Vision-Exp +VISION_CONFIG = dict( + vision_n_layers=24, + vision_dim=1024, + vision_n_heads=16, + vision_inter_dim=2816, + vision_patch_size=14, + vision_rope_theta=10000.0, + vision_downsample_ratio=3, + vision_max_n_token=384, + vision_min_pixels=147456, + vision_max_wh_ratio=8, +) class _ConfigCtx: @@ -34,7 +48,7 @@ def test_get_image_size_with_most_features_beats_wide_real_image(): A square under-counts because each aligner row pays an IMAGE_NEW_LINE. """ - config = DeepseekV4Config(vision_n_layers=24) + config = DeepseekV4Config(**VISION_CONFIG) info = DeepseekV4VLProcessingInfo(_ConfigCtx(config)) processor = DeepseekV4VLProcessor(config) @@ -47,7 +61,7 @@ def test_get_image_size_with_most_features_beats_wide_real_image(): def test_get_image_size_with_most_features_respects_max_wh_ratio(): - config = DeepseekV4Config(vision_n_layers=24) + config = DeepseekV4Config(**VISION_CONFIG) info = DeepseekV4VLProcessingInfo(_ConfigCtx(config)) dummy = info.get_image_size_with_most_features() assert dummy.width / dummy.height <= config.vision_max_wh_ratio diff --git a/tests/models/test_deepseek_v4_mega_moe.py b/tests/models/test_deepseek_v4_mega_moe.py index 6f1ff1afab..97dd1b376b 100644 --- a/tests/models/test_deepseek_v4_mega_moe.py +++ b/tests/models/test_deepseek_v4_mega_moe.py @@ -5,6 +5,7 @@ from types import SimpleNamespace import pytest import torch +from transformers import DeepseekV4Config from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import ( DeepGemmSm100MegaMoEBackend, @@ -28,7 +29,6 @@ from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_i from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE from vllm.platforms import current_platform -from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config from vllm.utils.deep_gemm import is_deep_gemm_supported from vllm.utils.torch_utils import set_default_torch_dtype @@ -979,7 +979,7 @@ def v4_dspark_config(dist_init): vocab_size=256, n_mtp_layers=2, enable_confidence_head=False, - compress_ratios=[1, 1], + layer_types=["sliding_attention"] * 2, q_lora_rank=32, o_lora_rank=32, qk_rope_head_dim=16, diff --git a/tests/test_config.py b/tests/test_config.py index c029b0a520..0e65d87150 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2005,9 +2005,7 @@ def test_draft_inherits_ep_only_for_moe( draft_is_moe: bool, ): """Validate final draft configs without loading weights or mocking validation.""" - from transformers import LlamaConfig, MixtralConfig - - from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config + from transformers import DeepseekV4Config, LlamaConfig, MixtralConfig common = dict( hidden_size=128, @@ -2024,7 +2022,7 @@ def test_draft_inherits_ep_only_for_moe( n_routed_experts=4, num_experts_per_tok=2, num_nextn_predict_layers=1, - compress_ratios=[1, 1], + layer_types=["sliding_attention"] * 2, head_dim=32, **common, ) diff --git a/tests/transformers_utils/test_deepseek_v4_utils.py b/tests/transformers_utils/test_deepseek_v4_utils.py new file mode 100644 index 0000000000..73e16a63c4 --- /dev/null +++ b/tests/transformers_utils/test_deepseek_v4_utils.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from transformers import DeepseekV4Config + +from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios + +# Legacy field as shipped by deepseek-ai/DeepSeek-V4-Flash-Vision-Exp +COMPRESS_RATIOS = [0, 0] + [4, 128] * 20 + [4, 0, 0, 0] + + +def test_compress_ratios_survive_upstream_config(): + """Upstream pops `compress_ratios`; the helper must recover it.""" + config = DeepseekV4Config( + num_hidden_layers=43, + num_nextn_predict_layers=3, + compress_ratios=COMPRESS_RATIOS, + ) + assert not hasattr(config, "compress_ratios") + assert get_compress_ratios(config) == COMPRESS_RATIOS diff --git a/tests/v1/attention/test_deepseek_v4_swa_visible.py b/tests/v1/attention/test_deepseek_v4_swa_visible.py index 5ccd721808..5dcea926f0 100644 --- a/tests/v1/attention/test_deepseek_v4_swa_visible.py +++ b/tests/v1/attention/test_deepseek_v4_swa_visible.py @@ -15,6 +15,7 @@ import inspect import pytest import torch +from transformers import DeepseekV4Config from typing_extensions import TypedDict from tests.v1.attention.utils import create_vllm_config @@ -532,26 +533,26 @@ def make_builder( builder_cls: type[ DeepseekSparseSWAMetadataBuilder ] = DeepseekSparseSWAMetadataBuilder, - mm_prefix_clamp_sliding_window: bool = True, + deepseek_v4_config: bool = True, bounded_replay: bool = False, ) -> DeepseekSparseSWAMetadataBuilder: - overrides: dict = {"sliding_window": window} - if vision: - # Emulate the V4 vision config, whose mm_prefix_clamp_sliding_window - # gates the in-kernel SWA widening. V4.1 (causal image tokens) leaves - # it off. - overrides.update( - vision_n_layers=2, - vision_max_n_token=max_image_tokens, - mm_prefix_clamp_sliding_window=mm_prefix_clamp_sliding_window, - ) + vision_fields = dict(vision_n_layers=2, vision_max_n_token=max_image_tokens) vllm_config = create_vllm_config( model_name=model_name, max_model_len=4096, max_num_batched_tokens=max_num_batched_tokens, max_num_seqs=8, - hf_config_override=overrides, + hf_config_override={"sliding_window": window} + | (vision_fields if vision and not deepseek_v4_config else {}), ) + if vision and deepseek_v4_config: + # Only V4 configs widen the SWA window in-kernel over image spans + vllm_config.model_config.hf_config = DeepseekV4Config( + num_hidden_layers=1, + layer_types=["sliding_attention"], + sliding_window=window, + **vision_fields, + ) spec = SlidingWindowMLASpec( block_size=BLOCK_SIZE, num_kv_heads=1, @@ -686,7 +687,7 @@ def test_v41_image_prefill_uses_causal_swa(compress_ratio, query_len): max_image_tokens, query_len, builder_cls=DeepseekV41SparseSWAMetadataBuilder, - mm_prefix_clamp_sliding_window=False, + deepseek_v4_config=False, ) assert builder.max_image_tokens == 0 md = build_metadata(builder, [seq_len], [query_len], {0: spans}) diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index bc6bca99ff..caf60b009f 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -261,6 +261,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( ) from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform +from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios from vllm.utils.flashinfer import has_flashinfer from vllm.utils.math_utils import cdiv, round_down, round_up from vllm.utils.torch_utils import ( @@ -1740,7 +1741,7 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims: hf_text_config = model_config.hf_text_config # Check if this is a DeepseekV4 config (uses unified head_dim + rope_head_dim) - if hasattr(hf_text_config, "compress_ratios"): + if get_compress_ratios(hf_text_config): # DeepseekV4 style config: unified head_dim with rope_head_dim head_dim = hf_text_config.head_dim rope_head_dim = hf_text_config.qk_rope_head_dim diff --git a/vllm/models/deepseek_v4/amd/model.py b/vllm/models/deepseek_v4/amd/model.py index 2facab7230..e7894f319f 100644 --- a/vllm/models/deepseek_v4/amd/model.py +++ b/vllm/models/deepseek_v4/amd/model.py @@ -608,7 +608,10 @@ class DeepseekV4MoE(nn.Module): self.image_sentinel_lo = ( IMAGE_SENTINEL_BASE_ID if getattr(config, "vision_n_layers", 0) > 0 else 0 ) - is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers + num_hash_layers = getattr(config, "num_hash_layers", None) # Set by V4.1 + if num_hash_layers is None: + num_hash_layers = config.mlp_layer_types.count("hash_moe") + is_hash_moe = extract_layer_index(prefix) < num_hash_layers self.hash_indices_dtype = torch.int32 if is_hash_moe: # hash MoE doesn't use e_score_correction_bias diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index 2542897ef8..764ae6e8d8 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -231,7 +231,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): # we do this for because MTP layer is not included # in the compress ratio list if layer_id < config.num_hidden_layers: - self.compress_ratio = max(1, config.compress_ratios[layer_id]) + layer_type = config.layer_types[layer_id] + self.compress_ratio = max(1, config.compress_rates.get(layer_type, 0)) else: self.compress_ratio = 1 self.eps = config.rms_norm_eps diff --git a/vllm/models/deepseek_v4/common/mm_preprocess.py b/vllm/models/deepseek_v4/common/mm_preprocess.py index 9c1a4664c8..2c7c5d8bd9 100644 --- a/vllm/models/deepseek_v4/common/mm_preprocess.py +++ b/vllm/models/deepseek_v4/common/mm_preprocess.py @@ -27,7 +27,7 @@ from typing import Any, assert_never import numpy as np import torch from PIL import Image, ImageOps -from transformers import BatchFeature +from transformers import BatchFeature, DeepseekV4Config from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict @@ -48,7 +48,6 @@ from vllm.multimodal.processing.processor import ( UpdateMode, _plan_prompt_updates, ) -from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config IMAGE_START, IMAGE_PAD, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(5) COMPRESS_PAD_TO = 4 diff --git a/vllm/models/deepseek_v4/common/ops/cache_utils.py b/vllm/models/deepseek_v4/common/ops/cache_utils.py index cea755bd1a..e190d95437 100644 --- a/vllm/models/deepseek_v4/common/ops/cache_utils.py +++ b/vllm/models/deepseek_v4/common/ops/cache_utils.py @@ -34,6 +34,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import ( triton_scalar_specialization_rep, ) from vllm.platforms import current_platform +from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios from vllm.triton_utils import tl, triton from vllm.utils.import_utils import has_cutedsl from vllm.utils.math_utils import next_power_of_2 @@ -393,7 +394,9 @@ class DequantizeAndGatherKCacheKernel( compress_ratios = frozenset( max(1, int(compress_ratio)) - for compress_ratio in vllm_config.model_config.hf_config.compress_ratios + for compress_ratio in get_compress_ratios( + vllm_config.model_config.hf_config + ) ) return self._trace_dispatch(self.dispatch)( zip_inputs( @@ -658,7 +661,7 @@ class ComputeGlobalTopkIndicesAndLensKernel( def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: index_topk = vllm_config.model_config.hf_config.index_topk - compress_ratios = vllm_config.model_config.hf_config.compress_ratios + compress_ratios = get_compress_ratios(vllm_config.model_config.hf_config) cache_block_size = vllm_config.cache_config.block_size if index_topk <= 0 or cache_block_size <= 0: return [] @@ -964,7 +967,7 @@ class CombineTopkSwaIndicesKernel( hf_config = vllm_config.model_config.hf_config index_topk = int(getattr(hf_config, "index_topk", 0) or 0) compress_ratios = tuple( - sorted({max(1, int(ratio)) for ratio in hf_config.compress_ratios}) + sorted({max(1, int(ratio)) for ratio in get_compress_ratios(hf_config)}) ) max_c128a_topk = ( ((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128 @@ -1592,7 +1595,7 @@ class BuildFlashinferMixedSparseIndicesKernel( image_widths = (0, max_image_tokens) if max_image_tokens > 0 else 0 index_topk = int(getattr(hf_config, "index_topk", 0) or 0) compress_ratios = tuple( - sorted({max(1, int(ratio)) for ratio in hf_config.compress_ratios}) + sorted({max(1, int(ratio)) for ratio in get_compress_ratios(hf_config)}) ) max_c128a_topk = ( ((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128 diff --git a/vllm/models/deepseek_v4/common/vl_model.py b/vllm/models/deepseek_v4/common/vl_model.py index 97701cc2d5..708a57d737 100644 --- a/vllm/models/deepseek_v4/common/vl_model.py +++ b/vllm/models/deepseek_v4/common/vl_model.py @@ -40,6 +40,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalKwargsItem from .mm_preprocess import ( + COMPRESS_PAD_TO, IMAGE_PLACEHOLDER, IMAGE_SENTINEL_BASE_ID, DeepseekV4VLDummyInputsBuilder, @@ -105,6 +106,10 @@ class DeepseekV4ForConditionalGeneration( # (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl. requires_raw_input_tokens = True supports_tower_connector_lora = True + # Image spans widen the sparse-SWA window in-kernel and the placeholder + # starts with a compressor-alignment pad + mm_prefix_clamp_sliding_window = True + mm_prefix_span_leading_pad_modulus = COMPRESS_PAD_TO @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: diff --git a/vllm/models/deepseek_v4/cpu/model.py b/vllm/models/deepseek_v4/cpu/model.py index af66c480fc..02ea09bcc4 100644 --- a/vllm/models/deepseek_v4/cpu/model.py +++ b/vllm/models/deepseek_v4/cpu/model.py @@ -145,7 +145,9 @@ class DeepseekV4MoE(nn.Module): self.gate.e_score_correction_bias = None self.gate.tid2eid = None - is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers + is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count( + "hash_moe" + ) self.hash_indices_dtype = torch.int32 if is_hash_moe: # hash MoE doesn't use e_score_correction_bias diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index c5d6a2d15a..ef24943f02 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -944,7 +944,7 @@ class DeepseekV4DecoderLayer(nn.Module): vllm_config, prefix=f"{prefix}.ffn", use_sequence_parallel=self.use_sequence_parallel, - num_hash_layers=config.num_hash_layers, + num_hash_layers=config.mlp_layer_types.count("hash_moe"), ) self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps) @@ -1207,7 +1207,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): prefix=f"{prefix}.layers", ) self.has_local_hash_moe = self.start_layer < min( - self.end_layer, config.num_hash_layers + self.end_layer, config.mlp_layer_types.count("hash_moe") ) if get_pp_group().is_last_rank: diff --git a/vllm/models/deepseek_v4/nvidia/ops/dequant_gather_k_cutedsl.py b/vllm/models/deepseek_v4/nvidia/ops/dequant_gather_k_cutedsl.py index 5920ed290c..d97c54a011 100644 --- a/vllm/models/deepseek_v4/nvidia/ops/dequant_gather_k_cutedsl.py +++ b/vllm/models/deepseek_v4/nvidia/ops/dequant_gather_k_cutedsl.py @@ -18,6 +18,7 @@ from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import ( CuTeDSLLaunchSpec, VllmCuTeDSLJitKernel, ) +from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios class DequantGatherKCacheKernel( @@ -329,7 +330,7 @@ class DequantGatherKCacheKernel( def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: block_size = vllm_config.cache_config.block_size - compress_ratios = vllm_config.model_config.hf_config.compress_ratios + compress_ratios = get_compress_ratios(vllm_config.model_config.hf_config) return self._trace_dispatch(self.dispatch)( zip_inputs( dict( diff --git a/vllm/models/deepseek_v4/nvidia/ops/sparse_attn_compress_cutedsl.py b/vllm/models/deepseek_v4/nvidia/ops/sparse_attn_compress_cutedsl.py index 74866b0960..16e612af01 100644 --- a/vllm/models/deepseek_v4/nvidia/ops/sparse_attn_compress_cutedsl.py +++ b/vllm/models/deepseek_v4/nvidia/ops/sparse_attn_compress_cutedsl.py @@ -436,7 +436,7 @@ class SparseAttnCompressNormRopeStoreC4Kernel( rope_head_dim=hf_config.qk_rope_head_dim, enabled=( cache_config.block_size > 0 - and 4 in hf_config.compress_ratios + and "compressed_sparse_attention" in hf_config.layer_types and cache_config.cache_dtype == "fp8_ds_mla" ), _when=lambda *, enabled: enabled, @@ -934,7 +934,7 @@ class SparseAttnCompressNormRopeStoreFullC4Kernel( rope_head_dim=hf_config.qk_rope_head_dim, enabled=( cache_config.block_size > 0 - and 4 in hf_config.compress_ratios + and "compressed_sparse_attention" in hf_config.layer_types and cache_config.cache_dtype != "fp8_ds_mla" ), _when=lambda *, enabled: enabled, @@ -1366,7 +1366,7 @@ class SparseAttnCompressC128Block8Kernel( return self._trace_dispatch(self.dispatch)( head_size=hf_config.head_dim, state_width=hf_config.head_dim, - enabled=128 in hf_config.compress_ratios, + enabled="heavily_compressed_attention" in hf_config.layer_types, _when=lambda *, enabled: enabled, ) @@ -2040,7 +2040,7 @@ class SparseAttnNormRopeStoreFullKernel( rope_head_dim=hf_config.qk_rope_head_dim, enabled=( cache_config.block_size > 0 - and 128 in hf_config.compress_ratios + and "heavily_compressed_attention" in hf_config.layer_types and cache_config.cache_dtype != "fp8_ds_mla" ), _when=lambda *, enabled: enabled, diff --git a/vllm/models/deepseek_v4/xpu/model.py b/vllm/models/deepseek_v4/xpu/model.py index 92093a15a2..80c579abae 100644 --- a/vllm/models/deepseek_v4/xpu/model.py +++ b/vllm/models/deepseek_v4/xpu/model.py @@ -638,7 +638,9 @@ class DeepseekV4MoE(nn.Module): self.gate.e_score_correction_bias = None self.gate.tid2eid = None - is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers + is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count( + "hash_moe" + ) self.hash_indices_dtype = torch.int64 if self.use_mega_moe else torch.int32 if is_hash_moe: # hash MoE doesn't use e_score_correction_bias diff --git a/vllm/models/deepseek_v41/common/ops/cache_utils.py b/vllm/models/deepseek_v41/common/ops/cache_utils.py index a6139c71f8..000fb03195 100644 --- a/vllm/models/deepseek_v41/common/ops/cache_utils.py +++ b/vllm/models/deepseek_v41/common/ops/cache_utils.py @@ -31,6 +31,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import ( TritonWarmupTensor, ) from vllm.platforms import current_platform +from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios from vllm.triton_utils import tl, triton from vllm.utils.import_utils import has_cutedsl from vllm.utils.math_utils import cdiv, next_power_of_2 @@ -1012,7 +1013,7 @@ def _combine_topk_swa_warmup_inputs(vllm_config: Any) -> _WarmupInputRows: """ model_config = getattr(vllm_config, "model_config", None) hf_config = getattr(model_config, "hf_config", None) - compress_ratios = getattr(hf_config, "compress_ratios", None) or [0] + compress_ratios = get_compress_ratios(hf_config) or [0] topk = _hf_config_int(vllm_config, "index_topk", 0) return zip_inputs( *( diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 892e00c43d..c67d468a13 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -127,7 +127,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict( cosmos3_edge="Cosmos3EdgeConfig", deepseek_vl_v2="DeepseekVLV2Config", deepseek_v32="DeepseekV3Config", - deepseek_v4="DeepseekV4Config", dots3_note="Dots3NoteConfig", k3_dspark="K3DSparkConfig", funaudiochat="FunAudioChatConfig", @@ -216,7 +215,10 @@ def is_rope_parameters_nested(rope_parameters: dict[str, Any]) -> bool: # Cannot be nested if rope_parameters is empty if not rope_parameters: return False - return set(rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES) + # DeepSeek-V4 nests by rope type ("main", "compress") instead + return set(rope_parameters.keys()).issubset( + {*ALLOWED_LAYER_TYPES, "main", "compress"} + ) @contextmanager diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index cf1d08d67c..68f82ead6b 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -29,7 +29,6 @@ _CLASS_TO_MODULE: dict[str, str] = { "Cosmos3EdgeTextConfig": "vllm.transformers_utils.configs.cosmos3_edge", "Cosmos3EdgeVisionConfig": "vllm.transformers_utils.configs.cosmos3_edge", "DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2", - "DeepseekV4Config": "vllm.transformers_utils.configs.deepseek_v4", "DeepseekV41Config": "vllm.transformers_utils.configs.deepseek_v41", "Dots3NoteConfig": "vllm.transformers_utils.configs.dots3_note", "K3DSparkConfig": "vllm.transformers_utils.configs.k3_dspark", @@ -103,7 +102,6 @@ __all__ = [ "Cosmos3EdgeVisionConfig", "DeepseekVLV2Config", "DeepseekV3Config", - "DeepseekV4Config", "Dots3NoteConfig", "K3DSparkConfig", "DotsOCRConfig", diff --git a/vllm/transformers_utils/configs/deepseek_v4.py b/vllm/transformers_utils/configs/deepseek_v4.py deleted file mode 100755 index 278f477864..0000000000 --- a/vllm/transformers_utils/configs/deepseek_v4.py +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any - -from transformers import PreTrainedConfig - - -class DeepseekV4Config(PreTrainedConfig): - model_type = "deepseek_v4" - - def __init__( - self, - max_position_embeddings: int = 1048576, - rope_scaling: dict[str, Any] | None = None, - rope_parameters: dict[str, Any] | None = None, - rope_theta: float = 10000.0, - vision_n_layers: int = 0, - vision_dim: int = 1024, - vision_n_heads: int = 16, - vision_inter_dim: int = 2816, - vision_patch_size: int = 14, - vision_rope_theta: float = 10000.0, - vision_downsample_ratio: int = 3, - vision_max_n_token: int = 384, - vision_min_pixels: int = 147456, - vision_max_wh_ratio: float = 8, - **kwargs, - ): - self.max_position_embeddings = max_position_embeddings - self.rope_scaling = rope_scaling - self.rope_theta = rope_theta - self.rope_parameters = rope_scaling or rope_parameters - # Vision tower config; vision_n_layers == 0 means text-only. - self.vision_n_layers = vision_n_layers - self.vision_dim = vision_dim - self.vision_n_heads = vision_n_heads - self.vision_inter_dim = vision_inter_dim - self.vision_patch_size = vision_patch_size - self.vision_rope_theta = vision_rope_theta - self.vision_downsample_ratio = vision_downsample_ratio - self.vision_max_n_token = vision_max_n_token - self.vision_min_pixels = vision_min_pixels - self.vision_max_wh_ratio = vision_max_wh_ratio - # The sparse-SWA index kernels widen the window within image spans - # in-kernel, so mm-prefix ranges longer than sliding_window must be - # kept (they are only consumed by those kernels). - self.mm_prefix_clamp_sliding_window = vision_n_layers > 0 - # The visibility span covers the sentinel block [IMAGE_START, - # IMAGE_END]; the mm placeholder additionally carries a leading - # compressor-alignment pad (see mm_preprocess.COMPRESS_PAD_TO). - self.mm_prefix_span_leading_pad_modulus = 4 if vision_n_layers > 0 else 0 - super().__init__(**kwargs) diff --git a/vllm/transformers_utils/deepseek_v4_utils.py b/vllm/transformers_utils/deepseek_v4_utils.py new file mode 100644 index 0000000000..f65f1bb526 --- /dev/null +++ b/vllm/transformers_utils/deepseek_v4_utils.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any + + +def get_compress_ratios(config: Any) -> list[int]: + """Per-layer compress ratios (0 = sliding window), MTP layers included. + + Reads the legacy `compress_ratios` of DeepSeek-V4.1 configs, or derives it + from `layer_types` of the upstream `DeepseekV4Config`, which drops the MTP + layers. Returns an empty list for other configs. + """ + if (compress_ratios := getattr(config, "compress_ratios", None)) is not None: + return compress_ratios + if (compress_rates := getattr(config, "compress_rates", None)) is None: + return [] + ratios = [compress_rates.get(t, 0) for t in config.layer_types] + return ratios + [0] * config.num_nextn_predict_layers diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index b16e0321c2..ed284342ac 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -18,6 +18,7 @@ from vllm.transformers_utils.config import ( ConfigFormat, get_safetensors_params_metadata, ) +from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios from vllm.transformers_utils.repo_utils import ( file_or_path_exists, get_hf_file_to_dict, @@ -90,7 +91,7 @@ class ModelArchConfigConvertorBase: def get_head_size(self) -> int: if self.is_deepseek_mla(): # special case for deepseek_v4 - if hasattr(self.hf_text_config, "compress_ratios"): + if get_compress_ratios(self.hf_text_config): return self.hf_text_config.head_dim qk_rope_head_dim = self._get_qk_rope_head_dim() if not envs.VLLM_MLA_DISABLE: @@ -318,7 +319,7 @@ class ModelArchConfigConvertorBase: "hy_v4_mtp", ): # check is deepseek_v4 model - if hasattr(self.hf_text_config, "compress_ratios"): + if get_compress_ratios(self.hf_text_config): return getattr(self.hf_text_config, "head_dim", None) is not None else: return getattr(self.hf_text_config, "kv_lora_rank", None) is not None diff --git a/vllm/v1/attention/backends/mla/compressor_utils.py b/vllm/v1/attention/backends/mla/compressor_utils.py index bf939dc049..31cfaab5dc 100644 --- a/vllm/v1/attention/backends/mla/compressor_utils.py +++ b/vllm/v1/attention/backends/mla/compressor_utils.py @@ -12,6 +12,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import ( VllmTritonJitKernel, triton_scalar_specialization_rep, ) +from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios from vllm.triton_utils import tl, triton from vllm.utils.math_utils import cdiv @@ -124,7 +125,7 @@ class CompressedSlotMappingKernel( def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: hf_text_config = vllm_config.model_config.hf_text_config configured_ratios = ( - *(getattr(hf_text_config, "compress_ratios", None) or ()), + *get_compress_ratios(hf_text_config), getattr(hf_text_config, "index_kpool", 1) or 1, ) compress_ratios = tuple( diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 04287ca1f6..4f18173343 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -19,6 +19,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import ( triton_scalar_specialization_rep, ) from vllm.platforms import current_platform +from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios from vllm.triton_utils import tl, triton from vllm.utils.deep_gemm import ( get_paged_mqa_logits_metadata, @@ -610,7 +611,7 @@ class BuildPrefillChunkMetadataKernel( dict.fromkeys( max(1, int(ratio)) for ratio in ( - *(getattr(hf_text_config, "compress_ratios", None) or (1,)), + *(get_compress_ratios(hf_text_config) or (1,)), getattr(hf_text_config, "index_kpool", 1) or 1, ) ) diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index a691efe8bf..65df2fcee4 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field from typing import Any, ClassVar, cast import torch +from transformers import DeepseekV4Config from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase @@ -18,6 +19,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import ( VllmTritonJitKernel, ) from vllm.platforms import current_platform +from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios from vllm.triton_utils import tl, triton from vllm.utils.math_utils import cdiv, next_power_of_2 from vllm.v1.attention.backend import ( @@ -471,13 +473,13 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): # V4 vision variant: image spans (up to vision_max_n_token tokens) are # visible bidirectionally, so prefill index rows widen from - # window_size to window_size + max_image_tokens. The V4 config sets - # mm_prefix_clamp_sliding_window exactly for these in-kernel-widened - # ranges; V4.1 image tokens use the plain causal window, and text-only - # models keep max_image_tokens == 0 everywhere. + # window_size to window_size + max_image_tokens. V4.1 image tokens use + # the plain causal window, and text-only models keep + # max_image_tokens == 0 everywhere. self.max_image_tokens = ( getattr(hf_config, "vision_max_n_token", 0) - if getattr(hf_config, "mm_prefix_clamp_sliding_window", False) + if isinstance(hf_config, DeepseekV4Config) + and getattr(hf_config, "vision_n_layers", 0) > 0 else 0 ) self.prefill_index_width = self.window_size + self.max_image_tokens @@ -485,7 +487,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): # Detect which DeepseekV4 layer types this model uses so we only build a # FlashMLA tile-scheduler plan for types that will actually be called. # Models without compress_ratios (pure SWA) fall back to swaonly. - compress_ratios = getattr(hf_config, "compress_ratios", None) or [1] + compress_ratios = get_compress_ratios(hf_config) or [1] self._layer_types: set[str] = set() for ratio in compress_ratios: self._layer_types.add(_layer_type_for(int(ratio))) diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index 79be2e8a4d..1ee04d1861 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -37,6 +37,7 @@ from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM from vllm.model_executor.models.qwen3_dflash import DFlashQwen3ForCausalLM from vllm.model_executor.models.qwen3_eagle3 import Eagle3Qwen3ForCausalLM from vllm.platforms import current_platform +from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -102,9 +103,7 @@ class SpecDecodeBaseProposer: # shape (T, hc_mult * hidden_size). Expand the hidden_states buffer # so target_hidden_states fits; detect DeepseekV4 via draft hf_config. draft_hf_config = self.draft_model_config.hf_config - if hasattr(draft_hf_config, "compress_ratios") and hasattr( - draft_hf_config, "hc_mult" - ): + if get_compress_ratios(draft_hf_config) and hasattr(draft_hf_config, "hc_mult"): self.hidden_size = self.hidden_size * draft_hf_config.hc_mult # Unifying eagle, draft model, and parallel drafting support. diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 229acdfcd2..f18e219d8e 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2344,7 +2344,7 @@ class GPUModelRunner( # TODO(Isotr0py): Refactor mm_prefix_lm implementation # for better readability and maintainability. _span_pad_modulus = getattr( - hf_text_config, "mm_prefix_span_leading_pad_modulus", 0 + self.model, "mm_prefix_span_leading_pad_modulus", 0 ) for req_id in self.input_batch.req_ids: image_doc_ranges = []