[Chore] Use upstream Transformers DeepseekV4Config (#61392)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Kevin H. Luu
parent
1c4c319792
commit
b965ee564d
@@ -261,6 +261,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.math_utils import cdiv, round_down, round_up
|
||||
from vllm.utils.torch_utils import (
|
||||
@@ -1740,7 +1741,7 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims:
|
||||
hf_text_config = model_config.hf_text_config
|
||||
|
||||
# Check if this is a DeepseekV4 config (uses unified head_dim + rope_head_dim)
|
||||
if hasattr(hf_text_config, "compress_ratios"):
|
||||
if get_compress_ratios(hf_text_config):
|
||||
# DeepseekV4 style config: unified head_dim with rope_head_dim
|
||||
head_dim = hf_text_config.head_dim
|
||||
rope_head_dim = hf_text_config.qk_rope_head_dim
|
||||
|
||||
@@ -608,7 +608,10 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.image_sentinel_lo = (
|
||||
IMAGE_SENTINEL_BASE_ID if getattr(config, "vision_n_layers", 0) > 0 else 0
|
||||
)
|
||||
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
|
||||
num_hash_layers = getattr(config, "num_hash_layers", None) # Set by V4.1
|
||||
if num_hash_layers is None:
|
||||
num_hash_layers = config.mlp_layer_types.count("hash_moe")
|
||||
is_hash_moe = extract_layer_index(prefix) < num_hash_layers
|
||||
self.hash_indices_dtype = torch.int32
|
||||
if is_hash_moe:
|
||||
# hash MoE doesn't use e_score_correction_bias
|
||||
|
||||
@@ -231,7 +231,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# we do this for because MTP layer is not included
|
||||
# in the compress ratio list
|
||||
if layer_id < config.num_hidden_layers:
|
||||
self.compress_ratio = max(1, config.compress_ratios[layer_id])
|
||||
layer_type = config.layer_types[layer_id]
|
||||
self.compress_ratio = max(1, config.compress_rates.get(layer_type, 0))
|
||||
else:
|
||||
self.compress_ratio = 1
|
||||
self.eps = config.rms_norm_eps
|
||||
|
||||
@@ -27,7 +27,7 @@ from typing import Any, assert_never
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image, ImageOps
|
||||
from transformers import BatchFeature
|
||||
from transformers import BatchFeature, DeepseekV4Config
|
||||
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
@@ -48,7 +48,6 @@ from vllm.multimodal.processing.processor import (
|
||||
UpdateMode,
|
||||
_plan_prompt_updates,
|
||||
)
|
||||
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
||||
|
||||
IMAGE_START, IMAGE_PAD, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(5)
|
||||
COMPRESS_PAD_TO = 4
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
triton_scalar_specialization_rep,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
@@ -393,7 +394,9 @@ class DequantizeAndGatherKCacheKernel(
|
||||
|
||||
compress_ratios = frozenset(
|
||||
max(1, int(compress_ratio))
|
||||
for compress_ratio in vllm_config.model_config.hf_config.compress_ratios
|
||||
for compress_ratio in get_compress_ratios(
|
||||
vllm_config.model_config.hf_config
|
||||
)
|
||||
)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
zip_inputs(
|
||||
@@ -658,7 +661,7 @@ class ComputeGlobalTopkIndicesAndLensKernel(
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
index_topk = vllm_config.model_config.hf_config.index_topk
|
||||
compress_ratios = vllm_config.model_config.hf_config.compress_ratios
|
||||
compress_ratios = get_compress_ratios(vllm_config.model_config.hf_config)
|
||||
cache_block_size = vllm_config.cache_config.block_size
|
||||
if index_topk <= 0 or cache_block_size <= 0:
|
||||
return []
|
||||
@@ -964,7 +967,7 @@ class CombineTopkSwaIndicesKernel(
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
index_topk = int(getattr(hf_config, "index_topk", 0) or 0)
|
||||
compress_ratios = tuple(
|
||||
sorted({max(1, int(ratio)) for ratio in hf_config.compress_ratios})
|
||||
sorted({max(1, int(ratio)) for ratio in get_compress_ratios(hf_config)})
|
||||
)
|
||||
max_c128a_topk = (
|
||||
((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128
|
||||
@@ -1592,7 +1595,7 @@ class BuildFlashinferMixedSparseIndicesKernel(
|
||||
image_widths = (0, max_image_tokens) if max_image_tokens > 0 else 0
|
||||
index_topk = int(getattr(hf_config, "index_topk", 0) or 0)
|
||||
compress_ratios = tuple(
|
||||
sorted({max(1, int(ratio)) for ratio in hf_config.compress_ratios})
|
||||
sorted({max(1, int(ratio)) for ratio in get_compress_ratios(hf_config)})
|
||||
)
|
||||
max_c128a_topk = (
|
||||
((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItem
|
||||
|
||||
from .mm_preprocess import (
|
||||
COMPRESS_PAD_TO,
|
||||
IMAGE_PLACEHOLDER,
|
||||
IMAGE_SENTINEL_BASE_ID,
|
||||
DeepseekV4VLDummyInputsBuilder,
|
||||
@@ -105,6 +106,10 @@ class DeepseekV4ForConditionalGeneration(
|
||||
# (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl.
|
||||
requires_raw_input_tokens = True
|
||||
supports_tower_connector_lora = True
|
||||
# Image spans widen the sparse-SWA window in-kernel and the placeholder
|
||||
# starts with a compressor-alignment pad
|
||||
mm_prefix_clamp_sliding_window = True
|
||||
mm_prefix_span_leading_pad_modulus = COMPRESS_PAD_TO
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
|
||||
@@ -145,7 +145,9 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
self.gate.e_score_correction_bias = None
|
||||
self.gate.tid2eid = None
|
||||
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
|
||||
is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count(
|
||||
"hash_moe"
|
||||
)
|
||||
self.hash_indices_dtype = torch.int32
|
||||
if is_hash_moe:
|
||||
# hash MoE doesn't use e_score_correction_bias
|
||||
|
||||
@@ -944,7 +944,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.ffn",
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
num_hash_layers=config.num_hash_layers,
|
||||
num_hash_layers=config.mlp_layer_types.count("hash_moe"),
|
||||
)
|
||||
|
||||
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
@@ -1207,7 +1207,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.has_local_hash_moe = self.start_layer < min(
|
||||
self.end_layer, config.num_hash_layers
|
||||
self.end_layer, config.mlp_layer_types.count("hash_moe")
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
|
||||
CuTeDSLLaunchSpec,
|
||||
VllmCuTeDSLJitKernel,
|
||||
)
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
|
||||
|
||||
class DequantGatherKCacheKernel(
|
||||
@@ -329,7 +330,7 @@ class DequantGatherKCacheKernel(
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
compress_ratios = vllm_config.model_config.hf_config.compress_ratios
|
||||
compress_ratios = get_compress_ratios(vllm_config.model_config.hf_config)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
zip_inputs(
|
||||
dict(
|
||||
|
||||
@@ -436,7 +436,7 @@ class SparseAttnCompressNormRopeStoreC4Kernel(
|
||||
rope_head_dim=hf_config.qk_rope_head_dim,
|
||||
enabled=(
|
||||
cache_config.block_size > 0
|
||||
and 4 in hf_config.compress_ratios
|
||||
and "compressed_sparse_attention" in hf_config.layer_types
|
||||
and cache_config.cache_dtype == "fp8_ds_mla"
|
||||
),
|
||||
_when=lambda *, enabled: enabled,
|
||||
@@ -934,7 +934,7 @@ class SparseAttnCompressNormRopeStoreFullC4Kernel(
|
||||
rope_head_dim=hf_config.qk_rope_head_dim,
|
||||
enabled=(
|
||||
cache_config.block_size > 0
|
||||
and 4 in hf_config.compress_ratios
|
||||
and "compressed_sparse_attention" in hf_config.layer_types
|
||||
and cache_config.cache_dtype != "fp8_ds_mla"
|
||||
),
|
||||
_when=lambda *, enabled: enabled,
|
||||
@@ -1366,7 +1366,7 @@ class SparseAttnCompressC128Block8Kernel(
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
head_size=hf_config.head_dim,
|
||||
state_width=hf_config.head_dim,
|
||||
enabled=128 in hf_config.compress_ratios,
|
||||
enabled="heavily_compressed_attention" in hf_config.layer_types,
|
||||
_when=lambda *, enabled: enabled,
|
||||
)
|
||||
|
||||
@@ -2040,7 +2040,7 @@ class SparseAttnNormRopeStoreFullKernel(
|
||||
rope_head_dim=hf_config.qk_rope_head_dim,
|
||||
enabled=(
|
||||
cache_config.block_size > 0
|
||||
and 128 in hf_config.compress_ratios
|
||||
and "heavily_compressed_attention" in hf_config.layer_types
|
||||
and cache_config.cache_dtype != "fp8_ds_mla"
|
||||
),
|
||||
_when=lambda *, enabled: enabled,
|
||||
|
||||
@@ -638,7 +638,9 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
self.gate.e_score_correction_bias = None
|
||||
self.gate.tid2eid = None
|
||||
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
|
||||
is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count(
|
||||
"hash_moe"
|
||||
)
|
||||
self.hash_indices_dtype = torch.int64 if self.use_mega_moe else torch.int32
|
||||
if is_hash_moe:
|
||||
# hash MoE doesn't use e_score_correction_bias
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
TritonWarmupTensor,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
@@ -1012,7 +1013,7 @@ def _combine_topk_swa_warmup_inputs(vllm_config: Any) -> _WarmupInputRows:
|
||||
"""
|
||||
model_config = getattr(vllm_config, "model_config", None)
|
||||
hf_config = getattr(model_config, "hf_config", None)
|
||||
compress_ratios = getattr(hf_config, "compress_ratios", None) or [0]
|
||||
compress_ratios = get_compress_ratios(hf_config) or [0]
|
||||
topk = _hf_config_int(vllm_config, "index_topk", 0)
|
||||
return zip_inputs(
|
||||
*(
|
||||
|
||||
@@ -127,7 +127,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
|
||||
cosmos3_edge="Cosmos3EdgeConfig",
|
||||
deepseek_vl_v2="DeepseekVLV2Config",
|
||||
deepseek_v32="DeepseekV3Config",
|
||||
deepseek_v4="DeepseekV4Config",
|
||||
dots3_note="Dots3NoteConfig",
|
||||
k3_dspark="K3DSparkConfig",
|
||||
funaudiochat="FunAudioChatConfig",
|
||||
@@ -216,7 +215,10 @@ def is_rope_parameters_nested(rope_parameters: dict[str, Any]) -> bool:
|
||||
# Cannot be nested if rope_parameters is empty
|
||||
if not rope_parameters:
|
||||
return False
|
||||
return set(rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES)
|
||||
# DeepSeek-V4 nests by rope type ("main", "compress") instead
|
||||
return set(rope_parameters.keys()).issubset(
|
||||
{*ALLOWED_LAYER_TYPES, "main", "compress"}
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
||||
@@ -29,7 +29,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"Cosmos3EdgeTextConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"Cosmos3EdgeVisionConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2",
|
||||
"DeepseekV4Config": "vllm.transformers_utils.configs.deepseek_v4",
|
||||
"DeepseekV41Config": "vllm.transformers_utils.configs.deepseek_v41",
|
||||
"Dots3NoteConfig": "vllm.transformers_utils.configs.dots3_note",
|
||||
"K3DSparkConfig": "vllm.transformers_utils.configs.k3_dspark",
|
||||
@@ -103,7 +102,6 @@ __all__ = [
|
||||
"Cosmos3EdgeVisionConfig",
|
||||
"DeepseekVLV2Config",
|
||||
"DeepseekV3Config",
|
||||
"DeepseekV4Config",
|
||||
"Dots3NoteConfig",
|
||||
"K3DSparkConfig",
|
||||
"DotsOCRConfig",
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
|
||||
class DeepseekV4Config(PreTrainedConfig):
|
||||
model_type = "deepseek_v4"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_position_embeddings: int = 1048576,
|
||||
rope_scaling: dict[str, Any] | None = None,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
rope_theta: float = 10000.0,
|
||||
vision_n_layers: int = 0,
|
||||
vision_dim: int = 1024,
|
||||
vision_n_heads: int = 16,
|
||||
vision_inter_dim: int = 2816,
|
||||
vision_patch_size: int = 14,
|
||||
vision_rope_theta: float = 10000.0,
|
||||
vision_downsample_ratio: int = 3,
|
||||
vision_max_n_token: int = 384,
|
||||
vision_min_pixels: int = 147456,
|
||||
vision_max_wh_ratio: float = 8,
|
||||
**kwargs,
|
||||
):
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.rope_scaling = rope_scaling
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_parameters = rope_scaling or rope_parameters
|
||||
# Vision tower config; vision_n_layers == 0 means text-only.
|
||||
self.vision_n_layers = vision_n_layers
|
||||
self.vision_dim = vision_dim
|
||||
self.vision_n_heads = vision_n_heads
|
||||
self.vision_inter_dim = vision_inter_dim
|
||||
self.vision_patch_size = vision_patch_size
|
||||
self.vision_rope_theta = vision_rope_theta
|
||||
self.vision_downsample_ratio = vision_downsample_ratio
|
||||
self.vision_max_n_token = vision_max_n_token
|
||||
self.vision_min_pixels = vision_min_pixels
|
||||
self.vision_max_wh_ratio = vision_max_wh_ratio
|
||||
# The sparse-SWA index kernels widen the window within image spans
|
||||
# in-kernel, so mm-prefix ranges longer than sliding_window must be
|
||||
# kept (they are only consumed by those kernels).
|
||||
self.mm_prefix_clamp_sliding_window = vision_n_layers > 0
|
||||
# The visibility span covers the sentinel block [IMAGE_START,
|
||||
# IMAGE_END]; the mm placeholder additionally carries a leading
|
||||
# compressor-alignment pad (see mm_preprocess.COMPRESS_PAD_TO).
|
||||
self.mm_prefix_span_leading_pad_modulus = 4 if vision_n_layers > 0 else 0
|
||||
super().__init__(**kwargs)
|
||||
@@ -0,0 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
|
||||
def get_compress_ratios(config: Any) -> list[int]:
|
||||
"""Per-layer compress ratios (0 = sliding window), MTP layers included.
|
||||
|
||||
Reads the legacy `compress_ratios` of DeepSeek-V4.1 configs, or derives it
|
||||
from `layer_types` of the upstream `DeepseekV4Config`, which drops the MTP
|
||||
layers. Returns an empty list for other configs.
|
||||
"""
|
||||
if (compress_ratios := getattr(config, "compress_ratios", None)) is not None:
|
||||
return compress_ratios
|
||||
if (compress_rates := getattr(config, "compress_rates", None)) is None:
|
||||
return []
|
||||
ratios = [compress_rates.get(t, 0) for t in config.layer_types]
|
||||
return ratios + [0] * config.num_nextn_predict_layers
|
||||
@@ -18,6 +18,7 @@ from vllm.transformers_utils.config import (
|
||||
ConfigFormat,
|
||||
get_safetensors_params_metadata,
|
||||
)
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
from vllm.transformers_utils.repo_utils import (
|
||||
file_or_path_exists,
|
||||
get_hf_file_to_dict,
|
||||
@@ -90,7 +91,7 @@ class ModelArchConfigConvertorBase:
|
||||
def get_head_size(self) -> int:
|
||||
if self.is_deepseek_mla():
|
||||
# special case for deepseek_v4
|
||||
if hasattr(self.hf_text_config, "compress_ratios"):
|
||||
if get_compress_ratios(self.hf_text_config):
|
||||
return self.hf_text_config.head_dim
|
||||
qk_rope_head_dim = self._get_qk_rope_head_dim()
|
||||
if not envs.VLLM_MLA_DISABLE:
|
||||
@@ -318,7 +319,7 @@ class ModelArchConfigConvertorBase:
|
||||
"hy_v4_mtp",
|
||||
):
|
||||
# check is deepseek_v4 model
|
||||
if hasattr(self.hf_text_config, "compress_ratios"):
|
||||
if get_compress_ratios(self.hf_text_config):
|
||||
return getattr(self.hf_text_config, "head_dim", None) is not None
|
||||
else:
|
||||
return getattr(self.hf_text_config, "kv_lora_rank", None) is not None
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
VllmTritonJitKernel,
|
||||
triton_scalar_specialization_rep,
|
||||
)
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
@@ -124,7 +125,7 @@ class CompressedSlotMappingKernel(
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
configured_ratios = (
|
||||
*(getattr(hf_text_config, "compress_ratios", None) or ()),
|
||||
*get_compress_ratios(hf_text_config),
|
||||
getattr(hf_text_config, "index_kpool", 1) or 1,
|
||||
)
|
||||
compress_ratios = tuple(
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
triton_scalar_specialization_rep,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import (
|
||||
get_paged_mqa_logits_metadata,
|
||||
@@ -610,7 +611,7 @@ class BuildPrefillChunkMetadataKernel(
|
||||
dict.fromkeys(
|
||||
max(1, int(ratio))
|
||||
for ratio in (
|
||||
*(getattr(hf_text_config, "compress_ratios", None) or (1,)),
|
||||
*(get_compress_ratios(hf_text_config) or (1,)),
|
||||
getattr(hf_text_config, "index_kpool", 1) or 1,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -4,6 +4,7 @@ from dataclasses import dataclass, field
|
||||
from typing import Any, ClassVar, cast
|
||||
|
||||
import torch
|
||||
from transformers import DeepseekV4Config
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
@@ -18,6 +19,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
VllmTritonJitKernel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
from vllm.v1.attention.backend import (
|
||||
@@ -471,13 +473,13 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
|
||||
# V4 vision variant: image spans (up to vision_max_n_token tokens) are
|
||||
# visible bidirectionally, so prefill index rows widen from
|
||||
# window_size to window_size + max_image_tokens. The V4 config sets
|
||||
# mm_prefix_clamp_sliding_window exactly for these in-kernel-widened
|
||||
# ranges; V4.1 image tokens use the plain causal window, and text-only
|
||||
# models keep max_image_tokens == 0 everywhere.
|
||||
# window_size to window_size + max_image_tokens. V4.1 image tokens use
|
||||
# the plain causal window, and text-only models keep
|
||||
# max_image_tokens == 0 everywhere.
|
||||
self.max_image_tokens = (
|
||||
getattr(hf_config, "vision_max_n_token", 0)
|
||||
if getattr(hf_config, "mm_prefix_clamp_sliding_window", False)
|
||||
if isinstance(hf_config, DeepseekV4Config)
|
||||
and getattr(hf_config, "vision_n_layers", 0) > 0
|
||||
else 0
|
||||
)
|
||||
self.prefill_index_width = self.window_size + self.max_image_tokens
|
||||
@@ -485,7 +487,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
# Detect which DeepseekV4 layer types this model uses so we only build a
|
||||
# FlashMLA tile-scheduler plan for types that will actually be called.
|
||||
# Models without compress_ratios (pure SWA) fall back to swaonly.
|
||||
compress_ratios = getattr(hf_config, "compress_ratios", None) or [1]
|
||||
compress_ratios = get_compress_ratios(hf_config) or [1]
|
||||
self._layer_types: set[str] = set()
|
||||
for ratio in compress_ratios:
|
||||
self._layer_types.add(_layer_type_for(int(ratio)))
|
||||
|
||||
@@ -37,6 +37,7 @@ from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
|
||||
from vllm.model_executor.models.qwen3_dflash import DFlashQwen3ForCausalLM
|
||||
from vllm.model_executor.models.qwen3_eagle3 import Eagle3Qwen3ForCausalLM
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -102,9 +103,7 @@ class SpecDecodeBaseProposer:
|
||||
# shape (T, hc_mult * hidden_size). Expand the hidden_states buffer
|
||||
# so target_hidden_states fits; detect DeepseekV4 via draft hf_config.
|
||||
draft_hf_config = self.draft_model_config.hf_config
|
||||
if hasattr(draft_hf_config, "compress_ratios") and hasattr(
|
||||
draft_hf_config, "hc_mult"
|
||||
):
|
||||
if get_compress_ratios(draft_hf_config) and hasattr(draft_hf_config, "hc_mult"):
|
||||
self.hidden_size = self.hidden_size * draft_hf_config.hc_mult
|
||||
|
||||
# Unifying eagle, draft model, and parallel drafting support.
|
||||
|
||||
@@ -2344,7 +2344,7 @@ class GPUModelRunner(
|
||||
# TODO(Isotr0py): Refactor mm_prefix_lm implementation
|
||||
# for better readability and maintainability.
|
||||
_span_pad_modulus = getattr(
|
||||
hf_text_config, "mm_prefix_span_leading_pad_modulus", 0
|
||||
self.model, "mm_prefix_span_leading_pad_modulus", 0
|
||||
)
|
||||
for req_id in self.input_batch.req_ids:
|
||||
image_doc_ranges = []
|
||||
|
||||
Reference in New Issue
Block a user