[Model][ROCm] Enable DeepSeek V4 Vision (#55107)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
127143e27f
commit
9dd969da09
@@ -20,7 +20,7 @@ if current_platform.is_rocm():
|
||||
)
|
||||
from .amd.model import DeepseekV4ForCausalLM
|
||||
from .amd.mtp import DeepSeekV4MTP
|
||||
from .vl_stub import ( # type: ignore[assignment]
|
||||
from .common.vl_model import ( # type: ignore[assignment]
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
elif current_platform.is_xpu():
|
||||
@@ -38,14 +38,14 @@ elif current_platform.is_cpu():
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
else:
|
||||
from .common.vl_model import ( # type: ignore[assignment]
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
from .nvidia.dspark import ( # type: ignore[assignment]
|
||||
DSparkDeepseekV4ForCausalLM,
|
||||
)
|
||||
from .nvidia.model import DeepseekV4ForCausalLM # type: ignore[assignment]
|
||||
from .nvidia.mtp import DeepSeekV4MTP # type: ignore[assignment]
|
||||
from .nvidia.vl_model import ( # type: ignore[assignment]
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DSparkDeepseekV4ForCausalLM",
|
||||
|
||||
@@ -79,6 +79,8 @@ from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -549,6 +551,10 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
self.gate.e_score_correction_bias = None
|
||||
self.gate.tid2eid = None
|
||||
self.gate.bias_vl = None
|
||||
self.image_sentinel_lo = (
|
||||
IMAGE_SENTINEL_BASE_ID if getattr(config, "vision_n_layers", 0) > 0 else 0
|
||||
)
|
||||
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
|
||||
self.hash_indices_dtype = torch.int32
|
||||
if is_hash_moe:
|
||||
@@ -564,12 +570,20 @@ class DeepseekV4MoE(nn.Module):
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
elif getattr(config, "topk_method", None) == "noaux_tc":
|
||||
if getattr(config, "topk_method", None) == "noaux_tc" and (
|
||||
not is_hash_moe or getattr(config, "vision_n_layers", 0) > 0
|
||||
):
|
||||
self.gate.e_score_correction_bias = nn.Parameter(
|
||||
torch.empty(config.n_routed_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
if getattr(config, "vision_n_layers", 0) > 0:
|
||||
self.gate.bias_vl = nn.Parameter(
|
||||
torch.empty(config.n_routed_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
self.n_shared_experts = config.n_shared_experts
|
||||
|
||||
# TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1`
|
||||
@@ -638,6 +652,8 @@ class DeepseekV4MoE(nn.Module):
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
hash_indices_table=self.gate.tid2eid,
|
||||
bias_vl=self.gate.bias_vl,
|
||||
image_sentinel_lo=self.image_sentinel_lo,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
router_logits_dtype=torch.float32,
|
||||
routed_experts_cls=(
|
||||
@@ -660,6 +676,8 @@ class DeepseekV4MoE(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
if self.gate.tid2eid is not None and input_ids is None:
|
||||
raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.")
|
||||
if self.gate.bias_vl is not None and input_ids is None:
|
||||
raise ValueError("DeepSeek V4 vision MoE routing requires input_ids.")
|
||||
|
||||
org_shape = hidden_states.shape
|
||||
final_hidden_states = self.experts(
|
||||
@@ -1351,6 +1369,7 @@ def _make_deepseek_v4_weights_mapper(
|
||||
|
||||
class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
model_cls = DeepseekV4Model
|
||||
finalizes_weights_during_load = False
|
||||
|
||||
# Default mapper assumes the original FP4-expert checkpoint layout.
|
||||
# Overridden per-instance in __init__ when expert_dtype != "fp4".
|
||||
@@ -1397,6 +1416,9 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.logits_processor(self.lm_head, hidden_states, skip_gather=True)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
|
||||
@@ -160,7 +160,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
inputs_embeds
|
||||
).unsqueeze(-2)
|
||||
hidden_states, residual, post_mix, res_mix = self.mtp_block(
|
||||
positions=positions, x=hidden_states, input_ids=None
|
||||
positions=positions, x=hidden_states, input_ids=input_ids
|
||||
)
|
||||
if self.mtp_block.use_fused_mhc:
|
||||
hidden_states = self.mtp_block.hc_post(
|
||||
|
||||
@@ -67,8 +67,13 @@ def weight_already_preshuffled(linear: torch.nn.Module) -> bool:
|
||||
The hand-shuffles below (fused_wqa_wkv, wo_b, gate_up_proj) must be skipped
|
||||
for those, since shuffle_weight is a permutation rather than an involution.
|
||||
"""
|
||||
kernel = getattr(getattr(linear, "quant_method", None), "fp8_linear", None)
|
||||
return bool(getattr(kernel, "preshuffles_weight", False))
|
||||
return any(
|
||||
getattr(getattr(method, "fp8_linear", None), "preshuffles_weight", False)
|
||||
for method in (
|
||||
getattr(linear, "quant_method", None),
|
||||
getattr(linear, "scheme", None),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def apply_pre_quantized_block_scaled_mm(
|
||||
@@ -116,13 +121,18 @@ def _combine_topk_swa_indices_kernel(
|
||||
query_start_loc_ptr,
|
||||
seq_lens_ptr,
|
||||
gather_lens_ptr,
|
||||
left_visible_ptr,
|
||||
right_visible_ptr,
|
||||
M,
|
||||
N,
|
||||
TOP_K: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
SWA_WIDTH: tl.constexpr,
|
||||
TOPK_WIDTH: tl.constexpr,
|
||||
PADDED_TOP_K: tl.constexpr,
|
||||
PADDED_SWA_WIDTH: tl.constexpr,
|
||||
HAS_IMAGE: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
@@ -141,7 +151,20 @@ def _combine_topk_swa_indices_kernel(
|
||||
token_idx_in_query = token_idx - query_start
|
||||
pos = start_pos + token_idx_in_query
|
||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
|
||||
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
|
||||
if HAS_IMAGE:
|
||||
left = tl.load(left_visible_ptr + token_idx)
|
||||
right = tl.load(right_visible_ptr + token_idx)
|
||||
else:
|
||||
left = 0
|
||||
right = 0
|
||||
left_add = tl.maximum(left - (WINDOW_SIZE - 1), 0)
|
||||
# Prefix caching can resume inside an image span. Do not generate
|
||||
# indices outside the SWA rows present in the gathered workspace.
|
||||
swa_start = tl.maximum(
|
||||
tl.maximum(pos - (WINDOW_SIZE - 1) - left_add, 0), gather_start
|
||||
)
|
||||
swa_end = tl.minimum(pos + right + 1, seq_len)
|
||||
swa_len = tl.maximum(swa_end - swa_start, 0)
|
||||
|
||||
topk_offset = tl.arange(0, PADDED_TOP_K)
|
||||
topk_mask = topk_offset < topk_len
|
||||
@@ -159,14 +182,14 @@ def _combine_topk_swa_indices_kernel(
|
||||
mask=topk_mask,
|
||||
)
|
||||
|
||||
swa_offset = tl.arange(0, WINDOW_SIZE)
|
||||
swa_offset = tl.arange(0, PADDED_SWA_WIDTH)
|
||||
tl.store(
|
||||
combined_indices_ptr
|
||||
+ token_idx * combined_indices_stride
|
||||
+ topk_len
|
||||
+ swa_offset,
|
||||
M * batch_idx + N + swa_offset + pos - swa_len + 1 - gather_start,
|
||||
mask=swa_offset < swa_len,
|
||||
M * batch_idx + N + swa_offset + swa_start - gather_start,
|
||||
mask=(swa_offset < swa_len) & (swa_offset < SWA_WIDTH),
|
||||
)
|
||||
|
||||
tl.store(combined_lens_ptr + token_idx, topk_len + swa_len)
|
||||
@@ -182,12 +205,21 @@ def combine_topk_swa_indices(
|
||||
topk: int,
|
||||
M: int,
|
||||
N: int,
|
||||
max_image_tokens: int = 0,
|
||||
left_visible: torch.Tensor | None = None,
|
||||
right_visible: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if (left_visible is None) != (right_visible is None):
|
||||
raise ValueError("left_visible and right_visible must be provided together")
|
||||
topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous()
|
||||
num_tokens = topk_indices.shape[0]
|
||||
num_reqs = seq_lens.shape[0]
|
||||
has_image = left_visible is not None
|
||||
# Keep the row shape fixed for a vision model even when a particular batch
|
||||
# has no image.
|
||||
swa_width = window_size + max_image_tokens
|
||||
combined_topk = (
|
||||
(topk + window_size + _SPARSE_PREFILL_TOPK_ALIGNMENT - 1)
|
||||
(topk + swa_width + _SPARSE_PREFILL_TOPK_ALIGNMENT - 1)
|
||||
// _SPARSE_PREFILL_TOPK_ALIGNMENT
|
||||
* _SPARSE_PREFILL_TOPK_ALIGNMENT
|
||||
)
|
||||
@@ -211,13 +243,18 @@ def combine_topk_swa_indices(
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
gather_lens,
|
||||
left_visible if left_visible is not None else topk_indices,
|
||||
right_visible if right_visible is not None else topk_indices,
|
||||
M,
|
||||
N,
|
||||
TOP_K=topk,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
WINDOW_SIZE=window_size,
|
||||
SWA_WIDTH=swa_width,
|
||||
TOPK_WIDTH=topk_indices.shape[-1],
|
||||
PADDED_TOP_K=triton.next_power_of_2(topk_indices.shape[-1]),
|
||||
PADDED_SWA_WIDTH=triton.next_power_of_2(swa_width),
|
||||
HAS_IMAGE=has_image,
|
||||
)
|
||||
return combined_indices, combined_lens
|
||||
|
||||
@@ -948,6 +985,12 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
assert query_start_loc_cpu is not None
|
||||
assert query_start_loc is not None
|
||||
prefill_token_base = query_start_loc_cpu[num_decodes]
|
||||
left_visible = swa_metadata.prefill_left_visible
|
||||
right_visible = swa_metadata.prefill_right_visible
|
||||
if left_visible is not None:
|
||||
left_visible = left_visible[num_decode_tokens:]
|
||||
assert right_visible is not None
|
||||
right_visible = right_visible[num_decode_tokens:]
|
||||
|
||||
if not swa_only:
|
||||
if self.compress_ratio == 4:
|
||||
@@ -1026,6 +1069,17 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
top_k,
|
||||
M,
|
||||
N,
|
||||
max_image_tokens=self.max_image_tokens,
|
||||
left_visible=(
|
||||
left_visible[query_start:query_end]
|
||||
if left_visible is not None
|
||||
else None
|
||||
),
|
||||
right_visible=(
|
||||
right_visible[query_start:query_end]
|
||||
if right_visible is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
rocm_sparse_attn_prefill(
|
||||
q=q[query_start:query_end],
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4 vision variant (e.g. DeepSeek-V4-Flash-Vision-Exp).
|
||||
|
||||
Thin multimodal wrapper around the text-only ``DeepseekV4ForCausalLM``:
|
||||
|
||||
- ``vision`` ViT + ``aligner`` produce per-image embeddings for the IMAGE
|
||||
sentinel positions; four learned vectors (``image_start`` / ``image_pad`` /
|
||||
``image_newline`` / ``image_end``) fill the remaining sentinel positions.
|
||||
- Image placeholders (``<|deepseek_image|>``) are expanded by the processor
|
||||
in ``common/mm_preprocess.py`` into sentinel blocks borrowing reserved
|
||||
in-vocab tokens ``<|place_holder_mm_span_0431|>``..``_0435|>``
|
||||
(see ``common/vision.py`` for the tower itself).
|
||||
- Merged embeddings enter the text model via ``inputs_embeds``, i.e. before
|
||||
its hyper-connection stream expansion. Raw ``input_ids`` still flow into
|
||||
the model so the MoE router can apply ``bias_vl`` to image tokens
|
||||
(``requires_raw_input_tokens``).
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from .mm_preprocess import (
|
||||
IMAGE_PLACEHOLDER,
|
||||
IMAGE_SENTINEL_BASE_ID,
|
||||
DeepseekV4VLDummyInputsBuilder,
|
||||
DeepseekV4VLMultiModalProcessor,
|
||||
DeepseekV4VLProcessingInfo,
|
||||
image_sentinel_mask,
|
||||
)
|
||||
from .vision import DeepseekV4Aligner, DeepseekV4ViT
|
||||
|
||||
|
||||
def _make_deepseek_v4_vl_weights_mapper(
|
||||
text_mapper: WeightsMapper, image_enabled: bool
|
||||
) -> WeightsMapper:
|
||||
"""Text-checkpoint mapping rules re-rooted under ``language_model.``."""
|
||||
orig_to_new_prefix = {
|
||||
src: None if dst is None else f"language_model.{dst}"
|
||||
for src, dst in text_mapper.orig_to_new_prefix.items()
|
||||
}
|
||||
if not image_enabled:
|
||||
orig_to_new_prefix.update({"vision.": None, "aligner.": None, "image_": None})
|
||||
return WeightsMapper(
|
||||
orig_to_new_renaming=text_mapper.orig_to_new_renaming,
|
||||
orig_to_new_prefix=orig_to_new_prefix,
|
||||
orig_to_new_regex=text_mapper.orig_to_new_regex,
|
||||
orig_to_new_stacked=text_mapper.orig_to_new_stacked,
|
||||
orig_to_new_suffix={
|
||||
**text_mapper.orig_to_new_suffix,
|
||||
"head.weight": "language_model.lm_head.weight",
|
||||
},
|
||||
orig_to_new_substr={
|
||||
**text_mapper.orig_to_new_substr,
|
||||
# Draft models load the checkpoint's MTP weights separately from
|
||||
# the target model.
|
||||
"mtp.": None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
DeepseekV4VLMultiModalProcessor,
|
||||
info=DeepseekV4VLProcessingInfo,
|
||||
dummy_inputs=DeepseekV4VLDummyInputsBuilder,
|
||||
)
|
||||
class DeepseekV4ForConditionalGeneration(
|
||||
nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3
|
||||
):
|
||||
"""Multimodal entry point for DeepSeek-V4 checkpoints with a vision tower.
|
||||
|
||||
``SupportsEagle3`` (aux hidden-state plumbing for MTP/DSpark drafters)
|
||||
delegates through ``language_model`` via the protocol defaults.
|
||||
"""
|
||||
|
||||
# The MoE router needs raw token ids to detect image sentinel tokens
|
||||
# (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl.
|
||||
requires_raw_input_tokens = True
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality == "image":
|
||||
return IMAGE_PLACEHOLDER
|
||||
raise ValueError(f"Unsupported modality: {modality!r}")
|
||||
|
||||
def __init__(self, *, vllm_config, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
model_config = vllm_config.model_config
|
||||
config = model_config.hf_config
|
||||
self.config = config
|
||||
self.multimodal_config = model_config.multimodal_config
|
||||
assert self.multimodal_config is not None
|
||||
|
||||
image_enabled = (
|
||||
config.vision_n_layers > 0
|
||||
and self.multimodal_config.get_limit_per_prompt("image") > 0
|
||||
)
|
||||
with self._mark_tower_model(vllm_config, {"image"}):
|
||||
self.vision: DeepseekV4ViT | None = None
|
||||
self.aligner: DeepseekV4Aligner | None = None
|
||||
self.image_start: nn.Parameter | None = None
|
||||
self.image_end: nn.Parameter | None = None
|
||||
self.image_newline: nn.Parameter | None = None
|
||||
self.image_pad: nn.Parameter | None = None
|
||||
if image_enabled:
|
||||
self.vision = DeepseekV4ViT(config)
|
||||
self.aligner = DeepseekV4Aligner(config)
|
||||
for name in (
|
||||
"image_start",
|
||||
"image_end",
|
||||
"image_newline",
|
||||
"image_pad",
|
||||
):
|
||||
setattr(
|
||||
self,
|
||||
name,
|
||||
nn.Parameter(
|
||||
torch.empty(config.hidden_size, dtype=torch.float32)
|
||||
),
|
||||
)
|
||||
self.vision.to(dtype=model_config.dtype)
|
||||
self.aligner.to(dtype=model_config.dtype)
|
||||
|
||||
with self._mark_language_model(vllm_config):
|
||||
# The arch convertor routes any config with a vision tower to
|
||||
# this wrapper class; mark the copy handed to the text backbone
|
||||
# so it resolves to DeepseekV4ForCausalLM instead of recursing
|
||||
# (with_hf_config deepcopies the config, the marker survives).
|
||||
config._dsv4_vl_inner = True # type: ignore[attr-defined]
|
||||
try:
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=config,
|
||||
prefix=maybe_prefix(prefix, "language_model"),
|
||||
architectures=["DeepseekV4ForCausalLM"],
|
||||
)
|
||||
finally:
|
||||
del config._dsv4_vl_inner # type: ignore[attr-defined]
|
||||
# The outer mapper (see load_weights) fully resolves HF names into
|
||||
# this wrapper's namespace before AutoWeightsLoader strips the
|
||||
# "language_model." prefix and delegates to the child's load_weights,
|
||||
# so the child's own mapper must be a no-op. Its suffix rules are not
|
||||
# idempotent (e.g. "lm_head.weight".endswith("head.weight") would
|
||||
# re-fire "head.weight" -> "lm_head.weight").
|
||||
text_mapper = self.language_model.hf_to_vllm_mapper
|
||||
self.language_model.hf_to_vllm_mapper = WeightsMapper()
|
||||
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
self.hf_to_vllm_mapper = _make_deepseek_v4_vl_weights_mapper(
|
||||
text_mapper, image_enabled
|
||||
)
|
||||
self._weights_finalized = False
|
||||
|
||||
def _parse_and_validate_image_input(self, **kwargs: object) -> dict | None:
|
||||
patches = kwargs.pop("patches", None)
|
||||
if patches is None:
|
||||
return None
|
||||
vit_grid = kwargs.pop("vit_grid", None)
|
||||
llm_grid = kwargs.pop("llm_grid", None)
|
||||
perm = kwargs.pop("perm", None)
|
||||
assert vit_grid is not None and llm_grid is not None and perm is not None
|
||||
return {
|
||||
"patches": patches,
|
||||
"vit_grid": vit_grid,
|
||||
"llm_grid": llm_grid,
|
||||
"perm": perm,
|
||||
}
|
||||
|
||||
def _encode_image(
|
||||
self,
|
||||
patches: torch.Tensor,
|
||||
n_vit_h: int,
|
||||
n_vit_w: int,
|
||||
perm: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
assert self.vision is not None and self.aligner is not None
|
||||
image_embeds = self.aligner(
|
||||
self.vision(patches, n_vit_h, n_vit_w), n_vit_h, n_vit_w
|
||||
)
|
||||
# Reorder into the N-layout block order used in the prompt.
|
||||
return image_embeds[perm.to(image_embeds.device)]
|
||||
|
||||
def _process_image_input(
|
||||
self,
|
||||
patches: torch.Tensor,
|
||||
vit_grid: torch.Tensor,
|
||||
llm_grid: torch.Tensor,
|
||||
perm: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
assert self.vision is not None and self.aligner is not None
|
||||
patches = patches.to(self.aligner.w1.weight.dtype)
|
||||
|
||||
embeds: list[torch.Tensor] = []
|
||||
vit_offset = 0
|
||||
llm_offset = 0
|
||||
for (n_vit_h, n_vit_w), (n_llm_h, n_llm_w) in zip(
|
||||
vit_grid.tolist(), llm_grid.tolist(), strict=True
|
||||
):
|
||||
n_vit = n_vit_h * n_vit_w
|
||||
n_llm = n_llm_h * n_llm_w
|
||||
embeds.append(
|
||||
self._encode_image(
|
||||
patches[vit_offset : vit_offset + n_vit],
|
||||
n_vit_h,
|
||||
n_vit_w,
|
||||
perm[llm_offset : llm_offset + n_llm],
|
||||
)
|
||||
)
|
||||
vit_offset += n_vit
|
||||
llm_offset += n_llm
|
||||
return tuple(embeds)
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
if image_input is None or self.vision is None:
|
||||
return []
|
||||
return self._process_image_input(
|
||||
image_input["patches"],
|
||||
image_input["vit_grid"],
|
||||
image_input["llm_grid"],
|
||||
image_input["perm"],
|
||||
)
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
|
||||
|
||||
# All ids are in-vocab here: image-block sentinels are borrowed
|
||||
# reserved tokens (their embedding rows are always overwritten below).
|
||||
inputs_embeds = self.language_model.embed_input_ids(input_ids)
|
||||
|
||||
if self.image_start is not None:
|
||||
# Branch-free sentinel overwrite: safe inside compiled/captured
|
||||
# regions (no data-dependent control flow).
|
||||
sentinel_mask = image_sentinel_mask(input_ids)
|
||||
if is_multimodal is not None:
|
||||
# IMAGE positions get vision embeddings via the merge below.
|
||||
sentinel_mask = sentinel_mask & ~is_multimodal.to(input_ids.device)
|
||||
table = torch.stack(
|
||||
[
|
||||
self.image_start,
|
||||
self.image_pad,
|
||||
self.image_pad,
|
||||
self.image_newline,
|
||||
self.image_end,
|
||||
]
|
||||
).to(inputs_embeds.dtype)
|
||||
idx = (input_ids - IMAGE_SENTINEL_BASE_ID).clamp(0, 4)
|
||||
inputs_embeds = torch.where(
|
||||
sentinel_mask.unsqueeze(-1), table[idx], inputs_embeds
|
||||
)
|
||||
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
return inputs_embeds
|
||||
|
||||
assert is_multimodal is not None
|
||||
return _merge_multimodal_embeddings(
|
||||
inputs_embeds=inputs_embeds,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
return self.language_model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.language_model.compute_logits_local(hidden_states)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.language_model.get_expert_mapping()
|
||||
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer for the MTP/DSpark draft model."""
|
||||
return self.language_model.get_mtp_target_hidden_states()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
child_finalizes = getattr(
|
||||
self.language_model, "finalizes_weights_during_load", True
|
||||
)
|
||||
mapped = self.hf_to_vllm_mapper.apply(weights)
|
||||
if child_finalizes:
|
||||
# A child which finalizes inside load_weights must see all of its
|
||||
# weights in one contiguous delegation from AutoWeightsLoader.
|
||||
mapped = iter(sorted(mapped, key=lambda x: x[0]))
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded_params = loader.load_weights(mapped)
|
||||
self._weights_finalized = child_finalizes
|
||||
return loaded_params
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
# Backbones such as the ROCm implementation require this to run only
|
||||
# after the loader's generic per-layer quantization finalization.
|
||||
if getattr(self, "_weights_finalized", False):
|
||||
return
|
||||
self.language_model.process_weights_after_loading()
|
||||
self._weights_finalized = True
|
||||
@@ -1790,6 +1790,7 @@ class DeepseekV4ForCausalLM(
|
||||
SupportsLoRA,
|
||||
DeepseekV4MixtureOfExperts,
|
||||
):
|
||||
finalizes_weights_during_load = True
|
||||
model_cls = DeepseekV4Model
|
||||
|
||||
# Default mapper assumes the original FP4-expert checkpoint layout.
|
||||
|
||||
@@ -1,333 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4 vision variant (e.g. DeepSeek-V4-Flash-Vision-Exp).
|
||||
"""Compatibility imports for the platform-neutral DeepSeek-V4 vision model."""
|
||||
|
||||
Thin multimodal wrapper around the text-only ``DeepseekV4ForCausalLM``:
|
||||
|
||||
- ``vision`` ViT + ``aligner`` produce per-image embeddings for the IMAGE
|
||||
sentinel positions; four learned vectors (``image_start`` / ``image_pad`` /
|
||||
``image_newline`` / ``image_end``) fill the remaining sentinel positions.
|
||||
- Image placeholders (``<|deepseek_image|>``) are expanded by the processor
|
||||
in ``common/mm_preprocess.py`` into sentinel blocks borrowing reserved
|
||||
in-vocab tokens ``<|place_holder_mm_span_0431|>``..``_0435|>``
|
||||
(see ``common/vision.py`` for the tower itself).
|
||||
- Merged embeddings enter the text model via ``inputs_embeds``, i.e. before
|
||||
its hyper-connection stream expansion. Raw ``input_ids`` still flow into
|
||||
the model so the MoE router can apply ``bias_vl`` to image tokens
|
||||
(``requires_raw_input_tokens``).
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
from ..common.vl_model import (
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
from ..common.vl_model import (
|
||||
_make_deepseek_v4_vl_weights_mapper as _make_common_vl_weights_mapper,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ..common.mm_preprocess import (
|
||||
IMAGE_PLACEHOLDER,
|
||||
IMAGE_SENTINEL_BASE_ID,
|
||||
DeepseekV4VLDummyInputsBuilder,
|
||||
DeepseekV4VLMultiModalProcessor,
|
||||
DeepseekV4VLProcessingInfo,
|
||||
image_sentinel_mask,
|
||||
)
|
||||
from ..common.vision import DeepseekV4Aligner, DeepseekV4ViT
|
||||
from .model import _make_deepseek_v4_weights_mapper
|
||||
|
||||
|
||||
def _make_deepseek_v4_vl_weights_mapper(
|
||||
expert_dtype: str, image_enabled: bool
|
||||
) -> WeightsMapper:
|
||||
"""Text-checkpoint mapping rules re-rooted under ``language_model.``."""
|
||||
base = _make_deepseek_v4_weights_mapper(expert_dtype)
|
||||
orig_to_new_prefix: dict[str, str | None] = {
|
||||
"layers.": "language_model.model.layers.",
|
||||
"embed.": "language_model.model.embed.",
|
||||
"norm.": "language_model.model.norm.",
|
||||
"hc_head": "language_model.model.hc_head",
|
||||
"mtp.": "language_model.model.mtp.",
|
||||
}
|
||||
if not image_enabled:
|
||||
orig_to_new_prefix.update({"vision.": None, "aligner.": None, "image_": None})
|
||||
return WeightsMapper(
|
||||
orig_to_new_prefix=orig_to_new_prefix,
|
||||
orig_to_new_regex=base.orig_to_new_regex,
|
||||
orig_to_new_suffix={
|
||||
"head.weight": "language_model.lm_head.weight",
|
||||
"embed.weight": "embed_tokens.weight",
|
||||
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
|
||||
},
|
||||
orig_to_new_substr={
|
||||
".shared_experts.w2": ".shared_experts.down_proj",
|
||||
# The MTP/DSpark draft heads are not supported for the vision
|
||||
# variant; drop their weights.
|
||||
"mtp.": None,
|
||||
},
|
||||
def _make_deepseek_v4_vl_weights_mapper(expert_dtype: str, image_enabled: bool):
|
||||
"""Retain the original NVIDIA helper signature for downstream imports."""
|
||||
return _make_common_vl_weights_mapper(
|
||||
_make_deepseek_v4_weights_mapper(expert_dtype), image_enabled
|
||||
)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
DeepseekV4VLMultiModalProcessor,
|
||||
info=DeepseekV4VLProcessingInfo,
|
||||
dummy_inputs=DeepseekV4VLDummyInputsBuilder,
|
||||
)
|
||||
class DeepseekV4ForConditionalGeneration(
|
||||
nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3
|
||||
):
|
||||
"""Multimodal entry point for DeepSeek-V4 checkpoints with a vision tower.
|
||||
|
||||
``SupportsEagle3`` (aux hidden-state plumbing for MTP/DSpark drafters)
|
||||
delegates through ``language_model`` via the protocol defaults.
|
||||
"""
|
||||
|
||||
# The MoE router needs raw token ids to detect image sentinel tokens
|
||||
# (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl.
|
||||
requires_raw_input_tokens = True
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality == "image":
|
||||
return IMAGE_PLACEHOLDER
|
||||
raise ValueError(f"Unsupported modality: {modality!r}")
|
||||
|
||||
def __init__(self, *, vllm_config, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
model_config = vllm_config.model_config
|
||||
config = model_config.hf_config
|
||||
self.config = config
|
||||
self.multimodal_config = model_config.multimodal_config
|
||||
assert self.multimodal_config is not None
|
||||
|
||||
image_enabled = (
|
||||
config.vision_n_layers > 0
|
||||
and self.multimodal_config.get_limit_per_prompt("image") > 0
|
||||
)
|
||||
with self._mark_tower_model(vllm_config, {"image"}):
|
||||
self.vision: DeepseekV4ViT | None = None
|
||||
self.aligner: DeepseekV4Aligner | None = None
|
||||
self.image_start: nn.Parameter | None = None
|
||||
self.image_end: nn.Parameter | None = None
|
||||
self.image_newline: nn.Parameter | None = None
|
||||
self.image_pad: nn.Parameter | None = None
|
||||
if image_enabled:
|
||||
self.vision = DeepseekV4ViT(config)
|
||||
self.aligner = DeepseekV4Aligner(config)
|
||||
for name in (
|
||||
"image_start",
|
||||
"image_end",
|
||||
"image_newline",
|
||||
"image_pad",
|
||||
):
|
||||
setattr(
|
||||
self,
|
||||
name,
|
||||
nn.Parameter(
|
||||
torch.empty(config.hidden_size, dtype=torch.float32)
|
||||
),
|
||||
)
|
||||
self.vision.to(dtype=model_config.dtype)
|
||||
self.aligner.to(dtype=model_config.dtype)
|
||||
|
||||
with self._mark_language_model(vllm_config):
|
||||
# The arch convertor routes any config with a vision tower to
|
||||
# this wrapper class; mark the copy handed to the text backbone
|
||||
# so it resolves to DeepseekV4ForCausalLM instead of recursing
|
||||
# (with_hf_config deepcopies the config, the marker survives).
|
||||
config._dsv4_vl_inner = True # type: ignore[attr-defined]
|
||||
try:
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=config,
|
||||
prefix=maybe_prefix(prefix, "language_model"),
|
||||
architectures=["DeepseekV4ForCausalLM"],
|
||||
)
|
||||
finally:
|
||||
del config._dsv4_vl_inner # type: ignore[attr-defined]
|
||||
# The outer mapper (see load_weights) fully resolves HF names into
|
||||
# this wrapper's namespace before AutoWeightsLoader strips the
|
||||
# "language_model." prefix and delegates to the child's load_weights,
|
||||
# so the child's own mapper must be a no-op. Its suffix rules are not
|
||||
# idempotent (e.g. "lm_head.weight".endswith("head.weight") would
|
||||
# re-fire "head.weight" -> "lm_head.weight").
|
||||
self.language_model.hf_to_vllm_mapper = WeightsMapper()
|
||||
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
expert_dtype = getattr(config, "expert_dtype", "fp4")
|
||||
self.hf_to_vllm_mapper = _make_deepseek_v4_vl_weights_mapper(
|
||||
expert_dtype, image_enabled
|
||||
)
|
||||
|
||||
def _parse_and_validate_image_input(self, **kwargs: object) -> dict | None:
|
||||
patches = kwargs.pop("patches", None)
|
||||
if patches is None:
|
||||
return None
|
||||
vit_grid = kwargs.pop("vit_grid", None)
|
||||
llm_grid = kwargs.pop("llm_grid", None)
|
||||
perm = kwargs.pop("perm", None)
|
||||
assert vit_grid is not None and llm_grid is not None and perm is not None
|
||||
return {
|
||||
"patches": patches,
|
||||
"vit_grid": vit_grid,
|
||||
"llm_grid": llm_grid,
|
||||
"perm": perm,
|
||||
}
|
||||
|
||||
def _encode_image(
|
||||
self,
|
||||
patches: torch.Tensor,
|
||||
n_vit_h: int,
|
||||
n_vit_w: int,
|
||||
perm: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
assert self.vision is not None and self.aligner is not None
|
||||
image_embeds = self.aligner(
|
||||
self.vision(patches, n_vit_h, n_vit_w), n_vit_h, n_vit_w
|
||||
)
|
||||
# Reorder into the N-layout block order used in the prompt.
|
||||
return image_embeds[perm.to(image_embeds.device)]
|
||||
|
||||
def _process_image_input(
|
||||
self,
|
||||
patches: torch.Tensor,
|
||||
vit_grid: torch.Tensor,
|
||||
llm_grid: torch.Tensor,
|
||||
perm: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
assert self.vision is not None and self.aligner is not None
|
||||
patches = patches.to(self.aligner.w1.weight.dtype)
|
||||
|
||||
embeds: list[torch.Tensor] = []
|
||||
vit_offset = 0
|
||||
llm_offset = 0
|
||||
for (n_vit_h, n_vit_w), (n_llm_h, n_llm_w) in zip(
|
||||
vit_grid.tolist(), llm_grid.tolist(), strict=True
|
||||
):
|
||||
n_vit = n_vit_h * n_vit_w
|
||||
n_llm = n_llm_h * n_llm_w
|
||||
embeds.append(
|
||||
self._encode_image(
|
||||
patches[vit_offset : vit_offset + n_vit],
|
||||
n_vit_h,
|
||||
n_vit_w,
|
||||
perm[llm_offset : llm_offset + n_llm],
|
||||
)
|
||||
)
|
||||
vit_offset += n_vit
|
||||
llm_offset += n_llm
|
||||
return tuple(embeds)
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
if image_input is None or self.vision is None:
|
||||
return []
|
||||
return self._process_image_input(
|
||||
image_input["patches"],
|
||||
image_input["vit_grid"],
|
||||
image_input["llm_grid"],
|
||||
image_input["perm"],
|
||||
)
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
|
||||
|
||||
# All ids are in-vocab here: image-block sentinels are borrowed
|
||||
# reserved tokens (their embedding rows are always overwritten below).
|
||||
inputs_embeds = self.language_model.embed_input_ids(input_ids)
|
||||
|
||||
if self.image_start is not None:
|
||||
# Branch-free sentinel overwrite: safe inside compiled/captured
|
||||
# regions (no data-dependent control flow).
|
||||
sentinel_mask = image_sentinel_mask(input_ids)
|
||||
if is_multimodal is not None:
|
||||
# IMAGE positions get vision embeddings via the merge below.
|
||||
sentinel_mask = sentinel_mask & ~is_multimodal.to(input_ids.device)
|
||||
table = torch.stack(
|
||||
[
|
||||
self.image_start,
|
||||
self.image_pad,
|
||||
self.image_pad,
|
||||
self.image_newline,
|
||||
self.image_end,
|
||||
]
|
||||
).to(inputs_embeds.dtype)
|
||||
idx = (input_ids - IMAGE_SENTINEL_BASE_ID).clamp(0, 4)
|
||||
inputs_embeds = torch.where(
|
||||
sentinel_mask.unsqueeze(-1), table[idx], inputs_embeds
|
||||
)
|
||||
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
return inputs_embeds
|
||||
|
||||
assert is_multimodal is not None
|
||||
return _merge_multimodal_embeddings(
|
||||
inputs_embeds=inputs_embeds,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
return self.language_model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.language_model.compute_logits_local(hidden_states)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.language_model.get_expert_mapping()
|
||||
|
||||
def get_mtp_target_hidden_states(self) -> torch.Tensor | None:
|
||||
"""Pre-hc_head residual stream buffer for the MTP/DSpark draft model."""
|
||||
return self.language_model.get_mtp_target_hidden_states()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Map HF names into this wrapper's namespace up front and sort, so
|
||||
# the "language_model." group reaches the child loader as one
|
||||
# contiguous block (AutoWeightsLoader delegates per contiguous group,
|
||||
# and the child's load_weights finalizes fused expert weights, which
|
||||
# must not run on a partially loaded model).
|
||||
mapped = sorted(self.hf_to_vllm_mapper.apply(weights), key=lambda x: x[0])
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded_params = loader.load_weights(mapped)
|
||||
# The child's load_weights already ran its post-load finalization.
|
||||
self._weights_finalized = True
|
||||
return loaded_params
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
# Model-level post-load hook (called by the loader after any load
|
||||
# format). Under DummyModelLoader the child's load_weights — and
|
||||
# hence its finalize step — is bypassed, so run it here instead.
|
||||
if getattr(self, "_weights_finalized", False):
|
||||
return
|
||||
self.language_model.process_weights_after_loading()
|
||||
__all__ = ["DeepseekV4ForConditionalGeneration"]
|
||||
|
||||
@@ -9,6 +9,6 @@ class DeepseekV4ForConditionalGeneration(nn.Module):
|
||||
def __init__(self, *, vllm_config, prefix: str = ""):
|
||||
super().__init__()
|
||||
raise NotImplementedError(
|
||||
"DeepSeek-V4 vision (DeepseekV4ForConditionalGeneration) is only "
|
||||
"supported on NVIDIA GPUs for now."
|
||||
"DeepSeek-V4 vision (DeepseekV4ForConditionalGeneration) is not "
|
||||
"supported on the current platform; use CUDA or ROCm."
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user