[DSV4.1] Remove compressor-aware image sentinel token padding (#56554)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-09-12 00:07:36 -07:00
committed by GitHub
parent 372477d2d9
commit 30118ba27d
4 changed files with 10 additions and 132 deletions
@@ -47,9 +47,7 @@ from ..common.mm_preprocess import (
IMAGE,
IMAGE_END,
IMAGE_NEW_LINE,
IMAGE_PAD_ID,
IMAGE_PLACEHOLDER,
IMAGE_SENTINEL_BASE_ID,
IMAGE_START,
DeepseekV4VLDummyInputsBuilder,
DeepseekV4VLMultiModalProcessor,
@@ -277,12 +275,6 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
) -> torch.Tensor:
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
# Compressor-alignment pads borrow a reserved id; embed them as the
# plain image token (the checkpoint has no image_pad vector).
# Branch-free: safe inside compiled/captured regions.
input_ids = input_ids.masked_fill(
input_ids == IMAGE_PAD_ID, IMAGE_SENTINEL_BASE_ID
)
inputs_embeds = self.language_model.embed_input_ids(input_ids)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
+10 -110
View File
@@ -12,14 +12,9 @@ to ``[IMAGE_START] + ([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h +
rows in reading order; the delimiters take the learned ``image_start`` /
``image_newline`` / ``image_end`` vectors.
One vLLM-side deviation from the reference token stream: a leading
compressor-alignment pad (``COMPRESS_PAD_TO - 1 - start % COMPRESS_PAD_TO``
positions, so the span always starts at the same compressor phase) is
prepended when the block is spliced into the final prompt. Pad positions
borrow the reserved in-vocab token ``<|place_holder_mm_span_0436|>`` so
they stay distinguishable from real span positions; they embed as the
plain image token (v4.1 has no ``image_pad`` vector) and are routed and
engram-deadened like image tokens.
The token stream matches the reference exactly: no compressor-alignment
pad is inserted (the reference pools image tokens across compressor-group
boundaries freely).
"""
import math
@@ -30,7 +25,6 @@ import numpy as np
import torch
from PIL import Image, ImageOps
from transformers import BatchFeature
from typing_extensions import assert_never
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.inputs import MultiModalDataDict
@@ -44,42 +38,26 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.processing.processor import (
MultiModalPromptUpdates,
MultiModalPromptUpdatesApplyResult,
PlaceholderFeaturesInfo,
UpdateMode,
_plan_prompt_updates,
)
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
IMAGE_START, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(4)
# Image spans are padded so they always start at the same compressor group
# phase; v4.1 uses ratio-2 compressors (v4.0 used 4).
COMPRESS_PAD_TO = 2
IMAGE_PLACEHOLDER = "<|deepseek_image|>"
# The checkpoint tokenizer's id for IMAGE_PLACEHOLDER (the config's
# ``image_token_id``). Every image-span position carries this id, exactly
# like the reference; the MoE router detects image tokens as the range
# [IMAGE_SENTINEL_BASE_ID, IMAGE_SENTINEL_BASE_ID + 5), which also covers
# IMAGE_PAD_ID below.
# [IMAGE_SENTINEL_BASE_ID, IMAGE_SENTINEL_BASE_ID + 5).
IMAGE_SENTINEL_BASE_ID = 129264
# Leading compressor-alignment pads borrow this reserved in-vocab token
# (``<|place_holder_mm_span_0436|>``) so they can be told apart from real
# span positions when the per-position role is decided from token ids.
IMAGE_PAD_ID = 129265
IMAGE_PAD_TOKEN_NAME = "<|place_holder_mm_span_0436|>"
def image_sentinel_mask(token_ids: torch.Tensor) -> torch.Tensor:
"""Boolean mask for image-span positions (span tokens and align pads)."""
return (token_ids == IMAGE_SENTINEL_BASE_ID) | (token_ids == IMAGE_PAD_ID)
"""Boolean mask for image-span positions."""
return token_ids == IMAGE_SENTINEL_BASE_ID
def validate_image_sentinel_ids(tokenizer) -> None:
"""Check the image/pad token ids against the tokenizer."""
"""Check the image token id against the tokenizer."""
image_id = tokenizer.convert_tokens_to_ids(IMAGE_PLACEHOLDER)
if image_id != IMAGE_SENTINEL_BASE_ID:
raise ValueError(
@@ -88,13 +66,6 @@ def validate_image_sentinel_ids(tokenizer) -> None:
"image_token_id); the DeepSeek-V4.1 vision path keys image "
"routing and engram masking off this id."
)
pad_id = tokenizer.convert_tokens_to_ids(IMAGE_PAD_TOKEN_NAME)
if pad_id != IMAGE_PAD_ID:
raise ValueError(
f"Image pad token {IMAGE_PAD_TOKEN_NAME!r} has id {pad_id}, "
f"expected {IMAGE_PAD_ID}; the DeepSeek-V4.1 vision path "
"borrows this reserved id for compressor-alignment pads."
)
def llm_grid(best_height, best_width, patch_size, downsample_ratio):
@@ -134,8 +105,7 @@ def safe_resize(
height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token
):
"""Shrink the pixel size until the image costs at most max_n_token LLM
tokens (minus the reservation for the compressor-alignment pad)."""
max_n_token -= COMPRESS_PAD_TO - 1
tokens."""
n_llm_h, n_llm_w = llm_grid(best_height, best_width, patch_size, downsample_ratio)
if num_image_tokens(n_llm_h, n_llm_w) > max_n_token:
best_height, best_width = solve_resize_ratio(
@@ -293,13 +263,7 @@ class DeepseekV4VLProcessingInfo(BaseProcessingInfo):
seq_len: int,
mm_counts: Mapping[str, int],
) -> Mapping[str, int]:
# ``safe_resize`` reserves COMPRESS_PAD_TO - 1 tokens of
# vision_max_n_token for the compressor-alignment pad, so the full
# image span is bounded by vision_max_n_token; the margin is kept
# in case that reservation changes.
return {
"image": self.get_hf_config().vision_max_n_token + COMPRESS_PAD_TO - 1,
}
return {"image": self.get_hf_config().vision_max_n_token}
def get_image_placeholder_token_id(self) -> int:
token_id = self.get_tokenizer().convert_tokens_to_ids(IMAGE_PLACEHOLDER)
@@ -314,7 +278,7 @@ class DeepseekV4VLProcessingInfo(BaseProcessingInfo):
# A square maximizes the ViT patch count (area) within the token
# budget; solve the budget-derived size directly to keep the dummy
# image small.
budget = hf_config.vision_max_n_token - (COMPRESS_PAD_TO - 1)
budget = hf_config.vision_max_n_token
side = budget * patch_size * downsample_ratio
best_h, best_w = solve_resize_ratio(
side, side, patch_size, downsample_ratio, budget
@@ -397,67 +361,3 @@ class DeepseekV4VLMultiModalProcessor(
replacement=get_image_replacement,
),
]
def _apply_token_matches_with_placeholders(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[
list[int],
MultiModalPromptUpdatesApplyResult,
Mapping[str, list[PlaceholderFeaturesInfo]],
]:
# Same as the base implementation, except that each image block gets
# its compressor-alignment pad (``COMPRESS_PAD_TO - 1 - start %
# COMPRESS_PAD_TO`` IMAGE_PAD_ID tokens) prepended while splicing:
# the pad depends on the block's final position in the prompt, which
# is unknown when the (cacheable) prompt updates are built. Pad
# positions are not embed positions (their id differs from
# image_token_id), so ``is_embed`` stays aligned with the pad-free
# replacement content.
matched_updates, result = _plan_prompt_updates(token_ids, mm_prompt_updates)
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
modality: [] for modality in mm_prompt_updates
}
new_token_ids = list[int]()
prev_end_idx = 0
for matched_update in matched_updates:
update = matched_update.update
match = matched_update.match
if update.mode == UpdateMode.INSERT:
end_idx_to_insert = match.end_idx
elif update.mode == UpdateMode.REPLACE:
end_idx_to_insert = match.start_idx
else:
assert_never(update.mode)
new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
start_idx = len(new_token_ids)
tokens = list(update.content.full)
if tokens and update.modality == "image":
compress_pad = COMPRESS_PAD_TO - 1 - start_idx % COMPRESS_PAD_TO
tokens = [IMAGE_PAD_ID] * compress_pad + tokens
if tokens:
content_is_embed = update.content.is_embed
is_embed = (
content_is_embed(tokens) if content_is_embed is not None else None
)
placeholders[update.modality].append(
PlaceholderFeaturesInfo(
modality=update.modality,
item_idx=update.item_idx,
start_idx=start_idx,
tokens=tokens,
is_embed=is_embed,
)
)
new_token_ids.extend(tokens)
prev_end_idx = match.end_idx
new_token_ids.extend(token_ids[prev_end_idx:])
return new_token_ids, result, placeholders
@@ -47,9 +47,7 @@ from ..common.mm_preprocess import (
IMAGE,
IMAGE_END,
IMAGE_NEW_LINE,
IMAGE_PAD_ID,
IMAGE_PLACEHOLDER,
IMAGE_SENTINEL_BASE_ID,
IMAGE_START,
DeepseekV4VLDummyInputsBuilder,
DeepseekV4VLMultiModalProcessor,
@@ -277,12 +275,6 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
) -> torch.Tensor:
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
# Compressor-alignment pads borrow a reserved id; embed them as the
# plain image token (the checkpoint has no image_pad vector).
# Branch-free: safe inside compiled/captured regions.
input_ids = input_ids.masked_fill(
input_ids == IMAGE_PAD_ID, IMAGE_SENTINEL_BASE_ID
)
inputs_embeds = self.language_model.embed_input_ids(input_ids)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
@@ -68,9 +68,3 @@ class DeepseekV41Config(PretrainedConfig):
# in-kernel, so mm-prefix ranges longer than sliding_window must be
# kept (they are only consumed by those kernels).
self.mm_prefix_clamp_sliding_window = vision_n_layers > 0
# The visibility span covers the image block [IMAGE_START,
# IMAGE_END]; the mm placeholder additionally carries a leading
# compressor-alignment pad of ``COMPRESS_PAD_TO - 1 - offset %
# COMPRESS_PAD_TO`` tokens (see common/mm_preprocess.py), with
# COMPRESS_PAD_TO = 2 for v4.1's ratio-2 compressors.
self.mm_prefix_span_leading_pad_modulus = 2 if vision_n_layers > 0 else 0