diff --git a/vllm/models/deepseek_v4_1/amd/vl_model.py b/vllm/models/deepseek_v4_1/amd/vl_model.py index 91a13bd725..3227b50a4e 100644 --- a/vllm/models/deepseek_v4_1/amd/vl_model.py +++ b/vllm/models/deepseek_v4_1/amd/vl_model.py @@ -47,9 +47,7 @@ from ..common.mm_preprocess import ( IMAGE, IMAGE_END, IMAGE_NEW_LINE, - IMAGE_PAD_ID, IMAGE_PLACEHOLDER, - IMAGE_SENTINEL_BASE_ID, IMAGE_START, DeepseekV4VLDummyInputsBuilder, DeepseekV4VLMultiModalProcessor, @@ -277,12 +275,6 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports ) -> torch.Tensor: from vllm.model_executor.models.utils import _merge_multimodal_embeddings - # Compressor-alignment pads borrow a reserved id; embed them as the - # plain image token (the checkpoint has no image_pad vector). - # Branch-free: safe inside compiled/captured regions. - input_ids = input_ids.masked_fill( - input_ids == IMAGE_PAD_ID, IMAGE_SENTINEL_BASE_ID - ) inputs_embeds = self.language_model.embed_input_ids(input_ids) if multimodal_embeddings is None or len(multimodal_embeddings) == 0: diff --git a/vllm/models/deepseek_v4_1/common/mm_preprocess.py b/vllm/models/deepseek_v4_1/common/mm_preprocess.py index 8bd8d70cae..4f87b816c9 100644 --- a/vllm/models/deepseek_v4_1/common/mm_preprocess.py +++ b/vllm/models/deepseek_v4_1/common/mm_preprocess.py @@ -12,14 +12,9 @@ to ``[IMAGE_START] + ([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h + rows in reading order; the delimiters take the learned ``image_start`` / ``image_newline`` / ``image_end`` vectors. -One vLLM-side deviation from the reference token stream: a leading -compressor-alignment pad (``COMPRESS_PAD_TO - 1 - start % COMPRESS_PAD_TO`` -positions, so the span always starts at the same compressor phase) is -prepended when the block is spliced into the final prompt. Pad positions -borrow the reserved in-vocab token ``<|place_holder_mm_span_0436|>`` so -they stay distinguishable from real span positions; they embed as the -plain image token (v4.1 has no ``image_pad`` vector) and are routed and -engram-deadened like image tokens. +The token stream matches the reference exactly: no compressor-alignment +pad is inserted (the reference pools image tokens across compressor-group +boundaries freely). """ import math @@ -30,7 +25,6 @@ import numpy as np import torch from PIL import Image, ImageOps from transformers import BatchFeature -from typing_extensions import assert_never from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions from vllm.inputs import MultiModalDataDict @@ -44,42 +38,26 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.processing.processor import ( - MultiModalPromptUpdates, - MultiModalPromptUpdatesApplyResult, - PlaceholderFeaturesInfo, - UpdateMode, - _plan_prompt_updates, -) from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config IMAGE_START, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(4) -# Image spans are padded so they always start at the same compressor group -# phase; v4.1 uses ratio-2 compressors (v4.0 used 4). -COMPRESS_PAD_TO = 2 IMAGE_PLACEHOLDER = "<|deepseek_image|>" # The checkpoint tokenizer's id for IMAGE_PLACEHOLDER (the config's # ``image_token_id``). Every image-span position carries this id, exactly # like the reference; the MoE router detects image tokens as the range -# [IMAGE_SENTINEL_BASE_ID, IMAGE_SENTINEL_BASE_ID + 5), which also covers -# IMAGE_PAD_ID below. +# [IMAGE_SENTINEL_BASE_ID, IMAGE_SENTINEL_BASE_ID + 5). IMAGE_SENTINEL_BASE_ID = 129264 -# Leading compressor-alignment pads borrow this reserved in-vocab token -# (``<|place_holder_mm_span_0436|>``) so they can be told apart from real -# span positions when the per-position role is decided from token ids. -IMAGE_PAD_ID = 129265 -IMAGE_PAD_TOKEN_NAME = "<|place_holder_mm_span_0436|>" def image_sentinel_mask(token_ids: torch.Tensor) -> torch.Tensor: - """Boolean mask for image-span positions (span tokens and align pads).""" - return (token_ids == IMAGE_SENTINEL_BASE_ID) | (token_ids == IMAGE_PAD_ID) + """Boolean mask for image-span positions.""" + return token_ids == IMAGE_SENTINEL_BASE_ID def validate_image_sentinel_ids(tokenizer) -> None: - """Check the image/pad token ids against the tokenizer.""" + """Check the image token id against the tokenizer.""" image_id = tokenizer.convert_tokens_to_ids(IMAGE_PLACEHOLDER) if image_id != IMAGE_SENTINEL_BASE_ID: raise ValueError( @@ -88,13 +66,6 @@ def validate_image_sentinel_ids(tokenizer) -> None: "image_token_id); the DeepSeek-V4.1 vision path keys image " "routing and engram masking off this id." ) - pad_id = tokenizer.convert_tokens_to_ids(IMAGE_PAD_TOKEN_NAME) - if pad_id != IMAGE_PAD_ID: - raise ValueError( - f"Image pad token {IMAGE_PAD_TOKEN_NAME!r} has id {pad_id}, " - f"expected {IMAGE_PAD_ID}; the DeepSeek-V4.1 vision path " - "borrows this reserved id for compressor-alignment pads." - ) def llm_grid(best_height, best_width, patch_size, downsample_ratio): @@ -134,8 +105,7 @@ def safe_resize( height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token ): """Shrink the pixel size until the image costs at most max_n_token LLM - tokens (minus the reservation for the compressor-alignment pad).""" - max_n_token -= COMPRESS_PAD_TO - 1 + tokens.""" n_llm_h, n_llm_w = llm_grid(best_height, best_width, patch_size, downsample_ratio) if num_image_tokens(n_llm_h, n_llm_w) > max_n_token: best_height, best_width = solve_resize_ratio( @@ -293,13 +263,7 @@ class DeepseekV4VLProcessingInfo(BaseProcessingInfo): seq_len: int, mm_counts: Mapping[str, int], ) -> Mapping[str, int]: - # ``safe_resize`` reserves COMPRESS_PAD_TO - 1 tokens of - # vision_max_n_token for the compressor-alignment pad, so the full - # image span is bounded by vision_max_n_token; the margin is kept - # in case that reservation changes. - return { - "image": self.get_hf_config().vision_max_n_token + COMPRESS_PAD_TO - 1, - } + return {"image": self.get_hf_config().vision_max_n_token} def get_image_placeholder_token_id(self) -> int: token_id = self.get_tokenizer().convert_tokens_to_ids(IMAGE_PLACEHOLDER) @@ -314,7 +278,7 @@ class DeepseekV4VLProcessingInfo(BaseProcessingInfo): # A square maximizes the ViT patch count (area) within the token # budget; solve the budget-derived size directly to keep the dummy # image small. - budget = hf_config.vision_max_n_token - (COMPRESS_PAD_TO - 1) + budget = hf_config.vision_max_n_token side = budget * patch_size * downsample_ratio best_h, best_w = solve_resize_ratio( side, side, patch_size, downsample_ratio, budget @@ -397,67 +361,3 @@ class DeepseekV4VLMultiModalProcessor( replacement=get_image_replacement, ), ] - - def _apply_token_matches_with_placeholders( - self, - token_ids: list[int], - mm_prompt_updates: MultiModalPromptUpdates, - ) -> tuple[ - list[int], - MultiModalPromptUpdatesApplyResult, - Mapping[str, list[PlaceholderFeaturesInfo]], - ]: - # Same as the base implementation, except that each image block gets - # its compressor-alignment pad (``COMPRESS_PAD_TO - 1 - start % - # COMPRESS_PAD_TO`` IMAGE_PAD_ID tokens) prepended while splicing: - # the pad depends on the block's final position in the prompt, which - # is unknown when the (cacheable) prompt updates are built. Pad - # positions are not embed positions (their id differs from - # image_token_id), so ``is_embed`` stays aligned with the pad-free - # replacement content. - matched_updates, result = _plan_prompt_updates(token_ids, mm_prompt_updates) - placeholders: dict[str, list[PlaceholderFeaturesInfo]] = { - modality: [] for modality in mm_prompt_updates - } - - new_token_ids = list[int]() - prev_end_idx = 0 - for matched_update in matched_updates: - update = matched_update.update - match = matched_update.match - - if update.mode == UpdateMode.INSERT: - end_idx_to_insert = match.end_idx - elif update.mode == UpdateMode.REPLACE: - end_idx_to_insert = match.start_idx - else: - assert_never(update.mode) - - new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert]) - start_idx = len(new_token_ids) - - tokens = list(update.content.full) - if tokens and update.modality == "image": - compress_pad = COMPRESS_PAD_TO - 1 - start_idx % COMPRESS_PAD_TO - tokens = [IMAGE_PAD_ID] * compress_pad + tokens - - if tokens: - content_is_embed = update.content.is_embed - is_embed = ( - content_is_embed(tokens) if content_is_embed is not None else None - ) - placeholders[update.modality].append( - PlaceholderFeaturesInfo( - modality=update.modality, - item_idx=update.item_idx, - start_idx=start_idx, - tokens=tokens, - is_embed=is_embed, - ) - ) - new_token_ids.extend(tokens) - - prev_end_idx = match.end_idx - - new_token_ids.extend(token_ids[prev_end_idx:]) - return new_token_ids, result, placeholders diff --git a/vllm/models/deepseek_v4_1/nvidia/vl_model.py b/vllm/models/deepseek_v4_1/nvidia/vl_model.py index 111936b431..bfbcaa89a1 100644 --- a/vllm/models/deepseek_v4_1/nvidia/vl_model.py +++ b/vllm/models/deepseek_v4_1/nvidia/vl_model.py @@ -47,9 +47,7 @@ from ..common.mm_preprocess import ( IMAGE, IMAGE_END, IMAGE_NEW_LINE, - IMAGE_PAD_ID, IMAGE_PLACEHOLDER, - IMAGE_SENTINEL_BASE_ID, IMAGE_START, DeepseekV4VLDummyInputsBuilder, DeepseekV4VLMultiModalProcessor, @@ -277,12 +275,6 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports ) -> torch.Tensor: from vllm.model_executor.models.utils import _merge_multimodal_embeddings - # Compressor-alignment pads borrow a reserved id; embed them as the - # plain image token (the checkpoint has no image_pad vector). - # Branch-free: safe inside compiled/captured regions. - input_ids = input_ids.masked_fill( - input_ids == IMAGE_PAD_ID, IMAGE_SENTINEL_BASE_ID - ) inputs_embeds = self.language_model.embed_input_ids(input_ids) if multimodal_embeddings is None or len(multimodal_embeddings) == 0: diff --git a/vllm/transformers_utils/configs/deepseek_v41.py b/vllm/transformers_utils/configs/deepseek_v41.py index 3a406d24bd..0be36d2480 100644 --- a/vllm/transformers_utils/configs/deepseek_v41.py +++ b/vllm/transformers_utils/configs/deepseek_v41.py @@ -68,9 +68,3 @@ class DeepseekV41Config(PretrainedConfig): # in-kernel, so mm-prefix ranges longer than sliding_window must be # kept (they are only consumed by those kernels). self.mm_prefix_clamp_sliding_window = vision_n_layers > 0 - # The visibility span covers the image block [IMAGE_START, - # IMAGE_END]; the mm placeholder additionally carries a leading - # compressor-alignment pad of ``COMPRESS_PAD_TO - 1 - offset % - # COMPRESS_PAD_TO`` tokens (see common/mm_preprocess.py), with - # COMPRESS_PAD_TO = 2 for v4.1's ratio-2 compressors. - self.mm_prefix_span_leading_pad_modulus = 2 if vision_n_layers > 0 else 0