[Model] Support Decision 3.0 decision models (vllm-sr/d3) (#61565)

Signed-off-by: Xunzhuo Liu <xunzhuo.liu@amd.com>
This commit is contained in:
Xunzhuo
2026-10-14 17:10:55 +00:00
committed by GitHub
parent 3895a8babe
commit 89cab55086
12 changed files with 683 additions and 2 deletions
+49
View File
@@ -1042,6 +1042,54 @@ class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
config.is_causal = is_causal
class D3ForSequenceClassificationConfig(Qwen3_5ForConditionalGenerationConfig):
"""Apply the contract declared by a Decision 3.0 `decision_config.json`."""
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
hf_config = model_config.hf_config
decision = getattr(hf_config, "decision_config", None)
if not isinstance(decision, dict):
raise ValueError("Decision 3.0 checkpoints need a decision_config.json")
if decision.get("format_version") != 1:
raise ValueError(
"unsupported decision_config.json format_version "
f"{decision.get('format_version')!r}; vLLM reads version 1"
)
if decision.get("pooling", "last") != "last":
raise ValueError(
f"unsupported decision_config.json pooling {decision['pooling']!r}"
)
attention_mode = decision.get("attention_mode", "causal")
if attention_mode not in ("causal", "noncausal_full_attention"):
raise ValueError(
f"unsupported decision_config.json attention_mode {attention_mode!r}"
)
codes = decision.get("codes")
token_ids = decision.get("token_ids")
if not codes or not isinstance(token_ids, list) or len(codes) != len(token_ids):
raise ValueError(
"decision_config.json must list one token id per answer code"
)
# The full attention layers read the whole prompt in both directions;
# the Gated DeltaNet layers stay causal.
is_causal = attention_mode == "causal"
for config in {
id(config): config for config in (hf_config, model_config.hf_text_config)
}.values():
config.is_causal = is_causal
# One readout row per answer code.
hf_config.id2label = dict(enumerate(codes))
# Probabilities only exist over a question's own options, so the head
# returns raw code logits unless activation is requested.
pooler_config = model_config.pooler_config
assert pooler_config is not None
if pooler_config.use_activation is None:
pooler_config.use_activation = False
class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
@@ -1089,6 +1137,7 @@ class LongcatFlashNgramForCausalLMConfig(VerifyAndUpdateConfig):
MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
"ColQwen3_5": ColQwen3_5Config,
"D3ForSequenceClassification": D3ForSequenceClassificationConfig,
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
"DeepseekV4ForConditionalGeneration": DeepseekV4ForCausalLMConfig,
"DeepseekV41ForCausalLM": DeepseekV4ForCausalLMConfig,
+83
View File
@@ -0,0 +1,83 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Decision 3.0 (vllm-sr/d3): a Qwen3.5 backbone with an answer-code readout.
The checkpoint is a bare `Qwen3_5Model` plus `readout.safetensors`, one row per
answer code, and `decision_config.json`. The readout scores the codes at the
last prompt token and is served as the `score` head of a classification model.
`D3ForSequenceClassificationConfig` applies the checkpoint's attention mode and
label space.
"""
from typing import TYPE_CHECKING
from transformers.models.qwen3_vl import Qwen3VLProcessor
from vllm.config import VllmConfig
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.multimodal import MULTIMODAL_REGISTRY
from .adapters import as_seq_cls_model
from .interfaces_base import default_pooling_type
from .qwen3_5 import Qwen3_5ForConditionalGeneration, Qwen3_5ProcessingInfo
from .qwen3_vl import Qwen3VLDummyInputsBuilder, Qwen3VLMultiModalProcessor
from .utils import WeightsMapper
READOUT_WEIGHTS = "readout.safetensors"
# The media budgets of the checkpoint's runtime (`d3_runtime.py`), which differ
# from its processor config: images at 65,536 to 1,638,400 pixels, video frames
# at up to 200,704 pixels and at most 32 frames per video.
IMAGE_SIZE = {"shortest_edge": 65_536, "longest_edge": 1_638_400}
VIDEO_FRAME_MAX_PIXELS = 200_704
VIDEO_FRAME_MIN_PIXELS = 4_096
VIDEO_MAX_FRAMES = 32
class D3ProcessingInfo(Qwen3_5ProcessingInfo):
def get_hf_processor(self, **kwargs: object) -> Qwen3VLProcessor:
processor = super().get_hf_processor(**kwargs)
processor.image_processor.size = dict(IMAGE_SIZE)
video = processor.video_processor
factor = video.patch_size * video.merge_size
video.size = {
"shortest_edge": VIDEO_FRAME_MIN_PIXELS,
"longest_edge": VIDEO_FRAME_MAX_PIXELS * VIDEO_MAX_FRAMES,
}
video.cap_pixels_per_frame = True
video.max_video_tokens = VIDEO_FRAME_MAX_PIXELS // factor**2
video.do_sample_frames = False
return processor
if TYPE_CHECKING:
_D3ForSequenceClassificationBase = Qwen3_5ForConditionalGeneration
else:
_D3ForSequenceClassificationBase = as_seq_cls_model(Qwen3_5ForConditionalGeneration)
@default_pooling_type(seq_pooling_type="LAST")
@MULTIMODAL_REGISTRY.register_processor(
Qwen3VLMultiModalProcessor,
info=D3ProcessingInfo,
dummy_inputs=Qwen3VLDummyInputsBuilder,
)
class D3ForSequenceClassification(_D3ForSequenceClassificationBase):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"language_model.": "language_model.model.",
"mtp.": None,
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
model_config = vllm_config.model_config
self.secondary_weights = [
DefaultModelLoader.Source(
model_or_path=model_config.model,
revision=model_config.revision,
prefix="score.",
allow_patterns_overrides=[READOUT_WEIGHTS],
)
]
+1
View File
@@ -329,6 +329,7 @@ _SEQUENCE_CLASSIFICATION_MODELS = {
"RobertaForSequenceClassification",
),
# [Multimodal]
"D3ForSequenceClassification": ("d3", "D3ForSequenceClassification"),
"JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),
"LlamaNemotronVLForSequenceClassification": (
"nemotron_vl",