89 lines
3.4 KiB
Python
89 lines
3.4 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Decision 3.0 (vllm-sr/d3): a Qwen3.5 backbone with an answer-code readout.
|
|
|
|
The checkpoint is a bare `Qwen3_5Model` plus `readout.safetensors`, one row per
|
|
answer code, and `decision_config.json`. The readout scores the codes at the
|
|
last prompt token and is served as the `score` head of a classification model.
|
|
`D3ForSequenceClassificationConfig` applies the checkpoint's attention mode and
|
|
label space.
|
|
"""
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
from transformers.models.qwen3_vl import Qwen3VLProcessor
|
|
|
|
from vllm.config import VllmConfig
|
|
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
|
|
|
from .adapters import as_seq_cls_model
|
|
from .interfaces_base import default_pooling_type
|
|
from .qwen3_5 import Qwen3_5ForConditionalGeneration, Qwen3_5ProcessingInfo
|
|
from .qwen3_vl import Qwen3VLDummyInputsBuilder, Qwen3VLMultiModalProcessor
|
|
from .utils import WeightsMapper
|
|
|
|
READOUT_WEIGHTS = "readout.safetensors"
|
|
|
|
# The media budgets of the checkpoint's runtime (`d3_runtime.py`), which differ
|
|
# from its processor config: images at 65,536 to 1,638,400 pixels, video frames
|
|
# at up to 200,704 pixels and at most 32 frames per video.
|
|
IMAGE_SIZE = {"shortest_edge": 65_536, "longest_edge": 1_638_400}
|
|
VIDEO_FRAME_MAX_PIXELS = 200_704
|
|
VIDEO_FRAME_MIN_PIXELS = 4_096
|
|
VIDEO_MAX_FRAMES = 32
|
|
|
|
|
|
class D3ProcessingInfo(Qwen3_5ProcessingInfo):
|
|
def get_hf_processor(self, **kwargs: object) -> Qwen3VLProcessor:
|
|
processor = super().get_hf_processor(**kwargs)
|
|
processor.image_processor.size = dict(IMAGE_SIZE)
|
|
video = processor.video_processor
|
|
factor = video.patch_size * video.merge_size
|
|
video.size = {
|
|
"shortest_edge": VIDEO_FRAME_MIN_PIXELS,
|
|
"longest_edge": VIDEO_FRAME_MAX_PIXELS * VIDEO_MAX_FRAMES,
|
|
}
|
|
video.cap_pixels_per_frame = True
|
|
video.max_video_tokens = VIDEO_FRAME_MAX_PIXELS // factor**2
|
|
video.do_sample_frames = False
|
|
return processor
|
|
|
|
|
|
if TYPE_CHECKING:
|
|
_D3ForSequenceClassificationBase = Qwen3_5ForConditionalGeneration
|
|
else:
|
|
_D3ForSequenceClassificationBase = as_seq_cls_model(Qwen3_5ForConditionalGeneration)
|
|
|
|
|
|
@default_pooling_type(seq_pooling_type="LAST")
|
|
@MULTIMODAL_REGISTRY.register_processor(
|
|
Qwen3VLMultiModalProcessor,
|
|
info=D3ProcessingInfo,
|
|
dummy_inputs=Qwen3VLDummyInputsBuilder,
|
|
)
|
|
class D3ForSequenceClassification(_D3ForSequenceClassificationBase):
|
|
hf_to_vllm_mapper = WeightsMapper(
|
|
orig_to_new_prefix={
|
|
"language_model.": "language_model.model.",
|
|
"mtp.": None,
|
|
}
|
|
)
|
|
|
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
|
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
|
model_config = vllm_config.model_config
|
|
self.secondary_weights = [
|
|
DefaultModelLoader.Source(
|
|
model_or_path=model_config.model,
|
|
revision=model_config.revision,
|
|
prefix="score.",
|
|
allow_patterns_overrides=[READOUT_WEIGHTS],
|
|
)
|
|
]
|
|
|
|
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
|
# The readout's only tensor, loaded through `secondary_weights`. A
|
|
# checkpoint without a weight index lists its file with the backbone's.
|
|
return name == "weight"
|