[2/N] Harden Transformers modelling backend multi-modal path (#51657)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-10 12:57:12 +00:00
committed by GitHub
parent 436be94e13
commit cf8f3a3bb2
3 changed files with 273 additions and 133 deletions
+54
View File
@@ -10,6 +10,13 @@ from typing import Any
import pytest
import torch
import torch.nn as nn
from transformers import AutoConfig, AutoModel
from vllm.config import ModelConfig, VllmConfig
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.transformers.base import Base
from vllm.model_executor.models.transformers.multimodal import MultiModalMixin
from vllm.model_executor.models.utils import StageMissingLayer
from ...conftest import HfRunner, VllmRunner
from ...utils import multi_gpu_test, prep_prompts
@@ -468,3 +475,50 @@ def test_replaced_embedding_exposes_one_vpe(vpe):
lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
assert lm_head.tie_weights(composed.embed).weight is composed.embed.weight
MULTIMODAL_MODEL = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
class MarkingStub(SupportsMultiModal, nn.Module):
"""Just enough of the backend to exercise component marking."""
_mark_model_components = MultiModalMixin._mark_model_components
_find_encoder_classes = MultiModalMixin._find_encoder_classes
_from_config_kwargs = Base._from_config_kwargs
_pre_trained_model_classes = Base._pre_trained_model_classes
def build_marked_model(image_limit: int, skip_tokenizer_init: bool = False):
"""Build the HF model inside the marking context and return it."""
model_config = ModelConfig(
model=MULTIMODAL_MODEL,
model_impl="transformers",
limit_mm_per_prompt={"image": image_limit},
)
# Set after construction: building the config itself needs the tokenizer
model_config.skip_tokenizer_init = skip_tokenizer_init
stub = MarkingStub()
stub.config = AutoConfig.from_pretrained(MULTIMODAL_MODEL)
stub.model_config = model_config
vllm_config = VllmConfig(model_config=model_config)
with stub._mark_model_components(vllm_config), torch.device("meta"):
stub.model = AutoModel.from_config(**stub._from_config_kwargs)
return stub.model
@pytest.mark.parametrize(("image_limit", "skipped"), [(0, True), (4, False)])
def test_tower_weights_skipped_when_modality_disabled(image_limit, skipped):
"""`--limit-mm-per-prompt image=0` should drop the vision tower's weights."""
vision_tower = build_marked_model(image_limit).vision_tower
assert isinstance(vision_tower, StageMissingLayer) is skipped
def test_marking_skipped_without_tokenizer():
"""Marking needs the HF processor, which needs a tokenizer, so it is skipped.
The tower is built as normal rather than the model failing to load.
"""
vision_tower = build_marked_model(0, skip_tokenizer_init=True).vision_tower
assert not isinstance(vision_tower, StageMissingLayer)