[Misc] Remove code paths for Transformers < 5.16.1 (#59762)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
8c60714da5
commit
5d20979fc9
@@ -5,16 +5,15 @@ from transformers import GraniteConfig
|
||||
|
||||
from vllm.model_executor.models.granite import granite_layer_attn_params
|
||||
|
||||
from ...utils import check_logprobs_close, check_transformers_version
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
# model -> minimum transformers version, or None if unconstrained
|
||||
MODELS = {
|
||||
MODELS = [
|
||||
# TODO(sang): Sliding window should be tested separately.
|
||||
"ibm/PowerLM-3b": None,
|
||||
"ibm/PowerMoE-3b": None,
|
||||
"ibm-granite/granite-swash-2b": "5.15.1",
|
||||
"ibm-granite/granite-swash-3b-a600m": "5.15.1",
|
||||
}
|
||||
"ibm/PowerLM-3b",
|
||||
"ibm/PowerMoE-3b",
|
||||
"ibm-granite/granite-swash-2b",
|
||||
"ibm-granite/granite-swash-3b-a600m",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@@ -31,8 +30,6 @@ def test_models(
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
check_transformers_version(model, min_transformers_version=MODELS[model])
|
||||
|
||||
with hf_runner(model, dtype=dtype) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
|
||||
@@ -4,11 +4,8 @@ import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from .transformers_backend import PROCESSOR_CLASSES, create_processor
|
||||
|
||||
AUDIO_MODEL_SETTINGS = {
|
||||
"ibm-granite/granite-speech-3.3-2b": {
|
||||
"prompt": (
|
||||
@@ -56,7 +53,6 @@ AUDIO_MODEL_SETTINGS = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
@@ -78,10 +74,12 @@ AUDIO_MODEL_SETTINGS = {
|
||||
"zai-org/GLM-ASR-Nano-2512",
|
||||
],
|
||||
)
|
||||
def test_audio_multimodal_processor(model_id, processor_cls):
|
||||
def test_audio_multimodal_processor(model_id):
|
||||
settings = AUDIO_MODEL_SETTINGS[model_id]
|
||||
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
audio = np.zeros(16000, dtype=np.float32)
|
||||
mm_data = {"audio": (audio, 16000)}
|
||||
@@ -112,13 +110,14 @@ def test_audio_multimodal_processor(model_id, processor_cls):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize("separator", [" and ", ""])
|
||||
def test_audio_multiple_inputs(separator, processor_cls):
|
||||
def test_audio_multiple_inputs(separator):
|
||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
audio_token = mm_processor.info.get_hf_processor().audio_token
|
||||
# One token per audio; the processor expands each to its placeholder run.
|
||||
@@ -128,21 +127,11 @@ def test_audio_multiple_inputs(separator, processor_cls):
|
||||
)
|
||||
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
|
||||
|
||||
def process():
|
||||
return mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
# The legacy path reads placeholders off contiguous runs of the audio token, so
|
||||
# it cannot tell adjacent ones apart and says so instead of merging them
|
||||
if processor_cls is LegacyMultiModalProcessor and not separator:
|
||||
with pytest.raises(ValueError, match="Separate them in the prompt"):
|
||||
process()
|
||||
return
|
||||
|
||||
result = process()
|
||||
result = mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert len(result["mm_placeholders"]["audio"]) == 2
|
||||
assert len(result["mm_kwargs"]["audio"]) == 2
|
||||
|
||||
@@ -5,23 +5,16 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.model_executor.models.transformers.multimodal import (
|
||||
LegacyMultiModalProcessor,
|
||||
OffsetsMultiModalProcessor,
|
||||
)
|
||||
|
||||
from .transformers_backend import (
|
||||
PROCESSOR_CLASSES,
|
||||
create_cached_processor,
|
||||
create_processor,
|
||||
offsets_only,
|
||||
)
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||
def test_multimodal_processor(model_id, processor_cls):
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
def test_multimodal_processor(model_id):
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
image_pil = ImageAsset("cherry_blossom").pil_image
|
||||
mm_data = {"image": image_pil}
|
||||
@@ -63,9 +56,11 @@ def test_multimodal_processor(model_id, processor_cls):
|
||||
)
|
||||
|
||||
|
||||
def _process_two_images(processor_cls, separator: str):
|
||||
def _process_two_images(separator: str):
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
prompt = (
|
||||
@@ -80,30 +75,29 @@ def _process_two_images(processor_cls, separator: str):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_image_multiple_inputs(processor_cls):
|
||||
def test_image_multiple_inputs():
|
||||
"""Multiple images per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
result = _process_two_images(processor_cls, separator="\n and ")
|
||||
result = _process_two_images(separator="\n and ")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_image_adjacent_inputs(processor_cls):
|
||||
def test_image_adjacent_inputs():
|
||||
"""Adjacent images stay separate placeholders rather than merging into one."""
|
||||
result = _process_two_images(processor_cls, separator="")
|
||||
result = _process_two_images(separator="")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_batch_padding_removed_from_image_items(processor_cls):
|
||||
def test_batch_padding_removed_from_image_items():
|
||||
"""Emu3 pads every image up to the largest in the batch, which would leave an
|
||||
item's data dependent on what it was processed with and so uncacheable."""
|
||||
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model="BAAI/Emu3-Chat-hf", model_impl="transformers")
|
||||
)
|
||||
image_token = mm_processor.info.get_hf_processor().image_token
|
||||
|
||||
images = [
|
||||
@@ -128,53 +122,39 @@ def test_batch_padding_removed_from_image_items(processor_cls):
|
||||
assert len(shapes) == 2
|
||||
|
||||
|
||||
def _process_one_gemma3_image(processor_cls):
|
||||
mm_processor = create_processor("google/gemma-3-4b-it", processor_cls)
|
||||
hf_processor = mm_processor.info.get_hf_processor()
|
||||
result = mm_processor(
|
||||
prompt=f"{hf_processor.boi_token} What is this?",
|
||||
def _process_one_gemma3_image():
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model="google/gemma-3-4b-it", model_impl="transformers")
|
||||
)
|
||||
boi_token = mm_processor.info.get_hf_processor().boi_token
|
||||
return mm_processor(
|
||||
prompt=f"{boi_token} What is this?",
|
||||
mm_items=mm_processor.info.parse_mm_data(
|
||||
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||
),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
return hf_processor, result
|
||||
|
||||
|
||||
@offsets_only
|
||||
def test_non_embedding_tokens_excluded_from_placeholders():
|
||||
"""Gemma3 wraps each image in text that carries no embeddings, which must be
|
||||
inside the placeholder range but masked out of it."""
|
||||
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
|
||||
result = _process_one_gemma3_image()
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
assert placeholder.is_embed is not None
|
||||
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
|
||||
|
||||
|
||||
def test_legacy_placeholders_hold_only_image_tokens():
|
||||
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
|
||||
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
|
||||
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
|
||||
marker in the unexpanded prompt, not the token the expansion repeats."""
|
||||
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
assert placeholder.length == hf_processor.image_seq_length
|
||||
prompt_ids = result["prompt_token_ids"]
|
||||
covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length]
|
||||
assert set(covered) == {hf_processor.tokenizer.image_token_id}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
|
||||
def test_tokens_structuring_an_image_are_masked_not_dropped():
|
||||
"""SmolVLM splits each image into tiles introduced by tokens carrying no
|
||||
embeddings. Those belong inside the placeholder and masked out, because the token
|
||||
count the processor reports is over the whole span. Idefics3 also refuses a prompt
|
||||
holding `<image>` when no images are passed, which is how the offsets path has to
|
||||
tokenize it before splicing in the expansion."""
|
||||
mm_processor = create_processor(
|
||||
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||
holding `<image>` when no images are passed, which is how the prompt has to be
|
||||
tokenized before splicing in the expansion."""
|
||||
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
result = mm_processor(
|
||||
prompt="<image>What is this?",
|
||||
@@ -189,12 +169,13 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
|
||||
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
|
||||
|
||||
|
||||
@offsets_only
|
||||
def test_missing_replacement_offsets_names_the_processor():
|
||||
"""A processor that reports no replacement offsets cannot be served, which must
|
||||
be said plainly rather than surfacing later as a field config mismatch."""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
hf_processor_cls = type(mm_processor.info.get_hf_processor())
|
||||
hf_call = hf_processor_cls.__call__
|
||||
|
||||
@@ -216,11 +197,12 @@ def test_missing_replacement_offsets_names_the_processor():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_text_only_prompt(processor_cls):
|
||||
def test_text_only_prompt():
|
||||
"""An image model still accepts a prompt with no images."""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
result = mm_processor(
|
||||
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
|
||||
@@ -232,12 +214,14 @@ def test_text_only_prompt(processor_cls):
|
||||
assert not result["mm_placeholders"]
|
||||
|
||||
|
||||
@offsets_only
|
||||
def test_repeated_image_hits_the_processor_cache():
|
||||
"""Check that mm caching is actually working."""
|
||||
mm_processor, cache = create_cached_processor(
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor
|
||||
model_config = ModelConfig(
|
||||
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", model_impl="transformers"
|
||||
)
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 4
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
|
||||
def process():
|
||||
@@ -255,7 +239,6 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
assert first["mm_hashes"] == second["mm_hashes"]
|
||||
|
||||
|
||||
@offsets_only
|
||||
@pytest.mark.parametrize(
|
||||
("model_id", "prompt"),
|
||||
[
|
||||
@@ -267,7 +250,7 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
"<image> and more text",
|
||||
marks=pytest.mark.xfail(
|
||||
reason="Emu3Processor prepends its BOS token only when images are "
|
||||
"passed, so the unexpanded prompt the offsets path tokenizes never "
|
||||
"passed, so the unexpanded prompt vLLM tokenizes never "
|
||||
"gets one. Fixed by huggingface/transformers#47924, unreleased.",
|
||||
strict=False,
|
||||
),
|
||||
@@ -275,33 +258,40 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
],
|
||||
)
|
||||
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
|
||||
"""The offsets path splices the expansion into a prompt tokenized without any
|
||||
multi-modal data, so its token ids have to come out the same as the ones the HF
|
||||
processor produces itself, which is what the legacy path returns."""
|
||||
prompt_ids = []
|
||||
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
prompt_ids.append(
|
||||
mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data(
|
||||
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||
),
|
||||
hf_processor_mm_kwargs={},
|
||||
)["prompt_token_ids"]
|
||||
)
|
||||
"""The prompt is tokenized without any multi-modal data and the expansion spliced
|
||||
in, so its token ids have to come out the same as the ones the HF processor
|
||||
produces itself."""
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
info = mm_processor.info
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
|
||||
legacy_ids, offsets_ids = prompt_ids
|
||||
assert legacy_ids == offsets_ids
|
||||
hf_processor = info.get_hf_processor()
|
||||
prompt_ids = info.get_tokenizer().encode(
|
||||
prompt, **info.default_tok_params.get_encode_kwargs()
|
||||
)
|
||||
hf_ids = info.ctx.call_hf_processor(
|
||||
hf_processor,
|
||||
dict(text=hf_processor.decode(prompt_ids), images=[image]),
|
||||
dict(truncation=False, add_special_tokens=False),
|
||||
)["input_ids"][0].tolist()
|
||||
|
||||
result = mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=info.parse_mm_data({"image": image}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert result["prompt_token_ids"] == hf_ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_nested_image_fields_split_per_image(processor_cls):
|
||||
def test_nested_image_fields_split_per_image():
|
||||
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
|
||||
belonging to each image one dimension further in. Slicing the batch dimension
|
||||
instead handed the first image every row and the second an empty tensor."""
|
||||
mm_processor = create_processor(
|
||||
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
result = mm_processor(
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Helpers for testing both processors of the Transformers modelling backend."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.transformers.multimodal import (
|
||||
LegacyMultiModalProcessor,
|
||||
MultiModalDummyInputsBuilder,
|
||||
MultiModalProcessingInfo,
|
||||
MultiModalProcessor,
|
||||
OffsetsMultiModalProcessor,
|
||||
)
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.processing import InputProcessingContext
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
|
||||
offsets_only = pytest.mark.skipif(
|
||||
MultiModalProcessor is not OffsetsMultiModalProcessor,
|
||||
reason="Replacement offsets are only used from transformers 5.15.0 onwards",
|
||||
)
|
||||
|
||||
PROCESSOR_CLASSES = [
|
||||
pytest.param(LegacyMultiModalProcessor, id="legacy"),
|
||||
pytest.param(OffsetsMultiModalProcessor, id="offsets", marks=offsets_only),
|
||||
]
|
||||
|
||||
|
||||
def create_processor(model_id: str, processor_cls):
|
||||
"""Build a processor directly, because the registry only ever builds the one the
|
||||
installed transformers version selects, leaving the other path untested."""
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
ctx = InputProcessingContext(
|
||||
model_config, cached_tokenizer_from_config(model_config)
|
||||
)
|
||||
info = MultiModalProcessingInfo(ctx)
|
||||
return processor_cls(info, MultiModalDummyInputsBuilder(info))
|
||||
|
||||
|
||||
def create_cached_processor(model_id: str, processor_cls):
|
||||
"""Build a processor backed by a real multi-modal processor cache, and hand the
|
||||
cache back so a test can check it was actually used."""
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 4
|
||||
ctx = InputProcessingContext(
|
||||
model_config, cached_tokenizer_from_config(model_config)
|
||||
)
|
||||
info = MultiModalProcessingInfo(ctx)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
return processor_cls(info, MultiModalDummyInputsBuilder(info)), cache
|
||||
+12
-74
@@ -233,9 +233,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"CohereLabs/North-Mini-Code",
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
min_transformers_version="5.9.0",
|
||||
),
|
||||
"CwmForCausalLM": _HfExamplesInfo("facebook/cwm", min_transformers_version="4.58"),
|
||||
"CwmForCausalLM": _HfExamplesInfo("facebook/cwm"),
|
||||
# FIXME: databricks/dbrx-instruct has been deleted
|
||||
"DbrxForCausalLM": _HfExamplesInfo(
|
||||
"databricks/dbrx-instruct", is_available_online=False
|
||||
@@ -266,9 +265,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True
|
||||
),
|
||||
"Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
|
||||
"ExaoneMoeForCausalLM": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/K-EXAONE-236B-A23B", min_transformers_version="5.1.0"
|
||||
),
|
||||
"ExaoneMoeForCausalLM": _HfExamplesInfo("LGAI-EXAONE/K-EXAONE-236B-A23B"),
|
||||
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
|
||||
"FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
|
||||
"FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
|
||||
@@ -279,7 +276,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
|
||||
"Gemma4ForCausalLM": _HfExamplesInfo(
|
||||
"google/gemma-4-E2B-it",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
|
||||
"GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
|
||||
@@ -287,11 +283,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"),
|
||||
"Glm4MoeLiteForCausalLM": _HfExamplesInfo(
|
||||
"zai-org/GLM-4.7-Flash",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"GlmMoeDsaForCausalLM": _HfExamplesInfo(
|
||||
"zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False
|
||||
),
|
||||
"GlmMoeDsaForCausalLM": _HfExamplesInfo("zai-org/GLM-5", is_available_online=False),
|
||||
"Glm5NextForCausalLM": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
|
||||
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2"),
|
||||
"GPTBigCodeForCausalLM": _HfExamplesInfo(
|
||||
@@ -316,15 +309,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"GraniteMoeSharedForCausalLM": _HfExamplesInfo(
|
||||
"ibm-research/moe-7b-1b-active-shared-experts"
|
||||
),
|
||||
"GraniteMoeSWAForCausalLM": _HfExamplesInfo(
|
||||
"ibm-granite/granite-swash-3b-a600m", min_transformers_version="5.15.1"
|
||||
),
|
||||
"GraniteSWAForCausalLM": _HfExamplesInfo(
|
||||
"ibm-granite/granite-swash-2b", min_transformers_version="5.15.1"
|
||||
),
|
||||
"GraniteMoeSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-3b-a600m"),
|
||||
"GraniteSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-2b"),
|
||||
"HrmTextForCausalLM": _HfExamplesInfo(
|
||||
"sapientinc/HRM-Text-1B",
|
||||
min_transformers_version="5.9.0",
|
||||
),
|
||||
"HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
|
||||
"HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct"),
|
||||
@@ -333,7 +321,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"HYV4ForCausalLM": _HfExamplesInfo("tencent/Hy4-preview", trust_remote_code=True),
|
||||
"HyperCLOVAXForCausalLM": _HfExamplesInfo(
|
||||
"naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
|
||||
min_transformers_version="5.9.0",
|
||||
),
|
||||
"InklingForCausalLM": _HfExamplesInfo(
|
||||
"thinkingmachines/Inkling-NVFP4",
|
||||
@@ -353,9 +340,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"IQuestLoopCoderForCausalLM": _HfExamplesInfo(
|
||||
"IQuestLab/IQuest-Coder-V1-40B-Loop-Instruct", trust_remote_code=True
|
||||
),
|
||||
"Jais2ForCausalLM": _HfExamplesInfo(
|
||||
"inceptionai/Jais-2-8B-Chat", min_transformers_version="4.58"
|
||||
),
|
||||
"Jais2ForCausalLM": _HfExamplesInfo("inceptionai/Jais-2-8B-Chat"),
|
||||
"JambaForCausalLM": _HfExamplesInfo(
|
||||
"ai21labs/AI21-Jamba-1.5-Mini",
|
||||
extras={
|
||||
@@ -375,7 +360,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
|
||||
"Lfm2MoeForCausalLM": _HfExamplesInfo(
|
||||
"LiquidAI/LFM2-8B-A1B",
|
||||
min_transformers_version="5.0.0",
|
||||
use_original_num_layers=True,
|
||||
# Initialize at least one MoE layer
|
||||
hf_overrides={"num_hidden_layers": 4},
|
||||
@@ -422,7 +406,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
),
|
||||
"MiniCPM4ForCausalLM": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM4.1-8B",
|
||||
min_transformers_version="4.56",
|
||||
max_transformers_version="4.57",
|
||||
transformers_version_reason={
|
||||
"hf": "HF remote code imports removed `is_torch_fx_available`; "
|
||||
@@ -523,7 +506,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Qwen3NextForCausalLM": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Next-80B-A3B-Instruct",
|
||||
extras={"tiny-random": "tiny-random/qwen3-next-moe"},
|
||||
min_transformers_version="4.56.3",
|
||||
),
|
||||
"Rnj1ForCausalLM": _HfExamplesInfo(
|
||||
"EssentialAI/rnj-1-instruct",
|
||||
@@ -722,7 +704,6 @@ _TOKEN_CLASSIFICATION_EXAMPLE_MODELS = {
|
||||
),
|
||||
"OpenAIPrivacyFilterForTokenClassification": _HfExamplesInfo(
|
||||
"openai/privacy-filter",
|
||||
min_transformers_version="5.6.0.dev0",
|
||||
),
|
||||
"RobertaForTokenClassification": _HfExamplesInfo(
|
||||
"Jean-Baptiste/roberta-large-ner-english"
|
||||
@@ -794,7 +775,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
|
||||
"AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/audio-flamingo-3-hf",
|
||||
min_transformers_version="5.3.0",
|
||||
transformers_version_reason={
|
||||
"vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
|
||||
},
|
||||
@@ -823,22 +803,18 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
|
||||
"CohereLabs/command-a-vision-07-2025",
|
||||
extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"},
|
||||
min_transformers_version="5.9.0",
|
||||
),
|
||||
"CohereCompassForConditionalGeneration": _HfExamplesInfo(
|
||||
"CohereLabs/North-Micro-Vision-Instruct",
|
||||
min_transformers_version="5.16.0",
|
||||
),
|
||||
"Cosmos3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/Cosmos3-Nano",
|
||||
extras={"super": "nvidia/Cosmos3-Super"},
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/Cosmos3-Edge",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="5.15",
|
||||
use_original_num_layers=True,
|
||||
hf_overrides={
|
||||
"text_config": {
|
||||
@@ -887,7 +863,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"Exaone4_5_ForConditionalGeneration": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/EXAONE-4.5-33B",
|
||||
min_transformers_version="5.6.0",
|
||||
),
|
||||
"FireRedASR2ForConditionalGeneration": _HfExamplesInfo(
|
||||
"allendou/FireRedASR2-LLM-vllm",
|
||||
@@ -918,19 +893,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"Gemma4ForConditionalGeneration": _HfExamplesInfo(
|
||||
"google/gemma-4-E2B-it",
|
||||
min_transformers_version="5.5.0",
|
||||
),
|
||||
# TODO: update min_transformers_version when Gemma4 Unified lands in
|
||||
# a stable transformers release.
|
||||
"Gemma4UnifiedForConditionalGeneration": _HfExamplesInfo(
|
||||
"google/gemma-4-12B-it",
|
||||
min_transformers_version="5.8.0",
|
||||
is_available_online=False,
|
||||
),
|
||||
"Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
|
||||
"GlmAsrForConditionalGeneration": _HfExamplesInfo(
|
||||
"zai-org/GLM-ASR-Nano-2512",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"Granite4VisionForConditionalGeneration": _HfExamplesInfo(
|
||||
"ibm-granite/granite-vision-4.1-4b",
|
||||
@@ -943,7 +913,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"GraniteSpeechPlusForConditionalGeneration": _HfExamplesInfo(
|
||||
"ibm-granite/granite-speech-4.1-2b-plus",
|
||||
min_transformers_version="5.8.0",
|
||||
),
|
||||
"GLM4VForCausalLM": _HfExamplesInfo(
|
||||
"zai-org/glm-4v-9b",
|
||||
@@ -957,7 +926,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
|
||||
"GlmOcrForConditionalGeneration": _HfExamplesInfo(
|
||||
"zai-org/GLM-OCR",
|
||||
min_transformers_version="5.1.0",
|
||||
),
|
||||
"Glm5NextForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
|
||||
"H2OVLChatModel": _HfExamplesInfo(
|
||||
@@ -974,7 +942,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"HunYuanVLForConditionalGeneration": _HfExamplesInfo(
|
||||
"tencent/HunyuanOCR",
|
||||
min_transformers_version="5.13",
|
||||
),
|
||||
"Idefics3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"HuggingFaceM4/Idefics3-8B-Llama3",
|
||||
@@ -1087,7 +1054,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"Lfm2VlForConditionalGeneration": _HfExamplesInfo(
|
||||
"LiquidAI/LFM2-VL-450M",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"Llama4ForConditionalGeneration": _HfExamplesInfo(
|
||||
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
@@ -1154,7 +1120,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM-V-4_6",
|
||||
min_transformers_version="5.7.0",
|
||||
),
|
||||
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
|
||||
"MiniMaxAI/MiniMax-M3",
|
||||
@@ -1201,7 +1166,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Molmo2ForConditionalGeneration": _HfExamplesInfo(
|
||||
"allenai/Molmo2-8B",
|
||||
extras={"olmo": "allenai/Molmo2-O-7B"},
|
||||
min_transformers_version="4.51",
|
||||
trust_remote_code=True,
|
||||
# required by current PrefixLM implementation
|
||||
max_num_batched_tokens=31872,
|
||||
@@ -1353,7 +1317,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"QianfanOCRForConditionalGeneration": _HfExamplesInfo(
|
||||
"baidu/Qianfan-OCR",
|
||||
min_transformers_version="5.6.0",
|
||||
),
|
||||
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen2-Audio-7B-Instruct"
|
||||
@@ -1368,12 +1331,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Qwen3VLForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-VL-4B-Instruct",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-VL-30B-A3B-Instruct",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Qwen3_5ForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3.5-0.8B",
|
||||
@@ -1391,23 +1352,19 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-ASR-0.6B",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Qwen3ASRRealtimeGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-ASR-0.6B",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]},
|
||||
),
|
||||
"Qwen3ASRForcedAlignerForTokenClassification": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-ForcedAligner-0.6B",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
hf_overrides={"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]},
|
||||
),
|
||||
"RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
|
||||
@@ -1456,7 +1413,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo(
|
||||
"microsoft/VibeVoice-ASR-HF",
|
||||
min_transformers_version="5.13.0",
|
||||
),
|
||||
"VoxtralForConditionalGeneration": _HfExamplesInfo(
|
||||
"mistralai/Voxtral-Mini-3B-2507",
|
||||
@@ -1530,7 +1486,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
use_original_num_layers=True, # DFlash requires all layers
|
||||
max_model_len=8192, # Reduce for CI
|
||||
max_num_seqs=32,
|
||||
min_transformers_version="4.56.3", # Required for Qwen3Next
|
||||
),
|
||||
"MuseGlimmerAssistantModel": _HfExamplesInfo(
|
||||
"meta-models/Muse-Glimmer-30B",
|
||||
@@ -1576,7 +1531,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
speculative_model="Qwen/Qwen3-Omni-30B-A3B-DSpark",
|
||||
is_available_online=False,
|
||||
use_original_num_layers=True,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Gemma4DSparkModel": _HfExamplesInfo(
|
||||
"google/gemma-4-12B-it",
|
||||
@@ -1737,7 +1691,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"google/gemma-4-E4B-it",
|
||||
speculative_model="google/gemma-4-E4B-it-assistant",
|
||||
trust_remote_code=True,
|
||||
min_transformers_version="5.8.0",
|
||||
),
|
||||
"ErnieMTPModel": _HfExamplesInfo(
|
||||
"baidu/ERNIE-4.5-21B-A3B-PT",
|
||||
@@ -1747,13 +1700,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"ExaoneMoeMTP": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/K-EXAONE-236B-A23B",
|
||||
speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B",
|
||||
min_transformers_version="5.1.0",
|
||||
enable_prefix_caching=False,
|
||||
),
|
||||
"Exaone4_5_MTP": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/EXAONE-4.5-33B",
|
||||
speculative_model="LGAI-EXAONE/EXAONE-4.5-33B",
|
||||
min_transformers_version="5.6.0",
|
||||
),
|
||||
"ExtractHiddenStatesModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-8B",
|
||||
@@ -1766,12 +1717,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"Glm4MoeLiteMTPModel": _HfExamplesInfo(
|
||||
"zai-org/GLM-4.7-Flash",
|
||||
speculative_model="zai-org/GLM-4.7-Flash",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"GlmOcrMTPModel": _HfExamplesInfo(
|
||||
"zai-org/GLM-OCR",
|
||||
speculative_model="zai-org/GLM-OCR",
|
||||
min_transformers_version="5.1.0",
|
||||
),
|
||||
"Glm5NextMTPModel": _HfExamplesInfo(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
@@ -1842,9 +1791,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"Qwen3NextMTP": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3"
|
||||
),
|
||||
"Qwen3NextMTP": _HfExamplesInfo("Qwen/Qwen3-Next-80B-A3B-Instruct"),
|
||||
"Qwen3_5MTP": _HfExamplesInfo(
|
||||
"Qwen/Qwen3.5-0.8B",
|
||||
speculative_model="Qwen/Qwen3.5-0.8B",
|
||||
@@ -1865,29 +1812,20 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
}
|
||||
|
||||
_TRANSFORMERS_BACKEND_MODELS = {
|
||||
"TransformersEmbeddingModel": _HfExamplesInfo(
|
||||
"BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0"
|
||||
),
|
||||
"TransformersEmbeddingModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
|
||||
"TransformersForSequenceClassification": _HfExamplesInfo(
|
||||
"papluca/xlm-roberta-base-language-detection",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"TransformersForCausalLM": _HfExamplesInfo(
|
||||
"hmellor/Ilama-3.2-1B", trust_remote_code=True
|
||||
),
|
||||
"TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
|
||||
"TransformersMoEForCausalLM": _HfExamplesInfo(
|
||||
"allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0"
|
||||
),
|
||||
"TransformersMoEForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924"),
|
||||
"TransformersMultiModalMoEForCausalLM": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0"
|
||||
),
|
||||
"TransformersMoEEmbeddingModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
|
||||
),
|
||||
"TransformersMoEForSequenceClassification": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
|
||||
"Qwen/Qwen3-VL-30B-A3B-Instruct"
|
||||
),
|
||||
"TransformersMoEEmbeddingModel": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
|
||||
"TransformersMoEForSequenceClassification": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
|
||||
"TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"),
|
||||
"TransformersMultiModalForSequenceClassification": _HfExamplesInfo(
|
||||
"google/gemma-3-4b-it"
|
||||
|
||||
@@ -185,17 +185,6 @@ def test_sinks(hf_runner: type[HfRunner], vllm_runner: type[VllmRunner]) -> None
|
||||
|
||||
|
||||
def test_mla(vllm_runner: type[VllmRunner], example_prompts: list[str]) -> None:
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
|
||||
installed = Version(transformers.__version__)
|
||||
required = Version("5.15.0.dev0")
|
||||
if installed < required:
|
||||
pytest.skip(
|
||||
"MLA models with the Transformers modeling backend require "
|
||||
f"transformers>={required}, but got {installed}"
|
||||
)
|
||||
|
||||
model = "hmellor/tiny-random-DeepseekV2ForCausalLM"
|
||||
args = (example_prompts, 32, 5)
|
||||
kwargs: dict[str, Any] = {"max_model_len": 2048, "enforce_eager": True}
|
||||
|
||||
@@ -609,17 +609,3 @@ def dummy_hf_overrides(
|
||||
)
|
||||
|
||||
return hf_config
|
||||
|
||||
|
||||
def check_transformers_version(
|
||||
model: str,
|
||||
min_transformers_version: str | None = None,
|
||||
max_transformers_version: str | None = None,
|
||||
):
|
||||
from .registry import _HfExamplesInfo
|
||||
|
||||
return _HfExamplesInfo(
|
||||
model,
|
||||
min_transformers_version=min_transformers_version,
|
||||
max_transformers_version=max_transformers_version,
|
||||
).check_transformers_version(on_fail="skip")
|
||||
|
||||
@@ -10,7 +10,6 @@ from transformers import AutoConfig
|
||||
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.hf import CachedHfTokenizer
|
||||
from vllm.tokenizers.registry import (
|
||||
TokenizerRegistry,
|
||||
cached_get_tokenizer,
|
||||
@@ -93,54 +92,6 @@ def test_resolve_tokenizer_args_rejects_hf_mistral_format_true():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("transformers_version", "model_type", "native_filename", "expect_error"),
|
||||
[
|
||||
pytest.param("5.14.0", "mistral", "tekken.json", True),
|
||||
pytest.param("5.15.0", "mistral", "tekken.json", False),
|
||||
pytest.param("5.14.0", "mistral3", "tekken.json", True),
|
||||
pytest.param("5.15.0", "mistral3", "tekken.json", False),
|
||||
pytest.param("5.14.0", "mixtral", "tokenizer.model.v1", True),
|
||||
pytest.param("5.15.0", "mixtral", "tokenizer.model.v1", False),
|
||||
pytest.param("5.14.0", "mistral", None, False),
|
||||
pytest.param("5.14.0", "qwen3_5_moe", "tekken.json", False),
|
||||
],
|
||||
)
|
||||
def test_get_tokenizer_handles_hf_mistral_transformers_compatibility(
|
||||
tmp_path: Path,
|
||||
transformers_version: str,
|
||||
model_type: str,
|
||||
native_filename: str | None,
|
||||
expect_error: bool,
|
||||
):
|
||||
if native_filename is not None:
|
||||
(tmp_path / native_filename).touch()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.tokenizers.registry.get_config",
|
||||
return_value=SimpleNamespace(model_type=model_type),
|
||||
),
|
||||
patch(
|
||||
"vllm.tokenizers.registry.transformers.__version__",
|
||||
transformers_version,
|
||||
),
|
||||
patch.object(
|
||||
CachedHfTokenizer,
|
||||
"from_pretrained",
|
||||
return_value=SimpleNamespace(is_fast=True),
|
||||
) as from_pretrained,
|
||||
):
|
||||
if expect_error:
|
||||
with pytest.raises(ValueError, match="requires transformers>=5.15.0"):
|
||||
get_tokenizer(str(tmp_path), tokenizer_mode="hf")
|
||||
else:
|
||||
tokenizer = get_tokenizer(str(tmp_path), tokenizer_mode="hf")
|
||||
assert tokenizer.is_fast is True
|
||||
|
||||
assert from_pretrained.called == (not expect_error)
|
||||
|
||||
|
||||
def test_customized_tokenizer():
|
||||
TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__)
|
||||
|
||||
|
||||
@@ -48,16 +48,6 @@ def check_mtp_correctness(
|
||||
elif "gemma-4" in model_name:
|
||||
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0}
|
||||
|
||||
if draft_model is not None and "gemma-4" in draft_model:
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
|
||||
if Version(transformers.__version__) < Version("5.8.0"):
|
||||
pytest.skip(
|
||||
"Gemma4 MTP assistant requires transformers>=5.8.0, "
|
||||
f"got {transformers.__version__}"
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
block_size=None,
|
||||
|
||||
@@ -7,8 +7,6 @@ from typing import Annotated, Any, Literal, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig
|
||||
from transformers.models.pixtral import PixtralProcessor
|
||||
|
||||
@@ -59,10 +57,6 @@ from .utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY = Version(transformers.__version__) >= Version(
|
||||
"5.15.0"
|
||||
)
|
||||
|
||||
|
||||
class Mistral3ImagePixelInputs(TensorSchema):
|
||||
"""Dimensions:
|
||||
@@ -284,13 +278,6 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
|
||||
|
||||
|
||||
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
|
||||
# PixtralProcessor supports image-only calls starting in transformers 5.15.
|
||||
# Older releases need dummy text and take the slower tokenization path.
|
||||
if not TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
return None
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
|
||||
@@ -691,10 +691,8 @@ class Base(
|
||||
# vLLM does not support encoder-decoder models, so if any encoder layer is
|
||||
# found in a text only model, we assume the whole model is an encoder model
|
||||
if has_encoder(self.model) and not is_multimodal(self.config):
|
||||
self.check_version("5.0.0", "encoder models support")
|
||||
return EncoderOnlyAttention
|
||||
if self.model_config.use_mla:
|
||||
self.check_version("5.15.0.dev0", "optimized MLA support")
|
||||
if any(
|
||||
isinstance(fuser, MLAFuser)
|
||||
for fusers in self.fusers.values()
|
||||
@@ -820,7 +818,6 @@ class Base(
|
||||
)
|
||||
|
||||
def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
|
||||
self.check_version("5.2.0", "Eagle3 support")
|
||||
from transformers.utils.output_capturing import (
|
||||
OutputRecorder,
|
||||
maybe_install_capturing_hooks,
|
||||
|
||||
@@ -156,7 +156,6 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
num_local_physical_experts: int
|
||||
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
self.check_version("5.0.0", "MoE models support")
|
||||
# Skip MixtureOfExperts.__init__ and call the next class in MRO
|
||||
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
|
||||
@@ -23,17 +23,11 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
from transformers.utils.generic import ModelOutput
|
||||
|
||||
from vllm.compilation.decorators import should_torch_compile_mm_encoder
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.inputs import (
|
||||
MultiModalDataBuiltins,
|
||||
MultiModalDataDict,
|
||||
MultiModalInput,
|
||||
mm_input,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataBuiltins, MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -42,26 +36,15 @@ from vllm.model_executor.models.interfaces import (
|
||||
)
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
ImageProcessorItems,
|
||||
ImageSize,
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
)
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
TimingContext,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -219,11 +202,13 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
|
||||
return data
|
||||
|
||||
|
||||
class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
"""Processing common to both Transformers backend processors: calling the HF
|
||||
processor, sizing images, and attributing its outputs to a modality.
|
||||
class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
"""Locates placeholders from the `text_replacement_offsets` the HF processor
|
||||
reports, expressing each one as a `PromptUpdate`.
|
||||
|
||||
Subclasses add the strategy for locating placeholders in the prompt.
|
||||
Stating the expansion as an update is what lets it be rebuilt from an
|
||||
unexpanded prompt, so this processor takes the base class's processing path
|
||||
and with it the multi-modal processor cache.
|
||||
"""
|
||||
|
||||
def _get_modality_field_names(self, modality: str) -> set[str]:
|
||||
@@ -415,301 +400,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
for image, (height, width) in zip(pixel_values, image_sizes.tolist())
|
||||
]
|
||||
|
||||
|
||||
class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"""Locates placeholders by searching the prompt the HF processor has already
|
||||
expanded for the tokens of each modality.
|
||||
|
||||
Serves transformers versions with no `return_text_replacement_offsets`.
|
||||
Placeholders found this way cannot be rebuilt from an unexpanded prompt, so
|
||||
this processor overrides `apply` and gets no multi-modal processor cache.
|
||||
Remove it once `requirements/common.txt` requires `transformers>=5.15.0`.
|
||||
"""
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""Empty, because `apply` writes the placeholder ranges itself rather than
|
||||
deriving them from updates."""
|
||||
return []
|
||||
|
||||
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
||||
"""Token ids marking where `modality` sits in the prompt, which for some
|
||||
processors differ from the placeholder written into it.
|
||||
|
||||
The expanded prompt is all this path has to go on, so it takes the
|
||||
processor at its word about which tokens belong to the modality.
|
||||
"""
|
||||
info = self.info
|
||||
processor = info.get_hf_processor()
|
||||
declared = getattr(processor, f"{modality}_token_ids", None) or ()
|
||||
if ids := [token_id for token_id in declared if token_id is not None]:
|
||||
return ids
|
||||
config = info.get_hf_config()
|
||||
names = (f"{modality}_token_id", f"{modality}_token_index")
|
||||
token_id = getattr(processor, names[0], getattr_iter(config, names))
|
||||
if token_id is None:
|
||||
token = getattr(processor, f"{modality}_token", None)
|
||||
if token is not None:
|
||||
token_id = info.get_tokenizer().get_vocab().get(token)
|
||||
if token_id is None:
|
||||
raise ValueError(
|
||||
f"Cannot find {modality}_token_id on processor or model config"
|
||||
)
|
||||
return [token_id]
|
||||
|
||||
def _apply_audio(
|
||||
self,
|
||||
prompt_ids: list[int],
|
||||
processed_data: "BatchFeature",
|
||||
num_audios: int,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
"""Take each contiguous run of the audio token as one item's placeholder,
|
||||
and record how many tokens the run holds."""
|
||||
audio_token_ids = self._get_mm_token_ids("audio")
|
||||
prompt_tensor = torch.tensor(prompt_ids)
|
||||
is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
|
||||
|
||||
if not is_audio.any():
|
||||
raise ValueError(
|
||||
f"{num_audios} audio item(s) were passed but the prompt "
|
||||
"contains no audio token. Add one placeholder per audio item."
|
||||
)
|
||||
|
||||
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
|
||||
transitions = padded.int().diff()
|
||||
offsets = torch.where(transitions == 1)[0]
|
||||
lengths = torch.where(transitions == -1)[0] - offsets
|
||||
|
||||
if len(offsets) != num_audios:
|
||||
raise ValueError(
|
||||
f"Found {len(offsets)} run(s) of the audio token in the prompt but "
|
||||
f"{num_audios} audio item(s) were passed. The Transformers backend "
|
||||
"locates audio placeholders by finding contiguous runs of the audio "
|
||||
"token, so placeholders with no text between them cannot yet be told "
|
||||
"apart. Separate them in the prompt to work around this."
|
||||
)
|
||||
|
||||
ranges = [
|
||||
PlaceholderRange(offset=offset.item(), length=length.item())
|
||||
for offset, length in zip(offsets, lengths)
|
||||
]
|
||||
processed_data["num_audio_tokens"] = lengths
|
||||
return {"audio": ranges}
|
||||
|
||||
def _get_num_multimodal_tokens(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, list[int]]:
|
||||
"""Ask the HF processor how many tokens and patches each image expands to."""
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = [
|
||||
(size.height, size.width)
|
||||
for size in map(images.get_image_size, range(len(images)))
|
||||
]
|
||||
return processor._get_num_multimodal_tokens(
|
||||
image_sizes=image_sizes,
|
||||
**self.info.ctx.get_merged_mm_kwargs({}),
|
||||
)
|
||||
|
||||
def _apply_vision(
|
||||
self,
|
||||
prompt_ids: list[int],
|
||||
processed_data: "BatchFeature",
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
mm_token_type_ids: torch.Tensor | None,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
"""Split the positions the processor marks as image into one placeholder per
|
||||
item, sized by the token count it reports for each image."""
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
if mm_token_type_ids is None:
|
||||
raise ValueError(
|
||||
f"{type(hf_processor).__name__} returned no `mm_token_type_ids`, so "
|
||||
"the Transformers modeling backend cannot locate the placeholder of "
|
||||
"each image."
|
||||
)
|
||||
|
||||
# These mark the tokens structuring an image as well as the image itself,
|
||||
# which is what the counts below are over
|
||||
mm_positions = torch.where(mm_token_type_ids[0] == 1)[0]
|
||||
mm_tokens_per_modality = self._get_num_multimodal_tokens(
|
||||
mm_items, hf_processor_mm_kwargs
|
||||
)
|
||||
|
||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
||||
split_sizes = mm_tokens_per_modality["num_image_tokens"]
|
||||
if sum(split_sizes) != len(mm_positions):
|
||||
raise ValueError(
|
||||
f"The expanded prompt holds {len(mm_positions)} image token(s) but "
|
||||
f"{type(hf_processor).__name__} accounts for {sum(split_sizes)} "
|
||||
f"across {mm_items.get_count('image')} image item(s)."
|
||||
)
|
||||
|
||||
if split_sizes:
|
||||
image_token_ids = torch.tensor(self._get_mm_token_ids("image"))
|
||||
mm_tokens = torch.tensor(prompt_ids)[mm_positions]
|
||||
ranges = [
|
||||
PlaceholderRange(
|
||||
offset=positions[0].item(),
|
||||
length=positions.shape[0],
|
||||
# Only some of the span carries embeddings
|
||||
is_embed=torch.isin(tokens, image_token_ids),
|
||||
)
|
||||
for positions, tokens in zip(
|
||||
torch.split(mm_positions, split_sizes),
|
||||
torch.split(mm_tokens, split_sizes),
|
||||
)
|
||||
]
|
||||
mm_placeholders = {"image": ranges}
|
||||
|
||||
processed_data["num_image_patches"] = torch.tensor(
|
||||
mm_tokens_per_modality["num_image_patches"]
|
||||
)
|
||||
return mm_placeholders
|
||||
|
||||
def apply(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalInput:
|
||||
"""Process the prompt and every multi-modal item in one HF processor call,
|
||||
then read the placeholder ranges out of the token ids it returns."""
|
||||
prompt = inputs.prompt
|
||||
mm_items = inputs.mm_data_items
|
||||
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
|
||||
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
prompt_text = hf_processor.decode(prompt)
|
||||
|
||||
# Bypass cached processor and always apply to the full set of mm inputs
|
||||
# NOTE: we can't just set caching=False because base class method
|
||||
# transforms outputs to `MultiModalKwargs` which is not going to
|
||||
# work for Transformers. The vision path has logic tied to
|
||||
# `mm_tokens_per_modality` in _apply_vision()
|
||||
hf_processor_mm_kwargs = {
|
||||
# vLLM needs the untruncated sequence to keep placeholder
|
||||
# tokens aligned. Note that the text inputs are just dummy
|
||||
# text, not the original prompt. The original prompt is
|
||||
# already tokenized by the renderer.
|
||||
"truncation": False,
|
||||
# HF processors only accept text, and the decoded string already
|
||||
# contains any special tokens, so don't let them be added again
|
||||
# (the HF processor call disables `add_special_tokens`).
|
||||
"add_special_tokens": False,
|
||||
**hf_processor_mm_kwargs,
|
||||
}
|
||||
|
||||
processor_data, _, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_processor_mm_kwargs
|
||||
)
|
||||
|
||||
# The real prompt is fed to the processor below and the placeholder
|
||||
# ranges are read back out of its output, so the dummy text meant for
|
||||
# media-only callers is discarded here.
|
||||
processor_data.pop("text", None)
|
||||
has_mm_data = any(processor_data.values())
|
||||
|
||||
# Ask which modality owns each token of the expanded prompt, which
|
||||
# is the only marking of the tokens an expansion adds around an item
|
||||
if has_mm_data:
|
||||
processor_data = {**processor_data, "return_mm_token_type_ids": True}
|
||||
|
||||
try:
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **processor_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
except ValueError:
|
||||
if has_mm_data:
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt_text)
|
||||
processed_data = transformers.BatchFeature(
|
||||
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(processed_data)
|
||||
self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
input_ids = processed_data.pop("input_ids")
|
||||
if not isinstance(input_ids, list):
|
||||
input_ids = input_ids.tolist()
|
||||
|
||||
(prompt_ids,) = input_ids
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
|
||||
# Gemma3 reports them under the key the model uses for its own token types
|
||||
mm_token_type_ids = processed_data.pop("token_type_ids", None)
|
||||
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
|
||||
|
||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
||||
if num_audios := mm_items.get_count("audio", strict=False):
|
||||
mm_placeholders.update(
|
||||
self._apply_audio(prompt_ids, processed_data, num_audios)
|
||||
)
|
||||
if mm_items.get_count("image", strict=False):
|
||||
mm_placeholders.update(
|
||||
self._apply_vision(
|
||||
prompt_ids,
|
||||
processed_data,
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_token_type_ids,
|
||||
)
|
||||
)
|
||||
|
||||
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
processed_data,
|
||||
self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
|
||||
)
|
||||
|
||||
# Bypassing `_maybe_apply_prompt_updates` also bypasses its validation.
|
||||
# `_validate_mm_placeholders` can't be reused because it is typed for the
|
||||
# `PlaceholderFeaturesInfo` the prompt update machinery produces.
|
||||
mm_item_counts = mm_items.get_all_counts()
|
||||
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
|
||||
for modality, item_count in mm_item_counts.items():
|
||||
num_placeholders = len(mm_placeholders.get(modality, []))
|
||||
if num_placeholders != item_count:
|
||||
raise RuntimeError(
|
||||
f"Expected there to be {item_count} prompt placeholders "
|
||||
f"corresponding to {item_count} {modality} items, but instead "
|
||||
f"found {num_placeholders} prompt placeholders! Make sure the "
|
||||
"prompt contains a placeholder token for each item."
|
||||
)
|
||||
|
||||
return mm_input(
|
||||
prompt_token_ids=prompt_ids,
|
||||
mm_kwargs=mm_kwargs,
|
||||
mm_hashes=mm_hashes,
|
||||
mm_placeholders=mm_placeholders,
|
||||
)
|
||||
|
||||
|
||||
class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"""Locates placeholders from the `text_replacement_offsets` the HF processor
|
||||
reports, expressing each one as a `PromptUpdate`.
|
||||
|
||||
Stating the expansion as an update is what lets it be rebuilt from an
|
||||
unexpanded prompt, so this processor takes the base class's processing path
|
||||
and with it the multi-modal processor cache.
|
||||
"""
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
@@ -800,9 +490,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
except (AttributeError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
@@ -856,8 +543,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"to reach `ProcessorMixin.get_text_with_replacements` with one "
|
||||
"replacement per item, which usually means implementing "
|
||||
"`replace_<modality>_token`. Please report this to transformers "
|
||||
"so it can be fixed, and install `transformers<5.15.0` in the "
|
||||
"meantime to locate placeholders in the expanded prompt instead."
|
||||
"so it can be fixed."
|
||||
)
|
||||
hf_inputs.pop("input_ids", None)
|
||||
return self._finalize_hf_mm_data(
|
||||
@@ -896,15 +582,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
)
|
||||
|
||||
|
||||
# From this version on, a processor reporting no offsets is an error rather than a
|
||||
# fallback to searching the expanded prompt
|
||||
MultiModalProcessor = (
|
||||
LegacyMultiModalProcessor
|
||||
if Version(transformers.__version__) < Version("5.15.0")
|
||||
else OffsetsMultiModalProcessor
|
||||
)
|
||||
|
||||
|
||||
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
# Skip SupportsMRoPE.__init__ and call the next class in MRO
|
||||
@@ -982,7 +659,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
|
||||
super()._decorate_for_torch_compile()
|
||||
# Decorate the encoder model classes to support torch compile if needed
|
||||
if self.compilation_config.compile_mm_encoder:
|
||||
self.check_version("5.0.0", "multimodal encoder compilation support")
|
||||
encoder_classes = self._pre_trained_model_classes.encoders
|
||||
if not encoder_classes:
|
||||
raise ValueError(
|
||||
@@ -1113,7 +789,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
|
||||
if input_features is None:
|
||||
return None
|
||||
|
||||
self.check_version("5.13.0", "audio models support")
|
||||
num_audio_tokens = kwargs.pop("num_audio_tokens")
|
||||
kwargs.pop("token_type_ids", None)
|
||||
kwargs.pop("mm_token_type_ids", None)
|
||||
|
||||
@@ -48,21 +48,18 @@ def detect_token_suffix(tokenizer: TokenizerLike) -> str:
|
||||
round-trips.
|
||||
|
||||
"""
|
||||
import transformers
|
||||
|
||||
if int(transformers.__version__.split(".")[0]) >= 5:
|
||||
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
|
||||
think_begin_as_special = init_kwargs.get(
|
||||
"model_specific_special_tokens", {}
|
||||
).get("think_begin_token", "")
|
||||
if think_begin_as_special:
|
||||
raise RuntimeError(
|
||||
"This checkpoint declares HYV4 structural tokens (think_begin_token"
|
||||
"/toolcalls_begin_token/argkey_begin_token) in "
|
||||
"tokenizer_config.json, which transformers 5 no longer supports. "
|
||||
"Remove those fields and keep the tokens in the tokenizer's own "
|
||||
"token definitions so the suffix can be read from the vocab."
|
||||
)
|
||||
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
|
||||
think_begin_as_special = init_kwargs.get("model_specific_special_tokens", {}).get(
|
||||
"think_begin_token", ""
|
||||
)
|
||||
if think_begin_as_special:
|
||||
raise RuntimeError(
|
||||
"This checkpoint declares HYV4 structural tokens (think_begin_token"
|
||||
"/toolcalls_begin_token/argkey_begin_token) in "
|
||||
"tokenizer_config.json, which transformers 5 no longer supports. "
|
||||
"Remove those fields and keep the tokens in the tokenizer's own "
|
||||
"token definitions so the suffix can be read from the vocab."
|
||||
)
|
||||
|
||||
structural_token_re = re.compile(
|
||||
r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>"
|
||||
|
||||
@@ -7,12 +7,6 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import huggingface_hub
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
from transformers.models.auto.tokenization_auto import (
|
||||
TOKENIZER_MAPPING_NAMES as _HF_TOKENIZER_MAPPING_NAMES,
|
||||
)
|
||||
from transformers.utils.import_utils import is_mistral_common_available
|
||||
from typing_extensions import TypeVar, assert_never
|
||||
|
||||
from vllm import envs
|
||||
@@ -32,8 +26,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL = Version("5.15.0")
|
||||
|
||||
|
||||
# Model types whose hub tokenizer_class is incorrect and should be overridden with
|
||||
# TokenizersBackend (the generic fast tokenizer). Adding a model type here is always a
|
||||
@@ -243,26 +235,6 @@ def get_tokenizer(
|
||||
token=kwargs.get("token"),
|
||||
)
|
||||
|
||||
# TODO: delete when Transformers version dependency is bumped >= 5.15.0
|
||||
if (
|
||||
tokenizer_mode == "hf"
|
||||
and Version(transformers.__version__) < _MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL
|
||||
and is_mistral_common_available()
|
||||
and _HF_TOKENIZER_MAPPING_NAMES.get(getattr(config, "model_type", None))
|
||||
== "MistralCommonBackend"
|
||||
and any_pattern_in_repo_files(
|
||||
model_name_or_path=str(tokenizer_name),
|
||||
allow_patterns=["tekken.json", "tokenizer.model.v*"],
|
||||
revision=revision,
|
||||
token=kwargs.get("token"),
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"Loading Mistral models with tokenizer_mode='hf' requires "
|
||||
f"transformers>={_MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL}. "
|
||||
"Please upgrade transformers or delete Mistral tokenizer files."
|
||||
)
|
||||
|
||||
# Some models have an incorrect tokenizer_class on the hub.
|
||||
# For these model types, bypass AutoTokenizer and use TokenizersBackend directly.
|
||||
model_type = getattr(config, "model_type", None) if config else None
|
||||
|
||||
@@ -104,21 +104,18 @@ def detect_token_suffix(tokenizer: TokenizerLike) -> str:
|
||||
round-trips.
|
||||
|
||||
"""
|
||||
import transformers
|
||||
|
||||
if int(transformers.__version__.split(".")[0]) >= 5:
|
||||
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
|
||||
think_begin_as_special = init_kwargs.get(
|
||||
"model_specific_special_tokens", {}
|
||||
).get("think_begin_token", "")
|
||||
if think_begin_as_special:
|
||||
raise RuntimeError(
|
||||
"This checkpoint declares HYV4 structural tokens (think_begin_token"
|
||||
"/toolcalls_begin_token/argkey_begin_token) in "
|
||||
"tokenizer_config.json, which transformers 5 no longer supports. "
|
||||
"Remove those fields and keep the tokens in the tokenizer's own "
|
||||
"token definitions so the suffix can be read from the vocab."
|
||||
)
|
||||
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
|
||||
think_begin_as_special = init_kwargs.get("model_specific_special_tokens", {}).get(
|
||||
"think_begin_token", ""
|
||||
)
|
||||
if think_begin_as_special:
|
||||
raise RuntimeError(
|
||||
"This checkpoint declares HYV4 structural tokens (think_begin_token"
|
||||
"/toolcalls_begin_token/argkey_begin_token) in "
|
||||
"tokenizer_config.json, which transformers 5 no longer supports. "
|
||||
"Remove those fields and keep the tokens in the tokenizer's own "
|
||||
"token definitions so the suffix can be read from the vocab."
|
||||
)
|
||||
|
||||
structural_token_re = re.compile(
|
||||
r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>"
|
||||
|
||||
@@ -7,7 +7,6 @@ from collections.abc import Callable, Iterator, Sequence
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import asdict
|
||||
from functools import cache, partial, wraps
|
||||
from importlib.metadata import version
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypeAlias
|
||||
|
||||
@@ -15,7 +14,6 @@ import huggingface_hub
|
||||
import torch
|
||||
import transformers.configuration_utils as hf_configuration_utils
|
||||
from huggingface_hub import constants
|
||||
from packaging.version import Version
|
||||
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
|
||||
from transformers import GenerationConfig, PreTrainedConfig
|
||||
from transformers.configuration_utils import ALLOWED_LAYER_TYPES
|
||||
@@ -69,12 +67,6 @@ _DENSE_MODULE_TYPES = {
|
||||
"pylate.models.Dense.Dense",
|
||||
}
|
||||
|
||||
if Version(version("transformers")) < Version("5.0.0"):
|
||||
raise ImportError(
|
||||
"Support for Transformers v4 is deprecated and was removed in vLLM v0.24.0. "
|
||||
"Please upgrade to Transformers v5: pip install --upgrade transformers"
|
||||
)
|
||||
|
||||
|
||||
class LazyConfigDict(dict):
|
||||
def __getitem__(self, key):
|
||||
|
||||
@@ -61,10 +61,8 @@ class MistralCommonFeatureExtractor(SequenceFeatureExtractor):
|
||||
"""HF-compatible duck-typed ``fetch_audio``.
|
||||
|
||||
Mirrors :meth:`transformers.SequenceFeatureExtractor.fetch_audio` so
|
||||
:class:`transformers.ProcessorMixin.prepare_inputs_layout` (added in
|
||||
transformers 5.10) works on this duck-typed feature extractor. Older
|
||||
transformers versions never invoke this method, so the addition is a
|
||||
no-op there.
|
||||
:class:`transformers.ProcessorMixin.prepare_inputs_layout` works on this
|
||||
duck-typed feature extractor.
|
||||
|
||||
Accepts the same shapes as ``SequenceFeatureExtractor.fetch_audio``:
|
||||
|
||||
@@ -77,11 +75,6 @@ class MistralCommonFeatureExtractor(SequenceFeatureExtractor):
|
||||
``ProcessorMixin.prepare_inputs_layout`` always passes already-decoded
|
||||
audio (numpy array or torch tensor), so the str / list-of-str branches
|
||||
exist only to keep the contract identical to the upstream method.
|
||||
|
||||
The semantics of ``transformers.audio_utils.is_valid_audio`` differ
|
||||
between transformers versions (5.9 only accepts ndarray/tensor; 5.10
|
||||
also accepts ``list[float]``). We detect ``list[float]`` explicitly to
|
||||
keep behavior identical across versions.
|
||||
"""
|
||||
from transformers.audio_utils import is_valid_audio
|
||||
|
||||
@@ -89,9 +82,6 @@ class MistralCommonFeatureExtractor(SequenceFeatureExtractor):
|
||||
if is_valid_audio(audio_url_or_urls):
|
||||
return audio_url_or_urls
|
||||
if isinstance(audio_url_or_urls, (list, tuple)):
|
||||
if audio_url_or_urls and isinstance(audio_url_or_urls[0], float):
|
||||
# A single audio represented as ``list[float]``.
|
||||
return audio_url_or_urls
|
||||
return [
|
||||
self.fetch_audio(x, sampling_rate=sampling_rate)
|
||||
for x in audio_url_or_urls
|
||||
|
||||
Reference in New Issue
Block a user