diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index fc0aa84a5b..186f722253 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -2946,7 +2946,6 @@ steps: - vllm/model_executor/models/qwen3_5_mtp.py - vllm/transformers_utils/configs/qwen3_5.py - vllm/transformers_utils/configs/qwen3_5_moe.py - - vllm/model_executor/models/qwen.py - vllm/model_executor/models/qwen2.py - vllm/model_executor/models/qwen3.py - vllm/model_executor/models/qwen3_next.py @@ -3184,7 +3183,6 @@ steps: - vllm/model_executor/models/qwen3_5_mtp.py - vllm/transformers_utils/configs/qwen3_5.py - vllm/transformers_utils/configs/qwen3_5_moe.py - - vllm/model_executor/models/qwen.py - vllm/model_executor/models/qwen2.py - vllm/model_executor/models/qwen3.py - vllm/model_executor/models/qwen3_next.py diff --git a/docs/configuration/optimization.md b/docs/configuration/optimization.md index 80aec64ee5..42458d5028 100644 --- a/docs/configuration/optimization.md +++ b/docs/configuration/optimization.md @@ -296,7 +296,7 @@ llm = LLM(model="Qwen/Qwen3-8B") The `fastokens` Python package (>= 0.2.0) must be installed; if it isn't, vLLM raises a clear `ImportError` at tokenizer load. The override applies to any `--tokenizer-mode` that ends up loading an HF fast tokenizer (`hf`, -`deepseek_v32`, `deepseek_v4`, `qwen_vl`, …). Models that don't use the HF +`deepseek_v32`, `deepseek_v4`, …). Models that don't use the HF fast tokenizer (`mistral`, `grok2`, `kimi_audio`) ignore the flag. Tokenizer-bound workloads — long shared prefixes, bursty short prompts, diff --git a/docs/contributing/model/multimodal.md b/docs/contributing/model/multimodal.md index 67cde8df98..b48258d539 100644 --- a/docs/contributing/model/multimodal.md +++ b/docs/contributing/model/multimodal.md @@ -884,4 +884,3 @@ Examples: - DeepSeek-VL2: [vllm/model_executor/models/deepseek_vl2.py](../../../vllm/model_executor/models/deepseek_vl2.py) - InternVL: [vllm/model_executor/models/internvl.py](../../../vllm/model_executor/models/internvl.py) -- Qwen-VL: [vllm/model_executor/models/qwen_vl.py](../../../vllm/model_executor/models/qwen_vl.py) diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 6fda22d136..6f7cc6dab4 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -469,7 +469,6 @@ th { | `PersimmonForCausalLM` | Persimmon | `adept/persimmon-8b-base`, `adept/persimmon-8b-chat`, etc. | | ✅︎ | | `Plamo2ForCausalLM` | PLaMo2 | `pfnet/plamo-2-1b`, `pfnet/plamo-2-8b`, etc. | ✅ | ✅︎ | | `Plamo3ForCausalLM` | PLaMo3 | `pfnet/plamo-3-nict-2b-base`, `pfnet/plamo-3-nict-8b-base`, etc. | ✅ | ✅︎ | -| `QWenLMHeadModel` | Qwen | `Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc. | ✅︎ | ✅︎ | | `Qwen2ForCausalLM` | QwQ, Qwen2 | `Qwen/QwQ-32B-Preview`, `Qwen/Qwen2-7B-Instruct`, `Qwen/Qwen2-7B`, etc. | ✅︎ | ✅︎ | | `Qwen2MoeForCausalLM` | Qwen2MoE | `Qwen/Qwen1.5-MoE-A2.7B`, `Qwen/Qwen1.5-MoE-A2.7B-Chat`, etc. | ✅︎ | ✅︎ | | `Qwen3ForCausalLM` | Qwen3 | `Qwen/Qwen3-8B`, etc. | ✅︎ | ✅︎ | @@ -620,7 +619,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `Phi4ForCausalLMV` | Phi-4-reasoning-vision | T + I+ | `microsoft/Phi-4-reasoning-vision-15B`, etc. | | ✅︎ | | `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I+ | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | ✅︎ | ✅︎ | | `QianfanOCRForConditionalGeneration` | QianfanOCR | T + IE+ | `baidu/Qianfan-OCR`, etc. | ✅︎ | ✅︎ | -| `QwenVLForConditionalGeneration`^ | Qwen-VL | T + IE+ | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ | | `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A+ | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ | | `Qwen2VLForConditionalGeneration` Q | QVQ, Qwen2-VL | T + IE+ + VE+ | `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ | | `Qwen2_5_VLForConditionalGeneration` Q | Qwen2.5-VL | T + IE+ + VE+ | `Qwen/Qwen2.5-VL-3B-Instruct`, `Qwen/Qwen2.5-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ | diff --git a/examples/generate/multimodal/vision_language_multi_image_offline.py b/examples/generate/multimodal/vision_language_multi_image_offline.py index 1b68a23b3b..0fb0da1ec9 100644 --- a/examples/generate/multimodal/vision_language_multi_image_offline.py +++ b/examples/generate/multimodal/vision_language_multi_image_offline.py @@ -1042,49 +1042,6 @@ def load_phi4siglip(question: str, image_urls: list[str]) -> ModelRequestData: ) -def load_qwen_vl_chat(question: str, image_urls: list[str]) -> ModelRequestData: - model_name = "Qwen/Qwen-VL-Chat" - engine_args = EngineArgs( - model=model_name, - trust_remote_code=True, - max_model_len=1024, - max_num_seqs=2, - hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]}, - limit_mm_per_prompt={"image": len(image_urls)}, - ) - placeholders = "".join( - f"Picture {i}: \n" for i, _ in enumerate(image_urls, start=1) - ) - - # This model does not have a chat_template attribute on its tokenizer, - # so we need to explicitly pass it. We use ChatML since it's used in the - # generation utils of the model: - # https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/qwen_generation_utils.py#L265 - tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) - - # Copied from: https://huggingface.co/docs/transformers/main/en/chat_templating - chat_template = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" # noqa: E501 - - messages = [{"role": "user", "content": f"{placeholders}\n{question}"}] - prompt = tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True, - chat_template=chat_template, - ) - - stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"] - stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens] - - return ModelRequestData( - engine_args=engine_args, - prompt=prompt, - stop_token_ids=stop_token_ids, - image_data=[fetch_image(url) for url in image_urls], - chat_template=chat_template, - ) - - def load_qwen2_vl(question: str, image_urls: list[str]) -> ModelRequestData: try: from qwen_vl_utils import smart_resize @@ -1544,7 +1501,6 @@ model_example_map = { "phi4_mm": load_phi4mm, "phi4_siglip": load_phi4siglip, "pixtral_hf": load_pixtral_hf, - "qwen_vl_chat": load_qwen_vl_chat, "qwen2_vl": load_qwen2_vl, "qwen2_5_vl": load_qwen2_5_vl, "rvl": load_r_vl, diff --git a/examples/generate/multimodal/vision_language_offline.py b/examples/generate/multimodal/vision_language_offline.py index 4b49d415c1..40a4b8ae6d 100644 --- a/examples/generate/multimodal/vision_language_offline.py +++ b/examples/generate/multimodal/vision_language_offline.py @@ -1938,27 +1938,6 @@ def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData: ) -# Qwen-VL -def run_qwen_vl(questions: list[str], modality: str) -> ModelRequestData: - assert modality == "image" - - engine_args = EngineArgs( - model="Qwen/Qwen-VL", - trust_remote_code=True, - max_model_len=1024, - max_num_seqs=2, - hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]}, - limit_mm_per_prompt={modality: 1}, - ) - - prompts = [f"{question}Picture 1: \n" for question in questions] - - return ModelRequestData( - engine_args=engine_args, - prompts=prompts, - ) - - # Qwen2-VL def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData: model_name = "Qwen/Qwen2-VL-7B-Instruct" @@ -2522,7 +2501,6 @@ model_example_map = { "phi4_mm": run_phi4mm, "phi4_siglip": run_phi4siglip, "pixtral_hf": run_pixtral_hf, - "qwen_vl": run_qwen_vl, "qwen2_vl": run_qwen2_vl, "qwen2_5_vl": run_qwen2_5_vl, "qwen2_5_omni": run_qwen2_5_omni, diff --git a/tests/distributed/test_pipeline_parallel.py b/tests/distributed/test_pipeline_parallel.py index c2dda1b51c..b495a9ed26 100644 --- a/tests/distributed/test_pipeline_parallel.py +++ b/tests/distributed/test_pipeline_parallel.py @@ -192,7 +192,6 @@ MULTIMODAL_MODELS = { "AIDC-AI/Ovis2.5-2B": PPTestSettings.fast(), "microsoft/Phi-3.5-vision-instruct": PPTestSettings.fast(), "mistralai/Pixtral-12B-2409": PPTestSettings.fast(load_format="dummy"), - "Qwen/Qwen-VL-Chat": PPTestSettings.fast(), "Qwen/Qwen2-Audio-7B-Instruct": PPTestSettings.fast(), "Qwen/Qwen2-VL-2B-Instruct": PPTestSettings.fast(), "fixie-ai/ultravox-v0_5-llama-3_2-1b": PPTestSettings.fast(), diff --git a/tests/models/multimodal/conftest.py b/tests/models/multimodal/conftest.py index 9283556d30..d00c3df786 100644 --- a/tests/models/multimodal/conftest.py +++ b/tests/models/multimodal/conftest.py @@ -5,21 +5,11 @@ import os import warnings -import pytest import torch -from tests.utils import prewarm_hf_cache from vllm.platforms import current_platform -@pytest.fixture(scope="session", autouse=True) -def _prewarm_hf_cache(): - # tokenization_qwen.py downloads SimSun.ttf from - # qianwen-res.oss-cn-beijing.aliyuncs.com; both Qwen/Qwen-VL and - # Qwen/Qwen-VL-Chat look it up from the Chat repo. - prewarm_hf_cache([("Qwen/Qwen-VL-Chat", "SimSun.ttf")]) - - def pytest_configure(config): """Early ROCm configuration that must happen before test collection.""" if not current_platform.is_rocm(): diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index 9ac0d4ab44..e2dd0d9de7 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -976,16 +976,6 @@ VLM_TEST_SETTINGS = { auto_cls=AutoModelForImageTextToText, hf_model_kwargs=model_utils.qianfan_ocr_hf_model_kwargs("baidu/Qianfan-OCR"), ), - "qwen_vl": VLMTestInfo( - models=["Qwen/Qwen-VL"], - test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE), - prompt_formatter=identity, - img_idx_to_prompt=lambda idx: f"Picture {idx}: \n", - max_model_len=1024, - max_num_seqs=2, - vllm_output_post_proc=model_utils.qwen_vllm_to_hf_output, - prompt_path_encoder=model_utils.qwen_prompt_path_encoder, - ), "qwen2_vl": VLMTestInfo( models=["Qwen/Qwen2-VL-2B-Instruct"], test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO), diff --git a/tests/models/registry.py b/tests/models/registry.py index e4b7851c88..d2d2794962 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -505,14 +505,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "pfnet/plamo-3-nict-2b-base", trust_remote_code=True, ), - "QWenLMHeadModel": _HfExamplesInfo( - "Qwen/Qwen-7B-Chat", - max_transformers_version="4.53", - transformers_version_reason={ - "hf": "HF model uses remote code that is not compatible with latest Transformers" # noqa: E501 - }, - trust_remote_code=True, - ), "Qwen2ForCausalLM": _HfExamplesInfo( "Qwen/Qwen2-0.5B-Instruct", extras={ @@ -1297,16 +1289,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { "baidu/Qianfan-OCR", min_transformers_version="5.6.0", ), - "QwenVLForConditionalGeneration": _HfExamplesInfo( - "Qwen/Qwen-VL", - extras={"chat": "Qwen/Qwen-VL-Chat"}, - trust_remote_code=True, - max_transformers_version="4.53.3", - transformers_version_reason={ - "hf": "HF model uses deprecated imports which have been removed." - }, # noqa: E501 - hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]}, - ), "Qwen2AudioForConditionalGeneration": _HfExamplesInfo( "Qwen/Qwen2-Audio-7B-Instruct" ), diff --git a/tests/renderers/conftest.py b/tests/renderers/conftest.py deleted file mode 100644 index c33ab35160..0000000000 --- a/tests/renderers/conftest.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest - -from tests.utils import prewarm_hf_cache - - -@pytest.fixture(scope="session", autouse=True) -def _prewarm_hf_cache(): - # tokenization_qwen.py downloads SimSun.ttf from - # qianwen-res.oss-cn-beijing.aliyuncs.com; both Qwen/Qwen-VL and - # Qwen/Qwen-VL-Chat look it up from the Chat repo. - prewarm_hf_cache([("Qwen/Qwen-VL-Chat", "SimSun.ttf")]) diff --git a/tests/renderers/test_hf.py b/tests/renderers/test_hf.py index 0545457eb7..f48a320840 100644 --- a/tests/renderers/test_hf.py +++ b/tests/renderers/test_hf.py @@ -428,8 +428,6 @@ def test_resolve_content_format_hf_defined(model, expected_format): ("deepseek-ai/deepseek-vl2-tiny", "string"), ("adept/fuyu-8b", "string"), ("google/paligemma-3b-mix-224", "string"), - ("Qwen/Qwen-VL", "string"), - ("Qwen/Qwen-VL-Chat", "string"), ], ) def test_resolve_content_format_fallbacks(model, expected_format): diff --git a/tests/tokenizers_/conftest.py b/tests/tokenizers_/conftest.py deleted file mode 100644 index c33ab35160..0000000000 --- a/tests/tokenizers_/conftest.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest - -from tests.utils import prewarm_hf_cache - - -@pytest.fixture(scope="session", autouse=True) -def _prewarm_hf_cache(): - # tokenization_qwen.py downloads SimSun.ttf from - # qianwen-res.oss-cn-beijing.aliyuncs.com; both Qwen/Qwen-VL and - # Qwen/Qwen-VL-Chat look it up from the Chat repo. - prewarm_hf_cache([("Qwen/Qwen-VL-Chat", "SimSun.ttf")]) diff --git a/tests/tokenizers_/test_basic.py b/tests/tokenizers_/test_basic.py index cf0d8f53c6..c3549e2c94 100644 --- a/tests/tokenizers_/test_basic.py +++ b/tests/tokenizers_/test_basic.py @@ -47,14 +47,6 @@ def test_tokenizer_like_protocol(): assert "DSV32" in tokenizer.__class__.__name__ _assert_tokenizer_like(tokenizer) - tokenizer = get_tokenizer( - "Qwen/Qwen-VL", - tokenizer_mode="qwen_vl", - trust_remote_code=True, - ) - assert isinstance(tokenizer, HfTokenizer) - assert "WithoutImagePad" in tokenizer.__class__.__name__ - @pytest.mark.parametrize("tokenizer_name", ["facebook/opt-125m", "gpt2"]) def test_tokenizer_revision(tokenizer_name: str): diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 8fa29f4df0..cbf7be44ae 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -1440,7 +1440,6 @@ def add_cli_args(parser: argparse.ArgumentParser): - "slow" will always use the slow tokenizer.\n - "mistral" will always use the tokenizer from `mistral_common`.\n - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n - - "qwen_vl" will always use the tokenizer from `qwen_vl`.\n - Other custom values can be supported via plugins.""", ) parser.add_argument("--use-beam-search", action="store_true") diff --git a/vllm/config/model.py b/vllm/config/model.py index 2f22140be9..015e75afac 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -135,14 +135,12 @@ class ModelConfig: - "mistral" will always use the tokenizer from `mistral_common`. - "deepseek_v32" will always use the tokenizer from `deepseek_v32`. - "deepseek_v4" will always use the tokenizer from `deepseek_v4`. - - "qwen_vl" will always use the tokenizer from `qwen_vl`. - Other custom values can be supported via plugins. To swap the Rust BPE backend that powers HF fast tokenizers for the [fastokens](https://github.com/crusoecloud/fastokens) implementation, set `VLLM_USE_FASTOKENS=1` instead — that override applies to any mode that - loads an HF fast tokenizer (`hf`, `deepseek_v32`, `deepseek_v4`, - `qwen_vl`, …).""" + loads an HF fast tokenizer (`hf`, `deepseek_v32`, `deepseek_v4`, …).""" trust_remote_code: bool = False """Trust remote code (e.g., from HuggingFace) when downloading the model and tokenizer.""" @@ -618,8 +616,6 @@ class ModelConfig: self.tokenizer_mode = "grok2" elif arch == "MoonshotKimiaForCausalLM": self.tokenizer_mode = "kimi_audio" - elif arch == "QwenVLForConditionalGeneration": - self.tokenizer_mode = "qwen_vl" elif arch == "DeepseekV32ForCausalLM": self.tokenizer_mode = "deepseek_v32" elif arch == "DeepseekV4ForCausalLM": diff --git a/vllm/envs.py b/vllm/envs.py index 58c348d7b3..17c3ffc2a8 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -686,7 +686,7 @@ environment_variables: dict[str, Callable[[], Any]] = { # If true, replace the Rust BPE backend that powers HF fast tokenizers # with the `fastokens` (https://github.com/crusoecloud/fastokens) shim. # Applies to any tokenizer mode that loads an HF fast tokenizer - # (`hf`, `deepseek_v32`, `deepseek_v4`, `qwen_vl`, …). The `fastokens` + # (`hf`, `deepseek_v32`, `deepseek_v4`, …). The `fastokens` # Python package must be installed. "VLLM_USE_FASTOKENS": lambda: bool(int(os.getenv("VLLM_USE_FASTOKENS", "0"))), # Interval in seconds to log a warning message when the ring buffer is full diff --git a/vllm/model_executor/models/qwen.py b/vllm/model_executor/models/qwen.py deleted file mode 100644 index b4526beac6..0000000000 --- a/vllm/model_executor/models/qwen.py +++ /dev/null @@ -1,377 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Adapted from -# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py -# Copyright (c) Alibaba Cloud. -# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE -"""Inference-only QWen model compatible with HuggingFace weights.""" - -import json -from collections.abc import Iterable -from itertools import islice -from typing import Any - -import torch -from torch import nn -from transformers import PretrainedConfig - -from vllm.compilation.decorators import support_torch_compile -from vllm.config import CacheConfig, VllmConfig -from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size -from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import ( - MergedColumnParallelLinear, - QKVParallelLinear, - RowParallelLinear, -) -from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.layers.vocab_parallel_embedding import ( - ParallelLMHead, - VocabParallelEmbedding, -) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader -from vllm.sequence import IntermediateTensors - -from .interfaces import SupportsLoRA, SupportsPP -from .utils import ( - is_pp_missing_parameter, - make_empty_intermediate_tensors_factory, - make_layers, - maybe_prefix, -) - - -class QWenMLP(nn.Module): - """MLP for the language component of the Qwen model, which contains a - MergedColumnParallelLinear merging 2 outputs via silu activation.""" - - def __init__( - self, - hidden_size: int, - intermediate_size: int, - hidden_act: str = "silu", - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - self.gate_up_proj = MergedColumnParallelLinear( - hidden_size, - [intermediate_size] * 2, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.gate_up_proj", - ) - self.c_proj = RowParallelLinear( - intermediate_size, - hidden_size, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.c_proj", - ) - if hidden_act != "silu": - raise ValueError( - f"Unsupported activation: {hidden_act}. Only silu is supported for now." - ) - self.act_fn = SiluAndMul() - - def forward(self, x: torch.Tensor) -> torch.Tensor: - gate_up, _ = self.gate_up_proj(x) - x = self.act_fn(gate_up) - x, _ = self.c_proj(x) - return x - - -class QWenAttention(nn.Module): - def __init__( - self, - hidden_size: int, - num_heads: int, - max_position_embeddings: int, - rope_parameters: dict[str, Any] | None = None, - cache_config: CacheConfig | None = None, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - self.hidden_size = hidden_size - tensor_model_parallel_world_size = get_tensor_model_parallel_world_size() - self.total_num_heads = num_heads - assert self.total_num_heads % tensor_model_parallel_world_size == 0 - self.num_heads = self.total_num_heads // tensor_model_parallel_world_size - self.head_dim = hidden_size // self.total_num_heads - self.c_attn = QKVParallelLinear( - hidden_size, - self.head_dim, - self.total_num_heads, - bias=True, - quant_config=quant_config, - prefix=f"{prefix}.c_attn", - ) - self.c_proj = RowParallelLinear( - self.total_num_heads * self.head_dim, - hidden_size, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.c_proj", - ) - self.scaling = self.head_dim**-0.5 - - self.rotary_emb = get_rope( - self.head_dim, - max_position=max_position_embeddings, - rope_parameters=rope_parameters, - ) - self.attn = Attention( - self.num_heads, - self.head_dim, - self.scaling, - cache_config=cache_config, - quant_config=quant_config, - prefix=f"{prefix}.attn", - ) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - ) -> torch.Tensor: - qkv, _ = self.c_attn(hidden_states) - q, k, v = qkv.chunk(chunks=3, dim=-1) - q, k = self.rotary_emb(positions, q, k) - attn_output = self.attn(q, k, v) - output, _ = self.c_proj(attn_output) - return output - - -class QWenBlock(nn.Module): - def __init__( - self, - config: PretrainedConfig, - cache_config: CacheConfig | None = None, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) - - self.attn = QWenAttention( - config.hidden_size, - config.num_attention_heads, - config.max_position_embeddings, - rope_parameters=config.rope_parameters, - cache_config=cache_config, - quant_config=quant_config, - prefix=f"{prefix}.attn", - ) - - self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) - - self.mlp = QWenMLP( - config.hidden_size, - config.intermediate_size // 2, - quant_config=quant_config, - prefix=f"{prefix}.mlp", - ) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - residual: torch.Tensor | None, - ) -> tuple[torch.Tensor, torch.Tensor]: - # Self Attention - if residual is None: - residual = hidden_states - hidden_states = self.ln_1(hidden_states) - else: - hidden_states, residual = self.ln_1(hidden_states, residual) - hidden_states = self.attn( - positions=positions, - hidden_states=hidden_states, - ) - - # Fully Connected - hidden_states, residual = self.ln_2(hidden_states, residual) - hidden_states = self.mlp(hidden_states) - return hidden_states, residual - - -@support_torch_compile -class QWenModel(nn.Module): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - - config = vllm_config.model_config.hf_config - cache_config = vllm_config.cache_config - quant_config = vllm_config.quant_config - - self.config = config - self.vocab_size = config.vocab_size - - self.wte = VocabParallelEmbedding( - config.vocab_size, - config.hidden_size, - ) - self.start_layer, self.end_layer, self.h = make_layers( - config.num_hidden_layers, - lambda prefix: QWenBlock(config, cache_config, quant_config, prefix=prefix), - prefix=f"{prefix}.h", - ) - self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) - self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( - ["hidden_states", "residual"], config.hidden_size - ) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.wte(input_ids) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None, - inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: - if get_pp_group().is_first_rank: - if inputs_embeds is not None: - hidden_states = inputs_embeds - else: - hidden_states = self.embed_input_ids(input_ids) - residual = None - else: - assert intermediate_tensors is not None - hidden_states = intermediate_tensors["hidden_states"] - residual = intermediate_tensors["residual"] - - for layer in islice(self.h, self.start_layer, self.end_layer): - hidden_states, residual = layer( - positions, - hidden_states, - residual, - ) - if not get_pp_group().is_last_rank: - return IntermediateTensors( - {"hidden_states": hidden_states, "residual": residual} - ) - hidden_states, _ = self.ln_f(hidden_states, residual) - return hidden_states - - -class QWenBaseModel(nn.Module): - def __init__( - self, - *, - vllm_config: VllmConfig, - prefix: str = "", - transformer_type: type[QWenModel] = QWenModel, - ) -> None: - super().__init__() - config = vllm_config.model_config.hf_config - quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config - self.config = config - self.multimodal_config = multimodal_config - self.quant_config = quant_config - self.transformer = transformer_type( - vllm_config=vllm_config, prefix=maybe_prefix(prefix, "transformer") - ) - self.lm_head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - quant_config=quant_config, - prefix=maybe_prefix(prefix, "lm_head"), - ) - if self.config.tie_word_embeddings: - self.lm_head.weight = self.transformer.wte.weight - self.logits_processor = LogitsProcessor(config.vocab_size) - self.make_empty_intermediate_tensors = ( - self.transformer.make_empty_intermediate_tensors - ) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.transformer.wte(input_ids) - - def compute_logits( - self, - hidden_states: torch.Tensor, - ) -> torch.Tensor | None: - logits = self.logits_processor(self.lm_head, hidden_states) - return logits - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("gate_up_proj", "w2", 0), - ("gate_up_proj", "w1", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - - -class QWenLMHeadModel(QWenBaseModel, SupportsPP, SupportsLoRA): - packed_modules_mapping = { - "c_attn": ["c_attn"], - "gate_up_proj": [ - "w2", - "w1", - ], - } - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - config = vllm_config.model_config.hf_config - if hasattr(config, "visual"): - hf_overrides = {"architectures": ["QwenVLForConditionalGeneration"]} - raise RuntimeError( - "The configuration of this model indicates that it supports " - "vision inputs, but you instantiated the text-only version " - "of this model. Please use the vision model by setting " - f"`--hf-overrides '{json.dumps(hf_overrides)}'`" - ) - - super().__init__(vllm_config=vllm_config, prefix=prefix) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: - hidden_states = self.transformer( - input_ids, positions, intermediate_tensors, inputs_embeds - ) - return hidden_states diff --git a/vllm/model_executor/models/qwen_vl.py b/vllm/model_executor/models/qwen_vl.py deleted file mode 100644 index e2232956ea..0000000000 --- a/vllm/model_executor/models/qwen_vl.py +++ /dev/null @@ -1,688 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Adapted from -# https://huggingface.co/Qwen/Qwen-VL/blob/main/modeling_qwen.py -# Copyright (c) Alibaba Cloud. -"""Inference-only Qwen-VL model compatible with HuggingFace weights.""" - -import math -from collections.abc import Callable, Mapping, Sequence -from functools import partial -from typing import Annotated, Literal, TypeAlias - -import regex as re -import torch -from torch import nn -from transformers import BatchFeature - -from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions -from vllm.inputs import MultiModalDataDict -from vllm.model_executor.layers.activation import get_act_fn -from vllm.model_executor.layers.conv import Conv2dLayer -from vllm.model_executor.layers.linear import ( - ColumnParallelLinear, - ReplicatedLinear, - RowParallelLinear, -) -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos -from vllm.model_executor.models.module_mapping import MultiModelKeys -from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import ( - MultiModalFieldConfig, - MultiModalKwargsItems, -) -from vllm.multimodal.parse import MultiModalDataItems -from vllm.multimodal.processing import ( - BaseDummyInputsBuilder, - BaseMultiModalProcessor, - BaseProcessingInfo, - PromptReplacement, - PromptUpdate, - PromptUpdateDetails, -) -from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.processors.qwen_vl import ( - QwenVLImageProcessorFast, - QwenVLProcessor, -) -from vllm.utils.tensor_schema import TensorSchema, TensorShape - -from .interfaces import ( - MultiModalEmbeddings, - SupportsLoRA, - SupportsMultiModal, - SupportsPP, -) -from .qwen import QWenBaseModel, QWenBlock, QWenModel - - -class QwenImagePixelInputs(TensorSchema): - """ - Dimensions: - - bn: Batch size * number of images - - c: Number of channels (3) - - h: Height - - w: Width - - Note that image_size is the value in the vision config to which we resize - the image to in the normalization transform. Currently multi-image support - can only be leveraged by passing image embeddings directly. - """ - - type: Literal["pixel_values"] = "pixel_values" - data: Annotated[torch.Tensor, TensorShape("bn", 3, "h", "w")] - - -class QwenImageEmbeddingInputs(TensorSchema): - """ - Dimensions: - - bn: Batch size * number of images - - ifs: Image feature size (256) - - hs: Hidden size - - `hidden_size` must match the hidden size of the language model backbone - and is stored in the visual config of the model if we have one. - """ - - type: Literal["image_embeds"] = "image_embeds" - data: Annotated[torch.Tensor, TensorShape("bn", 256, "hs")] - - -QwenImageInputs: TypeAlias = QwenImagePixelInputs | QwenImageEmbeddingInputs - - -class VisualAttention(nn.Module): - """self-attention layer class. - Self-attention layer takes input with size [s, b, h] - and returns output of the same size. - """ - - def __init__( - self, - embed_dim: int, - num_heads: int, - bias: bool = True, - kdim: int | None = None, - vdim: int | None = None, - prefix: str = "", - ): - super().__init__() - self.embed_dim = embed_dim - self.kdim = kdim if kdim is not None else embed_dim - self.vdim = vdim if vdim is not None else embed_dim - self._qkv_same_embed_dim = self.kdim == embed_dim and self.vdim == embed_dim - - self.num_heads = num_heads - - # Per attention head and per partition values. - assert embed_dim % num_heads == 0 - self.hidden_size_per_attention_head = embed_dim // num_heads - self.num_attention_heads_per_partition = num_heads - self.hidden_size_per_partition = embed_dim - - # Strided linear layer. - assert self._qkv_same_embed_dim, ( - "Visual Attention implementation only supports self-attention" - ) - self.in_proj = ReplicatedLinear( - embed_dim, 3 * embed_dim, prefix=f"{prefix}.in_proj" - ) - self.out_proj = ReplicatedLinear( - embed_dim, embed_dim, prefix=f"{prefix}.out_proj" - ) - self.norm_factor = math.sqrt(self.hidden_size_per_attention_head) - - def forward( - self, - x: torch.Tensor, - attn_mask: torch.Tensor | None = None, - ) -> torch.Tensor: - # query/key/value: [sq, b, h] - sq, b, _ = x.size() - mixed_x_layer, _ = self.in_proj(x) - - # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn] - new_tensor_shape = mixed_x_layer.size()[:-1] + ( - self.num_attention_heads_per_partition, - 3 * self.hidden_size_per_attention_head, - ) - mixed_x_layer = mixed_x_layer.view(*new_tensor_shape) - - # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn] - query_layer, key_layer, value_layer = mixed_x_layer.split( - self.hidden_size_per_attention_head, dim=-1 - ) - - # [sq, b, np, hn] -> [sq, b * np, hn] - query_layer = query_layer.view( - sq, - b * self.num_attention_heads_per_partition, - self.hidden_size_per_attention_head, - ).transpose(0, 1) - # [sk, b, np, hn] -> [sk, b * np, hn] - key_layer = key_layer.view( - sq, - b * self.num_attention_heads_per_partition, - self.hidden_size_per_attention_head, - ).transpose(0, 1) - - q_scaled = query_layer / self.norm_factor - if attn_mask is not None: - attention_probs = torch.baddbmm( - attn_mask, q_scaled, key_layer.transpose(-2, -1) - ) - else: - attention_probs = torch.bmm(q_scaled, key_layer.transpose(-2, -1)) - attention_probs = attention_probs.softmax(dim=-1) - - value_layer = value_layer.view( - sq, - b * self.num_attention_heads_per_partition, - self.hidden_size_per_attention_head, - ).transpose(0, 1) - - # matmul: [b * np, sq, hn] - context_layer = torch.bmm(attention_probs, value_layer) - - # change view [b, np, sq, hn] - context_layer = context_layer.view( - b, - self.num_attention_heads_per_partition, - sq, - self.hidden_size_per_attention_head, - ) - - # [b, np, sq, hn] --> [sq, b, np, hn] - context_layer = context_layer.permute(2, 0, 1, 3).contiguous() - - # [sq, b, np, hn] --> [sq, b, hp] - new_context_layer_shape = context_layer.size()[:-2] + ( - self.hidden_size_per_partition, - ) - context_layer = context_layer.view(*new_context_layer_shape) - - output, _ = self.out_proj(context_layer) - - return output - - -class QwenVLMLP(nn.Module): - """MLP for the visual component of the Qwen model.""" - - def __init__( - self, - hidden_size: int, - intermediate_size: int, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - self.c_fc = ColumnParallelLinear( - hidden_size, - intermediate_size, - bias=True, - quant_config=quant_config, - prefix=f"{prefix}.c_fc", - ) - self.act_fn = get_act_fn("gelu") - self.c_proj = RowParallelLinear( - intermediate_size, - hidden_size, - bias=True, - quant_config=quant_config, - prefix=f"{prefix}.c_proj", - ) - - def forward(self, x): - x, _ = self.c_fc(x) - x = self.act_fn(x) - x, _ = self.c_proj(x) - return x - - -class VisualAttentionBlock(nn.Module): - def __init__( - self, - d_model: int, - n_head: int, - mlp_ratio: float = 4.0, - norm_layer: Callable[[int], nn.Module] = nn.LayerNorm, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - - self.ln_1 = norm_layer(d_model) - self.ln_2 = norm_layer(d_model) - mlp_width = int(d_model * mlp_ratio) - self.attn = VisualAttention(d_model, n_head, prefix=f"{prefix}.attn") - self.mlp = QwenVLMLP( - hidden_size=d_model, - intermediate_size=mlp_width, - quant_config=quant_config, - prefix=f"{prefix}.mlp", - ) - - def attention( - self, - x: torch.Tensor, - attn_mask: torch.Tensor | None = None, - ) -> torch.Tensor: - attn_mask = attn_mask.to(x.dtype) if attn_mask is not None else None - return self.attn(x, attn_mask=attn_mask) - - def forward( - self, - x: torch.Tensor, - attn_mask: torch.Tensor | None = None, - ) -> torch.Tensor: - x = x + self.attention(self.ln_1(x), attn_mask=attn_mask) - x = x + self.mlp(self.ln_2(x)) - return x - - -class TransformerBlock(nn.Module): - def __init__( - self, - width: int, - layers: int, - heads: int, - mlp_ratio: float = 4.0, - norm_layer: Callable[[int], nn.Module] = nn.LayerNorm, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - self.width = width - self.layers = layers - - self.resblocks = nn.ModuleList( - [ - VisualAttentionBlock( - width, - heads, - mlp_ratio, - norm_layer=norm_layer, - quant_config=quant_config, - prefix=f"{prefix}.resblocks.{i}", - ) - for i in range(layers) - ] - ) - - def get_cast_dtype(self) -> torch.dtype: - return self.resblocks[0].mlp.c_fc.weight.dtype - - def get_cast_device(self) -> torch.device: - return self.resblocks[0].mlp.c_fc.weight.device - - def forward( - self, x: torch.Tensor, attn_mask: torch.Tensor | None = None - ) -> torch.Tensor: - for r in self.resblocks: - x = r(x, attn_mask=attn_mask) - return x - - -class VisionTransformer(nn.Module): - def __init__( - self, - image_size: int, - patch_size: int, - width: int, - layers: int, - heads: int, - mlp_ratio: float, - n_queries: int = 256, - output_dim: int = 512, - image_start_id: int = 151857, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - **kwargs, - ): - super().__init__() - image_height, image_width = self.image_size = (image_size, image_size) - patch_height, patch_width = self.patch_size = (patch_size, patch_size) - self.grid_size = (image_height // patch_height, image_width // patch_width) - self.output_dim = output_dim - self.conv1 = Conv2dLayer( - in_channels=3, - out_channels=width, - kernel_size=patch_size, - stride=patch_size, - bias=False, - ) - - # class embeddings and positional embeddings - scale = width**-0.5 - self.positional_embedding = nn.Parameter(scale * torch.randn(256, width)) - - norm_layer = partial(nn.LayerNorm, eps=1e-6) - - self.ln_pre = norm_layer(width) - self.transformer = TransformerBlock( - width, - layers, - heads, - mlp_ratio, - norm_layer=norm_layer, - quant_config=quant_config, - prefix=f"{prefix}.transformer", - ) - - self.attn_pool = Resampler2( - grid_size=int(math.sqrt(n_queries)), - embed_dim=output_dim, - num_heads=output_dim // 128, - kv_dim=width, - norm_layer=norm_layer, - adaptive=False, - do_post_projection=False, - prefix=f"{prefix}.attn_pool", - ).to( - device=self.positional_embedding.device, - dtype=self.positional_embedding.dtype, - ) - - self.ln_post = norm_layer(output_dim) - self.proj = nn.Parameter( - (output_dim**-0.5) * torch.randn(output_dim, output_dim) - ) - - self.image_start_id = image_start_id - self.image_end_id = image_start_id + 1 - self.image_pad_id = image_start_id + 2 - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = x.to( - dtype=self.transformer.get_cast_dtype(), - device=self.transformer.get_cast_device(), - ) - - # to patches - x = self.conv1(x) # shape = [*, width, grid, grid] - x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2] - x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] - - x = x + get_abs_pos(self.positional_embedding, int(math.sqrt(x.size(1)))) - - x = self.ln_pre(x) - - x = x.permute(1, 0, 2) # NLD -> LND - x = self.transformer(x) - x = x.permute(1, 0, 2) # LND -> NLD - - x = self.attn_pool(x) - x = self.ln_post(x) - x = x @ self.proj - - return x - - -class QwenVLModel(QWenModel): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__(vllm_config=vllm_config, prefix=prefix) - - config = vllm_config.model_config.hf_config - quant_config = vllm_config.quant_config - - self.visual = VisionTransformer( - **config.visual, quant_config=quant_config, prefix=f"{prefix}.visual" - ) - - -class QwenVLProcessingInfo(BaseProcessingInfo): - def get_image_processor(self, **kwargs): - config = self.get_hf_config() - vision_config = config.visual - - image_size = vision_config["image_size"] - kwargs = self.ctx.get_merged_mm_kwargs(kwargs) - kwargs.setdefault("size", {"width": image_size, "height": image_size}) - - return QwenVLImageProcessorFast(**kwargs) - - def get_hf_processor(self, **kwargs: object) -> QwenVLProcessor: - return QwenVLProcessor( - tokenizer=self.get_tokenizer(), - image_processor=self.get_image_processor(**kwargs), - ) - - def get_supported_mm_limits(self) -> Mapping[str, int | None]: - return {"image": None} - - def get_num_image_tokens(self) -> int: - hf_config = self.get_hf_config() - vision_config = hf_config.visual - - image_size = vision_config["image_size"] - patch_size = vision_config["patch_size"] - grid_length = image_size // patch_size // 2 - return grid_length * grid_length - - -class QwenVLDummyInputsBuilder(BaseDummyInputsBuilder[QwenVLProcessingInfo]): - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: - num_images = mm_counts.get("image", 0) - - hf_processor = self.info.get_hf_processor() - img_start = hf_processor.image_start_tag - img_end = hf_processor.image_end_tag - - return "".join( - f"Picture {i}: {img_start}{img_end}\n" for i in range(1, num_images + 1) - ) - - def get_dummy_mm_data( - self, - seq_len: int, - mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], - ) -> MultiModalDataDict: - hf_config = self.info.get_hf_config() - vision_config = hf_config.visual - - target_width = target_height = vision_config["image_size"] - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") - - return { - "image": self._get_dummy_images( - width=target_width, - height=target_height, - num_images=num_images, - overrides=image_overrides, - ) - } - - -class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], - ) -> BatchFeature: - # Drops anything between / tags; encoding with the tokenizer - # will automatically add the image pads for the context. - prompt, num_matched_images = re.subn( - r"(Picture \d*: ).*?(<\/img>\n)", - r"\1\2", - prompt, - ) - - image_data = mm_data.get("images") - if image_data is not None: - assert isinstance(image_data, list) - - num_images = len(image_data) - assert num_matched_images == num_images - - return super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, - ) - - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - return False - - def _get_mm_fields_config( - self, - hf_inputs: BatchFeature, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> Mapping[str, MultiModalFieldConfig]: - return dict( - pixel_values=MultiModalFieldConfig.batched("image"), - image_embeds=MultiModalFieldConfig.batched("image"), - ) - - def _get_prompt_updates( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - out_mm_kwargs: MultiModalKwargsItems, - ) -> Sequence[PromptUpdate]: - tokenizer = self.info.get_tokenizer() - special_tokens: dict[str, int] = tokenizer.special_tokens # type: ignore - - processor = self.info.get_hf_processor() - img_start_id = special_tokens[processor.image_start_tag] - img_end_id = special_tokens[processor.image_end_tag] - img_pad_id = special_tokens[processor.image_pad_tag] - - num_image_tokens = self.info.get_num_image_tokens() - image_tokens = [img_pad_id] * num_image_tokens - - return [ - PromptReplacement( - modality="image", - target=[img_start_id, img_end_id], - replacement=PromptUpdateDetails.select_token_id( - [img_start_id] + image_tokens + [img_end_id], - embed_token_id=img_pad_id, - ), - ) - ] - - -@MULTIMODAL_REGISTRY.register_processor( - QwenVLMultiModalProcessor, - info=QwenVLProcessingInfo, - dummy_inputs=QwenVLDummyInputsBuilder, -) -class QwenVLForConditionalGeneration( - QWenBaseModel, SupportsPP, SupportsLoRA, SupportsMultiModal -): - packed_modules_mapping = { - "c_attn": ["c_attn"], - "gate_up_proj": [ - "w2", - "w1", - ], - } - - embed_input_ids = SupportsMultiModal.embed_input_ids - - def get_mm_mapping(self) -> MultiModelKeys: - """ - Get the module prefix in multimodal models - """ - return MultiModelKeys.from_string_field( - language_model="transformer.h", - connector="transformer.visual.attn_pool", - tower_model="transformer.visual.transformer", - ) - - @classmethod - def get_placeholder_str(cls, modality: str, i: int) -> str | None: - if modality.startswith("image"): - return f"Picture {i}: " - - raise ValueError("Only image modality is supported") - - def __init__( - self, - *, - vllm_config: VllmConfig, - prefix: str = "", - transformer_type: type[QwenVLModel] = QwenVLModel, - ) -> None: - with self._mark_composite_model( - vllm_config, - language_targets=QWenBlock, - tower_targets={"image": VisionTransformer}, - ): - super().__init__( - vllm_config=vllm_config, - prefix=prefix, - transformer_type=transformer_type, - ) - - self.transformer: QwenVLModel - - def _parse_and_validate_image_input( - self, **kwargs: object - ) -> QwenImageInputs | None: - pixel_values = kwargs.pop("pixel_values", None) - image_embeds = kwargs.pop("image_embeds", None) - - if pixel_values is not None: - expected_h = expected_w = self.config.visual["image_size"] - resolve_bindings = {"h": expected_h, "w": expected_w} - - return QwenImagePixelInputs( - type="pixel_values", - data=pixel_values, - resolve_bindings=resolve_bindings, - ) - - if image_embeds is not None: - return QwenImageEmbeddingInputs( - type="image_embeds", - data=image_embeds, - ) - - return None - - def _process_image_input(self, image_input: QwenImageInputs) -> torch.Tensor: - if image_input["type"] == "image_embeds": - return image_input["data"] - - return self.transformer.visual(image_input["data"]) - - def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: - image_input = self._parse_and_validate_image_input(**kwargs) - if image_input is None: - return [] - - vision_embeddings = self._process_image_input(image_input) - return vision_embeddings - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - **kwargs: object, - ) -> torch.Tensor | IntermediateTensors: - if intermediate_tensors is not None: - inputs_embeds = None - - hidden_states = self.transformer( - input_ids, positions, intermediate_tensors, inputs_embeds - ) - return hidden_states diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index da7049eee6..e1ce0efae2 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -197,7 +197,6 @@ _TEXT_GENERATION_MODELS = { "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"), "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"), "Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"), - "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"), "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"), "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"), "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"), @@ -531,7 +530,6 @@ _MULTIMODAL_MODELS = { "qianfan_ocr", "QianfanOCRForConditionalGeneration", ), - "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"), "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), "Qwen2_5_VLForConditionalGeneration": ( "qwen2_5_vl", @@ -715,6 +713,8 @@ _PREVIOUSLY_SUPPORTED_MODELS = { "ErnieModel": "0.23.0", "ErnieForSequenceClassification": "0.23.0", "ErnieForTokenClassification": "0.23.0", + "QWenLMHeadModel": "0.23.0", + "QwenVLForConditionalGeneration": "0.23.0", # encoder-decoder models except whisper # have been removed for V0 deprecation. "DonutForConditionalGeneration": "0.10.2", diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index 8263dd713a..a6da9ec501 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -26,7 +26,6 @@ _VLLM_RENDERERS = { "hf": ("hf", "HfRenderer"), "kimi_audio": ("hf", "HfRenderer"), "mistral": ("mistral", "MistralRenderer"), - "qwen_vl": ("hf", "HfRenderer"), "terratorch": ("terratorch", "TerratorchRenderer"), } diff --git a/vllm/tokenizers/fastokens.py b/vllm/tokenizers/fastokens.py index 5f080a549d..8adf1d94f2 100644 --- a/vllm/tokenizers/fastokens.py +++ b/vllm/tokenizers/fastokens.py @@ -7,7 +7,7 @@ the inner Rust tokenizer of every HF fast tokenizer loaded afterwards with the fastokens shim and rebinds ``tokenizers.decoders.DecodeStream`` so the streaming detokenizer accepts the shim. The patch is process-global and idempotent, so it applies to any tokenizer mode that ends up loading an HF -fast tokenizer (`hf`, `deepseek_v32`, `deepseek_v4`, `qwen_vl`, …). +fast tokenizer (`hf`, `deepseek_v32`, `deepseek_v4`, …). """ from importlib.metadata import PackageNotFoundError, version diff --git a/vllm/tokenizers/qwen_vl.py b/vllm/tokenizers/qwen_vl.py deleted file mode 100644 index f36a22b025..0000000000 --- a/vllm/tokenizers/qwen_vl.py +++ /dev/null @@ -1,71 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import copy -import unicodedata -from collections.abc import Collection, Set - -from transformers import AutoTokenizer - -from .hf import HfTokenizer, get_cached_tokenizer -from .protocol import TokenizerLike - - -def get_qwen_vl_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: - """ - The logic of adding image pad tokens should only be applied in - `QwenVLProcessor`, so they are patched out here. - - The definition of the wrapped tokenizer can be found here: - https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py - """ - new_tokenizer = copy.copy(tokenizer) - - class TokenizerWithoutImagePad(tokenizer.__class__): # type: ignore - def tokenize( - self, - text: str, - allowed_special: Set[str] | str = "all", - disallowed_special: Collection[str] | str = (), - **kwargs, - ) -> list[bytes | str]: - text = unicodedata.normalize("NFC", text) - - return [ - self.decoder[t] - for t in self.tokenizer.encode( - text, - allowed_special=allowed_special, - disallowed_special=disallowed_special, - ) - ] - - def _decode( - self, - token_ids: int | list[int], - skip_special_tokens: bool = False, - errors: str | None = None, - **kwargs, - ) -> str: - if isinstance(token_ids, int): - token_ids = [token_ids] - - return self.tokenizer.decode( - token_ids, - errors=errors or self.errors, - ) - - TokenizerWithoutImagePad.__name__ = f"{tokenizer.__class__.__name__}WithoutImagePad" - - new_tokenizer.__class__ = TokenizerWithoutImagePad - return new_tokenizer - - -class QwenVLTokenizer(TokenizerLike): - image_start_tag: str - image_end_tag: str - image_pad_tag: str - - @classmethod - def from_pretrained(cls, *args, **kwargs) -> HfTokenizer: - tokenizer = AutoTokenizer.from_pretrained(*args, **kwargs) - return get_cached_tokenizer(get_qwen_vl_tokenizer(tokenizer)) diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 7578d3b43a..8e6c66f95a 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -47,7 +47,6 @@ _VLLM_TOKENIZERS = { "hf": ("hf", "CachedHfTokenizer"), "kimi_audio": ("kimi_audio", "KimiAudioTokenizer"), "mistral": ("mistral", "MistralTokenizer"), - "qwen_vl": ("qwen_vl", "QwenVLTokenizer"), } diff --git a/vllm/transformers_utils/chat_templates/registry.py b/vllm/transformers_utils/chat_templates/registry.py index 0c3d15f4db..a5f9bdac20 100644 --- a/vllm/transformers_utils/chat_templates/registry.py +++ b/vllm/transformers_utils/chat_templates/registry.py @@ -13,13 +13,6 @@ CHAT_TEMPLATES_DIR = Path(__file__).parent ChatTemplatePath: TypeAlias = Path | Callable[[str], Path | None] -def _get_qwen_chat_template_fallback(tokenizer_name_or_path: str) -> Path | None: - if tokenizer_name_or_path.endswith("-Chat"): - return CHAT_TEMPLATES_DIR / "template_chatml.jinja" - - return CHAT_TEMPLATES_DIR / "template_basic.jinja" - - def _get_minicpmv_chat_template_fallback(tokenizer_name_or_path: str) -> Path | None: # MiniCPM-V-4.5 version uses a dedicated template if "4.5" in tokenizer_name_or_path or "4_5" in tokenizer_name_or_path: @@ -41,7 +34,6 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = { "minicpmv": _get_minicpmv_chat_template_fallback, "minicpmv4_6": _get_minicpmv_chat_template_fallback, "paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja", - "qwen": _get_qwen_chat_template_fallback, "siglip": CHAT_TEMPLATES_DIR / "template_basic.jinja", "siglip2": CHAT_TEMPLATES_DIR / "template_basic.jinja", } diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py index b53dd87d60..a64be96189 100644 --- a/vllm/transformers_utils/processors/__init__.py +++ b/vllm/transformers_utils/processors/__init__.py @@ -40,7 +40,6 @@ __all__ = [ "OpenVLAProcessor", "OvisProcessor", "Ovis2_5Processor", - "QwenVLProcessor", "Qwen3ASRProcessor", "Step3VLProcessor", ] @@ -75,7 +74,6 @@ _CLASS_TO_MODULE: dict[str, str] = { "OpenVLAProcessor": "vllm.transformers_utils.processors.openvla", "OvisProcessor": "vllm.transformers_utils.processors.ovis", "Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5", - "QwenVLProcessor": "vllm.transformers_utils.processors.qwen_vl", "Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr", "Step3VLProcessor": "vllm.transformers_utils.processors.step3_vl", } diff --git a/vllm/transformers_utils/processors/qwen_vl.py b/vllm/transformers_utils/processors/qwen_vl.py deleted file mode 100644 index 7de9046d93..0000000000 --- a/vllm/transformers_utils/processors/qwen_vl.py +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Adapted from -# https://huggingface.co/Qwen/Qwen-VL/blob/main/modeling_qwen.py -# Copyright (c) Alibaba Cloud. -from transformers.image_processing_utils_fast import BaseImageProcessorFast -from transformers.image_utils import PILImageResampling -from transformers.processing_utils import ProcessorMixin - -from vllm.tokenizers.qwen_vl import QwenVLTokenizer - - -class QwenVLImageProcessorFast(BaseImageProcessorFast): - """ - Port of https://huggingface.co/Qwen/Qwen-VL/blob/main/visual.py#L354 - to HF Transformers. - """ - - resample = PILImageResampling.BICUBIC - image_mean = [0.48145466, 0.4578275, 0.40821073] - image_std = [0.26862954, 0.26130258, 0.27577711] - size = {"height": 448, "width": 448} - do_resize = True - do_rescale = True - do_normalize = True - - -class QwenVLProcessor(ProcessorMixin): - attributes = ["image_processor", "tokenizer"] - - def __init__( - self, - image_processor: QwenVLImageProcessorFast, - tokenizer: QwenVLTokenizer, - ) -> None: - self.image_processor = image_processor - self.tokenizer = tokenizer - - self.image_start_tag = tokenizer.image_start_tag - self.image_end_tag = tokenizer.image_end_tag - self.image_pad_tag = tokenizer.image_pad_tag