[Misc] Remove unnecessary Transformers version guards (#57556)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -11,8 +11,6 @@ from unittest.mock import Mock
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
from packaging.version import Version
|
|
||||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
|
||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm import LLM
|
from vllm import LLM
|
||||||
@@ -149,15 +147,6 @@ def test_models(
|
|||||||
if enable_prompt_embeds:
|
if enable_prompt_embeds:
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
prompt_embeds = hf_model.get_prompt_embeddings(example_prompts)
|
prompt_embeds = hf_model.get_prompt_embeddings(example_prompts)
|
||||||
if model == "hmellor/tiny-random-Gemma2ForCausalLM" and (
|
|
||||||
Version(TRANSFORMERS_VERSION) < Version("5.3.0.dev0")
|
|
||||||
):
|
|
||||||
# For Gemma 1/2 models with Transformers 5.4.0+, the prompt embeddings
|
|
||||||
# are normalised in `get_prompt_embeddings`, like Gemma 3.
|
|
||||||
# For older versions, we need to manually normalise.
|
|
||||||
embed_scale = hf_model.config.hidden_size**0.5
|
|
||||||
normalizer = torch.tensor(embed_scale, dtype=prompt_embeds[0].dtype)
|
|
||||||
prompt_embeds = [p_e * normalizer for p_e in prompt_embeds]
|
|
||||||
|
|
||||||
with VllmRunner(
|
with VllmRunner(
|
||||||
model,
|
model,
|
||||||
|
|||||||
@@ -3,8 +3,6 @@
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from packaging.version import Version
|
|
||||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
|
||||||
|
|
||||||
import vllm
|
import vllm
|
||||||
from tests.conftest import VllmRunner
|
from tests.conftest import VllmRunner
|
||||||
@@ -29,20 +27,12 @@ class TestConfig:
|
|||||||
|
|
||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
if self.mm_processor_kwargs is None:
|
if self.mm_processor_kwargs is None:
|
||||||
# There is a bug in transformers v4 where size is ignored by
|
self.mm_processor_kwargs = {
|
||||||
# `Qwen2VLProcessor.__call__`
|
"size": {
|
||||||
if Version(TRANSFORMERS_VERSION) < Version("5.2.0"):
|
"shortest_edge": 28 * 28,
|
||||||
self.mm_processor_kwargs = {
|
"longest_edge": 1280 * 28 * 28,
|
||||||
"min_pixels": 28 * 28,
|
|
||||||
"max_pixels": 1280 * 28 * 28,
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
self.mm_processor_kwargs = {
|
|
||||||
"size": {
|
|
||||||
"shortest_edge": 28 * 28,
|
|
||||||
"longest_edge": 1280 * 28 * 28,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class Qwen2VLTester:
|
class Qwen2VLTester:
|
||||||
|
|||||||
@@ -3,8 +3,6 @@
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
from packaging.version import Version
|
|
||||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
|
||||||
|
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
@@ -162,16 +160,6 @@ def test_models(
|
|||||||
if prompt_embeds is not None:
|
if prompt_embeds is not None:
|
||||||
embed = hf_model.model.get_input_embeddings()(token_ids)
|
embed = hf_model.model.get_input_embeddings()(token_ids)
|
||||||
|
|
||||||
if "gemma" in model.lower() and (
|
|
||||||
Version(TRANSFORMERS_VERSION) < Version("5.3.0.dev0")
|
|
||||||
):
|
|
||||||
# For Gemma 1/2 models with Transformers 5.4.0+, the prompt
|
|
||||||
# embeddings are normalised in `get_prompt_embeddings`,
|
|
||||||
# like Gemma 3. For older versions, we need to manually normalise.
|
|
||||||
embed_scale = hf_model.config.hidden_size**0.5
|
|
||||||
normalizer = torch.tensor(embed_scale, dtype=embed.dtype)
|
|
||||||
embed *= normalizer
|
|
||||||
|
|
||||||
# MiniCPM models apply scale_emb to embeddings internally.
|
# MiniCPM models apply scale_emb to embeddings internally.
|
||||||
# vLLM expects pre-scaled embeddings when using inputs_embeds.
|
# vLLM expects pre-scaled embeddings when using inputs_embeds.
|
||||||
if model in EMBED_SCALING_MODELS:
|
if model in EMBED_SCALING_MODELS:
|
||||||
|
|||||||
@@ -317,8 +317,6 @@ VLM_TEST_SETTINGS = {
|
|||||||
img_idx_to_prompt=lambda idx: "",
|
img_idx_to_prompt=lambda idx: "",
|
||||||
auto_cls=AutoModelForImageTextToText,
|
auto_cls=AutoModelForImageTextToText,
|
||||||
vllm_output_post_proc=model_utils.blip2_vllm_to_hf_output,
|
vllm_output_post_proc=model_utils.blip2_vllm_to_hf_output,
|
||||||
# FIXME: https://github.com/huggingface/transformers/pull/38510
|
|
||||||
marks=[pytest.mark.skip("Model is broken")],
|
|
||||||
),
|
),
|
||||||
"cosmos3": VLMTestInfo(
|
"cosmos3": VLMTestInfo(
|
||||||
models=["nvidia/Cosmos3-Nano"],
|
models=["nvidia/Cosmos3-Nano"],
|
||||||
@@ -819,10 +817,6 @@ VLM_TEST_SETTINGS = {
|
|||||||
patch_hf_runner=model_utils.paddleocr_vl_patch_hf_runner,
|
patch_hf_runner=model_utils.paddleocr_vl_patch_hf_runner,
|
||||||
image_size_factors=[(0.25,)],
|
image_size_factors=[(0.25,)],
|
||||||
marks=[
|
marks=[
|
||||||
pytest.mark.skipif(
|
|
||||||
Version(TRANSFORMERS_VERSION) == Version("4.57.3"),
|
|
||||||
reason="This model is broken in Transformers v4.57.3",
|
|
||||||
),
|
|
||||||
pytest.mark.skipif(
|
pytest.mark.skipif(
|
||||||
Version(TRANSFORMERS_VERSION) >= Version("5.0.0"),
|
Version(TRANSFORMERS_VERSION) >= Version("5.0.0"),
|
||||||
reason="Model's custom code uses ROPE_INIT_FUNCTIONS"
|
reason="Model's custom code uses ROPE_INIT_FUNCTIONS"
|
||||||
|
|||||||
@@ -308,16 +308,6 @@ def gemma3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
|||||||
|
|
||||||
hf_model.processor = processor
|
hf_model.processor = processor
|
||||||
|
|
||||||
orig_generate = hf_model.model.generate
|
|
||||||
|
|
||||||
def _generate(self, *args, **kwargs):
|
|
||||||
# FIXME: https://github.com/huggingface/transformers/issues/38333
|
|
||||||
kwargs["disable_compile"] = True
|
|
||||||
|
|
||||||
return orig_generate(*args, **kwargs)
|
|
||||||
|
|
||||||
hf_model.model.generate = types.MethodType(_generate, hf_model.model)
|
|
||||||
|
|
||||||
return hf_model
|
return hf_model
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import transformers
|
|
||||||
from packaging import version
|
|
||||||
from transformers import AutoModel
|
from transformers import AutoModel
|
||||||
|
|
||||||
from vllm.assets.base import VLLM_S3_BUCKET_URL
|
from vllm.assets.base import VLLM_S3_BUCKET_URL
|
||||||
@@ -246,10 +244,6 @@ def _run_test(
|
|||||||
|
|
||||||
@pytest.mark.parametrize("model", MODELS)
|
@pytest.mark.parametrize("model", MODELS)
|
||||||
@pytest.mark.parametrize("dtype", ["half"])
|
@pytest.mark.parametrize("dtype", ["half"])
|
||||||
@pytest.mark.skipif(
|
|
||||||
version.parse(transformers.__version__) == version.parse("4.57.5"),
|
|
||||||
reason="Skipped for transformers==4.57.5, https://github.com/huggingface/transformers/issues/43295",
|
|
||||||
)
|
|
||||||
def test_model_text_image(
|
def test_model_text_image(
|
||||||
hf_runner,
|
hf_runner,
|
||||||
vllm_runner,
|
vllm_runner,
|
||||||
@@ -269,10 +263,6 @@ def test_model_text_image(
|
|||||||
|
|
||||||
@pytest.mark.parametrize("model", MODELS)
|
@pytest.mark.parametrize("model", MODELS)
|
||||||
@pytest.mark.parametrize("dtype", ["half"])
|
@pytest.mark.parametrize("dtype", ["half"])
|
||||||
@pytest.mark.skipif(
|
|
||||||
version.parse(transformers.__version__) == version.parse("4.57.5"),
|
|
||||||
reason="Skipped for transformers==4.57.5, https://github.com/huggingface/transformers/issues/43295",
|
|
||||||
)
|
|
||||||
def test_model_text_text(
|
def test_model_text_text(
|
||||||
hf_runner,
|
hf_runner,
|
||||||
vllm_runner,
|
vllm_runner,
|
||||||
@@ -292,10 +282,6 @@ def test_model_text_text(
|
|||||||
|
|
||||||
@pytest.mark.parametrize("model", MODELS)
|
@pytest.mark.parametrize("model", MODELS)
|
||||||
@pytest.mark.parametrize("dtype", ["half"])
|
@pytest.mark.parametrize("dtype", ["half"])
|
||||||
@pytest.mark.skipif(
|
|
||||||
version.parse(transformers.__version__) == version.parse("4.57.5"),
|
|
||||||
reason="Skipped for transformers==4.57.5, https://github.com/huggingface/transformers/issues/43295",
|
|
||||||
)
|
|
||||||
def test_model_image_text(
|
def test_model_image_text(
|
||||||
hf_runner,
|
hf_runner,
|
||||||
vllm_runner,
|
vllm_runner,
|
||||||
@@ -315,10 +301,6 @@ def test_model_image_text(
|
|||||||
|
|
||||||
@pytest.mark.parametrize("model", MODELS)
|
@pytest.mark.parametrize("model", MODELS)
|
||||||
@pytest.mark.parametrize("dtype", ["half"])
|
@pytest.mark.parametrize("dtype", ["half"])
|
||||||
@pytest.mark.skipif(
|
|
||||||
version.parse(transformers.__version__) == version.parse("4.57.5"),
|
|
||||||
reason="Skipped for transformers==4.57.5, https://github.com/huggingface/transformers/issues/43295",
|
|
||||||
)
|
|
||||||
def test_model_image_image(
|
def test_model_image_image(
|
||||||
hf_runner,
|
hf_runner,
|
||||||
vllm_runner,
|
vllm_runner,
|
||||||
@@ -338,10 +320,6 @@ def test_model_image_image(
|
|||||||
|
|
||||||
@pytest.mark.parametrize("model", MODELS)
|
@pytest.mark.parametrize("model", MODELS)
|
||||||
@pytest.mark.parametrize("dtype", ["half"])
|
@pytest.mark.parametrize("dtype", ["half"])
|
||||||
@pytest.mark.skipif(
|
|
||||||
version.parse(transformers.__version__) == version.parse("4.57.5"),
|
|
||||||
reason="Skipped for transformers==4.57.5, https://github.com/huggingface/transformers/issues/43295",
|
|
||||||
)
|
|
||||||
def test_model_text_mixed_documents(
|
def test_model_text_mixed_documents(
|
||||||
hf_runner,
|
hf_runner,
|
||||||
vllm_runner,
|
vllm_runner,
|
||||||
|
|||||||
@@ -3,9 +3,7 @@
|
|||||||
"""Tests for Idefics3's multimodal preprocessing kwargs."""
|
"""Tests for Idefics3's multimodal preprocessing kwargs."""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from packaging.version import Version
|
|
||||||
from transformers import Idefics3Config
|
from transformers import Idefics3Config
|
||||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
|
||||||
|
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
|
|
||||||
@@ -13,10 +11,6 @@ from ....conftest import ImageTestAssets
|
|||||||
from ...utils import build_model_context
|
from ...utils import build_model_context
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(
|
|
||||||
Version(TRANSFORMERS_VERSION) < Version("5.2.0"),
|
|
||||||
reason="See https://github.com/huggingface/transformers/pull/43948",
|
|
||||||
)
|
|
||||||
@pytest.mark.parametrize("model_id", ["HuggingFaceM4/Idefics3-8B-Llama3"])
|
@pytest.mark.parametrize("model_id", ["HuggingFaceM4/Idefics3-8B-Llama3"])
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("mm_processor_kwargs", "expected_toks_per_img"),
|
("mm_processor_kwargs", "expected_toks_per_img"),
|
||||||
|
|||||||
@@ -3,9 +3,7 @@
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
from packaging.version import Version
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
|
||||||
|
|
||||||
from vllm.model_executor.models.vision import FusedInputNorm
|
from vllm.model_executor.models.vision import FusedInputNorm
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
@@ -133,12 +131,6 @@ def test_processor_override(
|
|||||||
kwargs_on_init: bool,
|
kwargs_on_init: bool,
|
||||||
):
|
):
|
||||||
"""Ensure Qwen2VLMultiModalProcessor handles min/max pixels properly."""
|
"""Ensure Qwen2VLMultiModalProcessor handles min/max pixels properly."""
|
||||||
if (
|
|
||||||
Version(TRANSFORMERS_VERSION) < Version("5.2.0")
|
|
||||||
and "size" in mm_processor_kwargs
|
|
||||||
):
|
|
||||||
pytest.skip("`size` ignored by `Qwen2VLProcessor.__call__`")
|
|
||||||
|
|
||||||
ctx = build_model_context(
|
ctx = build_model_context(
|
||||||
model_id,
|
model_id,
|
||||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||||
@@ -184,12 +176,6 @@ def test_get_image_size_with_most_features(
|
|||||||
model_id: str,
|
model_id: str,
|
||||||
mm_processor_kwargs: dict[str, object],
|
mm_processor_kwargs: dict[str, object],
|
||||||
):
|
):
|
||||||
if (
|
|
||||||
Version(TRANSFORMERS_VERSION) < Version("5.2.0")
|
|
||||||
and "size" in mm_processor_kwargs
|
|
||||||
):
|
|
||||||
pytest.skip("`size` ignored by `Qwen2VLProcessor.__call__`")
|
|
||||||
|
|
||||||
ctx = build_model_context(
|
ctx = build_model_context(
|
||||||
model_id,
|
model_id,
|
||||||
mm_processor_kwargs=mm_processor_kwargs,
|
mm_processor_kwargs=mm_processor_kwargs,
|
||||||
|
|||||||
@@ -3,9 +3,7 @@
|
|||||||
"""Tests for smolvlm's multimodal preprocessing kwargs."""
|
"""Tests for smolvlm's multimodal preprocessing kwargs."""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from packaging.version import Version
|
|
||||||
from transformers import SmolVLMConfig
|
from transformers import SmolVLMConfig
|
||||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
|
||||||
|
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
|
|
||||||
@@ -13,10 +11,6 @@ from ....conftest import ImageTestAssets
|
|||||||
from ...utils import build_model_context
|
from ...utils import build_model_context
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(
|
|
||||||
Version(TRANSFORMERS_VERSION) < Version("5.2.0"),
|
|
||||||
reason="See https://github.com/huggingface/transformers/pull/43948",
|
|
||||||
)
|
|
||||||
@pytest.mark.parametrize("model_id", ["HuggingFaceTB/SmolVLM2-2.2B-Instruct"])
|
@pytest.mark.parametrize("model_id", ["HuggingFaceTB/SmolVLM2-2.2B-Instruct"])
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("mm_processor_kwargs", "expected_toks_per_img"),
|
("mm_processor_kwargs", "expected_toks_per_img"),
|
||||||
|
|||||||
@@ -36,9 +36,7 @@ import numpy as np
|
|||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
import transformers
|
|
||||||
from einops import rearrange
|
from einops import rearrange
|
||||||
from packaging.version import Version
|
|
||||||
from transformers import BatchFeature, Glm4vProcessor
|
from transformers import BatchFeature, Glm4vProcessor
|
||||||
from transformers.image_processing_base import ImageProcessingMixin
|
from transformers.image_processing_base import ImageProcessingMixin
|
||||||
from transformers.models.glm4v.configuration_glm4v import (
|
from transformers.models.glm4v.configuration_glm4v import (
|
||||||
@@ -132,8 +130,6 @@ logger = init_logger(__name__)
|
|||||||
# For profile run
|
# For profile run
|
||||||
_MAX_FRAMES_PER_VIDEO = 600
|
_MAX_FRAMES_PER_VIDEO = 600
|
||||||
|
|
||||||
TRANSFORMERS_WITH_GA = Version(transformers.__version__) >= Version("5.10.0.dev0")
|
|
||||||
|
|
||||||
|
|
||||||
def _to_video_metadata(metadata: Mapping[str, Any]) -> VideoMetadata:
|
def _to_video_metadata(metadata: Mapping[str, Any]) -> VideoMetadata:
|
||||||
return VideoMetadata(
|
return VideoMetadata(
|
||||||
@@ -1389,17 +1385,6 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
|
|||||||
timestamps_list = full_second_idxs[::2]
|
timestamps_list = full_second_idxs[::2]
|
||||||
return list(timestamps_list)
|
return list(timestamps_list)
|
||||||
|
|
||||||
def _get_video_frame_embed_token_id(self, hf_processor: object) -> int:
|
|
||||||
attr = (
|
|
||||||
"image_token_id"
|
|
||||||
if isinstance(hf_processor, Glm4vProcessor) or TRANSFORMERS_WITH_GA
|
|
||||||
else "video_token_id"
|
|
||||||
)
|
|
||||||
token_id = getattr(hf_processor, attr, None)
|
|
||||||
if not isinstance(token_id, int):
|
|
||||||
raise ValueError(f"Processor has no valid {attr}")
|
|
||||||
return token_id
|
|
||||||
|
|
||||||
def _construct_video_placeholder(
|
def _construct_video_placeholder(
|
||||||
self,
|
self,
|
||||||
video_array: np.ndarray,
|
video_array: np.ndarray,
|
||||||
@@ -1437,7 +1422,7 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
|
|||||||
num_tokens_per_frame = int(H * W) // merge_length
|
num_tokens_per_frame = int(H * W) // merge_length
|
||||||
placeholder = []
|
placeholder = []
|
||||||
placeholder.append(bov_token_id)
|
placeholder.append(bov_token_id)
|
||||||
frame_embed_token_id = self._get_video_frame_embed_token_id(hf_processor)
|
frame_embed_token_id = hf_processor.image_token_id
|
||||||
for frame_idx in frames_idx_token:
|
for frame_idx in frames_idx_token:
|
||||||
placeholder.append(boi_token_id)
|
placeholder.append(boi_token_id)
|
||||||
placeholder.extend([frame_embed_token_id] * num_tokens_per_frame)
|
placeholder.extend([frame_embed_token_id] * num_tokens_per_frame)
|
||||||
@@ -1601,10 +1586,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
|
|
||||||
processor = self.info.get_hf_processor(**hf_kwargs)
|
processor = self.info.get_hf_processor(**hf_kwargs)
|
||||||
|
|
||||||
use_direct_path = (
|
if not isinstance(processor, Glm4vProcessor):
|
||||||
not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
|
|
||||||
)
|
|
||||||
if use_direct_path:
|
|
||||||
prepared_data, prepared_kwargs = self._get_direct_path_inputs(
|
prepared_data, prepared_kwargs = self._get_direct_path_inputs(
|
||||||
hf_data, hf_kwargs
|
hf_data, hf_kwargs
|
||||||
)
|
)
|
||||||
@@ -1627,8 +1609,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
del hf_data["videos"]
|
del hf_data["videos"]
|
||||||
video_grid_thw_lst = []
|
video_grid_thw_lst = []
|
||||||
pixel_values_videos_lst = []
|
pixel_values_videos_lst = []
|
||||||
frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor)
|
|
||||||
swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id
|
|
||||||
for item in videos:
|
for item in videos:
|
||||||
video_array, metadata = item
|
video_array, metadata = item
|
||||||
|
|
||||||
@@ -1651,10 +1631,10 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
video_mm_kwargs,
|
video_mm_kwargs,
|
||||||
)
|
)
|
||||||
input_ids = video_outputs.pop("input_ids")
|
input_ids = video_outputs.pop("input_ids")
|
||||||
if swap_video_frame_tokens:
|
|
||||||
input_ids[input_ids == processor.image_token_id] = (
|
input_ids[input_ids == processor.image_token_id] = (
|
||||||
processor.video_token_id
|
processor.video_token_id
|
||||||
)
|
)
|
||||||
video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
|
video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
|
||||||
prompt_text = prompt_text.replace(
|
prompt_text = prompt_text.replace(
|
||||||
"<|begin_of_video|><|video|><|end_of_video|>",
|
"<|begin_of_video|><|video|><|end_of_video|>",
|
||||||
@@ -1670,17 +1650,15 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
video_outputs = dict()
|
video_outputs = dict()
|
||||||
swap_video_frame_tokens = False
|
|
||||||
|
|
||||||
processed_data = self.info.ctx.call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
self.info.get_hf_processor(**hf_kwargs),
|
self.info.get_hf_processor(**hf_kwargs),
|
||||||
dict(text=prompt_text, **hf_data),
|
dict(text=prompt_text, **hf_data),
|
||||||
hf_kwargs,
|
hf_kwargs,
|
||||||
)
|
)
|
||||||
if swap_video_frame_tokens:
|
input_ids = processed_data["input_ids"]
|
||||||
input_ids = processed_data["input_ids"]
|
input_ids[input_ids == processor.video_token_id] = processor.image_token_id
|
||||||
input_ids[input_ids == processor.video_token_id] = processor.image_token_id
|
processed_data["input_ids"] = input_ids
|
||||||
processed_data["input_ids"] = input_ids
|
|
||||||
|
|
||||||
processed_data.update(video_outputs)
|
processed_data.update(video_outputs)
|
||||||
return self._finalize_hf_mm_data(
|
return self._finalize_hf_mm_data(
|
||||||
@@ -1728,7 +1706,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
)
|
)
|
||||||
return PromptUpdateDetails.select_token_id(
|
return PromptUpdateDetails.select_token_id(
|
||||||
placeholder,
|
placeholder,
|
||||||
embed_token_id=self.info._get_video_frame_embed_token_id(hf_processor),
|
embed_token_id=hf_processor.image_token_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
return [
|
return [
|
||||||
|
|||||||
@@ -1228,18 +1228,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
|||||||
) -> HFMultiModalInputs:
|
) -> HFMultiModalInputs:
|
||||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||||
|
|
||||||
if mm_items.get_count("audio", strict=False):
|
|
||||||
# TODO(Isotr0py): Remove this patch after upstream fix PR
|
|
||||||
# released and Transformers version update:
|
|
||||||
# https://github.com/huggingface/transformers/pull/41473
|
|
||||||
hf_inputs = hf_inputs._replace(
|
|
||||||
hf_kwargs={
|
|
||||||
**hf_inputs.hf_kwargs,
|
|
||||||
"audio_kwargs": dict(hf_inputs.hf_kwargs.get("audio_kwargs") or {}),
|
|
||||||
"text_kwargs": dict(hf_inputs.hf_kwargs.get("text_kwargs") or {}),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
audios = hf_inputs.hf_data.get("audio")
|
audios = hf_inputs.hf_data.get("audio")
|
||||||
if audios:
|
if audios:
|
||||||
feature_extractor = self.info.get_feature_extractor()
|
feature_extractor = self.info.get_feature_extractor()
|
||||||
|
|||||||
Reference in New Issue
Block a user