[Bugfix][Multimodal] Fix flat/scoped mm_processor_kwargs merge and resolution (#56372)
Signed-off-by: Daniel M. García-Ocaña Hernández <danielgarciaocana@gmail.com>
This commit is contained in:
@@ -188,6 +188,250 @@ def test_mm_processor_device_type_normalizes(device: object, expected: str | Non
|
||||
assert config.get_mm_processor_device_type() == expected
|
||||
|
||||
|
||||
def test_merge_mm_processor_kwargs_scoped_precedence():
|
||||
configured = {
|
||||
"size": {
|
||||
"shortest_edge": 64,
|
||||
"longest_edge": 512,
|
||||
},
|
||||
"do_resize": True,
|
||||
"padding": False,
|
||||
"images_kwargs": {
|
||||
"size": {"longest_edge": 1024},
|
||||
"do_resize": False,
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"size": {"longest_edge": 2048},
|
||||
},
|
||||
}
|
||||
inference = {
|
||||
"size": {"shortest_edge": 128},
|
||||
"do_resize": True,
|
||||
"images_kwargs": {
|
||||
"size": {"shortest_edge": 256},
|
||||
},
|
||||
}
|
||||
|
||||
config = MultiModalConfig(
|
||||
mm_processor_kwargs=configured,
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
|
||||
assert config.merge_mm_processor_kwargs(inference) == {
|
||||
"size": {
|
||||
"shortest_edge": 128,
|
||||
"longest_edge": 512,
|
||||
},
|
||||
"do_resize": True,
|
||||
"padding": False,
|
||||
"images_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": 256,
|
||||
"longest_edge": 1024,
|
||||
},
|
||||
"do_resize": True,
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": 128,
|
||||
"longest_edge": 2048,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_merge_mm_processor_kwargs_mapping_vs_replacement():
|
||||
config = MultiModalConfig(
|
||||
mm_processor_kwargs={
|
||||
"size": {"shortest_edge": 64},
|
||||
"images_kwargs": {"size": {"longest_edge": 1024}},
|
||||
},
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
|
||||
assert config.merge_mm_processor_kwargs({"size": 7}) == {
|
||||
"size": 7,
|
||||
"images_kwargs": {"size": 7},
|
||||
}
|
||||
assert config.merge_mm_processor_kwargs({"size": {}}) == {
|
||||
"size": {"shortest_edge": 64},
|
||||
"images_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": 64,
|
||||
"longest_edge": 1024,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
_EMPTY_IMAGE_SCOPE_KWARGS: tuple[dict[str, object], ...] = (
|
||||
{},
|
||||
{"images_kwargs": None},
|
||||
{"images_kwargs": {}},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured", _EMPTY_IMAGE_SCOPE_KWARGS)
|
||||
@pytest.mark.parametrize("inference", _EMPTY_IMAGE_SCOPE_KWARGS)
|
||||
def test_merge_mm_processor_kwargs_empty_scopes_are_absent(
|
||||
configured: dict[str, object],
|
||||
inference: dict[str, object],
|
||||
):
|
||||
config = MultiModalConfig(
|
||||
mm_processor_kwargs=configured,
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
|
||||
assert config.merge_mm_processor_kwargs(inference) == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("empty_inference", _EMPTY_IMAGE_SCOPE_KWARGS)
|
||||
def test_merge_mm_processor_kwargs_empty_inference_scope_preserves_config_merge(
|
||||
empty_inference: dict[str, object],
|
||||
):
|
||||
config = MultiModalConfig(
|
||||
mm_processor_kwargs={
|
||||
"images_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": 64,
|
||||
"longest_edge": 512,
|
||||
},
|
||||
},
|
||||
},
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
inference = {
|
||||
"size": {"shortest_edge": 128},
|
||||
**empty_inference,
|
||||
}
|
||||
|
||||
assert config.merge_mm_processor_kwargs(inference) == {
|
||||
"size": {"shortest_edge": 128},
|
||||
"images_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": 128,
|
||||
"longest_edge": 512,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("empty_config", _EMPTY_IMAGE_SCOPE_KWARGS)
|
||||
def test_merge_mm_processor_kwargs_empty_config_scope_allows_inference_merge(
|
||||
empty_config: dict[str, object],
|
||||
):
|
||||
config = MultiModalConfig(
|
||||
mm_processor_kwargs={
|
||||
"size": {"longest_edge": 512},
|
||||
**empty_config,
|
||||
},
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
|
||||
assert config.merge_mm_processor_kwargs(
|
||||
{
|
||||
"size": {"shortest_edge": 128},
|
||||
"images_kwargs": {"size": {"longest_edge": 1024}},
|
||||
}
|
||||
) == {
|
||||
"size": {
|
||||
"shortest_edge": 128,
|
||||
"longest_edge": 512,
|
||||
},
|
||||
"images_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": 128,
|
||||
"longest_edge": 1024,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("non_mapping_scope", [False, 0, "", [], 123])
|
||||
def test_merge_mm_processor_kwargs_falsey_non_mapping_scopes_are_not_empty(
|
||||
non_mapping_scope: object,
|
||||
):
|
||||
config = MultiModalConfig(
|
||||
mm_processor_kwargs={"images_kwargs": non_mapping_scope},
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
|
||||
for empty_inference in _EMPTY_IMAGE_SCOPE_KWARGS:
|
||||
assert config.merge_mm_processor_kwargs(empty_inference) == {
|
||||
"images_kwargs": non_mapping_scope
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured_scope", "inference_scope", "expected_scope"),
|
||||
[
|
||||
(123, {"size": {"shortest_edge": 128}}, {"size": {"shortest_edge": 128}}),
|
||||
({"size": {"shortest_edge": 64}}, 123, 123),
|
||||
(123, 456, 456),
|
||||
],
|
||||
)
|
||||
def test_merge_mm_processor_kwargs_non_mapping_scope_precedence(
|
||||
configured_scope: object,
|
||||
inference_scope: object,
|
||||
expected_scope: object,
|
||||
):
|
||||
config = MultiModalConfig(
|
||||
mm_processor_kwargs={"images_kwargs": configured_scope},
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
|
||||
merged = config.merge_mm_processor_kwargs({"images_kwargs": inference_scope})
|
||||
|
||||
assert merged["images_kwargs"] == expected_scope
|
||||
|
||||
|
||||
def test_merge_mm_processor_kwargs_none_leaf_is_explicit():
|
||||
config = MultiModalConfig(mm_device_do_normalize=False)
|
||||
|
||||
assert config.merge_mm_processor_kwargs({"images_kwargs": {"do_resize": None}}) == {
|
||||
"images_kwargs": {"do_resize": None}
|
||||
}
|
||||
|
||||
|
||||
def test_merge_mm_processor_kwargs_normalize_and_ownership():
|
||||
configured: dict[str, object] = {
|
||||
"images_kwargs": {
|
||||
"size": {"shortest_edge": 64},
|
||||
},
|
||||
}
|
||||
inference: dict[str, object] = {
|
||||
"do_normalize": True,
|
||||
"do_rescale": True,
|
||||
"images_kwargs": {
|
||||
"size": {"longest_edge": 1024},
|
||||
},
|
||||
}
|
||||
configured_before = copy.deepcopy(configured)
|
||||
inference_before = copy.deepcopy(inference)
|
||||
|
||||
config = MultiModalConfig(
|
||||
mm_processor_kwargs=configured,
|
||||
mm_device_do_normalize=True,
|
||||
)
|
||||
merged = config.merge_mm_processor_kwargs(inference)
|
||||
|
||||
assert merged["do_normalize"] is False
|
||||
assert merged["do_rescale"] is False
|
||||
assert configured == configured_before
|
||||
assert inference == inference_before
|
||||
assert config.mm_processor_kwargs == configured_before
|
||||
|
||||
images_kwargs = merged["images_kwargs"]
|
||||
assert isinstance(images_kwargs, dict)
|
||||
size = images_kwargs["size"]
|
||||
assert isinstance(size, dict)
|
||||
size["shortest_edge"] = 999
|
||||
|
||||
assert configured == configured_before
|
||||
assert inference == inference_before
|
||||
assert config.mm_processor_kwargs == configured_before
|
||||
|
||||
|
||||
def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None:
|
||||
ec_config = (
|
||||
None
|
||||
|
||||
@@ -202,19 +202,19 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
|
||||
)
|
||||
@pytest.mark.parametrize("video_uuid", [None, "same-video"])
|
||||
def test_video_cache_is_independent_of_image_kwargs(
|
||||
model_id: str,
|
||||
kwargs_on_init: bool,
|
||||
image_kwargs: dict[str, object],
|
||||
video_uuid: str | None,
|
||||
):
|
||||
"""Image overrides must not change video frames or depend on cache warmth."""
|
||||
kwargs = {"images_kwargs": image_kwargs}
|
||||
"""Request image overrides must not change video frames or make results
|
||||
depend on cache warmth.
|
||||
"""
|
||||
request_kwargs = {"images_kwargs": image_kwargs}
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
@@ -239,18 +239,11 @@ def test_video_cache_is_independent_of_image_kwargs(
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
baseline = process({}, cache=None)
|
||||
if kwargs_on_init:
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=kwargs,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
request_kwargs = {} if kwargs_on_init else kwargs
|
||||
process(request_kwargs, cache=cache)
|
||||
# Populate the video cache without image overrides, then change only
|
||||
# images_kwargs on the next request. Since images_kwargs are excluded from
|
||||
# the video hash, and must not affect the processed video, the cached result
|
||||
# must match a fresh recomputation.
|
||||
baseline = process({}, cache=cache)
|
||||
cached = process(request_kwargs, cache=cache)
|
||||
cache.clear_cache()
|
||||
fresh = process(request_kwargs, cache=cache)
|
||||
|
||||
@@ -100,6 +100,11 @@ def _expected_placeholder_tokens_per_image(
|
||||
[
|
||||
({}, (448, 448), 256),
|
||||
({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
|
||||
(
|
||||
{"images_kwargs": {"size": {"longest_edge": 1008}}},
|
||||
(1540, 1540),
|
||||
1296,
|
||||
),
|
||||
({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
|
||||
({"size": {"longest_edge": 1008}}, (29, 29), 4),
|
||||
({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
|
||||
@@ -148,6 +153,40 @@ def test_processor_size_override(
|
||||
assert prompt_update_tokens == hf_placeholder_tokens
|
||||
|
||||
|
||||
def test_scoped_request_size_overrides_configured_flat_size():
|
||||
ctx = build_model_context(
|
||||
_MODEL_ID,
|
||||
mm_processor_kwargs={"size": {"longest_edge": 1008}},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = AutoProcessor.from_pretrained(
|
||||
_MODEL_ID,
|
||||
fix_mistral_regex=True,
|
||||
)
|
||||
|
||||
hf_processor_mm_kwargs: dict[str, object] = {
|
||||
"images_kwargs": {"size": {"longest_edge": 1288}}
|
||||
}
|
||||
image = Image.new("RGB", (1536, 1187), color=(127, 127, 127))
|
||||
|
||||
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
|
||||
hf_processor, [image], merged_mm_kwargs
|
||||
)
|
||||
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
|
||||
processor, [image], pixel_values, hf_processor_mm_kwargs
|
||||
)
|
||||
|
||||
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
|
||||
hf_processor, pixel_values[0]
|
||||
)
|
||||
assert expected_from_pixel_values == 1656
|
||||
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656
|
||||
assert prompt_update_tokens == hf_placeholder_tokens
|
||||
|
||||
|
||||
def test_lightonocr_keeps_vision_config_image_size():
|
||||
ctx = build_model_context(
|
||||
_LIGHTON_MODEL_ID,
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
@@ -8,16 +10,238 @@ import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import batched_tensors_equal
|
||||
from vllm.multimodal.processing.context import InputProcessingContext
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mm_kwargs", "expected"),
|
||||
[
|
||||
(
|
||||
{"size": {"shortest_edge": 64, "longest_edge": 1024}},
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
"min_pixels": 64,
|
||||
"max_pixels": 1024,
|
||||
},
|
||||
),
|
||||
(
|
||||
{"min_pixels": 64, "max_pixels": 1024},
|
||||
{
|
||||
"min_pixels": 64,
|
||||
"max_pixels": 1024,
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
"min_pixels": 128,
|
||||
"max_pixels": 2048,
|
||||
},
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
"min_pixels": 128,
|
||||
"max_pixels": 2048,
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"min_pixels": None,
|
||||
"size": {"shortest_edge": 0},
|
||||
"images_kwargs": {"max_pixels": 4096},
|
||||
"videos_kwargs": {"size": {"longest_edge": 8192}},
|
||||
},
|
||||
{
|
||||
"min_pixels": None,
|
||||
"size": {"shortest_edge": 0},
|
||||
"images_kwargs": {
|
||||
"max_pixels": 4096,
|
||||
"size": {"longest_edge": 4096},
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"size": {"longest_edge": 8192},
|
||||
"max_pixels": 8192,
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
{"min_pixels": 0},
|
||||
{"min_pixels": 0, "size": {"shortest_edge": 0}},
|
||||
),
|
||||
(
|
||||
{"size": None, "min_pixels": 64},
|
||||
{"size": {"shortest_edge": 64}, "min_pixels": 64},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_complete_mm_processor_size_aliases(
|
||||
mm_kwargs: dict[str, object],
|
||||
expected: dict[str, object],
|
||||
) -> None:
|
||||
original = deepcopy(mm_kwargs)
|
||||
|
||||
actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs)
|
||||
|
||||
assert actual == expected
|
||||
assert mm_kwargs == original
|
||||
|
||||
|
||||
def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mm_config = MultiModalConfig(
|
||||
mm_processor_kwargs={
|
||||
"images_kwargs": {"min_pixels": 64},
|
||||
},
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
|
||||
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
|
||||
info = Qwen2VLProcessingInfo(ctx)
|
||||
monkeypatch.setattr(
|
||||
info,
|
||||
"get_supported_mm_processor_kwargs",
|
||||
lambda: {"images_kwargs": {"size", "min_pixels"}},
|
||||
)
|
||||
|
||||
assert info._merge_and_resolve_mm_processor_kwargs(
|
||||
{"size": {"shortest_edge": 128}}
|
||||
) == {
|
||||
"images_kwargs": {
|
||||
"size": {"shortest_edge": 128},
|
||||
"min_pixels": 128,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mm_kwargs", "default_size", "expected"),
|
||||
[
|
||||
(
|
||||
{},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
),
|
||||
(
|
||||
{"size": {"shortest_edge": 64}},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 64, "longest_edge": 1024},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 2048},
|
||||
"min_pixels": 128,
|
||||
"max_pixels": 4096,
|
||||
},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 128, "longest_edge": 4096},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64},
|
||||
"max_pixels": 4096,
|
||||
},
|
||||
None,
|
||||
{"shortest_edge": 64, "longest_edge": 4096},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 2048},
|
||||
"min_pixels": None,
|
||||
"max_pixels": 0,
|
||||
},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 64, "longest_edge": 0},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_vision_size(
|
||||
mm_kwargs: dict[str, object],
|
||||
default_size: dict[str, object] | None,
|
||||
expected: dict[str, object],
|
||||
) -> None:
|
||||
original_mm_kwargs = {
|
||||
key: dict(value) if isinstance(value, dict) else value
|
||||
for key, value in mm_kwargs.items()
|
||||
}
|
||||
original_default_size = dict(default_size) if default_size is not None else None
|
||||
|
||||
actual = Qwen2VLProcessingInfo._get_vision_size(
|
||||
mm_kwargs,
|
||||
default_size=default_size,
|
||||
)
|
||||
|
||||
assert actual == expected
|
||||
assert mm_kwargs == original_mm_kwargs
|
||||
assert default_size == original_default_size
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("modality", "scope"),
|
||||
[
|
||||
("image", "images_kwargs"),
|
||||
("video", "videos_kwargs"),
|
||||
],
|
||||
)
|
||||
def test_get_vision_info_reads_merged_modality_scope(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
modality: str,
|
||||
scope: str,
|
||||
) -> None:
|
||||
vision_config = SimpleNamespace(
|
||||
patch_size=14,
|
||||
spatial_merge_size=2,
|
||||
temporal_patch_size=2,
|
||||
)
|
||||
ctx = SimpleNamespace(
|
||||
get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config)
|
||||
)
|
||||
info = Qwen2VLProcessingInfo(ctx)
|
||||
merged = {
|
||||
"images_kwargs": {"size": {"shortest_edge": 111}},
|
||||
"videos_kwargs": {"size": {"shortest_edge": 222}},
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
info,
|
||||
"_merge_and_resolve_mm_processor_kwargs",
|
||||
lambda _: merged,
|
||||
)
|
||||
|
||||
seen: list[dict[str, object]] = []
|
||||
|
||||
def get_size(
|
||||
mm_kwargs: dict[str, object],
|
||||
default_size: dict[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
seen.append(mm_kwargs)
|
||||
return dict(default_size or {})
|
||||
|
||||
monkeypatch.setattr(info, "_get_vision_size", get_size)
|
||||
image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024})
|
||||
|
||||
info._get_vision_info(
|
||||
image_width=28,
|
||||
image_height=28,
|
||||
num_frames=2,
|
||||
do_resize=False,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs={},
|
||||
modality=modality, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert seen == [merged[scope]]
|
||||
|
||||
|
||||
def test_jina_vl_processing_order() -> None:
|
||||
"""Jina's document-first prompt keeps cached features and hashes aligned."""
|
||||
ctx = build_model_context(
|
||||
|
||||
@@ -1,19 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for Qwen3-VL processor.
|
||||
|
||||
Covers the fix for num_frames-based timestamp calculation
|
||||
(issue vllm-project/vllm#35909).
|
||||
"""
|
||||
"""Regression tests for Qwen3-VL multimodal processing."""
|
||||
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers.models.qwen3_vl import Qwen3VLVideoProcessor
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.processing.context import BaseProcessingInfo
|
||||
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import build_model_context
|
||||
@@ -21,6 +23,32 @@ from ...utils import build_model_context
|
||||
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
|
||||
|
||||
|
||||
def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__)
|
||||
assert "size" in native_video_kwargs
|
||||
assert "min_pixels" not in native_video_kwargs
|
||||
assert "max_pixels" not in native_video_kwargs
|
||||
|
||||
info = Qwen3VLProcessingInfo(SimpleNamespace())
|
||||
parent_supported = {
|
||||
"text_kwargs": {"padding"},
|
||||
"images_kwargs": {"size", "min_pixels", "max_pixels"},
|
||||
"videos_kwargs": set(native_video_kwargs),
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
BaseProcessingInfo,
|
||||
"get_supported_mm_processor_kwargs",
|
||||
lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()},
|
||||
)
|
||||
|
||||
supported = info.supported_mm_processor_kwargs
|
||||
|
||||
assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"]
|
||||
assert parent_supported["videos_kwargs"] == native_video_kwargs
|
||||
|
||||
|
||||
def _build_video_mm_data(
|
||||
num_frames: int,
|
||||
width: int = 128,
|
||||
@@ -45,6 +73,122 @@ def _build_video_mm_data(
|
||||
return {"video": [(video, metadata)]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_partial_scoped_image_size(model_id: str) -> None:
|
||||
"""Partial image ``size`` must be completed and applied before the HF call.
|
||||
|
||||
Keep video compatibility and per-video kwargs in an image-only call as well:
|
||||
Qwen3 must sanitize the video scope before HF validates it. Regression for
|
||||
#56363.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1, "video": 0},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
image = np.zeros((128, 128, 3), dtype=np.uint8)
|
||||
min_pixels = 256 * 32 * 32
|
||||
default_size = dict(processor.info.get_image_processor().size)
|
||||
complete_size = default_size | {"shortest_edge": min_pixels}
|
||||
|
||||
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data({"image": [image]})
|
||||
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
|
||||
"image_grid_thw"
|
||||
]
|
||||
|
||||
stock_grid = process({})
|
||||
complete_grid = process({"images_kwargs": {"size": complete_size}})
|
||||
partial_grid = process(
|
||||
{
|
||||
"images_kwargs": {
|
||||
"size": {"shortest_edge": min_pixels},
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"min_pixels": 4096,
|
||||
"max_pixels": 469_762_048,
|
||||
"fps": [2.0],
|
||||
"num_frames": [8],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert torch.equal(partial_grid, complete_grid)
|
||||
assert not torch.equal(partial_grid, stock_grid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_partial_scoped_video_size(model_id: str) -> None:
|
||||
"""Partial scoped video ``size`` must be completed and applied before HF."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
min_pixels = 512 * 32 * 32
|
||||
default_size = dict(processor.info.get_video_processor().size)
|
||||
complete_size = default_size | {"shortest_edge": min_pixels}
|
||||
|
||||
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
|
||||
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
|
||||
"video_grid_thw"
|
||||
]
|
||||
|
||||
stock_grid = process({})
|
||||
complete_grid = process({"videos_kwargs": {"size": complete_size}})
|
||||
partial_grid = process(
|
||||
{
|
||||
"videos_kwargs": {
|
||||
"size": {"shortest_edge": min_pixels},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert torch.equal(partial_grid, complete_grid)
|
||||
assert not torch.equal(partial_grid, stock_grid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None:
|
||||
"""Flat min/max pixel aliases keep their historical Qwen3 video effect."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
pixel_budget = 512 * 32 * 32
|
||||
|
||||
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
|
||||
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
|
||||
"video_grid_thw"
|
||||
]
|
||||
|
||||
stock_grid = process({})
|
||||
scoped_grid = process(
|
||||
{
|
||||
"videos_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": pixel_budget,
|
||||
"longest_edge": pixel_budget,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
flat_grid = process(
|
||||
{
|
||||
"min_pixels": pixel_budget,
|
||||
"max_pixels": pixel_budget,
|
||||
}
|
||||
)
|
||||
|
||||
assert torch.equal(flat_grid, scoped_grid)
|
||||
assert not torch.equal(flat_grid, stock_grid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
"num_frames",
|
||||
@@ -189,20 +333,21 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
"hf_mm_kwargs",
|
||||
[{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}],
|
||||
("hf_mm_kwargs", "scalar_kwargs"),
|
||||
[
|
||||
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
|
||||
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
|
||||
],
|
||||
)
|
||||
def test_processor_multi_video_list_kwargs(
|
||||
model_id: str,
|
||||
hf_mm_kwargs: dict[str, Any],
|
||||
scalar_kwargs: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Regression test: a multi-video request with list-valued per-video
|
||||
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
|
||||
crash.
|
||||
"""List-valued video kwargs must be scalarized per item without mutation.
|
||||
|
||||
Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
|
||||
video without slicing, so ``_get_video_second_idx`` received the list
|
||||
where a scalar was expected and raised ``TypeError``.
|
||||
Each video in the multi-video request must match processing the same input
|
||||
independently with its corresponding scalar ``fps``/``num_frames`` value.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
@@ -216,17 +361,36 @@ def test_processor_multi_video_list_kwargs(
|
||||
)
|
||||
mm_data = {
|
||||
"video": [
|
||||
_build_video_mm_data(num_frames=16)["video"][0],
|
||||
_build_video_mm_data(num_frames=32)["video"][0],
|
||||
_build_video_mm_data(num_frames=32)["video"][0],
|
||||
]
|
||||
}
|
||||
|
||||
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_mm_kwargs,
|
||||
)
|
||||
|
||||
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
single_mm_data = _build_video_mm_data(num_frames=32)
|
||||
single = processor(
|
||||
single_prompt,
|
||||
mm_items=processor.info.parse_mm_data(single_mm_data),
|
||||
hf_processor_mm_kwargs=hf_kwargs,
|
||||
)
|
||||
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
|
||||
|
||||
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
|
||||
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
|
||||
|
||||
assert not torch.equal(expected_grids[0], expected_grids[1])
|
||||
assert torch.equal(multi_grids[0], expected_grids[0])
|
||||
assert torch.equal(multi_grids[1], expected_grids[1])
|
||||
assert hf_mm_kwargs == hf_mm_kwargs_before
|
||||
|
||||
video_phs = processed["mm_placeholders"].get("video", [])
|
||||
assert len(video_phs) == 2, (
|
||||
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
|
||||
@@ -336,7 +500,12 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
|
||||
|
||||
capped_ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True},
|
||||
mm_processor_kwargs={
|
||||
"videos_kwargs": {
|
||||
"size": size,
|
||||
"cap_pixels_per_frame": True,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
|
||||
@@ -350,7 +519,7 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
|
||||
|
||||
uncapped_ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"size": size},
|
||||
mm_processor_kwargs={"videos_kwargs": {"size": size}},
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
|
||||
|
||||
@@ -3,18 +3,23 @@
|
||||
|
||||
import time
|
||||
from contextlib import nullcontext
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import TypedDict
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig, SchedulerConfig
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.hasher import MultiModalHasher
|
||||
from vllm.multimodal.parse import MultiModalDataParser
|
||||
from vllm.multimodal.processing.context import (
|
||||
BaseProcessingInfo,
|
||||
InputProcessingContext,
|
||||
overlay_modality_mm_kwargs,
|
||||
_resolve_mm_processor_kwargs,
|
||||
)
|
||||
from vllm.multimodal.processing.inputs import ProcessorInputs
|
||||
from vllm.multimodal.processing.processor import (
|
||||
@@ -1207,58 +1212,305 @@ def test_apply_prompt_updates_falls_back_with_index_targets():
|
||||
assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]]
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_overlay_modality_mm_kwargs_scoped_video_does_not_leak_to_image():
|
||||
"""HF-style videos_kwargs must overlay only when modality is video."""
|
||||
video_size = {"longest_edge": 469762048, "shortest_edge": 4096}
|
||||
kwargs = {"videos_kwargs": {"size": video_size}}
|
||||
|
||||
assert overlay_modality_mm_kwargs(kwargs, None) == kwargs
|
||||
assert "size" not in overlay_modality_mm_kwargs(kwargs, "image")
|
||||
assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == video_size
|
||||
class _TextProcessorKwargs(TypedDict, total=False):
|
||||
padding: bool
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_overlay_modality_mm_kwargs_flat_size_stays_shared():
|
||||
"""A flat size override keeps the current shared-namespace behavior."""
|
||||
size = {"longest_edge": 469762048, "shortest_edge": 4096}
|
||||
kwargs = {"size": size}
|
||||
|
||||
for modality in (None, "image", "video"):
|
||||
assert overlay_modality_mm_kwargs(kwargs, modality)["size"] == size
|
||||
class _AudioProcessorKwargs(TypedDict, total=False):
|
||||
sampling_rate: int
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_overlay_modality_mm_kwargs_scoped_wins_over_flat_for_modality():
|
||||
"""A nested videos_kwargs size wins over a flat size for video reads."""
|
||||
class _ProcessorKwargs(TypedDict, total=False):
|
||||
text_kwargs: _TextProcessorKwargs
|
||||
audio_kwargs: _AudioProcessorKwargs
|
||||
|
||||
|
||||
class _ImageProcessorKwargs(TypedDict, total=False):
|
||||
size: dict[str, int]
|
||||
min_pixels: int
|
||||
|
||||
|
||||
class _VideoProcessorKwargs(TypedDict, total=False):
|
||||
size: dict[str, int]
|
||||
fps: float
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("supported_mm_limits", "processor", "expected"),
|
||||
[
|
||||
(
|
||||
{"image": None, "video": None},
|
||||
SimpleNamespace(
|
||||
valid_processor_kwargs=_ProcessorKwargs,
|
||||
image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs),
|
||||
video_processor=SimpleNamespace(valid_kwargs=_VideoProcessorKwargs),
|
||||
),
|
||||
{
|
||||
"text_kwargs": {"padding"},
|
||||
"images_kwargs": {"size", "min_pixels"},
|
||||
"videos_kwargs": {"size", "fps"},
|
||||
},
|
||||
),
|
||||
(
|
||||
{"image": None},
|
||||
SimpleNamespace(
|
||||
valid_processor_kwargs=_ProcessorKwargs,
|
||||
image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs),
|
||||
),
|
||||
{
|
||||
"text_kwargs": {"padding"},
|
||||
"images_kwargs": {"size", "min_pixels"},
|
||||
},
|
||||
),
|
||||
(
|
||||
{"audio": None},
|
||||
SimpleNamespace(valid_processor_kwargs=_ProcessorKwargs),
|
||||
{
|
||||
"text_kwargs": {"padding"},
|
||||
"audio_kwargs": {"sampling_rate"},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_supported_mm_processor_kwargs_uses_supported_modalities(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
supported_mm_limits: dict[str, int | None],
|
||||
processor: SimpleNamespace,
|
||||
expected: dict[str, set[str]],
|
||||
) -> None:
|
||||
info = BaseProcessingInfo(SimpleNamespace())
|
||||
info.__dict__["supported_mm_limits"] = supported_mm_limits
|
||||
monkeypatch.setattr(info, "get_hf_processor", lambda **_: processor)
|
||||
|
||||
assert info.get_supported_mm_processor_kwargs() == expected
|
||||
|
||||
|
||||
def test_supported_mm_processor_kwargs_is_cached(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
info = BaseProcessingInfo(SimpleNamespace())
|
||||
expected = {"images_kwargs": {"size"}}
|
||||
calls = 0
|
||||
|
||||
def get_supported_mm_processor_kwargs() -> dict[str, set[str]]:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr(
|
||||
info,
|
||||
"get_supported_mm_processor_kwargs",
|
||||
get_supported_mm_processor_kwargs,
|
||||
)
|
||||
|
||||
first = info.supported_mm_processor_kwargs
|
||||
second = info.supported_mm_processor_kwargs
|
||||
|
||||
assert first is expected
|
||||
assert second is first
|
||||
assert calls == 1
|
||||
|
||||
|
||||
def test_resolve_mm_processor_kwargs_routes_supported_flat_kwargs():
|
||||
kwargs = {
|
||||
"size": {"longest_edge": 1},
|
||||
"videos_kwargs": {"size": {"longest_edge": 2}},
|
||||
"images_kwargs": {"size": {"longest_edge": 3}},
|
||||
"size": {"shortest_edge": 64},
|
||||
"padding": True,
|
||||
"sampling_rate": 16000,
|
||||
"unknown": "keep",
|
||||
"images_kwargs": {
|
||||
"size": {"longest_edge": 1024},
|
||||
"custom_image_kwarg": "keep",
|
||||
},
|
||||
}
|
||||
kwargs_before = deepcopy(kwargs)
|
||||
|
||||
resolved = _resolve_mm_processor_kwargs(
|
||||
kwargs,
|
||||
supported_mm_processor_kwargs={
|
||||
"text_kwargs": {"padding"},
|
||||
"images_kwargs": {"size"},
|
||||
"videos_kwargs": {"size"},
|
||||
"audio_kwargs": {"sampling_rate"},
|
||||
},
|
||||
)
|
||||
|
||||
assert resolved == {
|
||||
"unknown": "keep",
|
||||
"text_kwargs": {"padding": True},
|
||||
"images_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": 64,
|
||||
"longest_edge": 1024,
|
||||
},
|
||||
"custom_image_kwarg": "keep",
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"size": {"shortest_edge": 64},
|
||||
},
|
||||
"audio_kwargs": {"sampling_rate": 16000},
|
||||
}
|
||||
assert kwargs == kwargs_before
|
||||
|
||||
video_kwargs = resolved["videos_kwargs"]
|
||||
image_kwargs = resolved["images_kwargs"]
|
||||
assert isinstance(video_kwargs, dict)
|
||||
assert isinstance(image_kwargs, dict)
|
||||
video_size = video_kwargs["size"]
|
||||
image_size = image_kwargs["size"]
|
||||
assert isinstance(video_size, dict)
|
||||
assert isinstance(image_size, dict)
|
||||
video_size["shortest_edge"] = 999
|
||||
|
||||
assert image_size["shortest_edge"] == 64
|
||||
assert kwargs == kwargs_before
|
||||
|
||||
|
||||
def test_resolve_mm_processor_kwargs_rejects_non_mapping_destination_scope():
|
||||
with pytest.raises(TypeError, match="images_kwargs"):
|
||||
_resolve_mm_processor_kwargs(
|
||||
{
|
||||
"size": {"shortest_edge": 64},
|
||||
"images_kwargs": 123,
|
||||
},
|
||||
supported_mm_processor_kwargs={
|
||||
"images_kwargs": {"size"},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_mm_processor_kwargs_without_schema_deduplicates_existing_scopes():
|
||||
kwargs = {
|
||||
"size": {"shortest_edge": 64},
|
||||
"num_frames": 16,
|
||||
"fps": 4,
|
||||
"images_kwargs": {
|
||||
"size": {"longest_edge": 1024},
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"num_frames": 8,
|
||||
},
|
||||
"audio_kwargs": 123,
|
||||
"common_kwargs": {
|
||||
"fps": 2,
|
||||
},
|
||||
}
|
||||
kwargs_before = deepcopy(kwargs)
|
||||
|
||||
resolved = _resolve_mm_processor_kwargs(kwargs)
|
||||
|
||||
assert resolved == {
|
||||
"fps": 4,
|
||||
"images_kwargs": {
|
||||
"size": {"longest_edge": 1024},
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"num_frames": 8,
|
||||
},
|
||||
"audio_kwargs": 123,
|
||||
"common_kwargs": {
|
||||
"fps": 2,
|
||||
},
|
||||
}
|
||||
assert kwargs == kwargs_before
|
||||
assert "text_kwargs" not in resolved
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"inference_kwargs",
|
||||
[
|
||||
{"size": {"shortest_edge": 128}},
|
||||
{"size": {"shortest_edge": 128}, "images_kwargs": None},
|
||||
{"size": {"shortest_edge": 128}, "images_kwargs": {}},
|
||||
],
|
||||
)
|
||||
def test_get_merged_mm_kwargs_treats_empty_scopes_as_absent_before_routing(
|
||||
inference_kwargs: dict[str, object],
|
||||
):
|
||||
mm_config = MultiModalConfig(
|
||||
mm_processor_kwargs={
|
||||
"size": {
|
||||
"shortest_edge": 64,
|
||||
"longest_edge": 512,
|
||||
},
|
||||
"images_kwargs": {
|
||||
"size": {"longest_edge": 1024},
|
||||
},
|
||||
},
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
|
||||
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
|
||||
|
||||
merged = ctx.get_merged_mm_kwargs(
|
||||
inference_kwargs,
|
||||
supported_mm_processor_kwargs={
|
||||
"images_kwargs": {"size"},
|
||||
"videos_kwargs": {"size"},
|
||||
},
|
||||
)
|
||||
|
||||
assert merged == {
|
||||
"images_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": 128,
|
||||
"longest_edge": 1024,
|
||||
},
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": 128,
|
||||
"longest_edge": 512,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == {"longest_edge": 2}
|
||||
assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 3}
|
||||
assert overlay_modality_mm_kwargs(kwargs, None)["size"] == {"longest_edge": 1}
|
||||
|
||||
def test_get_merged_mm_kwargs_merges_before_routing():
|
||||
mm_config = MultiModalConfig(
|
||||
mm_processor_kwargs={
|
||||
"size": {
|
||||
"shortest_edge": 64,
|
||||
"longest_edge": 512,
|
||||
},
|
||||
"images_kwargs": {
|
||||
"size": {"longest_edge": 1024},
|
||||
},
|
||||
},
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
|
||||
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_overlay_modality_mm_kwargs_ignores_non_mapping_scoped_value():
|
||||
kwargs = {"images_kwargs": "not-a-dict", "size": {"longest_edge": 1}}
|
||||
assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 1}
|
||||
merged = ctx.get_merged_mm_kwargs(
|
||||
{
|
||||
"size": {
|
||||
"shortest_edge": 128,
|
||||
"longest_edge": 768,
|
||||
},
|
||||
"images_kwargs": {
|
||||
"size": {"longest_edge": 2048},
|
||||
},
|
||||
},
|
||||
supported_mm_processor_kwargs={
|
||||
"images_kwargs": {"size"},
|
||||
"videos_kwargs": {"size"},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping():
|
||||
"""Configured videos_kwargs overlay only for video reads after merge."""
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
|
||||
size = {"longest_edge": 469762048, "shortest_edge": 4096}
|
||||
mm_config = MultiModalConfig(mm_processor_kwargs={"videos_kwargs": {"size": size}})
|
||||
merged = mm_config.merge_mm_processor_kwargs({})
|
||||
assert overlay_modality_mm_kwargs(merged, "video")["size"] == size
|
||||
assert "size" not in overlay_modality_mm_kwargs(merged, "image")
|
||||
assert "size" not in overlay_modality_mm_kwargs(merged, None)
|
||||
assert merged == {
|
||||
"images_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": 128,
|
||||
"longest_edge": 2048,
|
||||
},
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": 128,
|
||||
"longest_edge": 768,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_processor_inputs_hashes_partial_uuids():
|
||||
|
||||
+187
-1
@@ -27,6 +27,186 @@ logger = init_logger(__name__)
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
# HuggingFace processors accept a shared flat namespace in addition to the
|
||||
# nested processor kwarg scopes ``text_kwargs`` / ``images_kwargs`` /
|
||||
# ``videos_kwargs`` / ``audio_kwargs``. vLLM recognizes these scopes when
|
||||
# resolving ``mm_processor_kwargs``.
|
||||
_HF_PROCESSOR_KWARG_SCOPES = (
|
||||
"text_kwargs",
|
||||
"images_kwargs",
|
||||
"videos_kwargs",
|
||||
"audio_kwargs",
|
||||
)
|
||||
|
||||
|
||||
def _recursively_merge_mm_processor_kwargs(
|
||||
defaults: Mapping[str, object],
|
||||
overrides: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""Merge two mappings recursively, with ``overrides`` taking precedence.
|
||||
|
||||
Nested values are merged only when both values are mappings; otherwise the
|
||||
override replaces the existing value.
|
||||
"""
|
||||
merged = dict(defaults)
|
||||
|
||||
for key, value in overrides.items():
|
||||
default_value = merged.get(key)
|
||||
if isinstance(default_value, Mapping) and isinstance(value, Mapping):
|
||||
merged[key] = _recursively_merge_mm_processor_kwargs(default_value, value)
|
||||
else:
|
||||
merged[key] = value
|
||||
|
||||
return merged
|
||||
|
||||
|
||||
def _merge_mm_processor_scope(
|
||||
config_flat: Mapping[str, object],
|
||||
config_scoped: Mapping[str, object],
|
||||
inference_flat: Mapping[str, object],
|
||||
inference_scoped: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""Merge kwargs for one existing HuggingFace processor kwarg scope.
|
||||
|
||||
Flat kwargs contribute only to keys present in either the configured or
|
||||
inference-time scope. For those keys, precedence is:
|
||||
configured flat < configured scoped < inference flat < inference scoped.
|
||||
|
||||
Nested mappings are merged recursively; otherwise the higher-priority value
|
||||
replaces the lower-priority value.
|
||||
"""
|
||||
# Flat kwargs contribute only to keys present in this scope in either input.
|
||||
scoped_keys = config_scoped.keys() | inference_scoped.keys()
|
||||
config_flat_for_scope = {
|
||||
key: value for key, value in config_flat.items() if key in scoped_keys
|
||||
}
|
||||
inference_flat_for_scope = {
|
||||
key: value for key, value in inference_flat.items() if key in scoped_keys
|
||||
}
|
||||
|
||||
# Merge from lowest to highest priority:
|
||||
# configured flat -> configured scoped -> inference flat -> inference scoped.
|
||||
merged = _recursively_merge_mm_processor_kwargs(
|
||||
config_flat_for_scope, config_scoped
|
||||
)
|
||||
merged = _recursively_merge_mm_processor_kwargs(merged, inference_flat_for_scope)
|
||||
merged = _recursively_merge_mm_processor_kwargs(merged, inference_scoped)
|
||||
return merged
|
||||
|
||||
|
||||
def _merge_mm_processor_kwargs(
|
||||
config_kwargs: Mapping[str, object],
|
||||
inference_kwargs: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""Merge configured and inference-time multi-modal processor kwargs.
|
||||
|
||||
The merge is performed in two parts:
|
||||
|
||||
1. The shared flat kwargs are merged recursively, with inference-time values
|
||||
taking precedence.
|
||||
2. Each existing HuggingFace processor kwarg scope is merged separately.
|
||||
For keys present in either the configured or inference-time scope,
|
||||
precedence is:
|
||||
configured flat < configured scoped < inference flat < inference scoped.
|
||||
|
||||
For example:
|
||||
|
||||
configured = {
|
||||
"size": {"shortest_edge": 256},
|
||||
"videos_kwargs": {
|
||||
"size": {"longest_edge": 1024},
|
||||
"num_frames": 16,
|
||||
},
|
||||
}
|
||||
inference = {
|
||||
"size": {"shortest_edge": 512},
|
||||
"videos_kwargs": {"fps": 4},
|
||||
}
|
||||
|
||||
produces:
|
||||
|
||||
{
|
||||
"size": {"shortest_edge": 512},
|
||||
"videos_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": 512,
|
||||
"longest_edge": 1024,
|
||||
},
|
||||
"num_frames": 16,
|
||||
"fps": 4,
|
||||
},
|
||||
}
|
||||
|
||||
Scoped values do not modify the shared flat kwargs or sibling scopes.
|
||||
Missing scopes, None-valued scopes, and empty scope mappings are treated as
|
||||
absent and do not create a scope.
|
||||
"""
|
||||
# Merge the shared flat kwargs first; nested scopes are handled separately
|
||||
# below.
|
||||
config_flat = {
|
||||
key: value
|
||||
for key, value in config_kwargs.items()
|
||||
if key not in _HF_PROCESSOR_KWARG_SCOPES
|
||||
}
|
||||
inference_flat = {
|
||||
key: value
|
||||
for key, value in inference_kwargs.items()
|
||||
if key not in _HF_PROCESSOR_KWARG_SCOPES
|
||||
}
|
||||
merged = _recursively_merge_mm_processor_kwargs(config_flat, inference_flat)
|
||||
|
||||
# Merge each nested processor kwarg scope present in either input.
|
||||
for scoped_key in _HF_PROCESSOR_KWARG_SCOPES:
|
||||
config_scope_value = config_kwargs.get(scoped_key)
|
||||
inference_scope_value = inference_kwargs.get(scoped_key)
|
||||
|
||||
# Treat missing scopes, None-valued scopes, and empty scope mappings as
|
||||
# absent.
|
||||
if isinstance(config_scope_value, Mapping) and not config_scope_value:
|
||||
config_scope_value = None
|
||||
if isinstance(inference_scope_value, Mapping) and not inference_scope_value:
|
||||
inference_scope_value = None
|
||||
|
||||
# Skip adding the scope to the merged result when it is absent from both
|
||||
# sources.
|
||||
if config_scope_value is None and inference_scope_value is None:
|
||||
continue
|
||||
|
||||
# Handle the inference scope first because it has higher precedence; only
|
||||
# mapping-valued scopes participate in the recursive scoped merge.
|
||||
inference_scope_kwargs: Mapping[str, object]
|
||||
if inference_scope_value is None:
|
||||
inference_scope_kwargs = {}
|
||||
elif not isinstance(inference_scope_value, Mapping):
|
||||
# Non-mapping scopes cannot participate in the recursive scoped merge;
|
||||
# the higher-priority inference value replaces the configured scope.
|
||||
merged[scoped_key] = inference_scope_value
|
||||
continue
|
||||
else:
|
||||
inference_scope_kwargs = inference_scope_value
|
||||
|
||||
# Handle the configured scope next because it has lower precedence; only
|
||||
# mapping-valued scopes participate in the recursive scoped merge.
|
||||
if isinstance(config_scope_value, Mapping):
|
||||
config_scope_kwargs = config_scope_value
|
||||
elif inference_scope_value is None:
|
||||
# Non-mapping scopes cannot participate in the recursive scoped merge;
|
||||
# preserve the configured value when inference contributes nothing.
|
||||
merged[scoped_key] = config_scope_value
|
||||
continue
|
||||
else:
|
||||
config_scope_kwargs = {}
|
||||
|
||||
# Merge this scope together with the matching flat kwargs.
|
||||
merged[scoped_key] = _merge_mm_processor_scope(
|
||||
config_flat,
|
||||
config_scope_kwargs,
|
||||
inference_flat,
|
||||
inference_scope_kwargs,
|
||||
)
|
||||
|
||||
return merged
|
||||
|
||||
|
||||
@dataclass
|
||||
class BaseDummyOptions:
|
||||
@@ -533,8 +713,14 @@ class MultiModalConfig:
|
||||
) -> dict[str, object]:
|
||||
"""Get the keyword arguments to pass to the multi-modal processor
|
||||
according to the extra arguments passed during inference.
|
||||
|
||||
Nested mappings are merged recursively, with inference-time values taking
|
||||
precedence over configured values.
|
||||
"""
|
||||
merged = dict(self.mm_processor_kwargs or {}) | dict(inference_kwargs)
|
||||
merged = _merge_mm_processor_kwargs(
|
||||
self.mm_processor_kwargs or {},
|
||||
inference_kwargs,
|
||||
)
|
||||
if self.mm_device_do_normalize:
|
||||
# The model expects raw pixels and normalises on device;
|
||||
# overriding these per request would silently double-normalise.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Literal
|
||||
from typing import Annotated, Any, Literal, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -212,8 +212,14 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
|
||||
) -> PixtralHFEncoderInfo:
|
||||
processor = self.get_hf_processor()
|
||||
size = processor.image_processor.size
|
||||
merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {})
|
||||
if override_size := merged_kwargs.get("size"):
|
||||
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(
|
||||
mm_processor_kwargs or {}
|
||||
)
|
||||
image_mm_kwargs = cast(
|
||||
Mapping[str, Any],
|
||||
merged_mm_kwargs.get("images_kwargs", {}),
|
||||
)
|
||||
if override_size := image_mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
|
||||
image_size = size["longest_edge"]
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
import math
|
||||
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
|
||||
from functools import partial
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
from typing import Annotated, Any, Literal, TypeAlias, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -865,6 +865,118 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"image": None, "video": None}
|
||||
|
||||
@staticmethod
|
||||
def _complete_mm_processor_size_aliases(
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""Complete missing Qwen-VL size aliases.
|
||||
|
||||
Qwen-VL accepts ``size.shortest_edge`` / ``min_pixels`` and
|
||||
``size.longest_edge`` / ``max_pixels`` as equivalent controls. When
|
||||
only one representation is provided with a non-``None`` value, add the
|
||||
missing equivalent representation. If both are already present, leave
|
||||
them unchanged even when their values conflict.
|
||||
|
||||
Apply this independently to the flat kwargs and to each existing
|
||||
``images_kwargs`` / ``videos_kwargs`` mapping. Values are not propagated
|
||||
between these mappings.
|
||||
"""
|
||||
|
||||
def add_missing_size_aliases(
|
||||
kwargs: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""Add missing size aliases within a kwargs mapping."""
|
||||
out_kwargs = dict(kwargs)
|
||||
# Copy `size` separately because `out_kwargs` is a shallow copy and
|
||||
# size edges may be added below.
|
||||
raw_size = cast(
|
||||
Mapping[str, object] | None,
|
||||
out_kwargs.get("size"),
|
||||
)
|
||||
size_kwargs = {} if raw_size is None else dict(raw_size)
|
||||
|
||||
for edge_key, pixel_key in (
|
||||
("shortest_edge", "min_pixels"),
|
||||
("longest_edge", "max_pixels"),
|
||||
):
|
||||
# Add the pixel alias from `size` when only the size edge is
|
||||
# present.
|
||||
if pixel_key not in out_kwargs:
|
||||
if edge_key in size_kwargs and size_kwargs[edge_key] is not None:
|
||||
out_kwargs[pixel_key] = size_kwargs[edge_key]
|
||||
continue
|
||||
|
||||
pixel_value = out_kwargs[pixel_key]
|
||||
# HF treats `min_pixels` / `max_pixels` set to None as no size
|
||||
# override, so do not propagate them to the equivalent size edge.
|
||||
if pixel_value is None:
|
||||
continue
|
||||
|
||||
# Add the size edge when only the pixel alias is present.
|
||||
if edge_key not in size_kwargs:
|
||||
size_kwargs[edge_key] = pixel_value
|
||||
out_kwargs["size"] = size_kwargs
|
||||
|
||||
return out_kwargs
|
||||
|
||||
prepared_mm_kwargs = add_missing_size_aliases(mm_kwargs)
|
||||
# Complete aliases independently inside existing image/video kwargs;
|
||||
# flat values are not copied into the nested mappings.
|
||||
for nested_kwargs_key in ("images_kwargs", "videos_kwargs"):
|
||||
nested_kwargs = prepared_mm_kwargs.get(nested_kwargs_key)
|
||||
if isinstance(nested_kwargs, Mapping):
|
||||
prepared_mm_kwargs[nested_kwargs_key] = add_missing_size_aliases(
|
||||
nested_kwargs
|
||||
)
|
||||
|
||||
return prepared_mm_kwargs
|
||||
|
||||
def _merge_and_resolve_mm_processor_kwargs(
|
||||
self,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> dict[str, Any]:
|
||||
"""Merge configured and request Qwen-VL ``mm_processor_kwargs``.
|
||||
|
||||
Request-side ``size.shortest_edge`` / ``min_pixels`` and
|
||||
``size.longest_edge`` / ``max_pixels`` aliases are completed before the
|
||||
merge so request precedence is preserved across equivalent
|
||||
representations.
|
||||
"""
|
||||
# Complete request-side Qwen-VL size aliases before merging with
|
||||
# configured kwargs.
|
||||
prepared_mm_kwargs = self._complete_mm_processor_size_aliases(mm_kwargs)
|
||||
|
||||
return super()._merge_and_resolve_mm_processor_kwargs(prepared_mm_kwargs)
|
||||
|
||||
@staticmethod
|
||||
def _get_vision_size(
|
||||
mm_kwargs: Mapping[str, object],
|
||||
default_size: Mapping[str, int] | None = None,
|
||||
) -> dict[str, int]:
|
||||
"""Resolve Qwen-VL vision size kwargs with optional processor defaults.
|
||||
|
||||
``size`` overrides any processor defaults, then ``min_pixels`` and
|
||||
``max_pixels`` override the corresponding size edges.
|
||||
"""
|
||||
size = dict(default_size) if default_size is not None else {}
|
||||
|
||||
override_size = cast(
|
||||
Mapping[str, int] | None,
|
||||
mm_kwargs.get("size"),
|
||||
)
|
||||
if override_size is not None:
|
||||
size.update(override_size)
|
||||
|
||||
min_pixels = cast(int | None, mm_kwargs.get("min_pixels"))
|
||||
if min_pixels is not None:
|
||||
size["shortest_edge"] = min_pixels
|
||||
|
||||
max_pixels = cast(int | None, mm_kwargs.get("max_pixels"))
|
||||
if max_pixels is not None:
|
||||
size["longest_edge"] = max_pixels
|
||||
|
||||
return size
|
||||
|
||||
def get_mm_max_tokens_per_item(
|
||||
self,
|
||||
seq_len: int,
|
||||
@@ -883,7 +995,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
do_resize: bool = True,
|
||||
image_processor: Qwen2VLImageProcessor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str | None = None,
|
||||
modality: Literal["image", "video"] | None = None,
|
||||
) -> tuple[ImageSize, int]:
|
||||
hf_config = self.get_hf_config()
|
||||
vision_config = hf_config.vision_config
|
||||
@@ -891,14 +1003,16 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
temporal_patch_size = vision_config.temporal_patch_size
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
|
||||
size = image_processor.size
|
||||
if override_size := mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
|
||||
size = size | {"shortest_edge": override_min_pixels}
|
||||
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
|
||||
size = size | {"longest_edge": override_max_pixels}
|
||||
if modality is None:
|
||||
modality = "image"
|
||||
|
||||
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
||||
scope = "images_kwargs" if modality == "image" else "videos_kwargs"
|
||||
scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
|
||||
size = self._get_vision_size(
|
||||
scoped_mm_kwargs,
|
||||
default_size=image_processor.size,
|
||||
)
|
||||
|
||||
if do_resize:
|
||||
resized_height, resized_width = smart_resize(
|
||||
@@ -986,14 +1100,12 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
if max_pixels is None:
|
||||
image_processor = self.get_image_processor()
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="image")
|
||||
size = image_processor.size
|
||||
if override_size := mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
|
||||
size = size | {"shortest_edge": override_min_pixels}
|
||||
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
|
||||
size = size | {"longest_edge": override_max_pixels}
|
||||
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
|
||||
image_mm_kwargs = merged_mm_kwargs.get("images_kwargs", {})
|
||||
size = self._get_vision_size(
|
||||
image_mm_kwargs,
|
||||
default_size=image_processor.size,
|
||||
)
|
||||
|
||||
max_pixels = size["longest_edge"]
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
|
||||
from functools import lru_cache, partial
|
||||
from itertools import islice
|
||||
from typing import Any
|
||||
from typing import Any, Literal
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -944,6 +944,16 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
|
||||
supported = super().get_supported_mm_processor_kwargs()
|
||||
|
||||
# vLLM has historically accepted `min_pixels` / `max_pixels` for
|
||||
# Qwen3-VL video. Include them in the routing schema even though
|
||||
# Qwen3VLVideoProcessor only accepts the equivalent `size`; they are
|
||||
# folded into `size` before calling HF.
|
||||
supported["videos_kwargs"].update({"min_pixels", "max_pixels"})
|
||||
return supported
|
||||
|
||||
def _get_vision_info(
|
||||
self,
|
||||
*,
|
||||
@@ -953,7 +963,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
do_resize: bool = True,
|
||||
image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str | None = None,
|
||||
modality: Literal["image", "video"] | None = None,
|
||||
) -> tuple[ImageSize, int]:
|
||||
is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
|
||||
|
||||
@@ -965,14 +975,14 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
|
||||
if modality is None:
|
||||
modality = "video" if is_video else "image"
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
|
||||
size = image_processor.size
|
||||
if override_size := mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
|
||||
size = size | {"shortest_edge": override_min_pixels}
|
||||
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
|
||||
size = size | {"longest_edge": override_max_pixels}
|
||||
|
||||
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
||||
scope = "videos_kwargs" if modality == "video" else "images_kwargs"
|
||||
scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
|
||||
size = self._get_vision_size(
|
||||
scoped_mm_kwargs,
|
||||
default_size=image_processor.size,
|
||||
)
|
||||
|
||||
if do_resize:
|
||||
if is_video:
|
||||
@@ -1030,9 +1040,13 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
) -> int:
|
||||
video_processor = self.get_video_processor()
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video")
|
||||
video_size = mm_kwargs.get("size", video_processor.size)
|
||||
temporal_patch_size = mm_kwargs.get(
|
||||
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
|
||||
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
|
||||
video_size = self._get_vision_size(
|
||||
video_mm_kwargs,
|
||||
default_size=video_processor.size,
|
||||
)
|
||||
temporal_patch_size = video_mm_kwargs.get(
|
||||
"temporal_patch_size", video_processor.temporal_patch_size
|
||||
)
|
||||
|
||||
@@ -1162,9 +1176,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
|
||||
|
||||
video_processor = self.info.get_video_processor()
|
||||
|
||||
mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}, modality="video")
|
||||
video_size = mm_kwargs.get("size", video_processor.size)
|
||||
temporal_patch_size = mm_kwargs.get(
|
||||
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs({})
|
||||
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
|
||||
video_size = self.info._get_vision_size(
|
||||
video_mm_kwargs,
|
||||
default_size=video_processor.size,
|
||||
)
|
||||
temporal_patch_size = video_mm_kwargs.get(
|
||||
"temporal_patch_size", video_processor.temporal_patch_size
|
||||
)
|
||||
|
||||
@@ -1180,13 +1198,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
|
||||
# over enough frames that the cap is not binding for the dummy.
|
||||
# This takes precedence over a shrinking num_frames override, which
|
||||
# would otherwise reintroduce the under-profiling.
|
||||
if mm_kwargs.get("cap_pixels_per_frame"):
|
||||
max_video_tokens = mm_kwargs.get(
|
||||
if video_mm_kwargs.get("cap_pixels_per_frame"):
|
||||
max_video_tokens = video_mm_kwargs.get(
|
||||
"max_video_tokens",
|
||||
getattr(video_processor, "max_video_tokens", 768),
|
||||
)
|
||||
patch_size = mm_kwargs.get("patch_size", video_processor.patch_size)
|
||||
merge_size = mm_kwargs.get("merge_size", video_processor.merge_size)
|
||||
patch_size = video_mm_kwargs.get("patch_size", video_processor.patch_size)
|
||||
merge_size = video_mm_kwargs.get("merge_size", video_processor.merge_size)
|
||||
per_frame_cap = max_video_tokens * (patch_size * merge_size) ** 2
|
||||
target_num_frames = max(
|
||||
target_num_frames,
|
||||
@@ -1348,6 +1366,47 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
if not hf_data:
|
||||
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
# Merge and resolve the multimodal processor kwargs once so every HF
|
||||
# call below starts from the same engine/request state and scoped form.
|
||||
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs(hf_kwargs)
|
||||
|
||||
if "images_kwargs" in merged_mm_kwargs:
|
||||
image_scoped_kwargs = dict(merged_mm_kwargs["images_kwargs"])
|
||||
|
||||
# Qwen2VLImageProcessor requires both `size` edges. If a partial size
|
||||
# override is passed, fill missing edges from the processor defaults
|
||||
# before any HF call.
|
||||
if image_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
|
||||
default_image_size = self.info.get_image_processor().size
|
||||
image_scoped_kwargs["size"] = self.info._get_vision_size(
|
||||
image_scoped_kwargs,
|
||||
default_size=default_image_size,
|
||||
)
|
||||
|
||||
merged_mm_kwargs["images_kwargs"] = image_scoped_kwargs
|
||||
|
||||
if "videos_kwargs" in merged_mm_kwargs:
|
||||
video_scoped_kwargs = dict(merged_mm_kwargs["videos_kwargs"])
|
||||
|
||||
# Qwen3VLVideoProcessor requires both `size` edges. If a partial size
|
||||
# override is passed, fill missing edges from the processor defaults
|
||||
# before any HF call.
|
||||
if video_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
|
||||
default_video_size = self.info.get_video_processor().size
|
||||
video_scoped_kwargs["size"] = self.info._get_vision_size(
|
||||
video_scoped_kwargs,
|
||||
default_size=default_video_size,
|
||||
)
|
||||
|
||||
# Qwen3VLVideoProcessor does not accept vLLM's min/max pixel
|
||||
# compatibility kwargs. They have already been folded into `size`,
|
||||
# so remove them before any HF call, including calls that contain
|
||||
# only text/images.
|
||||
video_scoped_kwargs.pop("min_pixels", None)
|
||||
video_scoped_kwargs.pop("max_pixels", None)
|
||||
|
||||
merged_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
|
||||
|
||||
# Separate video processing from image processing. Because the videos
|
||||
# are processed into several image patches
|
||||
video_input_ids_lst: list[list[int]] = []
|
||||
@@ -1373,32 +1432,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
# used to calculate the timestamps. Make sure that
|
||||
# do_sample_frames in mm_kwargs is false for presampled videos.
|
||||
|
||||
# NOTE: a copy of is created to update do_sample_frames,
|
||||
# otherwise mm_hash for the object will be incorrect.
|
||||
video_mm_kwargs: dict[str, Any] = dict(hf_kwargs)
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")
|
||||
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
|
||||
video_size = dict(self.info.get_video_processor().size)
|
||||
size_override = merged.get("size")
|
||||
if size_override is not None:
|
||||
video_size = video_size | size_override
|
||||
min_pixels = merged.get("min_pixels")
|
||||
if min_pixels is not None:
|
||||
video_size["shortest_edge"] = min_pixels
|
||||
max_pixels = merged.get("max_pixels")
|
||||
if max_pixels is not None:
|
||||
video_size["longest_edge"] = max_pixels
|
||||
video_mm_kwargs["size"] = video_size
|
||||
sampled_fps = video_mm_kwargs.get("fps")
|
||||
# The root copy is shallow, so give each item its own video
|
||||
# scope before modifying per-video values below.
|
||||
video_mm_kwargs = dict(merged_mm_kwargs)
|
||||
video_scoped_kwargs = dict(video_mm_kwargs.get("videos_kwargs", {}))
|
||||
video_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
|
||||
|
||||
sampled_fps = video_scoped_kwargs.get("fps")
|
||||
if is_list_of(sampled_fps, float):
|
||||
video_mm_kwargs["fps"] = sampled_fps[item_idx]
|
||||
sampled_num_frames = video_mm_kwargs.get("num_frames")
|
||||
video_scoped_kwargs["fps"] = sampled_fps[item_idx]
|
||||
sampled_num_frames = video_scoped_kwargs.get("num_frames")
|
||||
if is_list_of(sampled_num_frames, int):
|
||||
video_mm_kwargs["num_frames"] = sampled_num_frames[item_idx]
|
||||
if "do_sample_frames" not in video_mm_kwargs:
|
||||
video_scoped_kwargs["num_frames"] = sampled_num_frames[item_idx]
|
||||
if "do_sample_frames" not in video_scoped_kwargs:
|
||||
# qwen_vl_utils already has "do_sample_frames" in
|
||||
# mm_kwargs, don't overwrite it.
|
||||
video_mm_kwargs["do_sample_frames"] = metadata.get(
|
||||
video_scoped_kwargs["do_sample_frames"] = metadata.get(
|
||||
"do_sample_frames", False
|
||||
)
|
||||
|
||||
@@ -1409,9 +1458,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
# Compute timestamps here where we have access to metadata
|
||||
timestamps = self.info._get_video_second_idx(
|
||||
metadata=metadata,
|
||||
do_sample_frames=video_mm_kwargs["do_sample_frames"],
|
||||
sampled_fps=video_mm_kwargs.get("fps"),
|
||||
sampled_num_frames=video_mm_kwargs.get("num_frames"),
|
||||
do_sample_frames=video_scoped_kwargs["do_sample_frames"],
|
||||
sampled_fps=video_scoped_kwargs.get("fps"),
|
||||
sampled_num_frames=video_scoped_kwargs.get("num_frames"),
|
||||
)
|
||||
timestamps_per_video.append(timestamps)
|
||||
|
||||
@@ -1423,8 +1472,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
# to prevent HF's BaseVideoProcessor.preprocess() from
|
||||
# filling in the class default (fps=2) via setdefault(),
|
||||
# which would conflict with num_frames (mutually exclusive).
|
||||
if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
|
||||
video_mm_kwargs["fps"] = None
|
||||
if (
|
||||
"num_frames" in video_scoped_kwargs
|
||||
and "fps" not in video_scoped_kwargs
|
||||
):
|
||||
video_scoped_kwargs["fps"] = None
|
||||
|
||||
video_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**video_mm_kwargs),
|
||||
@@ -1485,11 +1537,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
else:
|
||||
video_outputs = dict()
|
||||
|
||||
# fps/num_frames are video-only kwargs already consumed by the loop;
|
||||
# exclude them so the text/image processor call below never gets a list.
|
||||
non_video_mm_kwargs = {
|
||||
k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames")
|
||||
}
|
||||
# The root copy is shallow, so copy the video scope before removing
|
||||
# per-video kwargs to avoid mutating `merged_mm_kwargs`.
|
||||
non_video_mm_kwargs = dict(merged_mm_kwargs)
|
||||
if "videos_kwargs" in non_video_mm_kwargs:
|
||||
non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"])
|
||||
non_video_scoped_kwargs.pop("fps", None)
|
||||
non_video_scoped_kwargs.pop("num_frames", None)
|
||||
non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**non_video_mm_kwargs),
|
||||
hf_data,
|
||||
|
||||
@@ -5,7 +5,7 @@ from collections.abc import Callable, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any, overload
|
||||
from typing import TYPE_CHECKING, Any, get_type_hints, overload
|
||||
|
||||
import torch
|
||||
from typing_extensions import TypeVar
|
||||
@@ -42,46 +42,129 @@ else:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# HuggingFace processors accept nested ``images_kwargs`` / ``videos_kwargs`` /
|
||||
# ``audio_kwargs`` in addition to a shared flat namespace. vLLM's token-budget
|
||||
# and dummy-input code only reads the flat keys, so scoped overrides have to
|
||||
# be overlaid per modality.
|
||||
_HF_MODALITY_PROCESSOR_KWARGS = {
|
||||
"image": "images_kwargs",
|
||||
"video": "videos_kwargs",
|
||||
"audio": "audio_kwargs",
|
||||
}
|
||||
# HuggingFace processors accept a shared flat namespace in addition to the
|
||||
# nested processor kwarg scopes ``text_kwargs`` / ``images_kwargs`` /
|
||||
# ``videos_kwargs`` / ``audio_kwargs``. vLLM recognizes these scopes when
|
||||
# resolving ``mm_processor_kwargs``.
|
||||
_HF_PROCESSOR_KWARG_SCOPES = (
|
||||
"text_kwargs",
|
||||
"images_kwargs",
|
||||
"videos_kwargs",
|
||||
"audio_kwargs",
|
||||
)
|
||||
|
||||
|
||||
def overlay_modality_mm_kwargs(
|
||||
kwargs: Mapping[str, object],
|
||||
modality: str | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Overlay HF-style nested processor kwargs onto the flat namespace.
|
||||
|
||||
Overlay only the requested modality so a video ``size`` bump does not leak
|
||||
into the image budget. Flat keys keep their current shared-namespace
|
||||
behavior when no scoped dict is present.
|
||||
|
||||
Args:
|
||||
kwargs: Merged multi-modal processor kwargs.
|
||||
modality: Target modality (``image``, ``video``, or ``audio``).
|
||||
When ``None``, ``kwargs`` is copied without overlay.
|
||||
|
||||
Returns:
|
||||
A new dict with the modality-scoped keys overlaid when present.
|
||||
def _merge_scoped_mm_processor_value(
|
||||
flat_value: object,
|
||||
scoped_value: object,
|
||||
) -> object:
|
||||
"""Merge a flat value with an existing, more specific scoped value.
|
||||
|
||||
Mappings are merged recursively; otherwise the scoped value replaces the
|
||||
flat value.
|
||||
"""
|
||||
merged = dict(kwargs)
|
||||
if modality is None:
|
||||
if isinstance(flat_value, Mapping) and isinstance(scoped_value, Mapping):
|
||||
# Merge recursively so scoped leaves override matching flat leaves
|
||||
# without dropping unrelated flat entries.
|
||||
merged = dict(flat_value)
|
||||
for key, value in scoped_value.items():
|
||||
if key in merged:
|
||||
merged[key] = _merge_scoped_mm_processor_value(merged[key], value)
|
||||
else:
|
||||
merged[key] = value
|
||||
return merged
|
||||
scoped_key = _HF_MODALITY_PROCESSOR_KWARGS.get(modality)
|
||||
if scoped_key is None:
|
||||
return merged
|
||||
scoped = merged.get(scoped_key)
|
||||
if not isinstance(scoped, Mapping):
|
||||
return merged
|
||||
return merged | dict(scoped)
|
||||
return scoped_value
|
||||
|
||||
|
||||
def _resolve_mm_processor_kwargs(
|
||||
kwargs: Mapping[str, object],
|
||||
supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Resolve flat processor kwargs into HuggingFace processor kwarg scopes.
|
||||
|
||||
With ``supported_mm_processor_kwargs``, each scope lists the flat keys it
|
||||
supports. Matching flat keys are routed to every scope that supports them,
|
||||
existing scoped values take precedence, and keys not supported by any scope
|
||||
remain flat.
|
||||
|
||||
Without ``supported_mm_processor_kwargs``, flat keys already represented in
|
||||
at least one existing mapping-valued scope are removed from the shared flat
|
||||
namespace; all other flat keys are left unchanged because there is no
|
||||
information to determine which scopes should receive them.
|
||||
"""
|
||||
resolved = dict(kwargs)
|
||||
|
||||
# Without ``supported_mm_processor_kwargs``, remove flat entries already
|
||||
# represented in at least one existing mapping-valued scope. There is no
|
||||
# information to route the remaining flat entries.
|
||||
if supported_mm_processor_kwargs is None:
|
||||
# Collect all keys represented in existing mapping-valued scopes.
|
||||
# Non-mapping scope values do not contain scoped kwargs and are left
|
||||
# untouched.
|
||||
represented_scoped_keys: set[str] = set()
|
||||
for scoped_key in _HF_PROCESSOR_KWARG_SCOPES:
|
||||
scoped_kwargs = resolved.get(scoped_key)
|
||||
if isinstance(scoped_kwargs, Mapping):
|
||||
represented_scoped_keys.update(scoped_kwargs)
|
||||
|
||||
# Remove flat entries for keys already represented in a scope.
|
||||
for key in represented_scoped_keys:
|
||||
if key not in _HF_PROCESSOR_KWARG_SCOPES:
|
||||
resolved.pop(key, None)
|
||||
return resolved
|
||||
|
||||
# With ``supported_mm_processor_kwargs``, collect the flat kwargs before
|
||||
# routing each one to the processor kwarg scopes that support it.
|
||||
flat_kwargs = {
|
||||
key: value
|
||||
for key, value in resolved.items()
|
||||
if key not in _HF_PROCESSOR_KWARG_SCOPES
|
||||
}
|
||||
|
||||
for key, flat_value in flat_kwargs.items():
|
||||
# Find every processor kwarg scope that supports this flat key. If none
|
||||
# supports it, keep the key in the shared flat namespace.
|
||||
supported_scopes = [
|
||||
scoped_key
|
||||
for scoped_key in _HF_PROCESSOR_KWARG_SCOPES
|
||||
if key in supported_mm_processor_kwargs.get(scoped_key, set())
|
||||
]
|
||||
if not supported_scopes:
|
||||
continue
|
||||
|
||||
# Add the flat value to each supported destination scope.
|
||||
for scoped_key in supported_scopes:
|
||||
# Start from a copy of the existing scoped kwargs, or an empty scope
|
||||
# if absent.
|
||||
if scoped_key in resolved:
|
||||
scoped_kwargs = resolved[scoped_key]
|
||||
# A scope must be a mapping before keys can be added to it. Do
|
||||
# not replace an explicit non-mapping value.
|
||||
if not isinstance(scoped_kwargs, Mapping):
|
||||
raise TypeError(f"`{scoped_key}` must be a mapping")
|
||||
scoped_kwargs = dict(scoped_kwargs)
|
||||
else:
|
||||
scoped_kwargs = {}
|
||||
|
||||
# Add the flat value to this scope. Existing scoped values take
|
||||
# precedence.
|
||||
if key in scoped_kwargs:
|
||||
scoped_kwargs[key] = _merge_scoped_mm_processor_value(
|
||||
flat_value, scoped_kwargs[key]
|
||||
)
|
||||
else:
|
||||
# Assign the flat value to this scope. For mappings, assign a
|
||||
# copy so the same mapping object is not shared between the flat
|
||||
# value and multiple scopes.
|
||||
scoped_kwargs[key] = (
|
||||
dict(flat_value) if isinstance(flat_value, Mapping) else flat_value
|
||||
)
|
||||
resolved[scoped_key] = scoped_kwargs
|
||||
|
||||
# After routing to every destination, remove the flat copy.
|
||||
resolved.pop(key)
|
||||
|
||||
return resolved
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -294,19 +377,27 @@ class InputProcessingContext:
|
||||
self,
|
||||
kwargs: Mapping[str, object],
|
||||
*,
|
||||
modality: str | None = None,
|
||||
supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Merge configured and request ``mm_processor_kwargs``.
|
||||
|
||||
When ``modality`` is set, HF-style nested
|
||||
``images_kwargs`` / ``videos_kwargs`` / ``audio_kwargs`` are overlaid
|
||||
onto the flat namespace for vLLM-side reads (token budgets, dummy
|
||||
inputs). Processor construction and HF ``__call__`` should omit
|
||||
``modality`` so the nested dicts still reach the HF processor.
|
||||
When ``supported_mm_processor_kwargs`` is provided, flat keys are matched
|
||||
against the keys supported by each HuggingFace processor kwarg scope.
|
||||
Matching flat keys are routed to every scope that supports them and
|
||||
removed from the shared flat namespace; keys not supported by any scope
|
||||
remain flat.
|
||||
|
||||
Without ``supported_mm_processor_kwargs``, flat keys already represented
|
||||
in at least one existing mapping-valued scope are removed from the shared
|
||||
flat namespace; all other flat keys are left unchanged because there is
|
||||
no information to determine which scopes should receive them.
|
||||
"""
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
merged = mm_config.merge_mm_processor_kwargs(kwargs)
|
||||
return overlay_modality_mm_kwargs(merged, modality)
|
||||
return _resolve_mm_processor_kwargs(
|
||||
merged,
|
||||
supported_mm_processor_kwargs=supported_mm_processor_kwargs,
|
||||
)
|
||||
|
||||
def call_hf_processor(
|
||||
self,
|
||||
@@ -380,6 +471,49 @@ class BaseProcessingInfo:
|
||||
"""
|
||||
return self.ctx.get_hf_processor(**kwargs)
|
||||
|
||||
def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
|
||||
"""Return supported kwarg names for each HF processor kwargs scope."""
|
||||
processor = self.get_hf_processor()
|
||||
processor_kwargs = get_type_hints(processor.valid_processor_kwargs)
|
||||
|
||||
supported = {
|
||||
"text_kwargs": set(processor_kwargs["text_kwargs"].__annotations__),
|
||||
}
|
||||
if "image" in self.supported_mm_limits:
|
||||
supported["images_kwargs"] = set(
|
||||
processor.image_processor.valid_kwargs.__annotations__
|
||||
)
|
||||
if "video" in self.supported_mm_limits:
|
||||
supported["videos_kwargs"] = set(
|
||||
processor.video_processor.valid_kwargs.__annotations__
|
||||
)
|
||||
if "audio" in self.supported_mm_limits:
|
||||
supported["audio_kwargs"] = set(
|
||||
processor_kwargs["audio_kwargs"].__annotations__
|
||||
)
|
||||
|
||||
return supported
|
||||
|
||||
@cached_property
|
||||
def supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
|
||||
"""Supported kwarg names for each HF processor kwargs scope."""
|
||||
return self.get_supported_mm_processor_kwargs()
|
||||
|
||||
def _merge_and_resolve_mm_processor_kwargs(
|
||||
self,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> dict[str, Any]:
|
||||
"""Merge configured and request ``mm_processor_kwargs``.
|
||||
|
||||
Flat kwargs are routed into the HuggingFace processor kwarg scopes that
|
||||
support them after the configured/request merge. When a routed flat value
|
||||
conflicts with an existing scoped value, the scoped value takes precedence.
|
||||
"""
|
||||
return self.ctx.get_merged_mm_kwargs(
|
||||
mm_kwargs,
|
||||
supported_mm_processor_kwargs=self.supported_mm_processor_kwargs,
|
||||
)
|
||||
|
||||
def get_default_tok_params(self) -> TokenizeParams:
|
||||
"""Construct the default parameters for tokenization."""
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
Reference in New Issue
Block a user