[MM] Move supports_multimodal_inputs and cache out of registry (#57913)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,12 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import copy
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from tests.models.utils import build_model_context
|
||||
from vllm.config.ec_transfer import ECRole, ECTransferConfig
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
@@ -17,6 +19,26 @@ from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id,limit_mm_per_prompt,expected",
|
||||
[
|
||||
("Qwen/Qwen2-0.5B-Instruct", {}, False),
|
||||
("Qwen/Qwen2.5-VL-3B-Instruct", {}, True),
|
||||
("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0, "video": 0}, False),
|
||||
("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0}, True),
|
||||
],
|
||||
)
|
||||
@pytest.mark.core_model
|
||||
def test_supports_multimodal_inputs(model_id, limit_mm_per_prompt, expected):
|
||||
"""Test supports_multimodal_inputs returns correct boolean for various
|
||||
configs."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
)
|
||||
assert ctx.model_config.supports_multimodal_inputs is expected
|
||||
|
||||
|
||||
def test_mm_encoder_attn_backend_str_conversion():
|
||||
config = MultiModalConfig(mm_encoder_attn_backend="FLASH_ATTN") # type: ignore[arg-type]
|
||||
assert config.mm_encoder_attn_backend == AttentionBackendEnum.FLASH_ATTN
|
||||
@@ -84,59 +106,33 @@ def test_mm_encoder_attn_dtype_hash_updates(tmp_path):
|
||||
assert fp8_hash != fp8_static_hash
|
||||
|
||||
|
||||
_MULTIMODAL_MODEL = "llava-hf/llava-1.5-7b-hf"
|
||||
_TEXT_ONLY_MODEL = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
|
||||
|
||||
|
||||
def _make_mm_prefix_model_config(
|
||||
model: str = _MULTIMODAL_MODEL,
|
||||
*,
|
||||
language_model_only: bool = False,
|
||||
) -> ModelConfig:
|
||||
model_config = MagicMock(spec=ModelConfig)
|
||||
model_config.multimodal_config = MultiModalConfig(
|
||||
language_model_only=language_model_only
|
||||
)
|
||||
# Bind real helper methods onto the mock.
|
||||
model_config._supports_multimodal_for_mm_prefix = (
|
||||
ModelConfig._supports_multimodal_for_mm_prefix.__get__(
|
||||
model_config, ModelConfig
|
||||
)
|
||||
)
|
||||
return model_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("supports_mm", [True, False])
|
||||
def test_supports_multimodal_for_mm_prefix_uses_registry(supports_mm: bool):
|
||||
model_config = _make_mm_prefix_model_config()
|
||||
|
||||
with patch(
|
||||
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
|
||||
return_value=supports_mm,
|
||||
) as mocked:
|
||||
assert model_config._supports_multimodal_for_mm_prefix() is supports_mm
|
||||
mocked.assert_called_once_with(model_config)
|
||||
|
||||
# Sticky cache — registry must not be consulted again.
|
||||
with patch(
|
||||
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
|
||||
side_effect=AssertionError("should use cache"),
|
||||
):
|
||||
assert model_config._supports_multimodal_for_mm_prefix() is supports_mm
|
||||
return ModelConfig(model, language_model_only=language_model_only)
|
||||
|
||||
|
||||
def test_supports_multimodal_for_mm_prefix_before_multimodal_config():
|
||||
model_config = _make_mm_prefix_model_config()
|
||||
model_config.multimodal_config = None
|
||||
"""A text-only model never builds a multimodal config, so the early
|
||||
return must neither clear mm_prefix nor write the sticky cache."""
|
||||
model_config = _make_mm_prefix_model_config(_TEXT_ONLY_MODEL)
|
||||
assert model_config.multimodal_config is None
|
||||
|
||||
assert model_config._supports_multimodal_for_mm_prefix() is True
|
||||
assert not hasattr(model_config, "_supports_multimodal_inputs_cached")
|
||||
assert not getattr(model_config, "_supports_multimodal_inputs_cache", None)
|
||||
|
||||
|
||||
def test_language_model_only_disables_via_supports_multimodal_inputs():
|
||||
"""language_model_only zeros all limits, so registry reports text-only."""
|
||||
model_config = _make_mm_prefix_model_config(language_model_only=True)
|
||||
|
||||
with patch(
|
||||
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
|
||||
return_value=False,
|
||||
):
|
||||
assert model_config._supports_multimodal_for_mm_prefix() is False
|
||||
assert model_config._supports_multimodal_for_mm_prefix() is False
|
||||
|
||||
|
||||
def test_convertor_clears_mm_prefix_when_multimodal_disabled():
|
||||
@@ -159,23 +155,19 @@ def test_convertor_clears_mm_prefix_when_multimodal_disabled():
|
||||
def test_sticky_cache_survives_text_subconfig_regeneration():
|
||||
"""with_hf_config deepcopies the cached decision onto text submodules."""
|
||||
model_config = _make_mm_prefix_model_config()
|
||||
with patch(
|
||||
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
|
||||
return_value=False,
|
||||
):
|
||||
assert model_config._supports_multimodal_for_mm_prefix() is False
|
||||
assert model_config._supports_multimodal_for_mm_prefix() is True
|
||||
|
||||
# Simulate deepcopy onto a Gemma4ForCausalLM-like config that would
|
||||
# otherwise fail registry lookup / return False incorrectly.
|
||||
text_config = _make_mm_prefix_model_config()
|
||||
text_config._supports_multimodal_inputs_cached = (
|
||||
model_config._supports_multimodal_inputs_cached
|
||||
)
|
||||
with patch(
|
||||
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
|
||||
side_effect=AssertionError("must not re-query registry"),
|
||||
):
|
||||
assert text_config._supports_multimodal_for_mm_prefix() is False
|
||||
# `with_hf_config` deep-copies this config and swaps `hf_config` for a
|
||||
# text-only submodule (e.g. Gemma4ForCausalLM).
|
||||
text_config = copy.deepcopy(model_config)
|
||||
text_config.hf_config = model_config.hf_text_config
|
||||
assert text_config._supports_multimodal_for_mm_prefix() is True
|
||||
|
||||
# Without the copied cache the submodule architecture has no registered
|
||||
# multimodal processor, so re-querying the registry wrongly settles on
|
||||
# text-only and would clear mm_prefix.
|
||||
del text_config._supports_multimodal_inputs_cache
|
||||
assert text_config._supports_multimodal_for_mm_prefix() is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -60,7 +60,10 @@ class MockModelConfig:
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
skip_tokenizer_init = False
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
vocab_size = 151936
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
|
||||
@@ -56,9 +56,10 @@ class MockModelConfig:
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
skip_tokenizer_init = False
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
|
||||
@@ -556,6 +556,7 @@ class MockModelConfig:
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
enable_prompt_embeds: bool = False
|
||||
|
||||
|
||||
@@ -61,9 +61,10 @@ class MockModelConfig:
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
skip_tokenizer_init = False
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
|
||||
@@ -54,6 +54,7 @@ class MockModelConfig:
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
|
||||
@@ -86,9 +86,10 @@ class MockModelConfig:
|
||||
generation_config: str = "auto"
|
||||
override_generation_config: dict[str, Any] = field(default_factory=dict)
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
skip_tokenizer_init = False
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
enable_prompt_embeds: bool = False
|
||||
|
||||
|
||||
@@ -61,9 +61,10 @@ class MockModelConfig:
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
skip_tokenizer_init = False
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
|
||||
@@ -56,9 +56,10 @@ class MockModelConfig:
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
skip_tokenizer_init = False
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
|
||||
@@ -22,7 +22,7 @@ import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorSenderCache
|
||||
from vllm.multimodal.cache import LruKeyReplicatedSenderCache
|
||||
|
||||
from ....multimodal.utils import random_audio, random_video
|
||||
from ...utils import build_model_context
|
||||
@@ -68,7 +68,7 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non
|
||||
# Sender cache: on a cache hit returns (None, prompt_updates) for each
|
||||
# item, setting mm_kwargs["video"] = [None] – the exact condition that
|
||||
# triggered the original bug.
|
||||
sender_cache = MultiModalProcessorSenderCache(ctx.model_config)
|
||||
sender_cache = LruKeyReplicatedSenderCache(ctx.model_config)
|
||||
|
||||
video_token_id = processor.info.get_hf_config().video_token_id
|
||||
|
||||
|
||||
@@ -10,20 +10,23 @@ import torch
|
||||
|
||||
from vllm.config import ModelConfig, ParallelConfig, VllmConfig
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import (
|
||||
BaseMultiModalProcessorCache,
|
||||
BaseMultiModalReceiverCache,
|
||||
MultiModalCache,
|
||||
LruKeyReplicatedReceiverCache,
|
||||
LruKeyReplicatedSenderCache,
|
||||
MultiModalCacheMissError,
|
||||
MultiModalProcessorOnlyCache,
|
||||
ShmObjectStoreReceiverCache,
|
||||
ShmObjectStoreSenderCache,
|
||||
engine_receiver_cache_from_config,
|
||||
processor_cache_from_config,
|
||||
)
|
||||
from vllm.multimodal.cache.base import (
|
||||
MultiModalCache,
|
||||
MultiModalProcessorCacheInItem,
|
||||
MultiModalProcessorCacheItem,
|
||||
MultiModalProcessorCacheItemMetadata,
|
||||
MultiModalProcessorOnlyCache,
|
||||
MultiModalProcessorSenderCache,
|
||||
MultiModalReceiverCache,
|
||||
ShmObjectStoreReceiverCache,
|
||||
ShmObjectStoreSenderCache,
|
||||
)
|
||||
from vllm.multimodal.hasher import MultiModalHasher
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -137,10 +140,10 @@ def _compare_caches(
|
||||
n_iter: int = 100,
|
||||
seed: int = 0,
|
||||
):
|
||||
cache_0_p0 = MULTIMODAL_REGISTRY.processor_cache_from_config(config_0)
|
||||
cache_0_p1 = MULTIMODAL_REGISTRY.engine_receiver_cache_from_config(config_0)
|
||||
cache_1_p0 = MULTIMODAL_REGISTRY.processor_cache_from_config(config_1)
|
||||
cache_1_p1 = MULTIMODAL_REGISTRY.engine_receiver_cache_from_config(config_1)
|
||||
cache_0_p0 = processor_cache_from_config(config_0)
|
||||
cache_0_p1 = engine_receiver_cache_from_config(config_0)
|
||||
cache_1_p0 = processor_cache_from_config(config_1)
|
||||
cache_1_p1 = engine_receiver_cache_from_config(config_1)
|
||||
|
||||
cache_size_gb = max(
|
||||
config_0.model_config.multimodal_config.mm_processor_cache_gb,
|
||||
@@ -275,8 +278,8 @@ def test_oversized_item_is_served_uncached():
|
||||
assert p0_only.get_and_update_item((small, []), "small")[0] is small
|
||||
assert p0_only.is_cached_item("small")
|
||||
|
||||
p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type]
|
||||
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
|
||||
p0 = LruKeyReplicatedSenderCache(model_config) # type: ignore[arg-type]
|
||||
p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type]
|
||||
assert p0.get_and_update_item((item, []), "big")[0] is item
|
||||
assert not p0.is_cached_item("big")
|
||||
assert p1.get_and_update_item(item, "big") is item
|
||||
@@ -296,8 +299,8 @@ def test_mm_cache_miss_raises_and_recovers():
|
||||
model or network.
|
||||
"""
|
||||
model_config = _StubModelConfig(mm_processor_cache_gb=1)
|
||||
p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type]
|
||||
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
|
||||
p0 = LruKeyReplicatedSenderCache(model_config) # type: ignore[arg-type]
|
||||
p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type]
|
||||
|
||||
mm_hash = "image_A"
|
||||
item = MultiModalKwargsItem.dummy(nbytes=64)
|
||||
@@ -338,7 +341,7 @@ def test_receiver_cache_replaces_stale_item_when_payload_resent():
|
||||
and kills EngineCore. Prefer the fresh payload and keep it for later hits.
|
||||
"""
|
||||
model_config = _StubModelConfig(mm_processor_cache_gb=1)
|
||||
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
|
||||
p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type]
|
||||
small = MultiModalKwargsItem.dummy(nbytes=64)
|
||||
large = MultiModalKwargsItem.dummy(nbytes=256)
|
||||
mm_hash = "shared-id"
|
||||
@@ -351,7 +354,7 @@ def test_receiver_cache_replaces_stale_item_when_payload_resent():
|
||||
def test_receiver_cache_features_keep_resent_payload():
|
||||
"""EngineCore updates features through get_and_update_features."""
|
||||
model_config = _StubModelConfig(mm_processor_cache_gb=1)
|
||||
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
|
||||
p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type]
|
||||
small = MultiModalKwargsItem.dummy(nbytes=64)
|
||||
large = MultiModalKwargsItem.dummy(nbytes=256)
|
||||
mm_hash = "shared-id"
|
||||
@@ -384,7 +387,7 @@ def test_mm_cache_miss_batches_all_drifted_hashes():
|
||||
non-drifted items in the same request are still ingested.
|
||||
"""
|
||||
model_config = _StubModelConfig(mm_processor_cache_gb=1)
|
||||
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
|
||||
p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type]
|
||||
item = MultiModalKwargsItem.dummy(nbytes=64)
|
||||
|
||||
def _feature(
|
||||
@@ -582,8 +585,8 @@ def test_cache_eviction_lru_cache():
|
||||
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
|
||||
mm_processor_cache_gb=6 / GiB_bytes,
|
||||
)
|
||||
sender_cache = MultiModalProcessorSenderCache(model_config)
|
||||
receiver_cache = MultiModalReceiverCache(model_config)
|
||||
sender_cache = LruKeyReplicatedSenderCache(model_config)
|
||||
receiver_cache = LruKeyReplicatedReceiverCache(model_config)
|
||||
|
||||
_run_test_cache_eviction_lru(sender_cache, receiver_cache, base_item_size=1)
|
||||
|
||||
@@ -775,7 +778,7 @@ def test_processor_cache_shared_across_loras():
|
||||
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
receiver_cache = MultiModalReceiverCache(model_config)
|
||||
receiver_cache = LruKeyReplicatedReceiverCache(model_config)
|
||||
|
||||
base_mm_hash = "image_hash_abc123"
|
||||
lora_a_identifier = f"12345:{base_mm_hash}"
|
||||
@@ -827,8 +830,8 @@ async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
|
||||
)
|
||||
model_config = ctx.model_config
|
||||
|
||||
sender = MultiModalProcessorSenderCache(model_config)
|
||||
receiver = MultiModalReceiverCache(model_config)
|
||||
sender = LruKeyReplicatedSenderCache(model_config)
|
||||
receiver = LruKeyReplicatedReceiverCache(model_config)
|
||||
item = _dummy_item({"pixel_values": 16})
|
||||
mm_hash = "image_A"
|
||||
payload, _ = sender.get_and_update_item((item, []), mm_hash)
|
||||
|
||||
@@ -10,31 +10,9 @@ import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ..models.utils import build_model_context
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id,limit_mm_per_prompt,expected",
|
||||
[
|
||||
("Qwen/Qwen2-0.5B-Instruct", {}, False),
|
||||
("Qwen/Qwen2.5-VL-3B-Instruct", {}, True),
|
||||
("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0, "video": 0}, False),
|
||||
("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0}, True),
|
||||
],
|
||||
)
|
||||
@pytest.mark.core_model
|
||||
def test_supports_multimodal_inputs(model_id, limit_mm_per_prompt, expected):
|
||||
"""Test supports_multimodal_inputs returns correct boolean for various
|
||||
configs."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
)
|
||||
assert MULTIMODAL_REGISTRY.supports_multimodal_inputs(ctx.model_config) is expected
|
||||
|
||||
|
||||
def test_create_processor_error_uses_served_model_name():
|
||||
model_config = SimpleNamespace(
|
||||
is_multimodal_model=False,
|
||||
|
||||
@@ -997,6 +997,7 @@ class _MockModelConfig:
|
||||
skip_tokenizer_init: bool = True
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ class MockModelConfig:
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
hidden_size: int = 768
|
||||
dtype: torch.dtype = torch.float32
|
||||
|
||||
@@ -40,6 +40,7 @@ class MockHFConfig:
|
||||
class MockModelConfig:
|
||||
runner_type: str = "generate"
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
multimodal_config: Any = None
|
||||
hf_config: MockHFConfig = field(default_factory=MockHFConfig)
|
||||
allowed_local_media_path: str = ""
|
||||
|
||||
@@ -37,6 +37,7 @@ class MockModelConfig:
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
|
||||
|
||||
@@ -79,11 +79,6 @@ def _run_engine_core_handshake(
|
||||
"resolve_kv_cache_block_sizes",
|
||||
lambda kv_cache_config, vllm_config: (16, 16),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
engine_core_module,
|
||||
"MULTIMODAL_REGISTRY",
|
||||
SimpleNamespace(engine_receiver_cache_from_config=lambda vllm_config: None),
|
||||
)
|
||||
monkeypatch.setattr(engine_core_module, "freeze_gc_heap", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
engine_core_module, "maybe_attach_gc_debug_callback", lambda: None
|
||||
@@ -105,7 +100,11 @@ def _run_engine_core_handshake(
|
||||
speculative_config=None,
|
||||
ec_transfer_config=None,
|
||||
max_concurrent_batches=1,
|
||||
model_config=SimpleNamespace(runner_type="generate", is_diffusion=False),
|
||||
model_config=SimpleNamespace(
|
||||
runner_type="generate",
|
||||
is_diffusion=False,
|
||||
supports_multimodal_inputs=False,
|
||||
),
|
||||
cache_config=SimpleNamespace(
|
||||
enable_prefix_caching=False,
|
||||
prefix_caching_hash_algo="builtin",
|
||||
|
||||
@@ -53,6 +53,7 @@ def create_scheduler() -> Scheduler:
|
||||
vllm_config.model_config = MagicMock()
|
||||
vllm_config.model_config.skip_tokenizer_init = True
|
||||
vllm_config.model_config.is_multimodal_model = False
|
||||
vllm_config.model_config.supports_multimodal_inputs = False
|
||||
vllm_config.model_config.is_encoder_decoder = False
|
||||
vllm_config.model_config.is_diffusion = False
|
||||
vllm_config.model_config.max_model_len = 1024
|
||||
|
||||
@@ -148,8 +148,16 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected)
|
||||
|
||||
|
||||
def test_mm_support_configured_after_model_load(monkeypatch):
|
||||
target_model_config = object()
|
||||
draft_model_config = object()
|
||||
checked_configs = []
|
||||
|
||||
class _ModelConfig:
|
||||
@property
|
||||
def supports_multimodal_inputs(self):
|
||||
checked_configs.append(self)
|
||||
return True
|
||||
|
||||
target_model_config = _ModelConfig()
|
||||
draft_model_config = _ModelConfig()
|
||||
vllm_config = SimpleNamespace(model_config=target_model_config)
|
||||
draft_model = _MultimodalDraftModel()
|
||||
|
||||
@@ -164,19 +172,8 @@ def test_mm_support_configured_after_model_load(monkeypatch):
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.use_acceptance_estimator = False
|
||||
|
||||
checked_configs = []
|
||||
|
||||
def supports_multimodal_inputs(model_config):
|
||||
checked_configs.append(model_config)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(DraftModelSpeculator, "__init__", init_base)
|
||||
_mock_base_model_load(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.MULTIMODAL_REGISTRY,
|
||||
"supports_multimodal_inputs",
|
||||
supports_multimodal_inputs,
|
||||
)
|
||||
|
||||
speculator = _TestSpeculator(vllm_config, torch.device("cpu"))
|
||||
|
||||
@@ -198,7 +195,9 @@ def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch):
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.inputs_embeds = None
|
||||
speculator.use_acceptance_estimator = False
|
||||
speculator.vllm_config = SimpleNamespace(model_config=object())
|
||||
speculator.vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(supports_multimodal_inputs=True)
|
||||
)
|
||||
speculator.max_num_tokens = 4
|
||||
speculator.hidden_size = 3
|
||||
speculator.dtype = torch.float32
|
||||
@@ -206,11 +205,6 @@ def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch):
|
||||
draft_model = _MultimodalDraftModel()
|
||||
speculator.test_draft_model = draft_model
|
||||
_mock_base_model_load(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.MULTIMODAL_REGISTRY,
|
||||
"supports_multimodal_inputs",
|
||||
lambda model_config: True,
|
||||
)
|
||||
|
||||
speculator.load_model(torch.nn.Module())
|
||||
|
||||
@@ -223,16 +217,13 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.inputs_embeds = None
|
||||
speculator.use_acceptance_estimator = False
|
||||
speculator.vllm_config = SimpleNamespace(model_config=object())
|
||||
speculator.vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(supports_multimodal_inputs=True)
|
||||
)
|
||||
draft_model = _TextOnlyDraftModel()
|
||||
speculator.test_draft_model = draft_model
|
||||
warning_messages = []
|
||||
_mock_base_model_load(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.MULTIMODAL_REGISTRY,
|
||||
"supports_multimodal_inputs",
|
||||
lambda model_config: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.logger,
|
||||
"warning_once",
|
||||
@@ -257,7 +248,9 @@ def test_multi_module_mm_support_configured_after_model_load(monkeypatch):
|
||||
speculator.inputs_embeds = None
|
||||
speculator.use_acceptance_estimator = False
|
||||
speculator.cached_draft_input_embeds = None
|
||||
speculator.vllm_config = SimpleNamespace(model_config=object())
|
||||
speculator.vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(supports_multimodal_inputs=True)
|
||||
)
|
||||
speculator.max_num_tokens = 4
|
||||
speculator.max_num_reqs = 2
|
||||
speculator.num_speculative_steps = 3
|
||||
@@ -271,11 +264,6 @@ def test_multi_module_mm_support_configured_after_model_load(monkeypatch):
|
||||
"load_draft_model",
|
||||
lambda self, target_model, target_attn_layer_names: draft_model,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.MULTIMODAL_REGISTRY,
|
||||
"supports_multimodal_inputs",
|
||||
lambda model_config: True,
|
||||
)
|
||||
|
||||
speculator.load_model(torch.nn.Module())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user