[Refactor] Use dynamic MM cache in processor (#57674)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -63,36 +63,32 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
|
||||
# Baseline: no cache, always processes from scratch.
|
||||
baseline_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config, cache=None
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Sender cache: on a cache hit returns (None, prompt_updates) for each
|
||||
# item, setting mm_kwargs["video"] = [None] – the exact condition that
|
||||
# triggered the original bug.
|
||||
sender_cache = MultiModalProcessorSenderCache(ctx.model_config)
|
||||
cached_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config, cache=sender_cache
|
||||
)
|
||||
|
||||
video_token_id = baseline_processor.info.get_hf_config().video_token_id
|
||||
video_token_id = processor.info.get_hf_config().video_token_id
|
||||
|
||||
mm_data = create_mm_data(num_videos)
|
||||
hf_processor_mm_kwargs = {"use_audio_in_video": True}
|
||||
|
||||
def run(processor):
|
||||
def run(processor, cache):
|
||||
return processor(
|
||||
[video_token_id] * num_videos,
|
||||
mm_items=baseline_processor.info.parse_mm_data(mm_data),
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
cache=cache,
|
||||
)["prompt_token_ids"]
|
||||
|
||||
baseline_ids = run(baseline_processor)
|
||||
baseline_ids = run(processor, cache=None)
|
||||
|
||||
# First call on the sender-cache processor: cache miss.
|
||||
# mm_kwargs["video"] items are real tensors; use_audio_in_video is
|
||||
# detected normally from the item data.
|
||||
first_ids = run(cached_processor)
|
||||
first_ids = run(processor, cache=sender_cache)
|
||||
assert first_ids == baseline_ids, (
|
||||
"Cache-miss call produced different prompt_token_ids than baseline.\n"
|
||||
f" baseline : {baseline_ids}\n"
|
||||
@@ -103,7 +99,7 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non
|
||||
# MultiModalProcessorSenderCache.get_and_update_item returns (None, …),
|
||||
# so mm_kwargs["video"] = [None]. Before the fix, use_audio_in_video was
|
||||
# not detected, yielding wrong token ids.
|
||||
second_ids = run(cached_processor)
|
||||
second_ids = run(processor, cache=sender_cache)
|
||||
assert second_ids == baseline_ids, (
|
||||
"Cache-hit call produced different prompt_token_ids than baseline.\n"
|
||||
"This is the regression introduced when use_audio_in_video detection\n"
|
||||
@@ -123,7 +119,7 @@ def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=None)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
video_token_id = processor.info.get_hf_config().video_token_id
|
||||
|
||||
rng = np.random.RandomState(0)
|
||||
|
||||
@@ -72,6 +72,8 @@ def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
|
||||
|
||||
|
||||
_IGNORE_MM_KEYS = {
|
||||
# Dithering causes minor divergence
|
||||
"cohere_asr": {"input_features"},
|
||||
# In Ultravox, the audio_features can be different depending on padding
|
||||
# The slight difference should not be a problem though, since
|
||||
# attention_mask lets us ignore the difference.
|
||||
@@ -286,8 +288,7 @@ def _test_processing_correctness(
|
||||
}
|
||||
)
|
||||
|
||||
baseline_processor = factories.build_processor(ctx, cache=None)
|
||||
cached_processor = factories.build_processor(ctx, cache=cache)
|
||||
processor = factories.build_processor(ctx)
|
||||
|
||||
rng = np.random.RandomState(0)
|
||||
|
||||
@@ -336,41 +337,42 @@ def _test_processing_correctness(
|
||||
_test_processing_correctness_one(
|
||||
model_config,
|
||||
mm_data,
|
||||
baseline_processor,
|
||||
cached_processor,
|
||||
processor,
|
||||
batch_idx,
|
||||
hit_rate,
|
||||
num_batches,
|
||||
simplify_rate,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
|
||||
def _test_processing_correctness_one(
|
||||
model_config: ModelConfig,
|
||||
mm_data: MultiModalDataDict,
|
||||
baseline_processor: BaseMultiModalProcessor,
|
||||
cached_processor: BaseMultiModalProcessor,
|
||||
processor: BaseMultiModalProcessor,
|
||||
batch_idx: int,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
cache: MultiModalProcessorOnlyCache,
|
||||
):
|
||||
model_type = model_config.hf_config.model_type
|
||||
|
||||
token_prompt = get_token_prompt(baseline_processor, mm_data)
|
||||
mm_items = baseline_processor.info.parse_mm_data(mm_data)
|
||||
token_prompt = get_token_prompt(processor, mm_data)
|
||||
mm_items = processor.info.parse_mm_data(mm_data)
|
||||
ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
|
||||
|
||||
baseline_tokenized_result = baseline_processor(
|
||||
baseline_tokenized_result = processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
cached_tokenized_result = cached_processor(
|
||||
cached_tokenized_result = processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
|
||||
@@ -221,7 +221,7 @@ def test_video_cache_is_independent_of_image_kwargs(
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=cache)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
image = PILImage.new("RGB", (48, 48), color=(128, 128, 128))
|
||||
frames = np.stack([np.asarray(image)] * 2)
|
||||
@@ -230,15 +230,16 @@ def test_video_cache_is_independent_of_image_kwargs(
|
||||
{"image": image, "video": [(frames, metadata)]}
|
||||
)
|
||||
|
||||
def process(mm_kwargs):
|
||||
def process(mm_kwargs, cache):
|
||||
return processor(
|
||||
hf_processor.image_token + hf_processor.video_token,
|
||||
mm_items,
|
||||
mm_uuid_items={"video": [video_uuid]},
|
||||
hf_processor_mm_kwargs=mm_kwargs,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
baseline = process({})
|
||||
baseline = process({}, cache=None)
|
||||
if kwargs_on_init:
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
@@ -247,12 +248,12 @@ def test_video_cache_is_independent_of_image_kwargs(
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=cache)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
request_kwargs = {} if kwargs_on_init else kwargs
|
||||
process(request_kwargs)
|
||||
cached = process(request_kwargs)
|
||||
process(request_kwargs, cache=cache)
|
||||
cached = process(request_kwargs, cache=cache)
|
||||
cache.clear_cache()
|
||||
fresh = process(request_kwargs)
|
||||
fresh = process(request_kwargs, cache=cache)
|
||||
|
||||
def pixels(result, modality, field):
|
||||
return result["mm_kwargs"][modality][0][field].data
|
||||
|
||||
@@ -23,11 +23,9 @@ def test_profiling(model_id: str, max_model_len: int):
|
||||
model_config_kwargs=model_config_kwargs,
|
||||
limit_mm_per_prompt=mm_counts,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
mm_inputs = MULTIMODAL_REGISTRY.get_dummy_mm_inputs(
|
||||
ctx.model_config,
|
||||
mm_counts=mm_counts,
|
||||
)
|
||||
mm_inputs = processor.get_dummy_mm_inputs(mm_counts)
|
||||
|
||||
hf_config = ctx.get_hf_config(Llama4Config)
|
||||
image_size = hf_config.vision_config.image_size
|
||||
|
||||
@@ -168,7 +168,6 @@ def _build_moss_audio_processor(cache=None):
|
||||
MossAudioMultiModalProcessor(
|
||||
info,
|
||||
MossAudioDummyInputsBuilder(info),
|
||||
cache=cache,
|
||||
),
|
||||
ctx,
|
||||
)
|
||||
@@ -259,21 +258,24 @@ def test_moss_audio_multimodal_processor_handles_token_and_cache_paths():
|
||||
)
|
||||
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
cached_processor, _ = _build_moss_audio_processor(cache=cache)
|
||||
cached_processor, _ = _build_moss_audio_processor()
|
||||
cached_text_miss = cached_processor(
|
||||
prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
cached_text_hit = cached_processor(
|
||||
prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
cached_token_hit = cached_processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
expected_audio_tokens = MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)
|
||||
|
||||
@@ -26,7 +26,6 @@ def test_jina_vl_processing_order() -> None:
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
@@ -37,6 +36,7 @@ def test_jina_vl_processing_order() -> None:
|
||||
return processor(
|
||||
placeholder * len(images),
|
||||
mm_items=processor.info.parse_mm_data({"image": images}),
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
query = process([query_image])
|
||||
|
||||
@@ -255,7 +255,7 @@ def test_model_tensor_schema(model_id: str):
|
||||
for modality, count in limit_mm_per_prompt.items()
|
||||
}
|
||||
)
|
||||
processor = factories.build_processor(ctx, cache=None)
|
||||
processor = factories.build_processor(ctx)
|
||||
|
||||
with initialize_dummy_model(model_cls, model_config) as model:
|
||||
for modality, _, mm_kwargs in create_batched_mm_kwargs(model_config, processor):
|
||||
|
||||
@@ -245,6 +245,7 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
prompt="<image>\nWhat is this?",
|
||||
mm_items=mm_processor.info.parse_mm_data({"image": image}),
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
first, second = process(), process()
|
||||
|
||||
@@ -48,4 +48,4 @@ def create_cached_processor(model_id: str, processor_cls):
|
||||
)
|
||||
info = MultiModalProcessingInfo(ctx)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
return processor_cls(info, MultiModalDummyInputsBuilder(info), cache=cache), cache
|
||||
return processor_cls(info, MultiModalDummyInputsBuilder(info)), cache
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import multiprocessing as mp
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -37,10 +35,13 @@ from vllm.multimodal.inputs import (
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.multimodal.processing import PromptInsertion
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.mem_constants import GiB_bytes, MiB_bytes
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
from ..models.utils import build_model_context
|
||||
from ..utils import create_new_process_for_each_test
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
@@ -773,12 +774,14 @@ def test_processor_cache_shared_across_loras():
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
|
||||
"""Release must not leave a sender hit pointing at a cleared receiver."""
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
get_multimodal_config=lambda: MultiModalConfig(mm_processor_cache_gb=1)
|
||||
ctx = build_model_context(
|
||||
"llava-hf/llava-v1.6-mistral-7b-hf",
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
model_config = ctx.model_config
|
||||
|
||||
sender = MultiModalProcessorSenderCache(model_config)
|
||||
receiver = MultiModalReceiverCache(model_config)
|
||||
item = _dummy_item({"pixel_values": 16})
|
||||
@@ -787,43 +790,39 @@ async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
|
||||
assert receiver.get_and_update_item(payload, mm_hash) is item
|
||||
assert sender.get_and_update_item(None, mm_hash)[0] is None
|
||||
|
||||
renderer = SimpleNamespace(mm_processor_cache=sender, _mm_cache_stats=None)
|
||||
renderer.clear_mm_cache = partial(BaseRenderer.clear_mm_cache, renderer)
|
||||
renderer = renderer_from_config(VllmConfig(model_config=model_config))
|
||||
renderer._mm_processor_cache = sender
|
||||
|
||||
def release():
|
||||
if release_error:
|
||||
raise RuntimeError(release_error)
|
||||
receiver.clear_cache()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
renderer._clear_mm_cache_async = make_async(
|
||||
renderer.clear_mm_cache, executor=executor
|
||||
)
|
||||
renderer.clear_mm_cache_async = partial(
|
||||
BaseRenderer.clear_mm_cache_async, renderer
|
||||
)
|
||||
engine = SimpleNamespace(
|
||||
renderer=renderer,
|
||||
engine_core=SimpleNamespace(
|
||||
release_kv_cache_memory=release,
|
||||
release_kv_cache_memory_async=make_async(release, executor=executor),
|
||||
engine = SimpleNamespace(
|
||||
renderer=renderer,
|
||||
engine_core=SimpleNamespace(
|
||||
release_kv_cache_memory=release,
|
||||
release_kv_cache_memory_async=make_async(
|
||||
release,
|
||||
executor=renderer._executor,
|
||||
),
|
||||
logger_manager=Mock(),
|
||||
)
|
||||
),
|
||||
logger_manager=Mock(),
|
||||
)
|
||||
|
||||
async def call_release():
|
||||
if use_async:
|
||||
await AsyncLLM.release_kv_cache_memory(engine)
|
||||
else:
|
||||
LLMEngine.release_kv_cache_memory(engine)
|
||||
|
||||
if release_error:
|
||||
with pytest.raises(RuntimeError, match=release_error):
|
||||
await call_release()
|
||||
engine.logger_manager.record_sleep_state.assert_not_called()
|
||||
async def call_release():
|
||||
if use_async:
|
||||
await AsyncLLM.release_kv_cache_memory(engine)
|
||||
else:
|
||||
LLMEngine.release_kv_cache_memory(engine)
|
||||
|
||||
if release_error:
|
||||
with pytest.raises(RuntimeError, match=release_error):
|
||||
await call_release()
|
||||
engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0)
|
||||
engine.logger_manager.record_sleep_state.assert_not_called()
|
||||
else:
|
||||
await call_release()
|
||||
engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0)
|
||||
|
||||
payload, _ = sender.get_and_update_item((item, []), mm_hash)
|
||||
assert payload is item
|
||||
|
||||
@@ -7,7 +7,7 @@ from contextlib import nullcontext
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config import ModelConfig, SchedulerConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.hasher import MultiModalHasher
|
||||
@@ -32,6 +32,7 @@ from vllm.multimodal.processing.processor import (
|
||||
)
|
||||
from vllm.utils.collection_utils import flatten_2d_lists
|
||||
|
||||
from ..models.utils import build_model_context
|
||||
from .utils import random_image
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
@@ -1361,3 +1362,37 @@ def test_processor_inputs_hashes_distinguish_kwargs_shapes(left, right):
|
||||
).get_mm_hashes("test-model", "blake3")["image"][0]
|
||||
|
||||
assert hash_with(left) != hash_with(right)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("chunked_prefill", "max_model_len", "expected_seq_len"),
|
||||
[(None, 491520, 491520), (True, 491520, 8192), (True, 128, 128), (False, 128, 128)],
|
||||
)
|
||||
def test_dummy_inputs_scheduler_budget(
|
||||
chunked_prefill, max_model_len, expected_seq_len
|
||||
):
|
||||
ctx = build_model_context(
|
||||
"llava-hf/llava-v1.6-mistral-7b-hf",
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
ctx.model_config.max_model_len = max_model_len
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
processor.apply = lambda *args, **kwargs: {"prompt_token_ids": [7]}
|
||||
|
||||
kwargs = {}
|
||||
if chunked_prefill is not None:
|
||||
kwargs["scheduler_config"] = SchedulerConfig(
|
||||
max_model_len=max_model_len,
|
||||
is_encoder_decoder=False,
|
||||
max_num_batched_tokens=8192,
|
||||
max_num_seqs=1,
|
||||
enable_chunked_prefill=chunked_prefill,
|
||||
)
|
||||
|
||||
result = processor.get_dummy_mm_inputs({"image": 1}, **kwargs)
|
||||
assert len(result["prompt_token_ids"]) == expected_seq_len
|
||||
|
||||
@@ -5,11 +5,9 @@ Qwen2.5-VL visual component loading behavior.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import SchedulerConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ..models.utils import build_model_context
|
||||
@@ -17,34 +15,6 @@ from ..models.utils import build_model_context
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("chunked_prefill", "max_model_len", "expected_seq_len"),
|
||||
[(None, 491520, 491520), (True, 491520, 8192), (True, 128, 128), (False, 128, 128)],
|
||||
)
|
||||
def test_dummy_inputs_scheduler_budget(
|
||||
chunked_prefill, max_model_len, expected_seq_len
|
||||
):
|
||||
model_config = MagicMock()
|
||||
model_config.max_model_len = max_model_len
|
||||
processor = MagicMock()
|
||||
processor.apply.return_value = {"prompt_token_ids": [7]}
|
||||
kwargs = {}
|
||||
if chunked_prefill is not None:
|
||||
kwargs["scheduler_config"] = SchedulerConfig(
|
||||
max_model_len=max_model_len,
|
||||
is_encoder_decoder=False,
|
||||
max_num_batched_tokens=8192,
|
||||
max_num_seqs=1,
|
||||
enable_chunked_prefill=chunked_prefill,
|
||||
)
|
||||
result = MULTIMODAL_REGISTRY.get_dummy_mm_inputs(
|
||||
model_config, {"image": 1}, processor=processor, **kwargs
|
||||
)
|
||||
get_inputs = processor.dummy_inputs.get_dummy_processor_inputs
|
||||
assert get_inputs.call_args.kwargs["seq_len"] == expected_seq_len
|
||||
assert len(result["prompt_token_ids"]) == expected_seq_len
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id,limit_mm_per_prompt,expected",
|
||||
[
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
These tests exercise:
|
||||
- Zero-limit modalities are filtered from mm_counts passed to
|
||||
get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...)
|
||||
- The warmup runs through the processor-only cache and clears it afterwards,
|
||||
so its dummy inputs never land in the sender cache used by the serving path
|
||||
- MM warmup is skipped entirely when mm_processor is None
|
||||
- The multimodal warmup is launched as a task on the single-worker
|
||||
_mm_executor to overlap engine-core init (future lifecycle, join by
|
||||
@@ -21,10 +23,13 @@ that acts as the renderer instance.
|
||||
"""
|
||||
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from types import MethodType, SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
from vllm.renderers.params import ChatParams
|
||||
|
||||
@@ -34,9 +39,11 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
|
||||
|
||||
render_chat is mocked to raise ChatTemplateResolutionError so the chat
|
||||
warmup block is skipped cleanly, keeping the test focused on MM warmup.
|
||||
"""
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
|
||||
|
||||
The processor is a mock at its I/O boundaries (``dummy_inputs`` and
|
||||
``apply``) but keeps the real ``get_dummy_mm_inputs``, since that is where
|
||||
the warmup's seq_len capping and cache plumbing actually live.
|
||||
"""
|
||||
renderer = MagicMock()
|
||||
|
||||
# chat warmup: make render_chat raise so we skip past it cleanly
|
||||
@@ -45,13 +52,17 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
|
||||
# MM processor with configurable limits
|
||||
mm_processor = MagicMock()
|
||||
mm_processor.info.allowed_mm_limits = mm_limits
|
||||
mm_processor.info.ctx.model_config = renderer.model_config
|
||||
mm_processor.apply.return_value = {"prompt_token_ids": [1]}
|
||||
mm_processor.get_dummy_mm_inputs = MethodType(
|
||||
BaseMultiModalProcessor.get_dummy_mm_inputs, mm_processor
|
||||
)
|
||||
renderer.mm_processor = mm_processor
|
||||
renderer._readonly_mm_processor = None
|
||||
renderer._mm_processor_cache = MagicMock(name="sender_cache")
|
||||
renderer._mm_processor_only_cache = MagicMock(name="processor_only_cache")
|
||||
renderer._warmup_mm_processor = BaseRenderer._warmup_mm_processor.__get__(
|
||||
renderer, BaseRenderer
|
||||
)
|
||||
renderer._clear_processor_cache = BaseRenderer._clear_processor_cache
|
||||
renderer.warmup_mm = BaseRenderer.warmup_mm.__get__(renderer, BaseRenderer)
|
||||
renderer.start_mm_warmup_in_background = (
|
||||
BaseRenderer.start_mm_warmup_in_background.__get__(renderer, BaseRenderer)
|
||||
@@ -64,7 +75,6 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
|
||||
renderer._mm_warmup_future = None
|
||||
# MM warmup has not run yet; warmup_mm must actually execute on the mock.
|
||||
renderer._mm_warmup_done = False
|
||||
renderer.clear_mm_cache = MagicMock()
|
||||
renderer.model_config.max_model_len = 128
|
||||
renderer.config.scheduler_config.max_num_batched_tokens = 8192
|
||||
renderer.config.scheduler_config.enable_chunked_prefill = True
|
||||
@@ -80,8 +90,7 @@ class TestMmWarmupZeroLimitFiltering:
|
||||
"""A modality with limit=0 must not appear in mm_counts."""
|
||||
renderer = _make_renderer_mock({"image": 1, "video": 0})
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
|
||||
get_inputs.assert_called_once()
|
||||
@@ -93,8 +102,7 @@ class TestMmWarmupZeroLimitFiltering:
|
||||
"""When all limits are 0, mm_counts must be empty."""
|
||||
renderer = _make_renderer_mock({"image": 0, "video": 0})
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
|
||||
get_inputs.assert_called_once()
|
||||
@@ -105,8 +113,7 @@ class TestMmWarmupZeroLimitFiltering:
|
||||
"""All modalities with limit > 0 must be present in mm_counts."""
|
||||
renderer = _make_renderer_mock({"image": 2, "video": 1})
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
|
||||
get_inputs.assert_called_once()
|
||||
@@ -125,8 +132,7 @@ class TestMmWarmupRunsNormally:
|
||||
renderer = _make_renderer_mock({"image": 1, "video": 1})
|
||||
renderer.model_config.max_model_len = max_model_len
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
|
||||
assert get_inputs.call_args.kwargs["seq_len"] == expected_seq_len
|
||||
@@ -134,28 +140,46 @@ class TestMmWarmupRunsNormally:
|
||||
def test_processor_apply_called(self):
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
renderer.mm_processor.apply.assert_called_once()
|
||||
|
||||
def test_mm_cache_cleared_after_warmup(self):
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
renderer.clear_mm_cache.assert_called_once()
|
||||
|
||||
def test_render_chat_called_with_warmup_message(self):
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
renderer.render_chat.assert_called_once()
|
||||
|
||||
|
||||
class TestMmWarmupCacheRouting:
|
||||
# The warmup runs through the processor-only cache and drops it afterwards,
|
||||
# so the dummy inputs never reach the sender cache used by the serving path.
|
||||
|
||||
def test_warmup_passes_processor_only_cache(self):
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
|
||||
with patch.object(
|
||||
renderer.mm_processor,
|
||||
"get_dummy_mm_inputs",
|
||||
wraps=renderer.mm_processor.get_dummy_mm_inputs,
|
||||
) as spy:
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
assert spy.call_args.kwargs["cache"] is renderer._mm_processor_only_cache
|
||||
assert (
|
||||
spy.call_args.kwargs["scheduler_config"] is renderer.config.scheduler_config
|
||||
)
|
||||
|
||||
def test_only_cache_cleared_and_sender_cache_untouched(self):
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
renderer._mm_processor_only_cache.clear_cache.assert_called_once()
|
||||
renderer._mm_processor_cache.clear_cache.assert_not_called()
|
||||
|
||||
|
||||
class TestMmWarmupSkippedWhenNoProcessor:
|
||||
"""MM warmup must be skipped when mm_processor is None (text-only model)."""
|
||||
|
||||
@@ -168,51 +192,30 @@ class TestMmWarmupSkippedWhenNoProcessor:
|
||||
renderer.model_config.get_multimodal_config.assert_not_called()
|
||||
|
||||
|
||||
class TestReadonlyMmWarmup:
|
||||
"""Readonly MM processor warmup must mirror the render path behavior."""
|
||||
|
||||
def test_readonly_processor_apply_called_and_cache_cleared(self):
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
readonly_mm_processor = MagicMock()
|
||||
readonly_mm_processor.info.allowed_mm_limits = {"image": 1}
|
||||
renderer._readonly_mm_processor = readonly_mm_processor
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
readonly_mm_processor.apply.assert_called_once()
|
||||
readonly_mm_processor.cache.clear_cache.assert_called_once()
|
||||
|
||||
|
||||
class TestWarmupFaultIsolation:
|
||||
# A failure during a multimodal processor warmup is caught so it does not
|
||||
# abort the remaining warmup steps; warmup itself must not raise.
|
||||
# A failure during a warmup step is caught so it does not abort the other
|
||||
# step; warmup itself must not raise.
|
||||
|
||||
def test_chat_failure_does_not_abort_mm_warmup(self):
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
renderer.render_chat.side_effect = RuntimeError("chat boom")
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams()) # must not raise
|
||||
BaseRenderer.warmup(renderer, ChatParams()) # must not raise
|
||||
|
||||
renderer.mm_processor.apply.assert_called_once()
|
||||
|
||||
def test_mm_failure_does_not_abort_readonly_warmup(self):
|
||||
def test_mm_failure_is_swallowed_and_cache_cleared(self):
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
readonly_mm_processor = MagicMock()
|
||||
readonly_mm_processor.info.allowed_mm_limits = {"image": 1}
|
||||
renderer._readonly_mm_processor = readonly_mm_processor
|
||||
# main processor warmup blows up before apply()
|
||||
# the warmup blows up before apply()
|
||||
renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs.side_effect = (
|
||||
RuntimeError("mm boom")
|
||||
)
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams()) # must not raise
|
||||
BaseRenderer.warmup(renderer, ChatParams()) # must not raise
|
||||
|
||||
readonly_mm_processor.apply.assert_called_once()
|
||||
# cache is still cleared in the failed task's finally
|
||||
renderer.clear_mm_cache.assert_called_once()
|
||||
renderer.mm_processor.apply.assert_not_called()
|
||||
# the cache is still cleared in the failed warmup's finally
|
||||
renderer._mm_processor_only_cache.clear_cache.assert_called_once()
|
||||
|
||||
|
||||
class TestBackgroundMmWarmup:
|
||||
@@ -234,16 +237,15 @@ class TestBackgroundMmWarmup:
|
||||
def test_start_submits_future_to_mm_executor(self):
|
||||
renderer = self._make_bg_renderer({"image": 1})
|
||||
try:
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
# Spy on the real executor's submit to confirm the warmup is
|
||||
# dispatched through _mm_executor (not a separate Thread).
|
||||
with patch.object(
|
||||
renderer._mm_executor, "submit", wraps=renderer._mm_executor.submit
|
||||
) as spy:
|
||||
renderer.start_mm_warmup_in_background()
|
||||
assert isinstance(renderer._mm_warmup_future, Future)
|
||||
assert spy.called
|
||||
renderer._join_mm_warmup()
|
||||
# Spy on the real executor's submit to confirm the warmup is
|
||||
# dispatched through _mm_executor (not a separate Thread).
|
||||
with patch.object(
|
||||
renderer._mm_executor, "submit", wraps=renderer._mm_executor.submit
|
||||
) as spy:
|
||||
renderer.start_mm_warmup_in_background()
|
||||
assert isinstance(renderer._mm_warmup_future, Future)
|
||||
assert spy.called
|
||||
renderer._join_mm_warmup()
|
||||
assert renderer._mm_warmup_future is None
|
||||
renderer.mm_processor.apply.assert_called_once()
|
||||
finally:
|
||||
@@ -252,7 +254,6 @@ class TestBackgroundMmWarmup:
|
||||
def test_start_noop_for_text_only_model(self):
|
||||
renderer = _make_renderer_mock({})
|
||||
renderer.mm_processor = None
|
||||
# _readonly_mm_processor is already None
|
||||
|
||||
renderer.start_mm_warmup_in_background()
|
||||
|
||||
@@ -261,12 +262,11 @@ class TestBackgroundMmWarmup:
|
||||
def test_start_is_run_at_most_once(self):
|
||||
renderer = self._make_bg_renderer({"image": 1})
|
||||
try:
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
renderer.start_mm_warmup_in_background()
|
||||
first = renderer._mm_warmup_future
|
||||
renderer.start_mm_warmup_in_background() # must not spawn a second
|
||||
assert renderer._mm_warmup_future is first
|
||||
renderer._join_mm_warmup()
|
||||
renderer.start_mm_warmup_in_background()
|
||||
first = renderer._mm_warmup_future
|
||||
renderer.start_mm_warmup_in_background() # must not spawn a second
|
||||
assert renderer._mm_warmup_future is first
|
||||
renderer._join_mm_warmup()
|
||||
finally:
|
||||
renderer._mm_executor.shutdown(wait=True)
|
||||
|
||||
@@ -275,9 +275,8 @@ class TestBackgroundMmWarmup:
|
||||
# run only the chat warmup — the MM warmup must not run twice.
|
||||
renderer = self._make_bg_renderer({"image": 1})
|
||||
try:
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
renderer.start_mm_warmup_in_background()
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
renderer.start_mm_warmup_in_background()
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
# MM apply called exactly once (by the background warmup task).
|
||||
renderer.mm_processor.apply.assert_called_once()
|
||||
@@ -294,13 +293,12 @@ class TestBackgroundMmWarmup:
|
||||
# re-run the MM warmup — the _mm_warmup_done flag survives the join.
|
||||
renderer = self._make_bg_renderer({"image": 1})
|
||||
try:
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
renderer.start_mm_warmup_in_background()
|
||||
# Simulate reset_mm_cache joining the background warmup first
|
||||
# (clears _mm_warmup_future, sets _mm_warmup_done via the join).
|
||||
renderer._join_mm_warmup()
|
||||
assert renderer._mm_warmup_future is None # joined & cleared
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
renderer.start_mm_warmup_in_background()
|
||||
# Simulate reset_mm_cache joining the background warmup first
|
||||
# (clears _mm_warmup_future, sets _mm_warmup_done via the join).
|
||||
renderer._join_mm_warmup()
|
||||
assert renderer._mm_warmup_future is None # joined & cleared
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
# MM apply called exactly once (by the background warmup task);
|
||||
# warmup() did not re-run warmup_mm despite the future being cleared.
|
||||
@@ -313,9 +311,8 @@ class TestBackgroundMmWarmup:
|
||||
# Direct repeated calls to warmup_mm run the MM warmup only once.
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
renderer.warmup_mm()
|
||||
renderer.warmup_mm() # second call must be a no-op
|
||||
renderer.warmup_mm()
|
||||
renderer.warmup_mm() # second call must be a no-op
|
||||
|
||||
renderer.mm_processor.apply.assert_called_once()
|
||||
|
||||
@@ -324,10 +321,9 @@ class TestBackgroundMmWarmup:
|
||||
# so the mm_processor_cache is never touched concurrently.
|
||||
renderer = self._make_bg_renderer({"image": 1})
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
renderer.start_mm_warmup_in_background()
|
||||
future = renderer._mm_warmup_future
|
||||
renderer.shutdown()
|
||||
renderer.start_mm_warmup_in_background()
|
||||
future = renderer._mm_warmup_future
|
||||
renderer.shutdown()
|
||||
|
||||
assert renderer._mm_warmup_future is None
|
||||
# The background warmup was allowed to complete (apply ran) and the
|
||||
@@ -344,10 +340,6 @@ class TestEngineStartWarmupHook:
|
||||
# These tests pin the renderer plumbing contract on EngineCoreClient.
|
||||
|
||||
def _mock_config(self):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.v1.engine.core_client import EngineCoreClient # noqa: F401
|
||||
|
||||
cfg = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=1,
|
||||
|
||||
Reference in New Issue
Block a user