[ModelRunnerV2] Support prompt embeds (#42963)
Signed-off-by: gcanlin <canlinguosdu@gmail.com> Signed-off-by: Canlin Guo <961750412@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -15,7 +15,13 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalSharedField,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
@@ -25,6 +31,25 @@ pytestmark = pytest.mark.cpu_test
|
||||
HIDDEN = 4
|
||||
|
||||
|
||||
def _model_state(cache: EncoderCache) -> MagicMock:
|
||||
"""A mock ModelState backed by a real EncoderCache."""
|
||||
state = MagicMock()
|
||||
state.encoder_cache = cache
|
||||
state.device = torch.device("cpu")
|
||||
return state
|
||||
|
||||
|
||||
def _embeds_item(embeds: torch.Tensor) -> MultiModalKwargsItem:
|
||||
"""A `prompt_embeds` kwargs item, as the HF renderer builds it."""
|
||||
return MultiModalKwargsItem(
|
||||
{
|
||||
"embedding": MultiModalFieldElem(
|
||||
data=embeds, field=MultiModalSharedField(batch_size=1)
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
return MultiModalFeatureSpec(
|
||||
data=None,
|
||||
@@ -197,8 +222,7 @@ def test_execute_mm_encoder_caches_outputs_without_gathering():
|
||||
items the connector already holds, and a producer has no load path).
|
||||
"""
|
||||
cache = EncoderCache()
|
||||
state = MagicMock()
|
||||
state.encoder_cache = cache
|
||||
state = _model_state(cache)
|
||||
embedding = torch.ones(2, HIDDEN)
|
||||
# (mm_hashes, [(modality, kwargs item), ...]), as prepare_mm_inputs returns.
|
||||
state.encoder_runner.prepare_mm_inputs.return_value = (
|
||||
@@ -216,8 +240,7 @@ def test_execute_mm_encoder_caches_outputs_without_gathering():
|
||||
def test_execute_mm_encoder_is_a_noop_without_scheduled_items():
|
||||
"""A step that schedules no encoder input must not touch the encoder."""
|
||||
cache = EncoderCache()
|
||||
state = MagicMock()
|
||||
state.encoder_cache = cache
|
||||
state = _model_state(cache)
|
||||
state.encoder_runner.prepare_mm_inputs.return_value = ([], [])
|
||||
|
||||
ModelState.execute_mm_encoder(state, {})
|
||||
@@ -226,6 +249,68 @@ def test_execute_mm_encoder_is_a_noop_without_scheduled_items():
|
||||
state.encoder_runner.execute_mm_encoder.assert_not_called()
|
||||
|
||||
|
||||
def _pe_feature(identifier: str, embeds: torch.Tensor, offset: int = 0):
|
||||
return MultiModalFeatureSpec(
|
||||
data=_embeds_item(embeds),
|
||||
modality="prompt_embeds",
|
||||
identifier=identifier,
|
||||
mm_position=PlaceholderRange(offset=offset, length=embeds.shape[0]),
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_mm_inputs_passes_prompt_embeds_through():
|
||||
"""`prompt_embeds` is already in embedding space, so no encoder may run.
|
||||
|
||||
The renderer delivers prompt_embeds mixed with real media as an ordinary MM
|
||||
modality. prepare_mm_inputs must cache the tensor directly and keep it out
|
||||
of the encoder batch -- the vision encoder cannot consume it, and a missing
|
||||
cache entry makes the subsequent gather raise "Encoder cache miss".
|
||||
"""
|
||||
prompt_embeds = torch.arange(2 * HIDDEN, dtype=torch.float32).view(2, HIDDEN)
|
||||
image_feature = MultiModalFeatureSpec(
|
||||
data=MagicMock(),
|
||||
modality="image",
|
||||
identifier="hash_img",
|
||||
mm_position=PlaceholderRange(offset=2, length=2),
|
||||
)
|
||||
runner = _make_runner(
|
||||
[_pe_feature("hash_pe", prompt_embeds), image_feature], cached=[]
|
||||
)
|
||||
|
||||
mm_hashes, mm_kwargs = runner.prepare_mm_inputs({"req0": [0, 1]})
|
||||
|
||||
# Only the image remains for the encoder; the embeds are already cached.
|
||||
assert mm_hashes == ["hash_img"]
|
||||
assert [modality for modality, _ in mm_kwargs] == ["image"]
|
||||
assert torch.equal(runner.encoder_cache.encoder_outputs["hash_pe"], prompt_embeds)
|
||||
|
||||
|
||||
def test_prepare_mm_inputs_skips_cached_prompt_embeds():
|
||||
"""A prompt_embeds item already in the cache must not be re-uploaded."""
|
||||
prompt_embeds = torch.ones(3, HIDDEN)
|
||||
feature = _pe_feature("hash_pe", prompt_embeds)
|
||||
runner = _make_runner([feature], cached=[feature])
|
||||
sentinel = runner.encoder_cache.encoder_outputs["hash_pe"]
|
||||
|
||||
mm_hashes, mm_kwargs = runner.prepare_mm_inputs({"req0": [0]})
|
||||
|
||||
assert mm_hashes == [] and mm_kwargs == []
|
||||
assert runner.encoder_cache.encoder_outputs["hash_pe"] is sentinel
|
||||
|
||||
|
||||
def test_execute_mm_encoder_skips_encoder_for_prompt_embeds_only():
|
||||
"""A batch of nothing but prompt_embeds must not invoke the encoder."""
|
||||
prompt_embeds = torch.ones(3, HIDDEN)
|
||||
runner = _make_runner([_pe_feature("hash_pe", prompt_embeds)], cached=[])
|
||||
state = _model_state(runner.encoder_cache)
|
||||
state.encoder_runner.prepare_mm_inputs.side_effect = runner.prepare_mm_inputs
|
||||
|
||||
ModelState.execute_mm_encoder(state, {"req0": [0]})
|
||||
|
||||
state.encoder_runner.execute_mm_encoder.assert_not_called()
|
||||
assert torch.equal(runner.encoder_cache.encoder_outputs["hash_pe"], prompt_embeds)
|
||||
|
||||
|
||||
def test_encoder_timing_stats_registry():
|
||||
runner = _make_runner([], [])
|
||||
runner.enable_timing = True
|
||||
|
||||
Reference in New Issue
Block a user