[ModelRunnerV2] Support prompt embeds (#42963)
Signed-off-by: gcanlin <canlinguosdu@gmail.com> Signed-off-by: Canlin Guo <961750412@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -15,7 +15,13 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalSharedField,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
@@ -25,6 +31,25 @@ pytestmark = pytest.mark.cpu_test
|
||||
HIDDEN = 4
|
||||
|
||||
|
||||
def _model_state(cache: EncoderCache) -> MagicMock:
|
||||
"""A mock ModelState backed by a real EncoderCache."""
|
||||
state = MagicMock()
|
||||
state.encoder_cache = cache
|
||||
state.device = torch.device("cpu")
|
||||
return state
|
||||
|
||||
|
||||
def _embeds_item(embeds: torch.Tensor) -> MultiModalKwargsItem:
|
||||
"""A `prompt_embeds` kwargs item, as the HF renderer builds it."""
|
||||
return MultiModalKwargsItem(
|
||||
{
|
||||
"embedding": MultiModalFieldElem(
|
||||
data=embeds, field=MultiModalSharedField(batch_size=1)
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
return MultiModalFeatureSpec(
|
||||
data=None,
|
||||
@@ -197,8 +222,7 @@ def test_execute_mm_encoder_caches_outputs_without_gathering():
|
||||
items the connector already holds, and a producer has no load path).
|
||||
"""
|
||||
cache = EncoderCache()
|
||||
state = MagicMock()
|
||||
state.encoder_cache = cache
|
||||
state = _model_state(cache)
|
||||
embedding = torch.ones(2, HIDDEN)
|
||||
# (mm_hashes, [(modality, kwargs item), ...]), as prepare_mm_inputs returns.
|
||||
state.encoder_runner.prepare_mm_inputs.return_value = (
|
||||
@@ -216,8 +240,7 @@ def test_execute_mm_encoder_caches_outputs_without_gathering():
|
||||
def test_execute_mm_encoder_is_a_noop_without_scheduled_items():
|
||||
"""A step that schedules no encoder input must not touch the encoder."""
|
||||
cache = EncoderCache()
|
||||
state = MagicMock()
|
||||
state.encoder_cache = cache
|
||||
state = _model_state(cache)
|
||||
state.encoder_runner.prepare_mm_inputs.return_value = ([], [])
|
||||
|
||||
ModelState.execute_mm_encoder(state, {})
|
||||
@@ -226,6 +249,68 @@ def test_execute_mm_encoder_is_a_noop_without_scheduled_items():
|
||||
state.encoder_runner.execute_mm_encoder.assert_not_called()
|
||||
|
||||
|
||||
def _pe_feature(identifier: str, embeds: torch.Tensor, offset: int = 0):
|
||||
return MultiModalFeatureSpec(
|
||||
data=_embeds_item(embeds),
|
||||
modality="prompt_embeds",
|
||||
identifier=identifier,
|
||||
mm_position=PlaceholderRange(offset=offset, length=embeds.shape[0]),
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_mm_inputs_passes_prompt_embeds_through():
|
||||
"""`prompt_embeds` is already in embedding space, so no encoder may run.
|
||||
|
||||
The renderer delivers prompt_embeds mixed with real media as an ordinary MM
|
||||
modality. prepare_mm_inputs must cache the tensor directly and keep it out
|
||||
of the encoder batch -- the vision encoder cannot consume it, and a missing
|
||||
cache entry makes the subsequent gather raise "Encoder cache miss".
|
||||
"""
|
||||
prompt_embeds = torch.arange(2 * HIDDEN, dtype=torch.float32).view(2, HIDDEN)
|
||||
image_feature = MultiModalFeatureSpec(
|
||||
data=MagicMock(),
|
||||
modality="image",
|
||||
identifier="hash_img",
|
||||
mm_position=PlaceholderRange(offset=2, length=2),
|
||||
)
|
||||
runner = _make_runner(
|
||||
[_pe_feature("hash_pe", prompt_embeds), image_feature], cached=[]
|
||||
)
|
||||
|
||||
mm_hashes, mm_kwargs = runner.prepare_mm_inputs({"req0": [0, 1]})
|
||||
|
||||
# Only the image remains for the encoder; the embeds are already cached.
|
||||
assert mm_hashes == ["hash_img"]
|
||||
assert [modality for modality, _ in mm_kwargs] == ["image"]
|
||||
assert torch.equal(runner.encoder_cache.encoder_outputs["hash_pe"], prompt_embeds)
|
||||
|
||||
|
||||
def test_prepare_mm_inputs_skips_cached_prompt_embeds():
|
||||
"""A prompt_embeds item already in the cache must not be re-uploaded."""
|
||||
prompt_embeds = torch.ones(3, HIDDEN)
|
||||
feature = _pe_feature("hash_pe", prompt_embeds)
|
||||
runner = _make_runner([feature], cached=[feature])
|
||||
sentinel = runner.encoder_cache.encoder_outputs["hash_pe"]
|
||||
|
||||
mm_hashes, mm_kwargs = runner.prepare_mm_inputs({"req0": [0]})
|
||||
|
||||
assert mm_hashes == [] and mm_kwargs == []
|
||||
assert runner.encoder_cache.encoder_outputs["hash_pe"] is sentinel
|
||||
|
||||
|
||||
def test_execute_mm_encoder_skips_encoder_for_prompt_embeds_only():
|
||||
"""A batch of nothing but prompt_embeds must not invoke the encoder."""
|
||||
prompt_embeds = torch.ones(3, HIDDEN)
|
||||
runner = _make_runner([_pe_feature("hash_pe", prompt_embeds)], cached=[])
|
||||
state = _model_state(runner.encoder_cache)
|
||||
state.encoder_runner.prepare_mm_inputs.side_effect = runner.prepare_mm_inputs
|
||||
|
||||
ModelState.execute_mm_encoder(state, {"req0": [0]})
|
||||
|
||||
state.encoder_runner.execute_mm_encoder.assert_not_called()
|
||||
assert torch.equal(runner.encoder_cache.encoder_outputs["hash_pe"], prompt_embeds)
|
||||
|
||||
|
||||
def test_encoder_timing_stats_registry():
|
||||
runner = _make_runner([], [])
|
||||
runner.enable_timing = True
|
||||
|
||||
@@ -68,6 +68,16 @@ NUM_BLOCKS = 10
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _restore_default_dtype():
|
||||
"""Several tests here set the process-wide default dtype to float16 and
|
||||
previously leaked it, corrupting later float-sensitive tests in the same
|
||||
pytest process (torch.randn silently produced fp16)."""
|
||||
old = torch.get_default_dtype()
|
||||
yield
|
||||
torch.set_default_dtype(old)
|
||||
|
||||
|
||||
def initialize_kv_cache(runner: GPUModelRunner):
|
||||
"""
|
||||
Only perform necessary steps in GPUModelRunner.initialize_kv_cache()
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the Model Runner V2 prompt-embeds overlay (PromptEmbedsState).
|
||||
|
||||
The overlay kernel reads each request's GPU-resident prompt embeddings through
|
||||
a per-request pointer table and writes the rows scheduled this step into
|
||||
`inputs_embeds`, honoring chunked prefill (`num_computed_tokens` offset), the
|
||||
prompt/decode boundary (rows past the embeds length untouched), and the
|
||||
mixed-mode `prompt_is_token_ids` mask (token-id rows keep the base embedding).
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
pytest.importorskip("triton")
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip(
|
||||
"CUDA required for prompt-embeds overlay tests", allow_module_level=True
|
||||
)
|
||||
|
||||
from vllm.v1.worker.gpu.model_states.prompt_embeds import PromptEmbedsState
|
||||
|
||||
HIDDEN = 24
|
||||
MAX_NUM_REQS = 8
|
||||
DEVICE = torch.device("cuda")
|
||||
|
||||
|
||||
@dataclass
|
||||
class _NewReqData:
|
||||
req_id: str
|
||||
prompt_embeds: torch.Tensor | None
|
||||
prompt_is_token_ids: list[bool] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Batch:
|
||||
num_reqs: int
|
||||
num_scheduled_tokens: torch.Tensor # np-like, only .max() is used
|
||||
idx_mapping: torch.Tensor
|
||||
query_start_loc: torch.Tensor
|
||||
|
||||
|
||||
def _make_state() -> PromptEmbedsState:
|
||||
return PromptEmbedsState(MAX_NUM_REQS, HIDDEN, torch.float32, DEVICE)
|
||||
|
||||
|
||||
def _batch(num_scheduled: list[int], idx_mapping: list[int]) -> _Batch:
|
||||
query_start_loc = [0]
|
||||
for n in num_scheduled:
|
||||
query_start_loc.append(query_start_loc[-1] + n)
|
||||
return _Batch(
|
||||
num_reqs=len(num_scheduled),
|
||||
num_scheduled_tokens=torch.tensor(num_scheduled, dtype=torch.int32),
|
||||
idx_mapping=torch.tensor(idx_mapping, dtype=torch.int64, device=DEVICE),
|
||||
query_start_loc=torch.tensor(query_start_loc, dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
|
||||
|
||||
def _apply(
|
||||
state: PromptEmbedsState,
|
||||
batch: _Batch,
|
||||
num_computed: list[int],
|
||||
num_tokens: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run the overlay on a fresh base buffer; return (result, base)."""
|
||||
num_computed_tokens = torch.zeros(MAX_NUM_REQS, dtype=torch.int32, device=DEVICE)
|
||||
for batch_idx, req_index in enumerate(batch.idx_mapping.tolist()):
|
||||
num_computed_tokens[req_index] = num_computed[batch_idx]
|
||||
base = torch.randn(num_tokens, HIDDEN, dtype=torch.float32, device=DEVICE)
|
||||
inputs_embeds = base.clone()
|
||||
state.apply(batch, num_computed_tokens, inputs_embeds)
|
||||
torch.accelerator.synchronize()
|
||||
return inputs_embeds, base
|
||||
|
||||
|
||||
def test_overlay_chunked_prefill_and_decode():
|
||||
"""Rows within the embeds range come from prompt_embeds at the
|
||||
num_computed offset; requests without embeds and requests past their
|
||||
embeds length (decode) keep the base embedding."""
|
||||
state = _make_state()
|
||||
embeds_a = torch.randn(6, HIDDEN, dtype=torch.float32)
|
||||
embeds_b = torch.randn(5, HIDDEN, dtype=torch.float32)
|
||||
state.add_request(0, _NewReqData("a", embeds_a))
|
||||
state.add_request(1, _NewReqData("b", embeds_b))
|
||||
state.add_request(2, _NewReqData("c", None))
|
||||
state.apply_staged_writes()
|
||||
|
||||
# a: chunk [2, 6) of its embeds; b: fully decoded; c: no embeds.
|
||||
batch = _batch(num_scheduled=[4, 1, 3], idx_mapping=[0, 1, 2])
|
||||
out, base = _apply(state, batch, num_computed=[2, 7, 1], num_tokens=8)
|
||||
|
||||
torch.testing.assert_close(out[0:4], embeds_a[2:6].to(DEVICE))
|
||||
torch.testing.assert_close(out[4:8], base[4:8])
|
||||
|
||||
|
||||
def test_overlay_clamps_to_embeds_length():
|
||||
"""A window straddling the end of the prompt embeds writes only the
|
||||
in-range rows (e.g. final prefill chunk + sampled token)."""
|
||||
state = _make_state()
|
||||
embeds = torch.randn(4, HIDDEN, dtype=torch.float32)
|
||||
state.add_request(3, _NewReqData("a", embeds))
|
||||
state.apply_staged_writes()
|
||||
|
||||
batch = _batch(num_scheduled=[3], idx_mapping=[3])
|
||||
out, base = _apply(state, batch, num_computed=[2], num_tokens=3)
|
||||
|
||||
torch.testing.assert_close(out[0:2], embeds[2:4].to(DEVICE))
|
||||
torch.testing.assert_close(out[2:3], base[2:3])
|
||||
|
||||
|
||||
def test_overlay_respects_is_token_ids_mask():
|
||||
"""Mixed mode: positions marked as real token ids keep the base
|
||||
embedding; only embed positions are overwritten."""
|
||||
state = _make_state()
|
||||
embeds = torch.randn(5, HIDDEN, dtype=torch.float32)
|
||||
is_token_ids = [True, False, False, True, False]
|
||||
state.add_request(0, _NewReqData("a", embeds, is_token_ids))
|
||||
state.apply_staged_writes()
|
||||
|
||||
batch = _batch(num_scheduled=[5], idx_mapping=[0])
|
||||
out, base = _apply(state, batch, num_computed=[0], num_tokens=5)
|
||||
|
||||
embeds_gpu = embeds.to(DEVICE)
|
||||
for pos, is_token in enumerate(is_token_ids):
|
||||
expected = base[pos] if is_token else embeds_gpu[pos]
|
||||
torch.testing.assert_close(out[pos], expected)
|
||||
|
||||
|
||||
def test_index_reuse_clears_stale_entry():
|
||||
"""A request added at a previously-used index without embeds must not
|
||||
inherit the prior occupant's pointer-table entry."""
|
||||
state = _make_state()
|
||||
state.add_request(0, _NewReqData("a", torch.randn(4, HIDDEN, dtype=torch.float32)))
|
||||
# A second live embeds request so the kernel actually launches (the
|
||||
# overlay is skipped entirely when no request holds embeds).
|
||||
other = torch.randn(2, HIDDEN, dtype=torch.float32)
|
||||
state.add_request(1, _NewReqData("other", other))
|
||||
state.apply_staged_writes()
|
||||
state.remove_request("a")
|
||||
state.add_request(0, _NewReqData("b", None))
|
||||
state.apply_staged_writes()
|
||||
|
||||
batch = _batch(num_scheduled=[2, 2], idx_mapping=[0, 1])
|
||||
out, base = _apply(state, batch, num_computed=[0, 0], num_tokens=4)
|
||||
|
||||
torch.testing.assert_close(out[0:2], base[0:2])
|
||||
torch.testing.assert_close(out[2:4], other.to(DEVICE))
|
||||
Reference in New Issue
Block a user