[ModelRunnerV2] Support prompt embeds (#42963)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Signed-off-by: Canlin Guo <961750412@qq.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Canlin Guo
2026-08-17 21:58:50 +00:00
committed by GitHub
co-authored by Nick Hill
parent e68fb75b25
commit 455edc022b
17 changed files with 551 additions and 54 deletions
+90 -5
View File
@@ -15,7 +15,13 @@ import numpy as np
import pytest
import torch
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldElem,
MultiModalKwargsItem,
MultiModalSharedField,
PlaceholderRange,
)
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
from vllm.v1.worker.gpu.model_states.interface import ModelState
@@ -25,6 +31,25 @@ pytestmark = pytest.mark.cpu_test
HIDDEN = 4
def _model_state(cache: EncoderCache) -> MagicMock:
"""A mock ModelState backed by a real EncoderCache."""
state = MagicMock()
state.encoder_cache = cache
state.device = torch.device("cpu")
return state
def _embeds_item(embeds: torch.Tensor) -> MultiModalKwargsItem:
"""A `prompt_embeds` kwargs item, as the HF renderer builds it."""
return MultiModalKwargsItem(
{
"embedding": MultiModalFieldElem(
data=embeds, field=MultiModalSharedField(batch_size=1)
)
}
)
def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec:
return MultiModalFeatureSpec(
data=None,
@@ -197,8 +222,7 @@ def test_execute_mm_encoder_caches_outputs_without_gathering():
items the connector already holds, and a producer has no load path).
"""
cache = EncoderCache()
state = MagicMock()
state.encoder_cache = cache
state = _model_state(cache)
embedding = torch.ones(2, HIDDEN)
# (mm_hashes, [(modality, kwargs item), ...]), as prepare_mm_inputs returns.
state.encoder_runner.prepare_mm_inputs.return_value = (
@@ -216,8 +240,7 @@ def test_execute_mm_encoder_caches_outputs_without_gathering():
def test_execute_mm_encoder_is_a_noop_without_scheduled_items():
"""A step that schedules no encoder input must not touch the encoder."""
cache = EncoderCache()
state = MagicMock()
state.encoder_cache = cache
state = _model_state(cache)
state.encoder_runner.prepare_mm_inputs.return_value = ([], [])
ModelState.execute_mm_encoder(state, {})
@@ -226,6 +249,68 @@ def test_execute_mm_encoder_is_a_noop_without_scheduled_items():
state.encoder_runner.execute_mm_encoder.assert_not_called()
def _pe_feature(identifier: str, embeds: torch.Tensor, offset: int = 0):
return MultiModalFeatureSpec(
data=_embeds_item(embeds),
modality="prompt_embeds",
identifier=identifier,
mm_position=PlaceholderRange(offset=offset, length=embeds.shape[0]),
)
def test_prepare_mm_inputs_passes_prompt_embeds_through():
"""`prompt_embeds` is already in embedding space, so no encoder may run.
The renderer delivers prompt_embeds mixed with real media as an ordinary MM
modality. prepare_mm_inputs must cache the tensor directly and keep it out
of the encoder batch -- the vision encoder cannot consume it, and a missing
cache entry makes the subsequent gather raise "Encoder cache miss".
"""
prompt_embeds = torch.arange(2 * HIDDEN, dtype=torch.float32).view(2, HIDDEN)
image_feature = MultiModalFeatureSpec(
data=MagicMock(),
modality="image",
identifier="hash_img",
mm_position=PlaceholderRange(offset=2, length=2),
)
runner = _make_runner(
[_pe_feature("hash_pe", prompt_embeds), image_feature], cached=[]
)
mm_hashes, mm_kwargs = runner.prepare_mm_inputs({"req0": [0, 1]})
# Only the image remains for the encoder; the embeds are already cached.
assert mm_hashes == ["hash_img"]
assert [modality for modality, _ in mm_kwargs] == ["image"]
assert torch.equal(runner.encoder_cache.encoder_outputs["hash_pe"], prompt_embeds)
def test_prepare_mm_inputs_skips_cached_prompt_embeds():
"""A prompt_embeds item already in the cache must not be re-uploaded."""
prompt_embeds = torch.ones(3, HIDDEN)
feature = _pe_feature("hash_pe", prompt_embeds)
runner = _make_runner([feature], cached=[feature])
sentinel = runner.encoder_cache.encoder_outputs["hash_pe"]
mm_hashes, mm_kwargs = runner.prepare_mm_inputs({"req0": [0]})
assert mm_hashes == [] and mm_kwargs == []
assert runner.encoder_cache.encoder_outputs["hash_pe"] is sentinel
def test_execute_mm_encoder_skips_encoder_for_prompt_embeds_only():
"""A batch of nothing but prompt_embeds must not invoke the encoder."""
prompt_embeds = torch.ones(3, HIDDEN)
runner = _make_runner([_pe_feature("hash_pe", prompt_embeds)], cached=[])
state = _model_state(runner.encoder_cache)
state.encoder_runner.prepare_mm_inputs.side_effect = runner.prepare_mm_inputs
ModelState.execute_mm_encoder(state, {"req0": [0]})
state.encoder_runner.execute_mm_encoder.assert_not_called()
assert torch.equal(runner.encoder_cache.encoder_outputs["hash_pe"], prompt_embeds)
def test_encoder_timing_stats_registry():
runner = _make_runner([], [])
runner.enable_timing = True
+10
View File
@@ -68,6 +68,16 @@ NUM_BLOCKS = 10
DEVICE_TYPE = current_platform.device_type
@pytest.fixture(autouse=True)
def _restore_default_dtype():
"""Several tests here set the process-wide default dtype to float16 and
previously leaked it, corrupting later float-sensitive tests in the same
pytest process (torch.randn silently produced fp16)."""
old = torch.get_default_dtype()
yield
torch.set_default_dtype(old)
def initialize_kv_cache(runner: GPUModelRunner):
"""
Only perform necessary steps in GPUModelRunner.initialize_kv_cache()
+149
View File
@@ -0,0 +1,149 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the Model Runner V2 prompt-embeds overlay (PromptEmbedsState).
The overlay kernel reads each request's GPU-resident prompt embeddings through
a per-request pointer table and writes the rows scheduled this step into
`inputs_embeds`, honoring chunked prefill (`num_computed_tokens` offset), the
prompt/decode boundary (rows past the embeds length untouched), and the
mixed-mode `prompt_is_token_ids` mask (token-id rows keep the base embedding).
"""
from dataclasses import dataclass
import pytest
import torch
pytest.importorskip("triton")
if not torch.cuda.is_available():
pytest.skip(
"CUDA required for prompt-embeds overlay tests", allow_module_level=True
)
from vllm.v1.worker.gpu.model_states.prompt_embeds import PromptEmbedsState
HIDDEN = 24
MAX_NUM_REQS = 8
DEVICE = torch.device("cuda")
@dataclass
class _NewReqData:
req_id: str
prompt_embeds: torch.Tensor | None
prompt_is_token_ids: list[bool] | None = None
@dataclass
class _Batch:
num_reqs: int
num_scheduled_tokens: torch.Tensor # np-like, only .max() is used
idx_mapping: torch.Tensor
query_start_loc: torch.Tensor
def _make_state() -> PromptEmbedsState:
return PromptEmbedsState(MAX_NUM_REQS, HIDDEN, torch.float32, DEVICE)
def _batch(num_scheduled: list[int], idx_mapping: list[int]) -> _Batch:
query_start_loc = [0]
for n in num_scheduled:
query_start_loc.append(query_start_loc[-1] + n)
return _Batch(
num_reqs=len(num_scheduled),
num_scheduled_tokens=torch.tensor(num_scheduled, dtype=torch.int32),
idx_mapping=torch.tensor(idx_mapping, dtype=torch.int64, device=DEVICE),
query_start_loc=torch.tensor(query_start_loc, dtype=torch.int32, device=DEVICE),
)
def _apply(
state: PromptEmbedsState,
batch: _Batch,
num_computed: list[int],
num_tokens: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run the overlay on a fresh base buffer; return (result, base)."""
num_computed_tokens = torch.zeros(MAX_NUM_REQS, dtype=torch.int32, device=DEVICE)
for batch_idx, req_index in enumerate(batch.idx_mapping.tolist()):
num_computed_tokens[req_index] = num_computed[batch_idx]
base = torch.randn(num_tokens, HIDDEN, dtype=torch.float32, device=DEVICE)
inputs_embeds = base.clone()
state.apply(batch, num_computed_tokens, inputs_embeds)
torch.accelerator.synchronize()
return inputs_embeds, base
def test_overlay_chunked_prefill_and_decode():
"""Rows within the embeds range come from prompt_embeds at the
num_computed offset; requests without embeds and requests past their
embeds length (decode) keep the base embedding."""
state = _make_state()
embeds_a = torch.randn(6, HIDDEN, dtype=torch.float32)
embeds_b = torch.randn(5, HIDDEN, dtype=torch.float32)
state.add_request(0, _NewReqData("a", embeds_a))
state.add_request(1, _NewReqData("b", embeds_b))
state.add_request(2, _NewReqData("c", None))
state.apply_staged_writes()
# a: chunk [2, 6) of its embeds; b: fully decoded; c: no embeds.
batch = _batch(num_scheduled=[4, 1, 3], idx_mapping=[0, 1, 2])
out, base = _apply(state, batch, num_computed=[2, 7, 1], num_tokens=8)
torch.testing.assert_close(out[0:4], embeds_a[2:6].to(DEVICE))
torch.testing.assert_close(out[4:8], base[4:8])
def test_overlay_clamps_to_embeds_length():
"""A window straddling the end of the prompt embeds writes only the
in-range rows (e.g. final prefill chunk + sampled token)."""
state = _make_state()
embeds = torch.randn(4, HIDDEN, dtype=torch.float32)
state.add_request(3, _NewReqData("a", embeds))
state.apply_staged_writes()
batch = _batch(num_scheduled=[3], idx_mapping=[3])
out, base = _apply(state, batch, num_computed=[2], num_tokens=3)
torch.testing.assert_close(out[0:2], embeds[2:4].to(DEVICE))
torch.testing.assert_close(out[2:3], base[2:3])
def test_overlay_respects_is_token_ids_mask():
"""Mixed mode: positions marked as real token ids keep the base
embedding; only embed positions are overwritten."""
state = _make_state()
embeds = torch.randn(5, HIDDEN, dtype=torch.float32)
is_token_ids = [True, False, False, True, False]
state.add_request(0, _NewReqData("a", embeds, is_token_ids))
state.apply_staged_writes()
batch = _batch(num_scheduled=[5], idx_mapping=[0])
out, base = _apply(state, batch, num_computed=[0], num_tokens=5)
embeds_gpu = embeds.to(DEVICE)
for pos, is_token in enumerate(is_token_ids):
expected = base[pos] if is_token else embeds_gpu[pos]
torch.testing.assert_close(out[pos], expected)
def test_index_reuse_clears_stale_entry():
"""A request added at a previously-used index without embeds must not
inherit the prior occupant's pointer-table entry."""
state = _make_state()
state.add_request(0, _NewReqData("a", torch.randn(4, HIDDEN, dtype=torch.float32)))
# A second live embeds request so the kernel actually launches (the
# overlay is skipped entirely when no request holds embeds).
other = torch.randn(2, HIDDEN, dtype=torch.float32)
state.add_request(1, _NewReqData("other", other))
state.apply_staged_writes()
state.remove_request("a")
state.add_request(0, _NewReqData("b", None))
state.apply_staged_writes()
batch = _batch(num_scheduled=[2, 2], idx_mapping=[0, 1])
out, base = _apply(state, batch, num_computed=[0, 0], num_tokens=4)
torch.testing.assert_close(out[0:2], base[0:2])
torch.testing.assert_close(out[2:4], other.to(DEVICE))