feat: Enable prompt_embeds Content Part Support in vLLM Chat Completions API (#40720)
Signed-off-by: Luis Robaina <luis@protopia.ai> Signed-off-by: Luis Robaina 🚀 <luisfabian1545@gmail.com> Signed-off-by: LuisRobaina <luis@protopia.ai> Co-authored-by: Andrew Sansom <qthequartermasterman@gmail.com>
This commit is contained in:
@@ -11,7 +11,9 @@ from vllm import LLM, SamplingParams
|
||||
|
||||
def _make_mock_llm() -> LLM:
|
||||
llm = object.__new__(LLM)
|
||||
llm.model_config = SimpleNamespace(runner_type="generate")
|
||||
llm.model_config = SimpleNamespace(
|
||||
runner_type="generate", enable_prompt_embeds=False
|
||||
)
|
||||
return llm
|
||||
|
||||
|
||||
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E test for mixing `prompt_embeds` with `audio_embeds` in a single
|
||||
Chat Completions request."""
|
||||
|
||||
import json
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import safetensors
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from huggingface_hub import hf_hub_download
|
||||
from transformers import AutoConfig, AutoTokenizer
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.utils.serial_utils import tensor2base64
|
||||
|
||||
QWEN2AUDIO_MODEL = "Qwen/Qwen2-Audio-7B-Instruct"
|
||||
|
||||
# Use the model's native dtype to avoid an implicit cast inside
|
||||
# `safe_load_prompt_embeds` (mismatched floating-point dtypes are cast to the
|
||||
# model's dtype automatically, matching here just skips the conversion).
|
||||
QWEN2AUDIO_DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen2audio_server_args() -> list[str]:
|
||||
return [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--enforce-eager",
|
||||
"--trust-remote-code",
|
||||
"--gpu-memory-utilization",
|
||||
"0.85",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"audio": 1}),
|
||||
"--enable-prompt-embeds",
|
||||
"--enable-mm-embeds",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen2audio_server(qwen2audio_server_args):
|
||||
with RemoteOpenAIServer(
|
||||
QWEN2AUDIO_MODEL,
|
||||
qwen2audio_server_args,
|
||||
max_wait_seconds=600,
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def qwen2audio_client(qwen2audio_server):
|
||||
async with qwen2audio_server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen2audio_hidden_size() -> int:
|
||||
config = AutoConfig.from_pretrained(QWEN2AUDIO_MODEL, trust_remote_code=True)
|
||||
return config.text_config.hidden_size
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen2audio_prompt_embeds_b64(qwen2audio_hidden_size: int) -> str:
|
||||
tensor = torch.randn(4, qwen2audio_hidden_size, dtype=QWEN2AUDIO_DTYPE)
|
||||
return tensor2base64(tensor)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen2audio_audio_embeds_b64(qwen2audio_hidden_size: int) -> str:
|
||||
# Shape matches the `audio_embeds` unit-test fixture.
|
||||
torch.manual_seed(0)
|
||||
tensor = torch.randn(1, 128, qwen2audio_hidden_size, dtype=QWEN2AUDIO_DTYPE)
|
||||
return tensor2base64(tensor)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_embeds_plus_audio_embeds(
|
||||
qwen2audio_client: openai.AsyncOpenAI,
|
||||
qwen2audio_prompt_embeds_b64: str,
|
||||
qwen2audio_audio_embeds_b64: str,
|
||||
):
|
||||
"""Single user message carrying both prompt_embeds and audio_embeds parts."""
|
||||
chat = await qwen2audio_client.chat.completions.create(
|
||||
model=QWEN2AUDIO_MODEL,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "prompt_embeds",
|
||||
"data": qwen2audio_prompt_embeds_b64,
|
||||
},
|
||||
{
|
||||
"type": "audio_embeds",
|
||||
"audio_embeds": qwen2audio_audio_embeds_b64,
|
||||
},
|
||||
{"type": "text", "text": "Continue."},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen2audio_aligned_content_and_embeds_b64() -> tuple[str, str]:
|
||||
"""Return `(content, base64_embeds)` where the embeddings are the model's
|
||||
embedding of `content` tokenized WITHOUT special tokens.
|
||||
|
||||
Loads only the `embed_tokens` shard from disk on CPU (~1.1 GB of host
|
||||
RAM) instead of the full 7B model on GPU.
|
||||
"""
|
||||
content = "Describe this audio."
|
||||
tokenizer = AutoTokenizer.from_pretrained(QWEN2AUDIO_MODEL, trust_remote_code=True)
|
||||
|
||||
index_path = hf_hub_download(QWEN2AUDIO_MODEL, "model.safetensors.index.json")
|
||||
with open(index_path) as f:
|
||||
weight_map = json.load(f)["weight_map"]
|
||||
embed_key = next(k for k in weight_map if k.endswith("embed_tokens.weight"))
|
||||
shard_path = hf_hub_download(QWEN2AUDIO_MODEL, weight_map[embed_key])
|
||||
with safetensors.safe_open(shard_path, framework="pt", device="cpu") as f:
|
||||
embed_weight = f.get_tensor(embed_key)
|
||||
embed_layer = nn.Embedding.from_pretrained(embed_weight.to(QWEN2AUDIO_DTYPE))
|
||||
|
||||
ids = tokenizer(content, add_special_tokens=False, return_tensors="pt").input_ids
|
||||
embeds = embed_layer(ids).squeeze(0)
|
||||
return content, tensor2base64(embeds)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"audio_first",
|
||||
[True, False],
|
||||
ids=["audio_embeds-then-text", "text-then-audio_embeds"],
|
||||
)
|
||||
async def test_text_content_and_prompt_embeds_match_with_audio_embeds(
|
||||
qwen2audio_client: openai.AsyncOpenAI,
|
||||
qwen2audio_audio_embeds_b64: str,
|
||||
qwen2audio_aligned_content_and_embeds_b64: tuple[str, str],
|
||||
audio_first: bool,
|
||||
):
|
||||
"""Same content as text vs `prompt_embeds` should yield identical Chat
|
||||
Completions output when mixed with `audio_embeds` in the same message.
|
||||
"""
|
||||
content, encoded_text_embeds = qwen2audio_aligned_content_and_embeds_b64
|
||||
|
||||
audio_part = {
|
||||
"type": "audio_embeds",
|
||||
"audio_embeds": qwen2audio_audio_embeds_b64,
|
||||
}
|
||||
text_part = {"type": "text", "text": content}
|
||||
embeds_part = {"type": "prompt_embeds", "data": encoded_text_embeds}
|
||||
|
||||
if audio_first:
|
||||
text_content = [audio_part, text_part]
|
||||
embeds_content = [audio_part, embeds_part]
|
||||
else:
|
||||
text_content = [text_part, audio_part]
|
||||
embeds_content = [embeds_part, audio_part]
|
||||
|
||||
text_resp = await qwen2audio_client.chat.completions.create(
|
||||
model=QWEN2AUDIO_MODEL,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": text_content}],
|
||||
)
|
||||
embeds_resp = await qwen2audio_client.chat.completions.create(
|
||||
model=QWEN2AUDIO_MODEL,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": embeds_content}],
|
||||
)
|
||||
|
||||
text_out = text_resp.choices[0].message.content
|
||||
embeds_out = embeds_resp.choices[0].message.content
|
||||
assert text_out is not None and len(text_out) > 0
|
||||
assert embeds_out is not None and len(embeds_out) > 0
|
||||
assert text_out == embeds_out
|
||||
+212
@@ -0,0 +1,212 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for mixing `prompt_embeds` with image content parts in a single
|
||||
Chat Completions request.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import safetensors
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from huggingface_hub import hf_hub_download
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
from vllm.utils.serial_utils import tensor2base64
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen2-VL-2B-Instruct"
|
||||
|
||||
# Use the model's native dtype to skip the implicit cast inside
|
||||
# `safe_load_prompt_embeds` (mismatched floating-point dtypes are cast to the
|
||||
# model's dtype automatically).
|
||||
MODEL_DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_args() -> list[str]:
|
||||
return [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.4",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"image": 1}),
|
||||
"--enable-prompt-embeds",
|
||||
"--enable-mm-embeds",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(server_args):
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
server_args,
|
||||
max_wait_seconds=600,
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def image_url() -> str:
|
||||
"""Stable real image as a data URL, kept identical across both the
|
||||
text and prompt_embeds requests so any output difference must come from
|
||||
how the text content is delivered."""
|
||||
return encode_image_url(ImageAsset("stop_sign").pil_image)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def aligned_content_and_embeds_b64() -> tuple[str, str]:
|
||||
"""`(content, base64_embeds)` where the embeddings are the model's
|
||||
embedding of `content` tokenized WITHOUT special tokens.
|
||||
|
||||
Loads only the `embed_tokens` shard from disk on CPU instead of the full
|
||||
model on GPU, so the fixture has zero VRAM footprint and won't contend
|
||||
with the running vLLM server.
|
||||
"""
|
||||
content = "Describe this image."
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
|
||||
|
||||
index_path = hf_hub_download(MODEL_NAME, "model.safetensors.index.json")
|
||||
with open(index_path) as f:
|
||||
weight_map = json.load(f)["weight_map"]
|
||||
embed_key = next(k for k in weight_map if k.endswith("embed_tokens.weight"))
|
||||
shard_path = hf_hub_download(MODEL_NAME, weight_map[embed_key])
|
||||
with safetensors.safe_open(shard_path, framework="pt", device="cpu") as f:
|
||||
embed_weight = f.get_tensor(embed_key)
|
||||
embed_layer = nn.Embedding.from_pretrained(embed_weight.to(MODEL_DTYPE))
|
||||
|
||||
ids = tokenizer(content, add_special_tokens=False, return_tensors="pt").input_ids
|
||||
embeds = embed_layer(ids).squeeze(0)
|
||||
return content, tensor2base64(embeds)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"image_first",
|
||||
[True, False],
|
||||
ids=["image_url-then-text", "text-then-image_url"],
|
||||
)
|
||||
async def test_text_content_and_prompt_embeds_match_with_image_url(
|
||||
client: openai.AsyncOpenAI,
|
||||
image_url: str,
|
||||
aligned_content_and_embeds_b64: tuple[str, str],
|
||||
image_first: bool,
|
||||
):
|
||||
"""Same content as text vs `prompt_embeds` should yield identical Chat
|
||||
Completions output when mixed with an `image_url` part in the same
|
||||
message under greedy decoding.
|
||||
"""
|
||||
content, encoded_text_embeds = aligned_content_and_embeds_b64
|
||||
|
||||
image_part = {"type": "image_url", "image_url": {"url": image_url}}
|
||||
text_part = {"type": "text", "text": content}
|
||||
embeds_part = {"type": "prompt_embeds", "data": encoded_text_embeds}
|
||||
|
||||
if image_first:
|
||||
text_content = [image_part, text_part]
|
||||
embeds_content = [image_part, embeds_part]
|
||||
else:
|
||||
text_content = [text_part, image_part]
|
||||
embeds_content = [embeds_part, image_part]
|
||||
|
||||
text_resp = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": text_content}],
|
||||
)
|
||||
embeds_resp = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": embeds_content}],
|
||||
)
|
||||
|
||||
text_out = text_resp.choices[0].message.content
|
||||
embeds_out = embeds_resp.choices[0].message.content
|
||||
assert text_out is not None and len(text_out) > 0
|
||||
assert embeds_out is not None and len(embeds_out) > 0
|
||||
assert text_out == embeds_out
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def image_embeds_b64() -> dict[str, str]:
|
||||
"""Synthetic but stable `image_embeds` for Qwen2-VL."""
|
||||
grid = (1, 4, 4)
|
||||
spatial_merge_size = 2
|
||||
num_patches = (grid[1] // spatial_merge_size) * (grid[2] // spatial_merge_size)
|
||||
text_hidden_size = 1536 # Qwen2-VL-2B
|
||||
torch.manual_seed(0)
|
||||
return {
|
||||
"image_embeds": tensor2base64(
|
||||
torch.randn(num_patches, text_hidden_size, dtype=MODEL_DTYPE)
|
||||
),
|
||||
"image_grid_thw": tensor2base64(torch.tensor(grid)),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"image_first",
|
||||
[True, False],
|
||||
ids=["image_embeds-then-text", "text-then-image_embeds"],
|
||||
)
|
||||
async def test_text_content_and_prompt_embeds_match_with_image_embeds(
|
||||
client: openai.AsyncOpenAI,
|
||||
image_embeds_b64: dict[str, str],
|
||||
aligned_content_and_embeds_b64: tuple[str, str],
|
||||
image_first: bool,
|
||||
):
|
||||
"""Same content as text vs `prompt_embeds` should yield identical Chat
|
||||
Completions output when mixed with a precomputed `image_embeds` part in
|
||||
the same message under greedy decoding.
|
||||
"""
|
||||
content, encoded_text_embeds = aligned_content_and_embeds_b64
|
||||
|
||||
image_part = {"type": "image_embeds", "image_embeds": image_embeds_b64}
|
||||
text_part = {"type": "text", "text": content}
|
||||
embeds_part = {"type": "prompt_embeds", "data": encoded_text_embeds}
|
||||
|
||||
if image_first:
|
||||
text_content = [image_part, text_part]
|
||||
embeds_content = [image_part, embeds_part]
|
||||
else:
|
||||
text_content = [text_part, image_part]
|
||||
embeds_content = [embeds_part, image_part]
|
||||
|
||||
text_resp = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": text_content}],
|
||||
)
|
||||
embeds_resp = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": embeds_content}],
|
||||
)
|
||||
|
||||
text_out = text_resp.choices[0].message.content
|
||||
embeds_out = embeds_resp.choices[0].message.content
|
||||
assert text_out is not None and len(text_out) > 0
|
||||
assert embeds_out is not None and len(embeds_out) > 0
|
||||
assert text_out == embeds_out
|
||||
@@ -0,0 +1,293 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for `prompt_embeds` content parts in the Chat Completions API."""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
|
||||
import openai
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import torch
|
||||
from openai import BadRequestError
|
||||
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
CHAT_TEMPLATE = VLLM_PATH / "examples/template_chatml.jinja"
|
||||
# Matches `--dtype` in `server_args` to avoid an implicit cast in
|
||||
# `safe_load_prompt_embeds` (mismatched floating-point dtypes are cast to the
|
||||
# model's dtype automatically, we match here just to skip the conversion).
|
||||
SERVER_DTYPE: torch.dtype = torch.bfloat16
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_args() -> list[str]:
|
||||
return [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
"--chat-template",
|
||||
str(CHAT_TEMPLATE),
|
||||
# Prompt Embeds server args
|
||||
"--enable-prompt-embeds",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(server_args):
|
||||
with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
def _encode_embeds(embeds: torch.Tensor) -> str:
|
||||
buf = io.BytesIO()
|
||||
torch.save(embeds, buf)
|
||||
return base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def prompt_embeds_b64(hf_runner) -> list[str]:
|
||||
"""Pre-compute embeddings for two short prompts and return as base64."""
|
||||
prompts = ["Hello, my name is", "What is an LLM?"]
|
||||
with hf_runner(MODEL_NAME) as hf_model:
|
||||
embeddings = hf_model.get_prompt_embeddings(prompts)
|
||||
# Cast to the server's dtype so `safe_load_prompt_embeds` doesn't need to
|
||||
# convert on its own, the function accepts any floating-point dtype and
|
||||
# will cast to the model's dtype, but matching up front skips the work.
|
||||
return [_encode_embeds(e.to(SERVER_DTYPE)) for e in embeddings]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_prompt_embeds_part(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""A user message with one prompt_embeds part + text."""
|
||||
b64 = prompt_embeds_b64[0]
|
||||
chat = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": "Continue:"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_prompt_embeds_parts(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""Multiple prompt_embeds parts in a single message."""
|
||||
b64_a, b64_b = prompt_embeds_b64
|
||||
chat = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64_a},
|
||||
{"type": "text", "text": " and "},
|
||||
{"type": "prompt_embeds", "data": b64_b},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_message_conversation(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""prompt_embeds in both system and user messages."""
|
||||
b64_sys, b64_usr = prompt_embeds_b64
|
||||
chat = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are helpful."},
|
||||
{"type": "prompt_embeds", "data": b64_sys},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64_usr},
|
||||
{"type": "text", "text": "Summarize."},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""Streaming chat completion with prompt_embeds."""
|
||||
b64 = prompt_embeds_b64[0]
|
||||
|
||||
# Non-streaming baseline.
|
||||
baseline = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": "Continue:"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
expected = baseline.choices[0].message.content
|
||||
|
||||
# Streaming.
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": "Continue:"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
chunks: list[str] = []
|
||||
async for chunk in stream:
|
||||
delta = chunk.choices[0].delta.content
|
||||
if delta:
|
||||
chunks.append(delta)
|
||||
assert "".join(chunks) == expected
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def aligned_content_and_embeds_b64(hf_runner) -> tuple[str, str]:
|
||||
"""Return `(content, base64_embeds)` where the embeddings are the model's
|
||||
embedding of `content` tokenized WITHOUT special tokens.
|
||||
"""
|
||||
content = "Hello, my name is"
|
||||
with hf_runner(MODEL_NAME) as hf_model:
|
||||
ids = hf_model.tokenizer(
|
||||
content, add_special_tokens=False, return_tensors="pt"
|
||||
).input_ids
|
||||
ids = hf_model.wrap_device({"input_ids": ids})["input_ids"]
|
||||
embed_layer = hf_model.model.get_input_embeddings()
|
||||
embeds = embed_layer(ids).squeeze(0).to(SERVER_DTYPE).cpu()
|
||||
return content, _encode_embeds(embeds)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_content_and_prompt_embeds_match(
|
||||
client: openai.AsyncOpenAI,
|
||||
aligned_content_and_embeds_b64: tuple[str, str],
|
||||
):
|
||||
"""Equal content in text and `prompt_embeds` should yield identical
|
||||
Chat Completions output under greedy decoding.
|
||||
"""
|
||||
content, encoded_embeds = aligned_content_and_embeds_b64
|
||||
|
||||
text_resp, embeds_resp = await asyncio.gather(
|
||||
client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": content}],
|
||||
),
|
||||
client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds", "data": encoded_embeds}],
|
||||
}
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
text_out = text_resp.choices[0].message.content
|
||||
embeds_out = embeds_resp.choices[0].message.content
|
||||
assert text_out is not None and len(text_out) > 0
|
||||
assert embeds_out is not None and len(embeds_out) > 0
|
||||
assert text_out == embeds_out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_data_field(
|
||||
client: openai.AsyncOpenAI,
|
||||
):
|
||||
"""A prompt_embeds part without `data` should return a clear error."""
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds"}],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_base64(
|
||||
client: openai.AsyncOpenAI,
|
||||
):
|
||||
"""Invalid base64 in the `data` field should return a clear error."""
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": "not_valid_base64!!"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
@@ -538,6 +538,7 @@ class MockModelConfig:
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
enable_prompt_embeds: bool = False
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
return self.diff_sampling_param or {}
|
||||
|
||||
@@ -62,6 +62,8 @@ def test_load_prompt_embeds(
|
||||
):
|
||||
model_config = Mock(spec=ModelConfig)
|
||||
model_config.enable_prompt_embeds = True
|
||||
model_config.get_hidden_size.return_value = hidden_size
|
||||
model_config.dtype = dtype
|
||||
|
||||
# construct arbitrary tensors of various dtypes, layouts, and sizes.
|
||||
# We need to check against different layouts to make sure that if a user
|
||||
|
||||
@@ -0,0 +1,576 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Offline unit tests for `prompt_embeds` chat-completion content parts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import io
|
||||
from typing import Final
|
||||
from unittest import mock
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import regex as re
|
||||
import torch
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
_ENABLE_PROMPT_EMBEDS_ERROR,
|
||||
_PROMPT_EMBEDS_MISSING_DATA_ERROR,
|
||||
_RESERVED_PLACEHOLDER_IN_TEXT_ERROR,
|
||||
MM_PARSER_MAP,
|
||||
MODALITY_PLACEHOLDERS_MAP,
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.renderers.hf import (
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR,
|
||||
_build_mixed_prompt_embeds,
|
||||
_build_prompt_embeds_positions,
|
||||
_build_prompt_embeds_updates,
|
||||
_ensure_prompt_embeds_placeholder_token,
|
||||
_expand_prompt_embeds_placeholders,
|
||||
)
|
||||
|
||||
# Cover distinct tokenizer families:
|
||||
# GPT2TokenizerFast (BPE, OpenAI-style)
|
||||
# Qwen2TokenizerFast (SentencePiece BPE variant)
|
||||
# BertTokenizerFast (WordPiece)
|
||||
TOKENIZER_IDS: Final[list[str]] = [
|
||||
"gpt2",
|
||||
"Qwen/Qwen2.5-1.5B-Instruct",
|
||||
"bert-base-uncased",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(params=TOKENIZER_IDS, ids=TOKENIZER_IDS)
|
||||
def tokenizer(request):
|
||||
"""A fresh tokenizer instance per tokenizer family."""
|
||||
return AutoTokenizer.from_pretrained(request.param)
|
||||
|
||||
|
||||
# Minimal chat template that works with any tokenizer. Iterates
|
||||
# `message.content` as either a string or a list of dicts (openai format).
|
||||
_SIMPLE_CHAT_TEMPLATE: Final[str] = (
|
||||
"{% for m in messages %}"
|
||||
"{% if m['content'] is string %}{{m['content']}}"
|
||||
"{% else %}{% for p in m['content'] %}{{p['text']}}{% endfor %}"
|
||||
"{% endif %}\n{% endfor %}"
|
||||
)
|
||||
|
||||
|
||||
async def _maybe_await(fn, *args, **kwargs):
|
||||
"""Call *fn* and `await` the result if it's a coroutine."""
|
||||
result = fn(*args, **kwargs)
|
||||
if inspect.iscoroutine(result):
|
||||
result = await result
|
||||
return result
|
||||
|
||||
|
||||
# Parametrize over sync / async parse paths so every end-to-end test
|
||||
# exercises both.
|
||||
_PARSE_FUNCTIONS = [parse_chat_messages, parse_chat_messages_async]
|
||||
|
||||
|
||||
@pytest.fixture(params=_PARSE_FUNCTIONS, ids=["sync", "async"])
|
||||
def parse_fn(request):
|
||||
"""Either the sync or async `parse_chat_messages` callable."""
|
||||
return request.param
|
||||
|
||||
|
||||
def _encode_tensor(t: torch.Tensor) -> str:
|
||||
buf = io.BytesIO()
|
||||
torch.save(t, buf)
|
||||
return base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
|
||||
|
||||
_MOCK_HIDDEN_SIZE: Final[int] = 8
|
||||
_MOCK_DTYPE: Final[torch.dtype] = torch.float32
|
||||
|
||||
|
||||
def _make_mock_model_config(*, enable_prompt_embeds: bool = True) -> mock.MagicMock:
|
||||
mc = mock.MagicMock()
|
||||
mc.enable_prompt_embeds = enable_prompt_embeds
|
||||
mc.multimodal_config = None
|
||||
mc.allowed_local_media_path = None
|
||||
mc.allowed_media_domains = None
|
||||
# Test text-only code path in `MultiModalItemTracker.resolve_items`.
|
||||
mc.is_multimodal_model = False
|
||||
# `safe_load_prompt_embeds` pins each tensor to the model's hidden_size
|
||||
# and dtype, so the mock must return concrete values.
|
||||
mc.get_hidden_size.return_value = _MOCK_HIDDEN_SIZE
|
||||
mc.dtype = _MOCK_DTYPE
|
||||
return mc
|
||||
|
||||
|
||||
def test_prompt_embeds_keys_registered():
|
||||
assert "prompt_embeds" in MODALITY_PLACEHOLDERS_MAP
|
||||
assert MODALITY_PLACEHOLDERS_MAP["prompt_embeds"] == "<##PROMPT_EMBEDS##>"
|
||||
assert "prompt_embeds" in MM_PARSER_MAP
|
||||
|
||||
|
||||
def test_ensure_placeholder_token_is_single_token_and_idempotent(tokenizer):
|
||||
"""Ensure the placeholder token is a single token and that multiple calls to
|
||||
"ensure" are idempotent, across all tokenizer families."""
|
||||
tid1 = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
tid2 = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
assert tid1 == tid2
|
||||
|
||||
ids = tokenizer.encode(PROMPT_EMBEDS_PLACEHOLDER_TOKEN, add_special_tokens=False)
|
||||
assert ids == [tid1]
|
||||
|
||||
# Repeating it in a string N times must produce exactly that many tokens.
|
||||
N = 5
|
||||
ids_rep = tokenizer.encode(
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN * N, add_special_tokens=False
|
||||
)
|
||||
assert ids_rep == [tid1] * N
|
||||
|
||||
|
||||
def test_parse_chat_messages_openai_format():
|
||||
NUM_TOKENS = 3
|
||||
t = torch.randn(NUM_TOKENS, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
b64 = _encode_tensor(t)
|
||||
mc = _make_mock_model_config()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Hello "},
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": " world"},
|
||||
],
|
||||
}
|
||||
]
|
||||
conv, mm_data, _ = parse_chat_messages(
|
||||
messages,
|
||||
mc,
|
||||
content_format="openai",
|
||||
)
|
||||
# The middle content part is rewritten to a single placeholder-token
|
||||
# sentinel.
|
||||
texts = [p["text"] for p in conv[0]["content"]]
|
||||
assert texts == [
|
||||
"Hello ",
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
|
||||
" world",
|
||||
]
|
||||
assert mm_data is not None and "prompt_embeds" in mm_data
|
||||
assert torch.equal(mm_data["prompt_embeds"][0], t)
|
||||
|
||||
|
||||
# Each layout entry is one content part:
|
||||
# ("text", "A") -> {"type": "text", "text": "A"}
|
||||
# ("embed", N) -> {"type": "prompt_embeds", "data": <base64 of (N, H) tensor>}
|
||||
@pytest.mark.parametrize(
|
||||
"layout",
|
||||
[
|
||||
# Case: Single embed only.
|
||||
[("embed", 2)],
|
||||
# Case: Embed at the start of the message.
|
||||
[("embed", 3), ("text", "B")],
|
||||
# Case: Embed at the end of the message.
|
||||
[("text", "A"), ("embed", 1)],
|
||||
# Case: Embed sandwiched between text spans.
|
||||
[("text", "A"), ("embed", 2), ("text", "B")],
|
||||
# Case: Multiple embeds with text in between.
|
||||
[("text", "A"), ("embed", 2), ("text", "B"), ("embed", 3)],
|
||||
# Case: Adjacent embeds with no separating text.
|
||||
[("embed", 1), ("embed", 2)],
|
||||
# Case: Multiple text spans before a trailing embed.
|
||||
[("text", "A"), ("text", "B"), ("embed", 1)],
|
||||
# Case: Long-ish run mixing both kinds.
|
||||
[
|
||||
("text", "head"),
|
||||
("embed", 4),
|
||||
("text", "mid"),
|
||||
("embed", 1),
|
||||
("embed", 2),
|
||||
("text", "tail"),
|
||||
],
|
||||
],
|
||||
ids=[
|
||||
"single-embed",
|
||||
"embed-then-text",
|
||||
"text-then-embed",
|
||||
"text-embed-text",
|
||||
"text-embed-text-embed",
|
||||
"adjacent-embeds",
|
||||
"text-text-embed",
|
||||
"long-mixed-run",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"interleave_mm_strings",
|
||||
# `None`: text-only path where `multimodal_config` is absent.
|
||||
# `False`: non-interleave multimodal path (the common default).
|
||||
# `True`: sentinel-substitution interleave path.
|
||||
# All three must preserve the request ordering of prompt_embeds
|
||||
# relative to surrounding text because prompt_embeds are spliced at the
|
||||
# token offset during rendering.
|
||||
[None, False, True],
|
||||
ids=["text-only", "interleave-off", "interleave-on"],
|
||||
)
|
||||
def test_parse_chat_messages_string_format_preserves_position(
|
||||
layout, interleave_mm_strings
|
||||
):
|
||||
mc = _make_mock_model_config()
|
||||
if interleave_mm_strings is not None:
|
||||
mm_cfg = mock.MagicMock()
|
||||
mm_cfg.interleave_mm_strings = interleave_mm_strings
|
||||
mc.multimodal_config = mm_cfg
|
||||
|
||||
content: list[dict] = []
|
||||
expected_parts: list[str] = []
|
||||
expected_embeds: list[torch.Tensor] = []
|
||||
for kind, value in layout:
|
||||
if kind == "text":
|
||||
content.append({"type": "text", "text": value})
|
||||
expected_parts.append(value)
|
||||
else: # prompt embeds
|
||||
num_tokens = value
|
||||
t = torch.randn(num_tokens, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
expected_embeds.append(t)
|
||||
content.append({"type": "prompt_embeds", "data": _encode_tensor(t)})
|
||||
# Parser emits ONE sentinel per part.
|
||||
expected_parts.append(PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
|
||||
|
||||
messages = [{"role": "user", "content": content}]
|
||||
conv, mm_data, _ = parse_chat_messages(
|
||||
messages,
|
||||
mc,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
assert conv[0]["content"] == "\n".join(expected_parts)
|
||||
assert mm_data is not None and "prompt_embeds" in mm_data
|
||||
assert len(mm_data["prompt_embeds"]) == len(expected_embeds)
|
||||
for got, want in zip(mm_data["prompt_embeds"], expected_embeds, strict=True):
|
||||
assert torch.equal(got, want)
|
||||
|
||||
|
||||
def test_parse_chat_messages_requires_flag():
|
||||
t = torch.randn(2, 4)
|
||||
b64 = _encode_tensor(t)
|
||||
mc = _make_mock_model_config(enable_prompt_embeds=False)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds", "data": b64}],
|
||||
}
|
||||
]
|
||||
with pytest.raises(ValueError, match=_ENABLE_PROMPT_EMBEDS_ERROR):
|
||||
parse_chat_messages(
|
||||
messages,
|
||||
mc,
|
||||
content_format="openai",
|
||||
)
|
||||
|
||||
|
||||
def test_parse_chat_messages_rejects_missing_data():
|
||||
# `data` is marked `Required` on `ChatCompletionContentPartPromptEmbedsParam`;
|
||||
# malformed requests without `data` must surface a clear validation error
|
||||
# rather than being silently dropped.
|
||||
mc = _make_mock_model_config()
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds"}], # no `data`
|
||||
}
|
||||
]
|
||||
with pytest.raises(ValueError, match=_PROMPT_EMBEDS_MISSING_DATA_ERROR):
|
||||
parse_chat_messages(
|
||||
messages,
|
||||
mc,
|
||||
content_format="openai",
|
||||
)
|
||||
|
||||
|
||||
# Reserved placeholder guard: when `enable_prompt_embeds=True` the tokenizer is
|
||||
# mutated to make `<prompt_embeds>` a single unsplittable token. Any user text
|
||||
# containing that literal sequence would tokenize to the same sentinel ID and
|
||||
# be mistaken for a splice point, so we reject it at parse time.
|
||||
_PLACEHOLDER_ERROR_PATTERN: Final[str] = re.sub(
|
||||
r"\\{[^}]*\\}", ".*", re.escape(_RESERVED_PLACEHOLDER_IN_TEXT_ERROR)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
# Case: Top-level string content (wrapped as a single text part).
|
||||
f"hello {PROMPT_EMBEDS_PLACEHOLDER_TOKEN} world",
|
||||
# Case: List with a typed text part containing the placeholder.
|
||||
[{"type": "text", "text": f"leading {PROMPT_EMBEDS_PLACEHOLDER_TOKEN}"}],
|
||||
# Case: List with a plain-string part (no wrapping dict).
|
||||
[f"raw string {PROMPT_EMBEDS_PLACEHOLDER_TOKEN}"],
|
||||
],
|
||||
ids=["top-level-string", "typed-text-part", "plain-string-part"],
|
||||
)
|
||||
def test_parse_chat_messages_rejects_placeholder_in_user_text(content):
|
||||
mc = _make_mock_model_config() # enable_prompt_embeds=True by default
|
||||
messages = [{"role": "user", "content": content}]
|
||||
with pytest.raises(ValueError, match=_PLACEHOLDER_ERROR_PATTERN):
|
||||
parse_chat_messages(messages, mc, content_format="openai")
|
||||
|
||||
|
||||
def test_parse_chat_messages_allows_placeholder_in_text_when_feature_disabled():
|
||||
# When `enable_prompt_embeds=False` the tokenizer is never mutated, so the
|
||||
# literal `<prompt_embeds>` is just ordinary text and must pass through.
|
||||
mc = _make_mock_model_config(enable_prompt_embeds=False)
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"benign mention of {PROMPT_EMBEDS_PLACEHOLDER_TOKEN} here",
|
||||
}
|
||||
]
|
||||
conv, mm_data, _ = parse_chat_messages(messages, mc, content_format="openai")
|
||||
assert mm_data is None or "prompt_embeds" not in mm_data
|
||||
# Text reaches the rendered conversation unchanged.
|
||||
texts = [p["text"] for p in conv[0]["content"]]
|
||||
assert PROMPT_EMBEDS_PLACEHOLDER_TOKEN in "".join(texts)
|
||||
|
||||
|
||||
# Token-stream spec: ints are regular token IDs, tuples `(N,)` expand to
|
||||
# a placeholder span of length N (creates corresponding `(N, H)` tensor).
|
||||
# `expected` lists the `(start_idx, length)` pairs that
|
||||
# `_build_prompt_embeds_positions` should return.
|
||||
@pytest.mark.parametrize(
|
||||
"stream, expected",
|
||||
[
|
||||
# Case: Single run in the middle.
|
||||
([10, 20, (3,), 30], [(2, 3)]),
|
||||
# Case: Single run at the start.
|
||||
([(2,), 10, 20], [(0, 2)]),
|
||||
# Case: Single run at the end.
|
||||
([10, 20, (4,)], [(2, 4)]),
|
||||
# Case: Two runs with tokens between.
|
||||
([1, (2,), 2, 3, (3,), 4], [(1, 2), (5, 3)]),
|
||||
# Case: Adjacent runs (no separating tokens).
|
||||
([(1,), (2,)], [(0, 1), (1, 2)]),
|
||||
# Case: Three runs.
|
||||
([5, (2,), 6, (1,), 7, (3,), 8], [(1, 2), (4, 1), (6, 3)]),
|
||||
],
|
||||
ids=[
|
||||
"single-middle",
|
||||
"single-start",
|
||||
"single-end",
|
||||
"two-runs-separated",
|
||||
"two-runs-adjacent",
|
||||
"three-runs",
|
||||
],
|
||||
)
|
||||
def test_build_positions(tokenizer, stream, expected):
|
||||
H = 4
|
||||
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
tensors: list[torch.Tensor] = []
|
||||
token_ids: list[int] = []
|
||||
for item in stream:
|
||||
if isinstance(item, tuple):
|
||||
length = item[0]
|
||||
tensors.append(torch.randn(length, H))
|
||||
token_ids.extend([tid] * length)
|
||||
else:
|
||||
token_ids.append(item)
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, tid)
|
||||
positions = _build_prompt_embeds_positions(token_ids, len(tensors), mm_updates)
|
||||
assert positions == expected
|
||||
|
||||
|
||||
def test_build_positions_length_mismatch(tokenizer):
|
||||
N1, H1 = 2, 4
|
||||
N2, H2 = 3, 4
|
||||
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
# 2 tensors expected but only a single placeholder run in the token
|
||||
# stream (simulating dropping the second one).
|
||||
tensors = [torch.randn(N1, H1), torch.randn(N2, H2)]
|
||||
token_ids = [1, tid, tid, 2, 3]
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, tid)
|
||||
# The error constant is a `str.format` template, escape it and turn
|
||||
# the `{field}` placeholders into `.*` so it matches any substitution.
|
||||
pattern = re.sub(
|
||||
r"\\{[^}]*\\}", ".*", re.escape(_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR)
|
||||
)
|
||||
with pytest.raises(ValueError, match=pattern):
|
||||
_build_prompt_embeds_positions(token_ids, len(tensors), mm_updates)
|
||||
|
||||
|
||||
# ints = regular token IDs (any value)
|
||||
# (N,) = embed span of length N
|
||||
@pytest.mark.parametrize(
|
||||
"stream",
|
||||
[
|
||||
[10, 20, (3,), 30],
|
||||
[(2,), 10, 20],
|
||||
[10, 20, (4,)],
|
||||
[1, (2,), 2, 3, (3,), 4],
|
||||
[(1,), (2,)],
|
||||
[5, (2,), 6, (1,), 7, (3,), 8],
|
||||
],
|
||||
ids=[
|
||||
"single-middle",
|
||||
"single-start",
|
||||
"single-end",
|
||||
"two-spans-separated",
|
||||
"two-spans-adjacent",
|
||||
"three-spans",
|
||||
],
|
||||
)
|
||||
def test_build_mixed_prompt_embeds(stream):
|
||||
H = 8
|
||||
_PLACEHOLDER = 0 # sentinel for embed positions in token_ids
|
||||
|
||||
tensors: list[torch.Tensor] = []
|
||||
token_ids: list[int] = []
|
||||
positions: list[tuple[int, int]] = []
|
||||
cursor = 0
|
||||
for item in stream:
|
||||
if isinstance(item, tuple):
|
||||
length = item[0]
|
||||
tensors.append(torch.randn(length, H))
|
||||
positions.append((cursor, length))
|
||||
token_ids.extend([_PLACEHOLDER] * length)
|
||||
cursor += length
|
||||
else:
|
||||
token_ids.append(item)
|
||||
cursor += 1
|
||||
|
||||
embeds, mask = _build_mixed_prompt_embeds(token_ids, tensors, positions)
|
||||
|
||||
assert embeds.shape == (len(token_ids), H)
|
||||
assert len(mask) == len(token_ids)
|
||||
|
||||
# Mask: False exactly at embed positions, True everywhere else.
|
||||
expected_mask = torch.ones(len(token_ids), dtype=torch.bool)
|
||||
for start, length in positions:
|
||||
expected_mask[start : start + length] = False
|
||||
assert mask == expected_mask.tolist()
|
||||
|
||||
# Embed rows match input tensors at the right positions.
|
||||
for tensor, (start, length) in zip(tensors, positions):
|
||||
assert torch.equal(embeds[start : start + length], tensor)
|
||||
|
||||
# Non-embed positions remain zero-filled.
|
||||
assert torch.all(embeds[expected_mask] == 0)
|
||||
|
||||
|
||||
# End-to-end tests: each runs both sync and async parse paths via the
|
||||
# `parse_fn` fixture.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("role", ["user", "system"])
|
||||
async def test_end_to_end_expand_and_build(tokenizer, parse_fn, role):
|
||||
"""Full renderer pipeline: parse -> chat template -> expand -> locate
|
||||
-> build mixed prompt, across tokenizers, roles, and sync/async."""
|
||||
tokenizer.chat_template = _SIMPLE_CHAT_TEMPLATE
|
||||
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
|
||||
LEN_A, LEN_B = 3, 2
|
||||
t_a = torch.randn(LEN_A, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
t_b = torch.randn(LEN_B, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
NUM_TENSORS = 2
|
||||
|
||||
mc = _make_mock_model_config()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": role,
|
||||
"content": [
|
||||
{"type": "text", "text": "Hello "},
|
||||
{"type": "prompt_embeds", "data": _encode_tensor(t_a)},
|
||||
{"type": "text", "text": " world "},
|
||||
{"type": "prompt_embeds", "data": _encode_tensor(t_b)},
|
||||
{"type": "text", "text": "!"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conv, mm_data, _ = await _maybe_await(
|
||||
parse_fn, messages, mc, content_format="openai"
|
||||
)
|
||||
tensors = list(mm_data["prompt_embeds"])
|
||||
assert len(tensors) == NUM_TENSORS
|
||||
|
||||
# Tokenize: each prompt_embeds part becomes 1 placeholder token.
|
||||
# `return_dict=False` to get a flat `list[int]` on transformers v5
|
||||
# (where the default flipped to True and yields a `BatchEncoding` dict).
|
||||
token_ids = tokenizer.apply_chat_template(conv, tokenize=True, return_dict=False)
|
||||
assert sum(t == tid for t in token_ids) == NUM_TENSORS
|
||||
|
||||
# Expand, locate, and build.
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, tid)
|
||||
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
|
||||
assert len(expanded) == len(token_ids) + LEN_A + LEN_B - NUM_TENSORS
|
||||
|
||||
positions = _build_prompt_embeds_positions(expanded, len(tensors), mm_updates)
|
||||
assert positions[0][1] == LEN_A
|
||||
assert positions[1][1] == LEN_B
|
||||
|
||||
embeds, mask = _build_mixed_prompt_embeds(expanded, tensors, positions)
|
||||
assert embeds.shape == (len(expanded), _MOCK_HIDDEN_SIZE)
|
||||
assert mask.count(False) == LEN_A + LEN_B
|
||||
assert torch.equal(embeds[positions[0][0] : positions[0][0] + LEN_A], t_a)
|
||||
assert torch.equal(embeds[positions[1][0] : positions[1][0] + LEN_B], t_b)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_end_to_end_multi_message_conversation(tokenizer, parse_fn):
|
||||
"""Full pipeline with prompt_embeds spread across system + user messages,
|
||||
verifying ordering and positioning in the final token stream."""
|
||||
tokenizer.chat_template = _SIMPLE_CHAT_TEMPLATE
|
||||
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
|
||||
LEN_SYS, LEN_USR = 4, 3
|
||||
t_sys = torch.randn(LEN_SYS, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
t_usr = torch.randn(LEN_USR, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
NUM_TENSORS = 2 # t_sys and t_usr.
|
||||
|
||||
mc = _make_mock_model_config()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are helpful."},
|
||||
{"type": "prompt_embeds", "data": _encode_tensor(t_sys)},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": _encode_tensor(t_usr)},
|
||||
{"type": "text", "text": "Summarize."},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
conv, mm_data, _ = await _maybe_await(
|
||||
parse_fn, messages, mc, content_format="openai"
|
||||
)
|
||||
tensors = list(mm_data["prompt_embeds"])
|
||||
assert len(tensors) == NUM_TENSORS
|
||||
|
||||
# Tokenize, expand, locate, and build.
|
||||
# `return_dict=False` to get a flat `list[int]` on transformers v5
|
||||
# (where the default flipped to True and yields a `BatchEncoding` dict).
|
||||
token_ids = tokenizer.apply_chat_template(conv, tokenize=True, return_dict=False)
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, tid)
|
||||
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
|
||||
positions = _build_prompt_embeds_positions(expanded, len(tensors), mm_updates)
|
||||
|
||||
assert positions[0][1] == LEN_SYS
|
||||
assert positions[1][1] == LEN_USR
|
||||
# System embed must appear before user embed in the token stream.
|
||||
assert positions[0][0] < positions[1][0]
|
||||
|
||||
embeds, mask = _build_mixed_prompt_embeds(expanded, tensors, positions)
|
||||
assert embeds.shape == (len(expanded), _MOCK_HIDDEN_SIZE)
|
||||
assert mask.count(False) == LEN_SYS + LEN_USR
|
||||
assert torch.equal(embeds[positions[0][0] : positions[0][0] + LEN_SYS], t_sys)
|
||||
assert torch.equal(embeds[positions[1][0] : positions[1][0] + LEN_USR], t_usr)
|
||||
@@ -39,6 +39,11 @@ class MockModelConfig:
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
hidden_size: int = 768
|
||||
dtype: torch.dtype = torch.float32
|
||||
|
||||
def get_hidden_size(self) -> int:
|
||||
return self.hidden_size
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -384,12 +389,13 @@ class TestRenderEmbedPrompt:
|
||||
assert torch.equal(results[0]["prompt_embeds"], tensor_input)
|
||||
|
||||
def test_multiple_prompt_embeds(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
hidden_size = 512
|
||||
renderer = _build_renderer(MockModelConfig(hidden_size=hidden_size))
|
||||
|
||||
# Create multiple test tensors
|
||||
tensor_inputs = [
|
||||
torch.randn(8, 512, dtype=torch.float32),
|
||||
torch.randn(12, 512, dtype=torch.float32),
|
||||
torch.randn(8, hidden_size, dtype=torch.float32),
|
||||
torch.randn(12, hidden_size, dtype=torch.float32),
|
||||
]
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
@@ -432,13 +438,15 @@ class TestRenderEmbedPrompt:
|
||||
assert torch.equal(results[0]["prompt_embeds"], expected)
|
||||
|
||||
def test_prompt_embed_different_dtypes(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
hidden_size = 256
|
||||
# Test different supported dtypes
|
||||
dtypes = [torch.float32, torch.float16, torch.bfloat16]
|
||||
|
||||
for dtype in dtypes:
|
||||
tensor_input = torch.randn(5, 256, dtype=dtype)
|
||||
renderer = _build_renderer(
|
||||
MockModelConfig(hidden_size=hidden_size, dtype=dtype)
|
||||
)
|
||||
tensor_input = torch.randn(5, hidden_size, dtype=dtype)
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(
|
||||
@@ -474,10 +482,11 @@ class TestRenderEmbedPrompt:
|
||||
assert results[0]["prompt_embeds"].shape == (10, 768)
|
||||
|
||||
def test_both_prompts_and_embeds(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
hidden_size = 256
|
||||
renderer = _build_renderer(MockModelConfig(hidden_size=hidden_size))
|
||||
|
||||
text_input = "Hello world"
|
||||
tensor_input = torch.randn(5, 256, dtype=torch.float32)
|
||||
tensor_input = torch.randn(5, hidden_size, dtype=torch.float32)
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(
|
||||
|
||||
@@ -12,6 +12,7 @@ import pybase64 as base64
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal.media import AudioEmbeddingMediaIO, ImageEmbeddingMediaIO
|
||||
from vllm.renderers.embed_utils import safe_load_prompt_embeds
|
||||
|
||||
@@ -53,8 +54,14 @@ def _create_malicious_sparse_tensor() -> torch.Tensor:
|
||||
values = torch.tensor([1.0])
|
||||
shape = (3, 3)
|
||||
|
||||
# Create sparse tensor (this will be invalid)
|
||||
sparse_tensor = torch.sparse_coo_tensor(indices, values, shape, dtype=torch.float32)
|
||||
# Create sparse tensor (this will be invalid). Pass `check_invariants=False`
|
||||
# explicitly so this fixture is robust to process-wide invariant-check state
|
||||
# left enabled by other tests (the global flag isn't thread-local, and
|
||||
# concurrent users of the `check_sparse_tensor_invariants` context manager
|
||||
# can leak the "enabled" state across tests).
|
||||
sparse_tensor = torch.sparse_coo_tensor(
|
||||
indices, values, shape, dtype=torch.float32, check_invariants=False
|
||||
)
|
||||
return sparse_tensor
|
||||
|
||||
|
||||
@@ -117,7 +124,7 @@ class TestPromptEmbedsValidation:
|
||||
shape = (10, 10)
|
||||
|
||||
malicious_tensor = torch.sparse_coo_tensor(
|
||||
indices, values, shape, dtype=torch.float32
|
||||
indices, values, shape, dtype=torch.float32, check_invariants=False
|
||||
)
|
||||
encoded = _encode_tensor(malicious_tensor)
|
||||
|
||||
@@ -132,13 +139,69 @@ class TestPromptEmbedsValidation:
|
||||
shape = (10, 10)
|
||||
|
||||
malicious_tensor = torch.sparse_coo_tensor(
|
||||
indices, values, shape, dtype=torch.float32
|
||||
indices, values, shape, dtype=torch.float32, check_invariants=False
|
||||
)
|
||||
encoded = _encode_tensor(malicious_tensor)
|
||||
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_hidden_size_mismatch_rejected(self, model_config):
|
||||
"""Tensors whose trailing dim doesn't match the model's hidden_size
|
||||
must be rejected at parse time."""
|
||||
# opt-125m has hidden_size=768, passing 512 triggers the check.
|
||||
wrong_hidden = torch.randn(10, 512, dtype=torch.float32)
|
||||
encoded = _encode_tensor(wrong_hidden)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="hidden_size"):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_float_dtype_mismatch_cast_to_model_dtype(self, model_config):
|
||||
"""Tensors whose dtype doesn't match the model's dtype but are still
|
||||
floating-point are cast, since API clients generally can't know the
|
||||
server's `--dtype` setting ahead of time."""
|
||||
# Fixture pins model dtype to float32, upload a bfloat16 tensor.
|
||||
mismatched_float = torch.randn(10, 768, dtype=torch.bfloat16)
|
||||
encoded = _encode_tensor(mismatched_float)
|
||||
|
||||
result = safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
assert result.dtype == torch.float32
|
||||
assert result.shape == mismatched_float.shape
|
||||
|
||||
def test_non_float_dtype_rejected(self, model_config):
|
||||
"""Non-floating-point dtypes cannot be safely cast for embeddings
|
||||
(e.g. integer tensors almost certainly indicate caller confusion),
|
||||
so they are rejected at parse time."""
|
||||
non_float = torch.randint(0, 100, (10, 768), dtype=torch.int32)
|
||||
encoded = _encode_tensor(non_float)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="floating-point"):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_non_2d_tensor_rejected(self, model_config):
|
||||
"""Tensors that aren't 2D (even after squeezing a leading dim)
|
||||
must be rejected with a clear error."""
|
||||
# A 1D tensor cannot be interpreted as (num_tokens, hidden_size).
|
||||
bad = torch.randn(768, dtype=torch.float32)
|
||||
encoded = _encode_tensor(bad)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="2D tensor"):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_non_tensor_payload_rejected(self, model_config):
|
||||
"""Deserializing to a non-Tensor object must raise a clear error
|
||||
instead of propagating an AssertionError."""
|
||||
# `torch.save` will serialize a plain dict; `weights_only=True` allows
|
||||
# loading built-in containers, so this exercises the isinstance check.
|
||||
buffer = io.BytesIO()
|
||||
torch.save({"not": "a tensor"}, buffer)
|
||||
buffer.seek(0)
|
||||
encoded = base64.b64encode(buffer.read())
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="torch.Tensor"):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
|
||||
class TestImageEmbedsValidation:
|
||||
"""Test sparse tensor validation in image embeddings (Chat API)."""
|
||||
|
||||
@@ -40,6 +40,7 @@ def _model_config():
|
||||
multimodal_config=None,
|
||||
allowed_local_media_path="",
|
||||
allowed_media_domains=None,
|
||||
enable_prompt_embeds=False,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -256,8 +256,10 @@ async def test_multi_abort(output_kind: RequestOutputKind):
|
||||
)
|
||||
)
|
||||
|
||||
# Let requests start
|
||||
await asyncio.sleep(0.5)
|
||||
# Let requests start generating, use a longer sleep to ensure all
|
||||
# requests have exited prefill and produced at least one
|
||||
# decode token before we abort.
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
# Use multi-abort to abort multiple requests at once
|
||||
abort_request_ids = [request_ids[i] for i in REQUEST_IDS_TO_ABORT]
|
||||
@@ -369,9 +371,10 @@ async def test_mid_stream_cancellation(
|
||||
# Wait for all tasks to complete
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
# Verify all tasks were cancelled at the expected point
|
||||
# Verify all tasks were cancelled at the expected point.
|
||||
# Uses >= because the cancel check is `count >= cancel_after`.
|
||||
for num_generated_tokens, request_id in results:
|
||||
assert num_generated_tokens == NUM_EXPECTED_TOKENS, (
|
||||
assert num_generated_tokens >= NUM_EXPECTED_TOKENS, (
|
||||
f"{request_id} generated {num_generated_tokens} tokens but "
|
||||
f"expected to cancel after {NUM_EXPECTED_TOKENS}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user