feat: Enable prompt_embeds Content Part Support in vLLM Chat Completions API (#40720)
Signed-off-by: Luis Robaina <luis@protopia.ai> Signed-off-by: Luis Robaina 🚀 <luisfabian1545@gmail.com> Signed-off-by: LuisRobaina <luis@protopia.ai> Co-authored-by: Andrew Sansom <qthequartermasterman@gmail.com>
This commit is contained in:
@@ -11,7 +11,9 @@ from vllm import LLM, SamplingParams
|
||||
|
||||
def _make_mock_llm() -> LLM:
|
||||
llm = object.__new__(LLM)
|
||||
llm.model_config = SimpleNamespace(runner_type="generate")
|
||||
llm.model_config = SimpleNamespace(
|
||||
runner_type="generate", enable_prompt_embeds=False
|
||||
)
|
||||
return llm
|
||||
|
||||
|
||||
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E test for mixing `prompt_embeds` with `audio_embeds` in a single
|
||||
Chat Completions request."""
|
||||
|
||||
import json
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import safetensors
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from huggingface_hub import hf_hub_download
|
||||
from transformers import AutoConfig, AutoTokenizer
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.utils.serial_utils import tensor2base64
|
||||
|
||||
QWEN2AUDIO_MODEL = "Qwen/Qwen2-Audio-7B-Instruct"
|
||||
|
||||
# Use the model's native dtype to avoid an implicit cast inside
|
||||
# `safe_load_prompt_embeds` (mismatched floating-point dtypes are cast to the
|
||||
# model's dtype automatically, matching here just skips the conversion).
|
||||
QWEN2AUDIO_DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen2audio_server_args() -> list[str]:
|
||||
return [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--enforce-eager",
|
||||
"--trust-remote-code",
|
||||
"--gpu-memory-utilization",
|
||||
"0.85",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"audio": 1}),
|
||||
"--enable-prompt-embeds",
|
||||
"--enable-mm-embeds",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen2audio_server(qwen2audio_server_args):
|
||||
with RemoteOpenAIServer(
|
||||
QWEN2AUDIO_MODEL,
|
||||
qwen2audio_server_args,
|
||||
max_wait_seconds=600,
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def qwen2audio_client(qwen2audio_server):
|
||||
async with qwen2audio_server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen2audio_hidden_size() -> int:
|
||||
config = AutoConfig.from_pretrained(QWEN2AUDIO_MODEL, trust_remote_code=True)
|
||||
return config.text_config.hidden_size
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen2audio_prompt_embeds_b64(qwen2audio_hidden_size: int) -> str:
|
||||
tensor = torch.randn(4, qwen2audio_hidden_size, dtype=QWEN2AUDIO_DTYPE)
|
||||
return tensor2base64(tensor)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen2audio_audio_embeds_b64(qwen2audio_hidden_size: int) -> str:
|
||||
# Shape matches the `audio_embeds` unit-test fixture.
|
||||
torch.manual_seed(0)
|
||||
tensor = torch.randn(1, 128, qwen2audio_hidden_size, dtype=QWEN2AUDIO_DTYPE)
|
||||
return tensor2base64(tensor)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_embeds_plus_audio_embeds(
|
||||
qwen2audio_client: openai.AsyncOpenAI,
|
||||
qwen2audio_prompt_embeds_b64: str,
|
||||
qwen2audio_audio_embeds_b64: str,
|
||||
):
|
||||
"""Single user message carrying both prompt_embeds and audio_embeds parts."""
|
||||
chat = await qwen2audio_client.chat.completions.create(
|
||||
model=QWEN2AUDIO_MODEL,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "prompt_embeds",
|
||||
"data": qwen2audio_prompt_embeds_b64,
|
||||
},
|
||||
{
|
||||
"type": "audio_embeds",
|
||||
"audio_embeds": qwen2audio_audio_embeds_b64,
|
||||
},
|
||||
{"type": "text", "text": "Continue."},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen2audio_aligned_content_and_embeds_b64() -> tuple[str, str]:
|
||||
"""Return `(content, base64_embeds)` where the embeddings are the model's
|
||||
embedding of `content` tokenized WITHOUT special tokens.
|
||||
|
||||
Loads only the `embed_tokens` shard from disk on CPU (~1.1 GB of host
|
||||
RAM) instead of the full 7B model on GPU.
|
||||
"""
|
||||
content = "Describe this audio."
|
||||
tokenizer = AutoTokenizer.from_pretrained(QWEN2AUDIO_MODEL, trust_remote_code=True)
|
||||
|
||||
index_path = hf_hub_download(QWEN2AUDIO_MODEL, "model.safetensors.index.json")
|
||||
with open(index_path) as f:
|
||||
weight_map = json.load(f)["weight_map"]
|
||||
embed_key = next(k for k in weight_map if k.endswith("embed_tokens.weight"))
|
||||
shard_path = hf_hub_download(QWEN2AUDIO_MODEL, weight_map[embed_key])
|
||||
with safetensors.safe_open(shard_path, framework="pt", device="cpu") as f:
|
||||
embed_weight = f.get_tensor(embed_key)
|
||||
embed_layer = nn.Embedding.from_pretrained(embed_weight.to(QWEN2AUDIO_DTYPE))
|
||||
|
||||
ids = tokenizer(content, add_special_tokens=False, return_tensors="pt").input_ids
|
||||
embeds = embed_layer(ids).squeeze(0)
|
||||
return content, tensor2base64(embeds)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"audio_first",
|
||||
[True, False],
|
||||
ids=["audio_embeds-then-text", "text-then-audio_embeds"],
|
||||
)
|
||||
async def test_text_content_and_prompt_embeds_match_with_audio_embeds(
|
||||
qwen2audio_client: openai.AsyncOpenAI,
|
||||
qwen2audio_audio_embeds_b64: str,
|
||||
qwen2audio_aligned_content_and_embeds_b64: tuple[str, str],
|
||||
audio_first: bool,
|
||||
):
|
||||
"""Same content as text vs `prompt_embeds` should yield identical Chat
|
||||
Completions output when mixed with `audio_embeds` in the same message.
|
||||
"""
|
||||
content, encoded_text_embeds = qwen2audio_aligned_content_and_embeds_b64
|
||||
|
||||
audio_part = {
|
||||
"type": "audio_embeds",
|
||||
"audio_embeds": qwen2audio_audio_embeds_b64,
|
||||
}
|
||||
text_part = {"type": "text", "text": content}
|
||||
embeds_part = {"type": "prompt_embeds", "data": encoded_text_embeds}
|
||||
|
||||
if audio_first:
|
||||
text_content = [audio_part, text_part]
|
||||
embeds_content = [audio_part, embeds_part]
|
||||
else:
|
||||
text_content = [text_part, audio_part]
|
||||
embeds_content = [embeds_part, audio_part]
|
||||
|
||||
text_resp = await qwen2audio_client.chat.completions.create(
|
||||
model=QWEN2AUDIO_MODEL,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": text_content}],
|
||||
)
|
||||
embeds_resp = await qwen2audio_client.chat.completions.create(
|
||||
model=QWEN2AUDIO_MODEL,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": embeds_content}],
|
||||
)
|
||||
|
||||
text_out = text_resp.choices[0].message.content
|
||||
embeds_out = embeds_resp.choices[0].message.content
|
||||
assert text_out is not None and len(text_out) > 0
|
||||
assert embeds_out is not None and len(embeds_out) > 0
|
||||
assert text_out == embeds_out
|
||||
+212
@@ -0,0 +1,212 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for mixing `prompt_embeds` with image content parts in a single
|
||||
Chat Completions request.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import safetensors
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from huggingface_hub import hf_hub_download
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
from vllm.utils.serial_utils import tensor2base64
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen2-VL-2B-Instruct"
|
||||
|
||||
# Use the model's native dtype to skip the implicit cast inside
|
||||
# `safe_load_prompt_embeds` (mismatched floating-point dtypes are cast to the
|
||||
# model's dtype automatically).
|
||||
MODEL_DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_args() -> list[str]:
|
||||
return [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.4",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"image": 1}),
|
||||
"--enable-prompt-embeds",
|
||||
"--enable-mm-embeds",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(server_args):
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
server_args,
|
||||
max_wait_seconds=600,
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def image_url() -> str:
|
||||
"""Stable real image as a data URL, kept identical across both the
|
||||
text and prompt_embeds requests so any output difference must come from
|
||||
how the text content is delivered."""
|
||||
return encode_image_url(ImageAsset("stop_sign").pil_image)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def aligned_content_and_embeds_b64() -> tuple[str, str]:
|
||||
"""`(content, base64_embeds)` where the embeddings are the model's
|
||||
embedding of `content` tokenized WITHOUT special tokens.
|
||||
|
||||
Loads only the `embed_tokens` shard from disk on CPU instead of the full
|
||||
model on GPU, so the fixture has zero VRAM footprint and won't contend
|
||||
with the running vLLM server.
|
||||
"""
|
||||
content = "Describe this image."
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
|
||||
|
||||
index_path = hf_hub_download(MODEL_NAME, "model.safetensors.index.json")
|
||||
with open(index_path) as f:
|
||||
weight_map = json.load(f)["weight_map"]
|
||||
embed_key = next(k for k in weight_map if k.endswith("embed_tokens.weight"))
|
||||
shard_path = hf_hub_download(MODEL_NAME, weight_map[embed_key])
|
||||
with safetensors.safe_open(shard_path, framework="pt", device="cpu") as f:
|
||||
embed_weight = f.get_tensor(embed_key)
|
||||
embed_layer = nn.Embedding.from_pretrained(embed_weight.to(MODEL_DTYPE))
|
||||
|
||||
ids = tokenizer(content, add_special_tokens=False, return_tensors="pt").input_ids
|
||||
embeds = embed_layer(ids).squeeze(0)
|
||||
return content, tensor2base64(embeds)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"image_first",
|
||||
[True, False],
|
||||
ids=["image_url-then-text", "text-then-image_url"],
|
||||
)
|
||||
async def test_text_content_and_prompt_embeds_match_with_image_url(
|
||||
client: openai.AsyncOpenAI,
|
||||
image_url: str,
|
||||
aligned_content_and_embeds_b64: tuple[str, str],
|
||||
image_first: bool,
|
||||
):
|
||||
"""Same content as text vs `prompt_embeds` should yield identical Chat
|
||||
Completions output when mixed with an `image_url` part in the same
|
||||
message under greedy decoding.
|
||||
"""
|
||||
content, encoded_text_embeds = aligned_content_and_embeds_b64
|
||||
|
||||
image_part = {"type": "image_url", "image_url": {"url": image_url}}
|
||||
text_part = {"type": "text", "text": content}
|
||||
embeds_part = {"type": "prompt_embeds", "data": encoded_text_embeds}
|
||||
|
||||
if image_first:
|
||||
text_content = [image_part, text_part]
|
||||
embeds_content = [image_part, embeds_part]
|
||||
else:
|
||||
text_content = [text_part, image_part]
|
||||
embeds_content = [embeds_part, image_part]
|
||||
|
||||
text_resp = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": text_content}],
|
||||
)
|
||||
embeds_resp = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": embeds_content}],
|
||||
)
|
||||
|
||||
text_out = text_resp.choices[0].message.content
|
||||
embeds_out = embeds_resp.choices[0].message.content
|
||||
assert text_out is not None and len(text_out) > 0
|
||||
assert embeds_out is not None and len(embeds_out) > 0
|
||||
assert text_out == embeds_out
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def image_embeds_b64() -> dict[str, str]:
|
||||
"""Synthetic but stable `image_embeds` for Qwen2-VL."""
|
||||
grid = (1, 4, 4)
|
||||
spatial_merge_size = 2
|
||||
num_patches = (grid[1] // spatial_merge_size) * (grid[2] // spatial_merge_size)
|
||||
text_hidden_size = 1536 # Qwen2-VL-2B
|
||||
torch.manual_seed(0)
|
||||
return {
|
||||
"image_embeds": tensor2base64(
|
||||
torch.randn(num_patches, text_hidden_size, dtype=MODEL_DTYPE)
|
||||
),
|
||||
"image_grid_thw": tensor2base64(torch.tensor(grid)),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"image_first",
|
||||
[True, False],
|
||||
ids=["image_embeds-then-text", "text-then-image_embeds"],
|
||||
)
|
||||
async def test_text_content_and_prompt_embeds_match_with_image_embeds(
|
||||
client: openai.AsyncOpenAI,
|
||||
image_embeds_b64: dict[str, str],
|
||||
aligned_content_and_embeds_b64: tuple[str, str],
|
||||
image_first: bool,
|
||||
):
|
||||
"""Same content as text vs `prompt_embeds` should yield identical Chat
|
||||
Completions output when mixed with a precomputed `image_embeds` part in
|
||||
the same message under greedy decoding.
|
||||
"""
|
||||
content, encoded_text_embeds = aligned_content_and_embeds_b64
|
||||
|
||||
image_part = {"type": "image_embeds", "image_embeds": image_embeds_b64}
|
||||
text_part = {"type": "text", "text": content}
|
||||
embeds_part = {"type": "prompt_embeds", "data": encoded_text_embeds}
|
||||
|
||||
if image_first:
|
||||
text_content = [image_part, text_part]
|
||||
embeds_content = [image_part, embeds_part]
|
||||
else:
|
||||
text_content = [text_part, image_part]
|
||||
embeds_content = [embeds_part, image_part]
|
||||
|
||||
text_resp = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": text_content}],
|
||||
)
|
||||
embeds_resp = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": embeds_content}],
|
||||
)
|
||||
|
||||
text_out = text_resp.choices[0].message.content
|
||||
embeds_out = embeds_resp.choices[0].message.content
|
||||
assert text_out is not None and len(text_out) > 0
|
||||
assert embeds_out is not None and len(embeds_out) > 0
|
||||
assert text_out == embeds_out
|
||||
@@ -0,0 +1,293 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for `prompt_embeds` content parts in the Chat Completions API."""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
|
||||
import openai
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import torch
|
||||
from openai import BadRequestError
|
||||
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
CHAT_TEMPLATE = VLLM_PATH / "examples/template_chatml.jinja"
|
||||
# Matches `--dtype` in `server_args` to avoid an implicit cast in
|
||||
# `safe_load_prompt_embeds` (mismatched floating-point dtypes are cast to the
|
||||
# model's dtype automatically, we match here just to skip the conversion).
|
||||
SERVER_DTYPE: torch.dtype = torch.bfloat16
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_args() -> list[str]:
|
||||
return [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
"--chat-template",
|
||||
str(CHAT_TEMPLATE),
|
||||
# Prompt Embeds server args
|
||||
"--enable-prompt-embeds",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(server_args):
|
||||
with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
def _encode_embeds(embeds: torch.Tensor) -> str:
|
||||
buf = io.BytesIO()
|
||||
torch.save(embeds, buf)
|
||||
return base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def prompt_embeds_b64(hf_runner) -> list[str]:
|
||||
"""Pre-compute embeddings for two short prompts and return as base64."""
|
||||
prompts = ["Hello, my name is", "What is an LLM?"]
|
||||
with hf_runner(MODEL_NAME) as hf_model:
|
||||
embeddings = hf_model.get_prompt_embeddings(prompts)
|
||||
# Cast to the server's dtype so `safe_load_prompt_embeds` doesn't need to
|
||||
# convert on its own, the function accepts any floating-point dtype and
|
||||
# will cast to the model's dtype, but matching up front skips the work.
|
||||
return [_encode_embeds(e.to(SERVER_DTYPE)) for e in embeddings]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_prompt_embeds_part(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""A user message with one prompt_embeds part + text."""
|
||||
b64 = prompt_embeds_b64[0]
|
||||
chat = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": "Continue:"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_prompt_embeds_parts(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""Multiple prompt_embeds parts in a single message."""
|
||||
b64_a, b64_b = prompt_embeds_b64
|
||||
chat = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64_a},
|
||||
{"type": "text", "text": " and "},
|
||||
{"type": "prompt_embeds", "data": b64_b},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_message_conversation(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""prompt_embeds in both system and user messages."""
|
||||
b64_sys, b64_usr = prompt_embeds_b64
|
||||
chat = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are helpful."},
|
||||
{"type": "prompt_embeds", "data": b64_sys},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64_usr},
|
||||
{"type": "text", "text": "Summarize."},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""Streaming chat completion with prompt_embeds."""
|
||||
b64 = prompt_embeds_b64[0]
|
||||
|
||||
# Non-streaming baseline.
|
||||
baseline = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": "Continue:"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
expected = baseline.choices[0].message.content
|
||||
|
||||
# Streaming.
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": "Continue:"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
chunks: list[str] = []
|
||||
async for chunk in stream:
|
||||
delta = chunk.choices[0].delta.content
|
||||
if delta:
|
||||
chunks.append(delta)
|
||||
assert "".join(chunks) == expected
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def aligned_content_and_embeds_b64(hf_runner) -> tuple[str, str]:
|
||||
"""Return `(content, base64_embeds)` where the embeddings are the model's
|
||||
embedding of `content` tokenized WITHOUT special tokens.
|
||||
"""
|
||||
content = "Hello, my name is"
|
||||
with hf_runner(MODEL_NAME) as hf_model:
|
||||
ids = hf_model.tokenizer(
|
||||
content, add_special_tokens=False, return_tensors="pt"
|
||||
).input_ids
|
||||
ids = hf_model.wrap_device({"input_ids": ids})["input_ids"]
|
||||
embed_layer = hf_model.model.get_input_embeddings()
|
||||
embeds = embed_layer(ids).squeeze(0).to(SERVER_DTYPE).cpu()
|
||||
return content, _encode_embeds(embeds)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_content_and_prompt_embeds_match(
|
||||
client: openai.AsyncOpenAI,
|
||||
aligned_content_and_embeds_b64: tuple[str, str],
|
||||
):
|
||||
"""Equal content in text and `prompt_embeds` should yield identical
|
||||
Chat Completions output under greedy decoding.
|
||||
"""
|
||||
content, encoded_embeds = aligned_content_and_embeds_b64
|
||||
|
||||
text_resp, embeds_resp = await asyncio.gather(
|
||||
client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": content}],
|
||||
),
|
||||
client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds", "data": encoded_embeds}],
|
||||
}
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
text_out = text_resp.choices[0].message.content
|
||||
embeds_out = embeds_resp.choices[0].message.content
|
||||
assert text_out is not None and len(text_out) > 0
|
||||
assert embeds_out is not None and len(embeds_out) > 0
|
||||
assert text_out == embeds_out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_data_field(
|
||||
client: openai.AsyncOpenAI,
|
||||
):
|
||||
"""A prompt_embeds part without `data` should return a clear error."""
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds"}],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_base64(
|
||||
client: openai.AsyncOpenAI,
|
||||
):
|
||||
"""Invalid base64 in the `data` field should return a clear error."""
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": "not_valid_base64!!"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
@@ -538,6 +538,7 @@ class MockModelConfig:
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
enable_prompt_embeds: bool = False
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
return self.diff_sampling_param or {}
|
||||
|
||||
@@ -62,6 +62,8 @@ def test_load_prompt_embeds(
|
||||
):
|
||||
model_config = Mock(spec=ModelConfig)
|
||||
model_config.enable_prompt_embeds = True
|
||||
model_config.get_hidden_size.return_value = hidden_size
|
||||
model_config.dtype = dtype
|
||||
|
||||
# construct arbitrary tensors of various dtypes, layouts, and sizes.
|
||||
# We need to check against different layouts to make sure that if a user
|
||||
|
||||
Reference in New Issue
Block a user