feat: Enable prompt_embeds Content Part Support in vLLM Chat Completions API (#40720)

Signed-off-by: Luis Robaina <luis@protopia.ai>
Signed-off-by: Luis Robaina 🚀 <luisfabian1545@gmail.com>
Signed-off-by: LuisRobaina <luis@protopia.ai>
Co-authored-by: Andrew Sansom <qthequartermasterman@gmail.com>
This commit is contained in:
Luis 🚀
2026-05-01 10:05:55 +08:00
committed by GitHub
co-authored by Andrew Sansom
parent 1adaa5056b
commit 14043dfecd
28 changed files with 2301 additions and 82 deletions
+45 -6
View File
@@ -7,6 +7,7 @@ import pybase64
import torch
from vllm.exceptions import VLLMValidationError
from vllm.utils.async_utils import make_async
if TYPE_CHECKING:
from vllm.config import ModelConfig
@@ -30,15 +31,53 @@ def safe_load_prompt_embeds(
weights_only=True,
map_location=torch.device("cpu"),
)
assert isinstance(tensor, torch.Tensor) and tensor.dtype in (
torch.float32,
torch.bfloat16,
torch.float16,
)
if not isinstance(tensor, torch.Tensor):
raise VLLMValidationError(
"`prompt_embeds` payload did not deserialize to a torch.Tensor.",
parameter="prompt_embeds",
)
tensor = tensor.to_dense()
if tensor.dim() > 2:
tensor = tensor.squeeze(0)
assert tensor.dim() == 2
if tensor.dim() != 2:
raise VLLMValidationError(
"`prompt_embeds` must be a 2D tensor of shape "
f"(num_tokens, hidden_size); got shape {tuple(tensor.shape)}.",
parameter="prompt_embeds",
)
# Pin each tensor to the model's hidden_size. Validating here
# also transitively guarantees cross-tensor consistency for requests that
# include multiple `prompt_embeds` parts, which is required by downstream
# concatenation in `_build_mixed_prompt_embeds`.
expected_hidden_size = model_config.get_hidden_size()
if tensor.shape[1] != expected_hidden_size:
raise VLLMValidationError(
f"`prompt_embeds` hidden_size {tensor.shape[1]} does not match "
f"the model's hidden_size {expected_hidden_size}.",
parameter="prompt_embeds",
)
# Cast to the model's dtype so API clients don't need to know the server's
# `--dtype` setting ahead of time. Only floating-point source dtypes are
# allowed. integer / bool / complex inputs almost certainly indicate caller
# error (e.g. quantized payloads, wrong tensor), and a silent `.to()`
# could hide a real mistake.
expected_dtype = model_config.dtype
if tensor.dtype != expected_dtype:
if not tensor.is_floating_point():
raise VLLMValidationError(
f"`prompt_embeds` dtype {tensor.dtype} is not a floating-point "
f"type, cannot safely cast to the model's dtype {expected_dtype}.",
parameter="prompt_embeds",
)
tensor = tensor.to(expected_dtype)
return tensor
safe_load_prompt_embeds_async = make_async(safe_load_prompt_embeds)
"""Async variant of `safe_load_prompt_embeds` that defers the decode to a
thread-pool executor, so the asyncio event loop is not blocked by the base64
decode + `torch.load` work."""