feat: Enable prompt_embeds Content Part Support in vLLM Chat Completions API (#40720)

Signed-off-by: Luis Robaina <luis@protopia.ai>
Signed-off-by: Luis Robaina 🚀 <luisfabian1545@gmail.com>
Signed-off-by: LuisRobaina <luis@protopia.ai>
Co-authored-by: Andrew Sansom <qthequartermasterman@gmail.com>
This commit is contained in:
Luis 🚀
2026-05-01 10:05:55 +08:00
committed by GitHub
co-authored by Andrew Sansom
parent 1adaa5056b
commit 14043dfecd
28 changed files with 2301 additions and 82 deletions
+10 -1
View File
@@ -50,6 +50,10 @@ class CachedRequestState:
lora_request: LoRARequest | None = None
prompt_embeds: torch.Tensor | None = None
# Per-position mask for mixed-mode inputs (e.g chat completion with
# prompt_embeds content parts). See `Request.prompt_is_token_ids`.
prompt_is_token_ids: list[bool] | None = None
# Used when both async_scheduling and spec_decode are enabled.
prev_num_draft_len: int = 0
@@ -356,7 +360,12 @@ class InputBatch:
end_idx = start_idx + len(request.output_token_ids)
if request.prompt_token_ids is not None:
self.token_ids_cpu[req_index, :num_prompt_tokens] = request.prompt_token_ids
self.is_token_ids[req_index, :num_prompt_tokens] = True
if request.prompt_is_token_ids is not None:
self.is_token_ids[req_index, :num_prompt_tokens] = (
request.prompt_is_token_ids
)
else:
self.is_token_ids[req_index, :num_prompt_tokens] = True
else:
self.is_token_ids[req_index, :num_prompt_tokens] = False
if request.prompt_embeds is not None: