feat: Enable prompt_embeds Content Part Support in vLLM Chat Completions API (#40720)
Signed-off-by: Luis Robaina <luis@protopia.ai> Signed-off-by: Luis Robaina 🚀 <luisfabian1545@gmail.com> Signed-off-by: LuisRobaina <luis@protopia.ai> Co-authored-by: Andrew Sansom <qthequartermasterman@gmail.com>
This commit is contained in:
@@ -94,6 +94,12 @@ class EngineCoreRequest(
|
||||
data_parallel_rank: int | None
|
||||
prompt_embeds: torch.Tensor | None = None
|
||||
|
||||
# Per-position mask for mixed-mode inputs (e.g chat completion with
|
||||
# prompt_embeds content parts). `True` means the position is a real
|
||||
# token ID; `False` means the position uses a pre-computed entry from
|
||||
# `prompt_embeds`. `None` for pure-tokens and pure-embeds requests.
|
||||
prompt_is_token_ids: list[bool] | None = None
|
||||
|
||||
# Index of the client, used to ensure outputs are sent back to the same
|
||||
# client for this request when scaling out the front-end.
|
||||
client_index: int = 0
|
||||
|
||||
@@ -292,11 +292,13 @@ class InputProcessor:
|
||||
|
||||
# Mypy can be conservative for TypedDict unions; normalize access.
|
||||
if decoder_inputs["type"] == "embeds":
|
||||
prompt_token_ids = None
|
||||
prompt_embeds = decoder_inputs["prompt_embeds"]
|
||||
prompt_token_ids = decoder_inputs.get("prompt_token_ids")
|
||||
prompt_is_token_ids = decoder_inputs.get("is_token_ids")
|
||||
else:
|
||||
prompt_token_ids = decoder_inputs["prompt_token_ids"]
|
||||
prompt_embeds = None
|
||||
prompt_is_token_ids = None
|
||||
|
||||
sampling_params = None
|
||||
pooling_params = None
|
||||
@@ -361,6 +363,7 @@ class InputProcessor:
|
||||
request_id=request_id,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
prompt_embeds=prompt_embeds,
|
||||
prompt_is_token_ids=prompt_is_token_ids,
|
||||
mm_features=mm_features,
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=pooling_params,
|
||||
|
||||
Reference in New Issue
Block a user