[CORE] Prompt Embeddings Support for v1 Engine (#24278)
Signed-off-by: Andrew Sansom <andrew@protopia.ai> Signed-off-by: Andrew Sansom <qthequartermasterman@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
co-authored by
Cyrus Leung
parent
9fac6aa30b
commit
9a4600e4dc
@@ -3443,3 +3443,30 @@ def decorate_logs(process_name: Optional[str] = None) -> None:
|
||||
pid = os.getpid()
|
||||
_add_prefix(sys.stdout, process_name, pid)
|
||||
_add_prefix(sys.stderr, process_name, pid)
|
||||
|
||||
|
||||
def length_from_prompt_token_ids_or_embeds(
|
||||
prompt_token_ids: Optional[list[int]],
|
||||
prompt_embeds: Optional[torch.Tensor],
|
||||
) -> int:
|
||||
"""Calculate the request length (in number of tokens) give either
|
||||
prompt_token_ids or prompt_embeds.
|
||||
"""
|
||||
prompt_token_len = None if prompt_token_ids is None else len(
|
||||
prompt_token_ids)
|
||||
prompt_embeds_len = \
|
||||
None if prompt_embeds is None else len(prompt_embeds)
|
||||
|
||||
if prompt_token_len is None:
|
||||
if prompt_embeds_len is None:
|
||||
raise ValueError(
|
||||
"Neither prompt_token_ids nor prompt_embeds were defined.")
|
||||
return prompt_embeds_len
|
||||
else:
|
||||
if (prompt_embeds_len is not None
|
||||
and prompt_embeds_len != prompt_token_len):
|
||||
raise ValueError(
|
||||
"Prompt token ids and prompt embeds had different lengths"
|
||||
f" prompt_token_ids={prompt_token_len}"
|
||||
f" prompt_embeds={prompt_embeds_len}")
|
||||
return prompt_token_len
|
||||
|
||||
Reference in New Issue
Block a user