[Core][Bugfix] Support prompt_logprobs returned with speculative decoding (#8047)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
This commit is contained in:
Travis Johnson
2024-09-24 17:29:56 -07:00
committed by GitHub
parent 13f9f7a3d0
commit 01b6f9e1f0
14 changed files with 486 additions and 128 deletions
+2
View File
@@ -26,6 +26,8 @@ if TYPE_CHECKING:
VLLM_TOKEN_ID_ARRAY_TYPE = "l"
VLLM_INVALID_TOKEN_ID = -1
# We use dataclass for now because it is used for
# openai server output, and msgspec is not serializable.