[Render] Add /derender endpoints for disaggregated postprocessing (#43606)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
co-authored by
Isotr0py
parent
1033ffac2e
commit
43f0e024bc
@@ -46,6 +46,7 @@ from vllm.entrypoints.openai.engine.serving import (
|
||||
GenerationError,
|
||||
OpenAIServing,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
@@ -1129,7 +1130,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
step_top_logprobs = top_logprobs[i]
|
||||
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
|
||||
if should_return_as_token_id:
|
||||
token = f"token_id:{token_id}"
|
||||
token = format_token_id_placeholder(token_id)
|
||||
else:
|
||||
if tokenizer is None:
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user