[Render] Add /derender endpoints for disaggregated postprocessing (#43606)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
co-authored by
Isotr0py
parent
1033ffac2e
commit
43f0e024bc
@@ -46,6 +46,7 @@ from vllm.entrypoints.openai.engine.serving import (
|
||||
GenerationError,
|
||||
OpenAIServing,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
@@ -1129,7 +1130,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
step_top_logprobs = top_logprobs[i]
|
||||
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
|
||||
if should_return_as_token_id:
|
||||
token = f"token_id:{token_id}"
|
||||
token = format_token_id_placeholder(token_id)
|
||||
else:
|
||||
if tokenizer is None:
|
||||
raise ValueError(
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.entrypoints.openai.engine.serving import (
|
||||
GenerationError,
|
||||
OpenAIServing,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
@@ -628,7 +629,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
step_top_logprobs = top_logprobs[i]
|
||||
if step_top_logprobs is None:
|
||||
if should_return_as_token_id:
|
||||
token = f"token_id:{token_id}"
|
||||
token = format_token_id_placeholder(token_id)
|
||||
else:
|
||||
if tokenizer is None:
|
||||
raise VLLMValidationError(
|
||||
|
||||
@@ -452,7 +452,7 @@ class OpenAIServing(BeamSearchOnlineMixin):
|
||||
return_as_token_id: bool = False,
|
||||
) -> str:
|
||||
if return_as_token_id:
|
||||
return f"token_id:{token_id}"
|
||||
return format_token_id_placeholder(token_id)
|
||||
|
||||
if logprob.decoded_token is not None:
|
||||
return logprob.decoded_token
|
||||
@@ -472,6 +472,38 @@ class OpenAIServing(BeamSearchOnlineMixin):
|
||||
return self.models.is_base_model(model_name)
|
||||
|
||||
|
||||
def format_token_id_placeholder(token_id: int) -> str:
|
||||
return f"token_id:{token_id}"
|
||||
|
||||
|
||||
def resolve_token_id_placeholder(
|
||||
token: str, tokenizer: TokenizerLike
|
||||
) -> tuple[str, list[int] | None]:
|
||||
"""Decode a 'token_id:N' placeholder back to a token string and UTF-8 bytes.
|
||||
|
||||
Returns (token, None) unchanged if token is not a placeholder.
|
||||
This is the inverse of format_token_id_placeholder / _get_decoded_token
|
||||
when return_as_token_id=True.
|
||||
"""
|
||||
suffix = token.removeprefix("token_id:")
|
||||
if suffix == token:
|
||||
return token, None
|
||||
try:
|
||||
token_id = int(suffix)
|
||||
except ValueError:
|
||||
return token, None
|
||||
token_repr = tokenizer.convert_ids_to_tokens([token_id])[0]
|
||||
if token_repr is None:
|
||||
logger.warning_once(
|
||||
"resolve_token_id_placeholder: token_id %d has no vocab entry; "
|
||||
"substituting empty string",
|
||||
token_id,
|
||||
)
|
||||
return "", None
|
||||
token_str = tokenizer.convert_tokens_to_string([token_repr])
|
||||
return token_str, list(token_str.encode("utf-8", errors="replace"))
|
||||
|
||||
|
||||
def clamp_prompt_logprobs(
|
||||
prompt_logprobs: PromptLogprobs | None,
|
||||
) -> PromptLogprobs | None:
|
||||
|
||||
Reference in New Issue
Block a user