[Render] Add /derender endpoints for disaggregated postprocessing (#43606)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
co-authored by
Isotr0py
parent
1033ffac2e
commit
43f0e024bc
@@ -452,7 +452,7 @@ class OpenAIServing(BeamSearchOnlineMixin):
|
||||
return_as_token_id: bool = False,
|
||||
) -> str:
|
||||
if return_as_token_id:
|
||||
return f"token_id:{token_id}"
|
||||
return format_token_id_placeholder(token_id)
|
||||
|
||||
if logprob.decoded_token is not None:
|
||||
return logprob.decoded_token
|
||||
@@ -472,6 +472,38 @@ class OpenAIServing(BeamSearchOnlineMixin):
|
||||
return self.models.is_base_model(model_name)
|
||||
|
||||
|
||||
def format_token_id_placeholder(token_id: int) -> str:
|
||||
return f"token_id:{token_id}"
|
||||
|
||||
|
||||
def resolve_token_id_placeholder(
|
||||
token: str, tokenizer: TokenizerLike
|
||||
) -> tuple[str, list[int] | None]:
|
||||
"""Decode a 'token_id:N' placeholder back to a token string and UTF-8 bytes.
|
||||
|
||||
Returns (token, None) unchanged if token is not a placeholder.
|
||||
This is the inverse of format_token_id_placeholder / _get_decoded_token
|
||||
when return_as_token_id=True.
|
||||
"""
|
||||
suffix = token.removeprefix("token_id:")
|
||||
if suffix == token:
|
||||
return token, None
|
||||
try:
|
||||
token_id = int(suffix)
|
||||
except ValueError:
|
||||
return token, None
|
||||
token_repr = tokenizer.convert_ids_to_tokens([token_id])[0]
|
||||
if token_repr is None:
|
||||
logger.warning_once(
|
||||
"resolve_token_id_placeholder: token_id %d has no vocab entry; "
|
||||
"substituting empty string",
|
||||
token_id,
|
||||
)
|
||||
return "", None
|
||||
token_str = tokenizer.convert_tokens_to_string([token_repr])
|
||||
return token_str, list(token_str.encode("utf-8", errors="replace"))
|
||||
|
||||
|
||||
def clamp_prompt_logprobs(
|
||||
prompt_logprobs: PromptLogprobs | None,
|
||||
) -> PromptLogprobs | None:
|
||||
|
||||
Reference in New Issue
Block a user