[RL][Frontend] Return weight-version spans on OpenAI completions and chat (#61347)
Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -40,6 +40,7 @@ from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import WeightVersionSpan
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
@@ -129,6 +130,8 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||
# ``None`` if (a) the request was aborted before any forward pass,
|
||||
# or (b) ``enable_return_routed_experts`` is off server-side.
|
||||
routed_experts: str | None = None
|
||||
# Contiguous weight version spans over output-token indexes.
|
||||
weight_versions: list[WeightVersionSpan] | None = None
|
||||
|
||||
|
||||
class ChatCompletionResponse(OpenAIBaseModel):
|
||||
@@ -168,6 +171,8 @@ class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
stop_reason: int | str | None = None
|
||||
# not part of the OpenAI spec but for tracing the tokens
|
||||
token_ids: list[int] | None = None
|
||||
# Set on the final chunk only; covers the whole output.
|
||||
weight_versions: list[WeightVersionSpan] | None = None
|
||||
|
||||
|
||||
class ChatCompletionStreamResponse(OpenAIBaseModel):
|
||||
@@ -430,6 +435,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"need to map generated text back to input tokens."
|
||||
),
|
||||
)
|
||||
return_weight_versions: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, each choice includes weight_versions: the weight-version "
|
||||
"label that sampled each half-open range of its output tokens. "
|
||||
"When streaming, only the last chunk of each choice includes it. "
|
||||
"Beam search does not return it."
|
||||
),
|
||||
)
|
||||
routed_experts_prompt_start: int = Field(
|
||||
default=0,
|
||||
ge=0,
|
||||
|
||||
@@ -651,17 +651,21 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
|
||||
delta_text = output.text
|
||||
|
||||
if (
|
||||
nothing_generated = (
|
||||
not delta_text
|
||||
and not output.token_ids
|
||||
and not previous_num_tokens[i]
|
||||
):
|
||||
)
|
||||
if nothing_generated and output.finish_reason is None:
|
||||
# Chunked prefill case, don't return empty chunks
|
||||
continue
|
||||
|
||||
delta_message: DeltaMessage | None
|
||||
|
||||
if parser is not None:
|
||||
if nothing_generated:
|
||||
# Finished without output: send the final chunk unparsed.
|
||||
delta_message = DeltaMessage()
|
||||
elif parser is not None:
|
||||
delta_message = parser.parse_delta(
|
||||
delta_text=delta_text,
|
||||
delta_token_ids=as_list(output.token_ids),
|
||||
@@ -773,6 +777,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
token_ids=(
|
||||
as_list(output.token_ids) if include_token_ids else None
|
||||
),
|
||||
weight_versions=(
|
||||
output.weight_versions
|
||||
if request.return_weight_versions
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
finish_reason_sent[i] = True
|
||||
@@ -1097,6 +1106,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
else None
|
||||
),
|
||||
routed_experts=routed_experts_b64,
|
||||
weight_versions=(
|
||||
output.weight_versions if request.return_weight_versions else None
|
||||
),
|
||||
)
|
||||
choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import WeightVersionSpan
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sampling_params import (
|
||||
BeamSearchParams,
|
||||
@@ -178,6 +179,15 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
"need to map generated text back to input tokens."
|
||||
),
|
||||
)
|
||||
return_weight_versions: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, each choice includes weight_versions: the weight-version "
|
||||
"label that sampled each half-open range of its output tokens. "
|
||||
"When streaming, only the last chunk of each choice includes it. "
|
||||
"Beam search does not return it."
|
||||
),
|
||||
)
|
||||
routed_experts_prompt_start: int = Field(
|
||||
default=0,
|
||||
ge=0,
|
||||
@@ -677,6 +687,8 @@ class CompletionResponseChoice(OpenAIBaseModel):
|
||||
# ``None`` if (a) the request was aborted before any forward pass,
|
||||
# or (b) ``enable_return_routed_experts`` is off server-side.
|
||||
routed_experts: str | None = None
|
||||
# Contiguous weight version spans over output-token indexes.
|
||||
weight_versions: list[WeightVersionSpan] | None = None
|
||||
|
||||
|
||||
class CompletionResponse(OpenAIBaseModel):
|
||||
@@ -716,6 +728,8 @@ class CompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
# prompt tokens is put into choice to align with CompletionResponseChoice
|
||||
prompt_token_ids: list[int] | None = None
|
||||
token_ids: list[int] | None = None
|
||||
# Set on the final chunk only; covers the whole output.
|
||||
weight_versions: list[WeightVersionSpan] | None = None
|
||||
|
||||
|
||||
class CompletionStreamResponse(OpenAIBaseModel):
|
||||
|
||||
@@ -376,6 +376,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
not delta_text
|
||||
and not delta_token_ids
|
||||
and not previous_num_tokens[i]
|
||||
and finish_reason is None
|
||||
):
|
||||
# Chunked prefill case, don't return empty chunks
|
||||
continue
|
||||
@@ -421,6 +422,11 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
if request.return_token_ids
|
||||
else None
|
||||
),
|
||||
weight_versions=(
|
||||
output.weight_versions
|
||||
if request.return_weight_versions
|
||||
else None
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
@@ -594,6 +600,11 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
as_list(output.token_ids) if request.return_token_ids else None
|
||||
),
|
||||
routed_experts=routed_experts_b64,
|
||||
weight_versions=(
|
||||
output.weight_versions
|
||||
if request.return_weight_versions
|
||||
else None
|
||||
),
|
||||
)
|
||||
choices.append(choice_data)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user