[RL][Frontend] Return token-level weight-version spans on final outputs (#59582)
Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com> Signed-off-by: Shuolei Wang <948904026@qq.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -31,6 +31,7 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import WeightVersionSpan
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import random_uuid
|
||||
@@ -411,6 +412,8 @@ class GenerateChoiceBase(BaseModel):
|
||||
# or (b) `enable_return_routed_experts` is off server-side.
|
||||
routed_experts: str | None = None
|
||||
sampling_mask: list[list[int]] | None = None
|
||||
# Contiguous weight version spans over output-token indexes.
|
||||
weight_versions: list[WeightVersionSpan] | None = None
|
||||
|
||||
@field_validator("token_ids")
|
||||
@classmethod
|
||||
@@ -443,6 +446,8 @@ class GenerateStreamChoiceBase(BaseModel):
|
||||
token_ids: list[int] | None = None
|
||||
routed_experts: str | None = None
|
||||
sampling_mask: list[list[int]] | None = None
|
||||
# Set on the final chunk only; covers the whole output.
|
||||
weight_versions: list[WeightVersionSpan] | None = None
|
||||
|
||||
|
||||
class GenerateTokensStreamChoice(GenerateStreamChoiceBase):
|
||||
|
||||
@@ -453,6 +453,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
token_ids=as_list(output.token_ids),
|
||||
routed_experts=routed_experts_b64,
|
||||
sampling_mask=sampling_mask,
|
||||
weight_versions=output.weight_versions,
|
||||
)
|
||||
if text_mode:
|
||||
text_choices.append(
|
||||
@@ -625,6 +626,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
token_ids=as_list(delta_token_ids),
|
||||
routed_experts=routed_experts_b64,
|
||||
sampling_mask=sampling_mask,
|
||||
weight_versions=output.weight_versions,
|
||||
)
|
||||
chunk: GenerateTokensStreamResponse | GenerateTextStreamResponse
|
||||
if text_mode:
|
||||
|
||||
Reference in New Issue
Block a user