[RL][Frontend] Return weight-version spans on OpenAI completions and chat (#61347)

Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Shuolei Wang
2026-10-17 02:08:07 +00:00
committed by GitHub
co-authored by aoshen02
parent 6b6a6c5c98
commit b3a7e2ff6e
7 changed files with 139 additions and 4 deletions
@@ -1188,6 +1188,15 @@ async def test_streaming_reasoning_usage_lifecycle_edges(
await iterator.aclose()
error_chunks = [c for c in chunks if isinstance(c, dict) and "error" in c]
if name == "empty-finish":
# A choice that finishes without output still gets its last chunk, unparsed.
assert any(
choice.get("finish_reason") == "stop"
for item in chunks
if isinstance(item, dict)
for choice in item.get("choices", [])
)
parsers[0].parse_delta.assert_not_called()
call_count = sum(parser.count_reasoning_tokens.call_count for parser in parsers)
if expected_tokens is not None:
assert outcome == "exhausted" and chunks[-1] == "[DONE]" and not error_chunks
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from dataclasses import dataclass, field
from typing import Any
from unittest.mock import AsyncMock, MagicMock
@@ -17,7 +18,7 @@ from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.exceptions import GenerationError, VLLMValidationError
from vllm.logprobs import Logprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.outputs import CompletionOutput, RequestOutput, WeightVersionSpan
from vllm.renderers.deepseek_v4 import DeepseekV4Renderer
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
@@ -310,6 +311,76 @@ def test_completion_metrics_carries_both_timing_and_spec_decode():
assert response.metrics.speculative_decoding.num_spec_steps == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("opt_in", [False, True])
async def test_completion_weight_versions(stream: bool, opt_in: bool):
"""Each choice gets its spans on its final output, even with no tokens."""
expected = [{"version": "v1", "start": 0, "end": 2}]
spans = [WeightVersionSpan(**span) for span in expected]
def output(index, token_ids, finish_reason=None, weight_versions=None):
return CompletionOutput(
index=index,
text="x" * len(token_ids),
token_ids=token_ids,
cumulative_logprob=None,
logprobs=None,
finish_reason=finish_reason,
weight_versions=weight_versions,
)
# Choice 1 is aborted before its first token.
final = [output(1, [], "abort", [])]
if stream:
steps = [[output(0, [100])], [output(0, [101], "length", spans), *final]]
else:
steps = [[output(0, [100, 101], "length", spans), *final]]
async def mock_generate(*args, **kwargs):
for i, outputs in enumerate(steps):
yield RequestOutput(
request_id="test-id",
prompt="Test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=outputs,
finished=i == len(steps) - 1,
)
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
mock_engine.generate = MagicMock(side_effect=mock_generate)
request = CompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
max_tokens=10,
n=2,
stream=stream,
return_weight_versions=opt_in,
)
response = await _build_serving_completion(mock_engine).create_completion(request)
if stream:
finals = {}
async for line in response:
payload = line.removeprefix("data: ").strip()
if payload == "[DONE]":
continue
for choice in json.loads(payload)["choices"]:
if choice.get("finish_reason"):
finals[choice["index"]] = choice
else:
assert choice.get("weight_versions") is None
got = [finals[i].get("weight_versions") for i in (0, 1)]
else:
got = [choice["weight_versions"] for choice in response.model_dump()["choices"]]
assert got == ([expected, []] if opt_in else [None, None])
def _build_online_renderer_for_completion(
model_config: MockModelConfig,
renderer: Any | None = None,