[RL][Frontend] Return weight-version spans on OpenAI completions and chat (#61347)
Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
@@ -17,7 +18,7 @@ from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.outputs import CompletionOutput, RequestOutput, WeightVersionSpan
|
||||
from vllm.renderers.deepseek_v4 import DeepseekV4Renderer
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
@@ -310,6 +311,76 @@ def test_completion_metrics_carries_both_timing_and_spec_decode():
|
||||
assert response.metrics.speculative_decoding.num_spec_steps == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize("opt_in", [False, True])
|
||||
async def test_completion_weight_versions(stream: bool, opt_in: bool):
|
||||
"""Each choice gets its spans on its final output, even with no tokens."""
|
||||
expected = [{"version": "v1", "start": 0, "end": 2}]
|
||||
spans = [WeightVersionSpan(**span) for span in expected]
|
||||
|
||||
def output(index, token_ids, finish_reason=None, weight_versions=None):
|
||||
return CompletionOutput(
|
||||
index=index,
|
||||
text="x" * len(token_ids),
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason=finish_reason,
|
||||
weight_versions=weight_versions,
|
||||
)
|
||||
|
||||
# Choice 1 is aborted before its first token.
|
||||
final = [output(1, [], "abort", [])]
|
||||
if stream:
|
||||
steps = [[output(0, [100])], [output(0, [101], "length", spans), *final]]
|
||||
else:
|
||||
steps = [[output(0, [100, 101], "length", spans), *final]]
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
for i, outputs in enumerate(steps):
|
||||
yield RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=outputs,
|
||||
finished=i == len(steps) - 1,
|
||||
)
|
||||
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
mock_engine.generate = MagicMock(side_effect=mock_generate)
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="Test prompt",
|
||||
max_tokens=10,
|
||||
n=2,
|
||||
stream=stream,
|
||||
return_weight_versions=opt_in,
|
||||
)
|
||||
response = await _build_serving_completion(mock_engine).create_completion(request)
|
||||
|
||||
if stream:
|
||||
finals = {}
|
||||
async for line in response:
|
||||
payload = line.removeprefix("data: ").strip()
|
||||
if payload == "[DONE]":
|
||||
continue
|
||||
for choice in json.loads(payload)["choices"]:
|
||||
if choice.get("finish_reason"):
|
||||
finals[choice["index"]] = choice
|
||||
else:
|
||||
assert choice.get("weight_versions") is None
|
||||
got = [finals[i].get("weight_versions") for i in (0, 1)]
|
||||
else:
|
||||
got = [choice["weight_versions"] for choice in response.model_dump()["choices"]]
|
||||
assert got == ([expected, []] if opt_in else [None, None])
|
||||
|
||||
|
||||
def _build_online_renderer_for_completion(
|
||||
model_config: MockModelConfig,
|
||||
renderer: Any | None = None,
|
||||
|
||||
Reference in New Issue
Block a user