[Frontend] Expose per-request spec decode metrics in generate API (#43310)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Hongxin Xu
2026-09-20 12:21:33 +00:00
committed by GitHub
co-authored by aoshen02 Codex
parent bf01fc4a31
commit 10e6a7f210
12 changed files with 516 additions and 9 deletions
@@ -52,6 +52,7 @@ def _make_generate_response(
logprobs: dict | None = None,
prompt_logprobs: list | None = None,
kv_transfer_params: dict | None = None,
metrics: dict | None = None,
) -> dict:
choice: dict = {
"index": 0,
@@ -64,6 +65,7 @@ def _make_generate_response(
"choices": [choice],
"prompt_logprobs": prompt_logprobs,
"kv_transfer_params": kv_transfer_params,
"metrics": metrics,
}
@@ -266,6 +268,49 @@ async def test_derender_chat_kv_transfer_params_passthrough(client):
assert response.json()["kv_transfer_params"] == kv
@pytest.mark.asyncio
async def test_derender_chat_metrics_passthrough(client):
gen_req = await _render_chat(client)
metrics = {
"speculative_decoding": {
"mean_acceptance_length": 2.0,
"draft_acceptance_rate": 0.5,
"acceptance_histogram": [0, 1],
"num_spec_steps": 1,
"num_accepted_draft_tokens": 1,
"num_draft_tokens": 2,
"num_spec_tokens": 1,
}
}
response = await client.post(
"/v1/chat/completions/derender",
json={
"model": MODEL_NAME,
"generate_response": _make_generate_response(
gen_req["token_ids"][:3], metrics=metrics
),
},
)
assert response.status_code == 200
actual = response.json()["metrics"]
assert set(actual) == {
"time_to_first_token_ms",
"generation_time_ms",
"queue_time_ms",
"mean_itl_ms",
"tokens_per_second",
"speculative_decoding",
}
assert all(
actual[name] is None for name in actual if name != "speculative_decoding"
)
assert actual["speculative_decoding"] == {
**metrics["speculative_decoding"],
"per_step_accepted": None,
"per_step_drafted": None,
}
@pytest.mark.asyncio
async def test_derender_chat_empty_token_ids(client):
"""Empty token_ids list returns 400."""
@@ -27,6 +27,8 @@ from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
PerRequestMetrics,
SpeculativeDecodingMetrics,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderStreamState,
@@ -164,6 +166,7 @@ def _make_stream_chunk(
finish_reason: str | None = None,
request_id: str = "test-req",
usage: dict | None = None,
metrics: PerRequestMetrics | None = None,
) -> GenerateStreamResponse:
"""Build a GenerateStreamResponse SSE chunk."""
return GenerateStreamResponse(
@@ -176,6 +179,7 @@ def _make_stream_chunk(
)
],
usage=UsageInfo(**usage) if usage else None,
metrics=metrics,
)
@@ -512,6 +516,26 @@ class TestDerenderChatStream:
assert chunk1.choices[0].delta.role == "assistant"
assert chunk2.choices[0].delta.role is None
@pytest.mark.asyncio
async def test_metrics_passthrough(self, derenderer):
metrics = PerRequestMetrics(
speculative_decoding=SpeculativeDecodingMetrics(
mean_acceptance_length=2.0,
draft_acceptance_rate=0.5,
acceptance_histogram=[0, 1],
num_spec_steps=1,
num_accepted_draft_tokens=1,
num_draft_tokens=2,
num_spec_tokens=1,
)
)
chunk, _ = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk([], metrics=metrics),
state=DerenderStreamState(),
)
assert chunk.metrics == metrics
@pytest.mark.asyncio
async def test_chunked_equals_oneshot(self, derenderer, tokenizer):
"""Sum of streaming content deltas == one shot decode."""
@@ -28,6 +28,7 @@ from vllm.renderers import renderer_from_config
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.metrics.stats import RequestSpecDecodeMetrics
MODEL_NAME = "openai-community/gpt2"
BASE_MODEL_PATHS = [
@@ -130,6 +131,7 @@ def _make_request_output(
logprobs: list[dict[int, Any] | None] | None = None,
num_cached_tokens: int | None = None,
index: int = 0,
spec_decode_metrics: RequestSpecDecodeMetrics | None = None,
) -> RequestOutput:
return RequestOutput(
request_id=request_id,
@@ -144,6 +146,7 @@ def _make_request_output(
cumulative_logprob=None,
logprobs=logprobs,
finish_reason=finish_reason,
spec_decode_metrics=spec_decode_metrics,
)
],
finished=finished,
@@ -321,6 +324,145 @@ async def test_stream_basic():
assert data_chunks[2]["choices"][0]["finish_reason"] == "stop"
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_serve_tokens_returns_spec_decode_metrics(stream: bool):
engine = _mock_engine()
metrics = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
metrics.observe(num_draft_tokens=3, num_accepted=2)
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10],
finish_reason="stop",
finished=True,
spec_decode_metrics=metrics,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
stream=stream,
stream_options=StreamOptions(include_usage=True) if stream else None,
)
response = await serving.serve_tokens(request)
if stream:
chunks = [chunk async for chunk in response]
payload = next(
chunk
for chunk in _parse_sse_chunks(chunks)
if isinstance(chunk, dict) and chunk.get("metrics")
)["metrics"]["speculative_decoding"]
else:
assert isinstance(response, GenerateResponse)
assert response.metrics is not None
assert response.metrics.speculative_decoding is not None
payload = response.metrics.speculative_decoding.model_dump()
assert payload["num_draft_tokens"] == 3
assert payload["num_accepted_draft_tokens"] == 2
assert payload["num_spec_steps"] == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("include_usage", [False, True])
async def test_stream_returns_spec_decode_metrics_on_empty_terminal_output(
include_usage,
):
engine = _mock_engine()
metrics = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
metrics.observe(num_draft_tokens=3, num_accepted=2)
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10])
yield _make_request_output(
"req-1",
token_ids=[],
finish_reason="stop",
finished=True,
spec_decode_metrics=metrics,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
stream=True,
stream_options=StreamOptions(include_usage=include_usage),
)
chunks = [chunk async for chunk in await serving.serve_tokens(request)]
data_chunks = [chunk for chunk in _parse_sse_chunks(chunks) if chunk != "[DONE]"]
assert "metrics" not in data_chunks[0]
assert len(data_chunks) == 1 + int(include_usage)
if include_usage:
assert data_chunks[1]["choices"] == []
assert (
data_chunks[1]["metrics"]["speculative_decoding"]["num_draft_tokens"] == 3
)
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_serve_tokens_omits_spec_decode_metrics_for_parallel_sampling(
stream: bool,
):
engine = _mock_engine()
metrics = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
metrics.observe(num_draft_tokens=3, num_accepted=2)
result = _make_request_output(
"req-1",
token_ids=[10],
finish_reason="stop",
finished=True,
spec_decode_metrics=metrics,
)
result.outputs.append(
CompletionOutput(
index=1,
text="",
token_ids=[20],
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
spec_decode_metrics=metrics,
)
)
async def mock_generate(*args, **kwargs):
yield result
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1, n=2),
model=MODEL_NAME,
stream=stream,
stream_options=StreamOptions(include_usage=True) if stream else None,
)
response = await serving.serve_tokens(request)
if stream:
chunks = [chunk async for chunk in response]
assert all(
"metrics" not in chunk
for chunk in _parse_sse_chunks(chunks)
if isinstance(chunk, dict)
)
else:
assert isinstance(response, GenerateResponse)
assert response.metrics is None
@pytest.mark.asyncio
async def test_non_stream_returns_final_prompt_token_ids_when_requested():
engine = _mock_engine()