[Frontend] Expose per-request spec decode metrics in generate API (#43310)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
aoshen02
Codex
parent
bf01fc4a31
commit
10e6a7f210
@@ -52,6 +52,7 @@ def _make_generate_response(
|
||||
logprobs: dict | None = None,
|
||||
prompt_logprobs: list | None = None,
|
||||
kv_transfer_params: dict | None = None,
|
||||
metrics: dict | None = None,
|
||||
) -> dict:
|
||||
choice: dict = {
|
||||
"index": 0,
|
||||
@@ -64,6 +65,7 @@ def _make_generate_response(
|
||||
"choices": [choice],
|
||||
"prompt_logprobs": prompt_logprobs,
|
||||
"kv_transfer_params": kv_transfer_params,
|
||||
"metrics": metrics,
|
||||
}
|
||||
|
||||
|
||||
@@ -266,6 +268,49 @@ async def test_derender_chat_kv_transfer_params_passthrough(client):
|
||||
assert response.json()["kv_transfer_params"] == kv
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_metrics_passthrough(client):
|
||||
gen_req = await _render_chat(client)
|
||||
metrics = {
|
||||
"speculative_decoding": {
|
||||
"mean_acceptance_length": 2.0,
|
||||
"draft_acceptance_rate": 0.5,
|
||||
"acceptance_histogram": [0, 1],
|
||||
"num_spec_steps": 1,
|
||||
"num_accepted_draft_tokens": 1,
|
||||
"num_draft_tokens": 2,
|
||||
"num_spec_tokens": 1,
|
||||
}
|
||||
}
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
gen_req["token_ids"][:3], metrics=metrics
|
||||
),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
actual = response.json()["metrics"]
|
||||
assert set(actual) == {
|
||||
"time_to_first_token_ms",
|
||||
"generation_time_ms",
|
||||
"queue_time_ms",
|
||||
"mean_itl_ms",
|
||||
"tokens_per_second",
|
||||
"speculative_decoding",
|
||||
}
|
||||
assert all(
|
||||
actual[name] is None for name in actual if name != "speculative_decoding"
|
||||
)
|
||||
assert actual["speculative_decoding"] == {
|
||||
**metrics["speculative_decoding"],
|
||||
"per_step_accepted": None,
|
||||
"per_step_drafted": None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_empty_token_ids(client):
|
||||
"""Empty token_ids list returns 400."""
|
||||
|
||||
@@ -27,6 +27,8 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
PerRequestMetrics,
|
||||
SpeculativeDecodingMetrics,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderStreamState,
|
||||
@@ -164,6 +166,7 @@ def _make_stream_chunk(
|
||||
finish_reason: str | None = None,
|
||||
request_id: str = "test-req",
|
||||
usage: dict | None = None,
|
||||
metrics: PerRequestMetrics | None = None,
|
||||
) -> GenerateStreamResponse:
|
||||
"""Build a GenerateStreamResponse SSE chunk."""
|
||||
return GenerateStreamResponse(
|
||||
@@ -176,6 +179,7 @@ def _make_stream_chunk(
|
||||
)
|
||||
],
|
||||
usage=UsageInfo(**usage) if usage else None,
|
||||
metrics=metrics,
|
||||
)
|
||||
|
||||
|
||||
@@ -512,6 +516,26 @@ class TestDerenderChatStream:
|
||||
assert chunk1.choices[0].delta.role == "assistant"
|
||||
assert chunk2.choices[0].delta.role is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metrics_passthrough(self, derenderer):
|
||||
metrics = PerRequestMetrics(
|
||||
speculative_decoding=SpeculativeDecodingMetrics(
|
||||
mean_acceptance_length=2.0,
|
||||
draft_acceptance_rate=0.5,
|
||||
acceptance_histogram=[0, 1],
|
||||
num_spec_steps=1,
|
||||
num_accepted_draft_tokens=1,
|
||||
num_draft_tokens=2,
|
||||
num_spec_tokens=1,
|
||||
)
|
||||
)
|
||||
chunk, _ = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk([], metrics=metrics),
|
||||
state=DerenderStreamState(),
|
||||
)
|
||||
assert chunk.metrics == metrics
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chunked_equals_oneshot(self, derenderer, tokenizer):
|
||||
"""Sum of streaming content deltas == one shot decode."""
|
||||
|
||||
@@ -28,6 +28,7 @@ from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.metrics.stats import RequestSpecDecodeMetrics
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
BASE_MODEL_PATHS = [
|
||||
@@ -130,6 +131,7 @@ def _make_request_output(
|
||||
logprobs: list[dict[int, Any] | None] | None = None,
|
||||
num_cached_tokens: int | None = None,
|
||||
index: int = 0,
|
||||
spec_decode_metrics: RequestSpecDecodeMetrics | None = None,
|
||||
) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id=request_id,
|
||||
@@ -144,6 +146,7 @@ def _make_request_output(
|
||||
cumulative_logprob=None,
|
||||
logprobs=logprobs,
|
||||
finish_reason=finish_reason,
|
||||
spec_decode_metrics=spec_decode_metrics,
|
||||
)
|
||||
],
|
||||
finished=finished,
|
||||
@@ -321,6 +324,145 @@ async def test_stream_basic():
|
||||
assert data_chunks[2]["choices"][0]["finish_reason"] == "stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_serve_tokens_returns_spec_decode_metrics(stream: bool):
|
||||
engine = _mock_engine()
|
||||
metrics = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
|
||||
metrics.observe(num_draft_tokens=3, num_accepted=2)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
spec_decode_metrics=metrics,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
model=MODEL_NAME,
|
||||
stream=stream,
|
||||
stream_options=StreamOptions(include_usage=True) if stream else None,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
if stream:
|
||||
chunks = [chunk async for chunk in response]
|
||||
payload = next(
|
||||
chunk
|
||||
for chunk in _parse_sse_chunks(chunks)
|
||||
if isinstance(chunk, dict) and chunk.get("metrics")
|
||||
)["metrics"]["speculative_decoding"]
|
||||
else:
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert response.metrics is not None
|
||||
assert response.metrics.speculative_decoding is not None
|
||||
payload = response.metrics.speculative_decoding.model_dump()
|
||||
|
||||
assert payload["num_draft_tokens"] == 3
|
||||
assert payload["num_accepted_draft_tokens"] == 2
|
||||
assert payload["num_spec_steps"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_usage", [False, True])
|
||||
async def test_stream_returns_spec_decode_metrics_on_empty_terminal_output(
|
||||
include_usage,
|
||||
):
|
||||
engine = _mock_engine()
|
||||
metrics = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
|
||||
metrics.observe(num_draft_tokens=3, num_accepted=2)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10])
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
spec_decode_metrics=metrics,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
stream_options=StreamOptions(include_usage=include_usage),
|
||||
)
|
||||
|
||||
chunks = [chunk async for chunk in await serving.serve_tokens(request)]
|
||||
data_chunks = [chunk for chunk in _parse_sse_chunks(chunks) if chunk != "[DONE]"]
|
||||
|
||||
assert "metrics" not in data_chunks[0]
|
||||
assert len(data_chunks) == 1 + int(include_usage)
|
||||
if include_usage:
|
||||
assert data_chunks[1]["choices"] == []
|
||||
assert (
|
||||
data_chunks[1]["metrics"]["speculative_decoding"]["num_draft_tokens"] == 3
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_serve_tokens_omits_spec_decode_metrics_for_parallel_sampling(
|
||||
stream: bool,
|
||||
):
|
||||
engine = _mock_engine()
|
||||
metrics = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
|
||||
metrics.observe(num_draft_tokens=3, num_accepted=2)
|
||||
result = _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
spec_decode_metrics=metrics,
|
||||
)
|
||||
result.outputs.append(
|
||||
CompletionOutput(
|
||||
index=1,
|
||||
text="",
|
||||
token_ids=[20],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
spec_decode_metrics=metrics,
|
||||
)
|
||||
)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield result
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=1, n=2),
|
||||
model=MODEL_NAME,
|
||||
stream=stream,
|
||||
stream_options=StreamOptions(include_usage=True) if stream else None,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
if stream:
|
||||
chunks = [chunk async for chunk in response]
|
||||
assert all(
|
||||
"metrics" not in chunk
|
||||
for chunk in _parse_sse_chunks(chunks)
|
||||
if isinstance(chunk, dict)
|
||||
)
|
||||
else:
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert response.metrics is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_stream_returns_final_prompt_token_ids_when_requested():
|
||||
engine = _mock_engine()
|
||||
|
||||
Reference in New Issue
Block a user