[Bugfix] Fix --enable-prompt-tokens-details omitting zero cached tokens (#44383)

Signed-off-by: Sasindharan Sankar <sasindharansankar@email.com>
Co-authored-by: Sasindharan Sankar <sasindharansankar@email.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
sasindharan
2026-06-11 20:37:42 -07:00
committed by GitHub
co-authored by Sasindharan Sankar Chauncey
parent 2263f8a3de
commit 42ae5e7ac6
5 changed files with 61 additions and 9 deletions
@@ -58,9 +58,12 @@ async def test_single_completion(client: openai.AsyncOpenAI, model_name: str) ->
choice = completion.choices[0]
assert len(choice.text) >= 5
assert choice.finish_reason == "length"
assert completion.usage == openai.types.CompletionUsage(
completion_tokens=5, prompt_tokens=6, total_tokens=11
)
assert completion.usage is not None
assert completion.usage.completion_tokens == 5
assert completion.usage.prompt_tokens == 6
assert completion.usage.total_tokens == 11
assert completion.usage.prompt_tokens_details is not None
assert completion.usage.prompt_tokens_details.cached_tokens == 0
# test using token IDs
completion = await client.completions.create(
@@ -512,3 +512,46 @@ async def test_stream_prompt_tokens_details():
usage_chunk = parsed[-2]
assert usage_chunk["choices"] == []
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 2
@pytest.mark.asyncio
async def test_stream_prompt_tokens_details_zero_cached():
"""enable_prompt_tokens_details includes cached_tokens=0 in final usage.
Regression test for https://github.com/vllm-project/vllm/issues/44377:
zero cached tokens must not be treated as falsy and omitted.
"""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10],
finish_reason="stop",
finished=True,
num_cached_tokens=0,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine, enable_prompt_tokens_details=True)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
stream=True,
stream_options=StreamOptions(include_usage=True),
)
response = await serving.serve_tokens(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
parsed = _parse_sse_chunks(chunks)
# Usage-only chunk (before [DONE])
usage_chunk = parsed[-2]
assert usage_chunk["choices"] == []
# Zero cached tokens must be present, not omitted
assert usage_chunk["usage"]["prompt_tokens_details"] is not None
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 0
@@ -732,7 +732,7 @@ class OpenAIServingChat(OpenAIServing):
completion_tokens=completion_tokens,
total_tokens=num_prompt_tokens + completion_tokens,
)
if self.enable_prompt_tokens_details and num_cached_tokens:
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
final_usage.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=num_cached_tokens
)
@@ -1023,7 +1023,10 @@ class OpenAIServingChat(OpenAIServing):
completion_tokens=num_generated_tokens,
total_tokens=num_prompt_tokens + num_generated_tokens,
)
if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
if (
self.enable_prompt_tokens_details
and final_res.num_cached_tokens is not None
):
usage.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=final_res.num_cached_tokens
)
@@ -443,7 +443,7 @@ class OpenAIServingCompletion(OpenAIServing):
total_tokens=total_prompt_tokens + total_completion_tokens,
)
if self.enable_prompt_tokens_details and num_cached_tokens:
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=num_cached_tokens
)
@@ -583,7 +583,7 @@ class OpenAIServingCompletion(OpenAIServing):
if (
self.enable_prompt_tokens_details
and last_final_res
and last_final_res.num_cached_tokens
and last_final_res.num_cached_tokens is not None
):
usage.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=last_final_res.num_cached_tokens
+5 -2
View File
@@ -307,7 +307,10 @@ class ServingTokens(OpenAIServing):
completion_tokens=num_generated_tokens,
total_tokens=num_prompt_tokens + num_generated_tokens,
)
if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
if (
self.enable_prompt_tokens_details
and final_res.num_cached_tokens is not None
):
# This info is not available at the /coordinator level
usage.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=final_res.num_cached_tokens
@@ -424,7 +427,7 @@ class ServingTokens(OpenAIServing):
total_tokens=num_prompt_tokens + total_completion_tokens,
)
if self.enable_prompt_tokens_details and num_cached_tokens:
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=num_cached_tokens
)