[Bugfix] Fix --enable-prompt-tokens-details omitting zero cached tokens (#44383)
Signed-off-by: Sasindharan Sankar <sasindharansankar@email.com> Co-authored-by: Sasindharan Sankar <sasindharansankar@email.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
Sasindharan Sankar
Chauncey
parent
2263f8a3de
commit
42ae5e7ac6
@@ -58,9 +58,12 @@ async def test_single_completion(client: openai.AsyncOpenAI, model_name: str) ->
|
||||
choice = completion.choices[0]
|
||||
assert len(choice.text) >= 5
|
||||
assert choice.finish_reason == "length"
|
||||
assert completion.usage == openai.types.CompletionUsage(
|
||||
completion_tokens=5, prompt_tokens=6, total_tokens=11
|
||||
)
|
||||
assert completion.usage is not None
|
||||
assert completion.usage.completion_tokens == 5
|
||||
assert completion.usage.prompt_tokens == 6
|
||||
assert completion.usage.total_tokens == 11
|
||||
assert completion.usage.prompt_tokens_details is not None
|
||||
assert completion.usage.prompt_tokens_details.cached_tokens == 0
|
||||
|
||||
# test using token IDs
|
||||
completion = await client.completions.create(
|
||||
|
||||
@@ -512,3 +512,46 @@ async def test_stream_prompt_tokens_details():
|
||||
usage_chunk = parsed[-2]
|
||||
assert usage_chunk["choices"] == []
|
||||
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_prompt_tokens_details_zero_cached():
|
||||
"""enable_prompt_tokens_details includes cached_tokens=0 in final usage.
|
||||
|
||||
Regression test for https://github.com/vllm-project/vllm/issues/44377:
|
||||
zero cached tokens must not be treated as falsy and omitted.
|
||||
"""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
num_cached_tokens=0,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine, enable_prompt_tokens_details=True)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
stream_options=StreamOptions(include_usage=True),
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
parsed = _parse_sse_chunks(chunks)
|
||||
# Usage-only chunk (before [DONE])
|
||||
usage_chunk = parsed[-2]
|
||||
assert usage_chunk["choices"] == []
|
||||
# Zero cached tokens must be present, not omitted
|
||||
assert usage_chunk["usage"]["prompt_tokens_details"] is not None
|
||||
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 0
|
||||
|
||||
@@ -732,7 +732,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=num_prompt_tokens + completion_tokens,
|
||||
)
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens:
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
|
||||
final_usage.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=num_cached_tokens
|
||||
)
|
||||
@@ -1023,7 +1023,10 @@ class OpenAIServingChat(OpenAIServing):
|
||||
completion_tokens=num_generated_tokens,
|
||||
total_tokens=num_prompt_tokens + num_generated_tokens,
|
||||
)
|
||||
if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
|
||||
if (
|
||||
self.enable_prompt_tokens_details
|
||||
and final_res.num_cached_tokens is not None
|
||||
):
|
||||
usage.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=final_res.num_cached_tokens
|
||||
)
|
||||
|
||||
@@ -443,7 +443,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
total_tokens=total_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens:
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
|
||||
final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=num_cached_tokens
|
||||
)
|
||||
@@ -583,7 +583,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
if (
|
||||
self.enable_prompt_tokens_details
|
||||
and last_final_res
|
||||
and last_final_res.num_cached_tokens
|
||||
and last_final_res.num_cached_tokens is not None
|
||||
):
|
||||
usage.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=last_final_res.num_cached_tokens
|
||||
|
||||
@@ -307,7 +307,10 @@ class ServingTokens(OpenAIServing):
|
||||
completion_tokens=num_generated_tokens,
|
||||
total_tokens=num_prompt_tokens + num_generated_tokens,
|
||||
)
|
||||
if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
|
||||
if (
|
||||
self.enable_prompt_tokens_details
|
||||
and final_res.num_cached_tokens is not None
|
||||
):
|
||||
# This info is not available at the /coordinator level
|
||||
usage.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=final_res.num_cached_tokens
|
||||
@@ -424,7 +427,7 @@ class ServingTokens(OpenAIServing):
|
||||
total_tokens=num_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens:
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
|
||||
final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=num_cached_tokens
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user