[Front-end] [Messages] Populate num_cache_creation_tokens (#48535)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-07-18 13:04:35 -04:00
committed by GitHub
parent e94243893d
commit a287eb163f
15 changed files with 201 additions and 127 deletions
@@ -23,7 +23,6 @@ from vllm.entrypoints.anthropic.protocol import (
from vllm.entrypoints.anthropic.serving import (
AnthropicServingMessages,
_build_anthropic_usage,
_get_cached_tokens,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionResponse,
@@ -668,42 +667,6 @@ class TestThinkingBlockConversion:
# ======================================================================
class TestGetCachedTokens:
"""Tests for _get_cached_tokens helper."""
def test_none_usage(self):
assert _get_cached_tokens(None) is None
def test_no_prompt_tokens_details(self):
usage = UsageInfo(prompt_tokens=100, completion_tokens=10)
assert _get_cached_tokens(usage) is None
def test_cached_tokens_present(self):
usage = UsageInfo(
prompt_tokens=100,
completion_tokens=10,
prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=80),
)
assert _get_cached_tokens(usage) == 80
def test_cached_tokens_zero(self):
"""Zero cached tokens should return 0, not None."""
usage = UsageInfo(
prompt_tokens=100,
completion_tokens=10,
prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=0),
)
assert _get_cached_tokens(usage) == 0
def test_cached_tokens_none_in_details(self):
usage = UsageInfo(
prompt_tokens=100,
completion_tokens=10,
prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=None),
)
assert _get_cached_tokens(usage) is None
class TestBuildAnthropicUsage:
"""Tests for _build_anthropic_usage helper.
@@ -711,36 +674,32 @@ class TestBuildAnthropicUsage:
vLLM's prompt_tokens is the total.
"""
def test_no_cache_info(self):
"""When cache info is unavailable, return raw prompt_tokens."""
result = _build_anthropic_usage(100, 10, None)
assert result.input_tokens == 100
assert result.output_tokens == 10
assert result.cache_read_input_tokens is None
assert result.cache_creation_input_tokens is None
def test_cache_hit(self):
"""When cache is hit, input_tokens excludes cached tokens."""
usage = UsageInfo(
prompt_tokens=100,
completion_tokens=10,
prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=80),
prompt_tokens_details=PromptTokenUsageInfo(
cached_tokens=80, created_cache_tokens=10
),
)
result = _build_anthropic_usage(100, 10, usage)
assert result.input_tokens == 20 # 100 - 80
result = _build_anthropic_usage(usage)
assert result.input_tokens == 10 # 100 - 80 - 10
assert result.output_tokens == 10
assert result.cache_read_input_tokens == 80
assert result.cache_creation_input_tokens == 0
assert result.cache_creation_input_tokens == 10
def test_zero_cached_tokens(self):
"""Zero cached tokens should still set cache_creation to 0."""
usage = UsageInfo(
prompt_tokens=100,
completion_tokens=10,
prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=0),
prompt_tokens_details=PromptTokenUsageInfo(
cached_tokens=0, created_cache_tokens=0
),
)
result = _build_anthropic_usage(100, 10, usage)
assert result.input_tokens == 100 # 100 - 0
result = _build_anthropic_usage(usage)
assert result.input_tokens == 100 # 100 - 0 - 0
assert result.cache_read_input_tokens == 0
assert result.cache_creation_input_tokens == 0
@@ -749,9 +708,11 @@ class TestBuildAnthropicUsage:
usage = UsageInfo(
prompt_tokens=100,
completion_tokens=10,
prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=100),
prompt_tokens_details=PromptTokenUsageInfo(
cached_tokens=100, created_cache_tokens=0
),
)
result = _build_anthropic_usage(100, 10, usage)
result = _build_anthropic_usage(usage)
assert result.input_tokens == 0
assert result.cache_read_input_tokens == 100
assert result.cache_creation_input_tokens == 0
@@ -759,7 +720,7 @@ class TestBuildAnthropicUsage:
def test_no_prompt_tokens_details(self):
"""UsageInfo without prompt_tokens_details returns no cache info."""
usage = UsageInfo(prompt_tokens=100, completion_tokens=10)
result = _build_anthropic_usage(100, 10, usage)
result = _build_anthropic_usage(usage)
assert result.input_tokens == 100
assert result.cache_read_input_tokens is None
assert result.cache_creation_input_tokens is None
@@ -1241,7 +1202,9 @@ class TestStreamingCacheUsageSemantics:
prompt_tokens=100,
completion_tokens=5,
total_tokens=105,
prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=80),
prompt_tokens_details=PromptTokenUsageInfo(
cached_tokens=80, created_cache_tokens=10
),
),
)
yield "data: [DONE]"
@@ -1263,9 +1226,9 @@ class TestStreamingCacheUsageSemantics:
delta_usage = next(
data["usage"] for ev, data in events if ev == "message_delta"
)
assert delta_usage["input_tokens"] == 20 # 100 - 80
assert delta_usage["input_tokens"] == 10 # 100 - 80 - 10
assert delta_usage["cache_read_input_tokens"] == 80
assert delta_usage["cache_creation_input_tokens"] == 0
assert delta_usage["cache_creation_input_tokens"] == 10
@pytest.mark.asyncio
async def test_streaming_no_cache_hit(self):
@@ -1284,7 +1247,9 @@ class TestStreamingCacheUsageSemantics:
prompt_tokens=50,
completion_tokens=5,
total_tokens=55,
prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=0),
prompt_tokens_details=PromptTokenUsageInfo(
cached_tokens=0, created_cache_tokens=0
),
),
)
yield "data: [DONE]"
@@ -1302,7 +1267,7 @@ class TestStreamingCacheUsageSemantics:
assert start_usage["input_tokens"] == 50
assert "cache_read_input_tokens" not in start_usage
assert "cache_creation_input_tokens" not in start_usage
assert delta_usage["input_tokens"] == 50 # 50 - 0
assert delta_usage["input_tokens"] == 50 # 50 - 0 - 0
assert delta_usage["cache_read_input_tokens"] == 0
assert delta_usage["cache_creation_input_tokens"] == 0
@@ -18,6 +18,7 @@ def server():
"--max-model-len",
"2048",
"--enforce-eager",
"--enable-prompt-tokens-details",
"--enable-auto-tool-choice",
"--tool-call-parser",
"hermes",
@@ -191,3 +192,54 @@ async def test_anthropic_structured_output(client: anthropic.AsyncAnthropic):
json_obj = json.loads(response.content[0].text)
for key in ["name", "email", "plan_interest", "demo_requested"]:
assert key in json_obj, f"Missing key in output: {key}"
@pytest.mark.asyncio
async def test_anthropic_streaming_cache_usage(client: anthropic.AsyncAnthropic):
async def get_stream_usage(resp):
prompt_tokens = None
usage = None
async for chunk in resp:
if (
chunk.type == "message_start"
and chunk.message is not None
and chunk.message.usage is not None
):
prompt_tokens = chunk.message.usage.input_tokens
elif chunk.type == "message_delta" and chunk.usage is not None:
usage = chunk.usage
assert usage is not None
assert usage.input_tokens >= 0
assert usage.output_tokens >= 0
cache_created = usage.cache_creation_input_tokens
cache_read = usage.cache_read_input_tokens
assert cache_read is not None
assert cache_created is not None
assert cache_created >= 0
assert cache_read >= 0
assert prompt_tokens == usage.input_tokens + cache_created + cache_read
return usage
request = dict(
model="claude-3-7-sonnet-latest",
max_tokens=1,
temperature=0.0,
messages=[
{
"role": "user",
"content": "Cache coverage sentinel. " * 256
+ "Answer with exactly one word: ok.",
}
],
stream=True,
)
cold_usage = await get_stream_usage(await client.messages.create(**request))
assert cold_usage.cache_read_input_tokens == 0
assert cold_usage.cache_creation_input_tokens is not None
assert cold_usage.cache_creation_input_tokens > 0
warm_usage = await get_stream_usage(await client.messages.create(**request))
assert warm_usage.cache_read_input_tokens is not None
assert warm_usage.cache_read_input_tokens > 0
@@ -831,16 +831,23 @@ def test_mm_prompt_tokens_details():
assert counts == {"image": 600, "video": 1200}
# Gated off, or nothing to report -> no details.
assert _make_prompt_tokens_details(False, 5, counts) is None
assert _make_prompt_tokens_details(True, None, None) is None
assert _make_prompt_tokens_details(False, 5, 0, counts) is None
assert _make_prompt_tokens_details(True, None, None, None) is None
# Zero cached_tokens is still reported (not None), matching the cached-only
# behavior; multimodal counts ride alongside even when cached_tokens is None.
assert _make_prompt_tokens_details(True, 0, None).cached_tokens == 0
details = _make_prompt_tokens_details(True, None, counts)
details = _make_prompt_tokens_details(True, 0, 0, None)
assert details.cached_tokens == 0
assert details.created_cache_tokens == 0
assert details.multimodal_tokens is None
details = _make_prompt_tokens_details(True, None, None, counts)
assert details.cached_tokens is None
assert details.created_cache_tokens is None
assert details.multimodal_tokens == {"image": 600, "video": 1200}
details = _make_prompt_tokens_details(True, 3, 0, counts)
assert details.cached_tokens == 3
assert details.created_cache_tokens == 0
assert details.multimodal_tokens == {"image": 600, "video": 1200}
assert _make_prompt_tokens_details(True, 3, counts).cached_tokens == 3
@pytest.mark.asyncio