diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index 4663a6565d..b3447387c8 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -8,6 +8,8 @@ AnthropicServingMessages._convert_anthropic_to_openai_request(). Also covers extended-thinking edge cases such as ``redacted_thinking`` blocks echoed back by Anthropic clients, and streaming conversion in ``message_stream_converter``. + +Also covers cache usage computation in ``_build_anthropic_usage``. """ import json @@ -18,7 +20,11 @@ import pytest from vllm.entrypoints.anthropic.protocol import ( AnthropicMessagesRequest, ) -from vllm.entrypoints.anthropic.serving import AnthropicServingMessages +from vllm.entrypoints.anthropic.serving import ( + AnthropicServingMessages, + _build_anthropic_usage, + _get_cached_tokens, +) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse, @@ -27,6 +33,7 @@ from vllm.entrypoints.openai.engine.protocol import ( DeltaFunctionCall, DeltaMessage, DeltaToolCall, + PromptTokenUsageInfo, UsageInfo, ) @@ -653,6 +660,108 @@ class TestThinkingBlockConversion: assert asst.get("content") == "Hi!" +# ====================================================================== +# Cache usage computation +# ====================================================================== + + +class TestGetCachedTokens: + """Tests for _get_cached_tokens helper.""" + + def test_none_usage(self): + assert _get_cached_tokens(None) is None + + def test_no_prompt_tokens_details(self): + usage = UsageInfo(prompt_tokens=100, completion_tokens=10) + assert _get_cached_tokens(usage) is None + + def test_cached_tokens_present(self): + usage = UsageInfo( + prompt_tokens=100, + completion_tokens=10, + prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=80), + ) + assert _get_cached_tokens(usage) == 80 + + def test_cached_tokens_zero(self): + """Zero cached tokens should return 0, not None.""" + usage = UsageInfo( + prompt_tokens=100, + completion_tokens=10, + prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=0), + ) + assert _get_cached_tokens(usage) == 0 + + def test_cached_tokens_none_in_details(self): + usage = UsageInfo( + prompt_tokens=100, + completion_tokens=10, + prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=None), + ) + assert _get_cached_tokens(usage) is None + + +class TestBuildAnthropicUsage: + """Tests for _build_anthropic_usage helper. + + Anthropic defines: total_input = input_tokens + cache_read + cache_creation + vLLM's prompt_tokens is the total. + """ + + def test_no_cache_info(self): + """When cache info is unavailable, return raw prompt_tokens.""" + result = _build_anthropic_usage(100, 10, None) + assert result.input_tokens == 100 + assert result.output_tokens == 10 + assert result.cache_read_input_tokens is None + assert result.cache_creation_input_tokens is None + + def test_cache_hit(self): + """When cache is hit, input_tokens excludes cached tokens.""" + usage = UsageInfo( + prompt_tokens=100, + completion_tokens=10, + prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=80), + ) + result = _build_anthropic_usage(100, 10, usage) + assert result.input_tokens == 20 # 100 - 80 + assert result.output_tokens == 10 + assert result.cache_read_input_tokens == 80 + assert result.cache_creation_input_tokens == 0 + + def test_zero_cached_tokens(self): + """Zero cached tokens should still set cache_creation to 0.""" + usage = UsageInfo( + prompt_tokens=100, + completion_tokens=10, + prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=0), + ) + result = _build_anthropic_usage(100, 10, usage) + assert result.input_tokens == 100 # 100 - 0 + assert result.cache_read_input_tokens == 0 + assert result.cache_creation_input_tokens == 0 + + def test_all_tokens_cached(self): + """When all tokens are cached, input_tokens should be 0.""" + usage = UsageInfo( + prompt_tokens=100, + completion_tokens=10, + prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=100), + ) + result = _build_anthropic_usage(100, 10, usage) + assert result.input_tokens == 0 + assert result.cache_read_input_tokens == 100 + assert result.cache_creation_input_tokens == 0 + + def test_no_prompt_tokens_details(self): + """UsageInfo without prompt_tokens_details returns no cache info.""" + usage = UsageInfo(prompt_tokens=100, completion_tokens=10) + result = _build_anthropic_usage(100, 10, usage) + assert result.input_tokens == 100 + assert result.cache_read_input_tokens is None + assert result.cache_creation_input_tokens is None + + class TestInlineSystemMessageInMessagesArray: """Verify that ``role: system`` messages embedded inside the ``messages`` array are preserved in their original position. @@ -1098,6 +1207,135 @@ class TestMessageStartIncludesTypeAndRole: assert message["role"] == "assistant" +class TestStreamingCacheUsageSemantics: + """Locks in the documented streaming behavior of cache usage fields. + + vLLM's OpenAI chat completion streaming only attaches + ``prompt_tokens_details`` to the terminal usage chunk. The Anthropic layer + mirrors that contract: cache fields are omitted on ``message_start`` (key + absence signals "unknown") and populated on ``message_delta`` (the final + cumulative count). This is intentionally consistent with vLLM's OpenAI + behavior, even though Anthropic's upstream API populates cache fields on + ``message_start``; closing that gap requires plumbing cache info into the + first chunk at the OpenAI layer, which is out of scope here. + """ + + @pytest.mark.asyncio + async def test_streaming_cache_fields_absent_then_populated(self): + """First chunk lacks prompt_tokens_details (vLLM contract); + message_start omits cache fields. The final chunk carries + prompt_tokens_details, so message_delta carries resolved values.""" + + async def sse_input(): + yield _make_stream_chunk( + delta=DeltaMessage(role="assistant", content="hi"), + usage=UsageInfo(prompt_tokens=100, total_tokens=100), + ) + yield _make_stream_chunk(finish_reason="stop") + yield _make_stream_chunk( + choices=[], + usage=UsageInfo( + prompt_tokens=100, + completion_tokens=5, + total_tokens=105, + prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=80), + ), + ) + yield "data: [DONE]" + + converter = _make_stream_converter() + output = [] + async for event in converter.message_stream_converter(sse_input()): + output.append(event) + events = _parse_sse_events(output) + + # message_start: cache fields unknown → omitted from JSON entirely. + start_usage = events[0][1]["message"]["usage"] + assert events[0][0] == "message_start" + assert start_usage["input_tokens"] == 100 + assert "cache_read_input_tokens" not in start_usage + assert "cache_creation_input_tokens" not in start_usage + + # message_delta: authoritative usage with cache fields populated. + delta_usage = next( + data["usage"] for ev, data in events if ev == "message_delta" + ) + assert delta_usage["input_tokens"] == 20 # 100 - 80 + assert delta_usage["cache_read_input_tokens"] == 80 + assert delta_usage["cache_creation_input_tokens"] == 0 + + @pytest.mark.asyncio + async def test_streaming_no_cache_hit(self): + """When the final chunk reports cached_tokens=0, message_delta carries + cache fields = 0 (cache miss); message_start still omits them.""" + + async def sse_input(): + yield _make_stream_chunk( + delta=DeltaMessage(role="assistant"), + usage=UsageInfo(prompt_tokens=50, total_tokens=50), + ) + yield _make_stream_chunk(finish_reason="stop") + yield _make_stream_chunk( + choices=[], + usage=UsageInfo( + prompt_tokens=50, + completion_tokens=5, + total_tokens=55, + prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=0), + ), + ) + yield "data: [DONE]" + + converter = _make_stream_converter() + output = [] + async for event in converter.message_stream_converter(sse_input()): + output.append(event) + events = _parse_sse_events(output) + + start_usage = events[0][1]["message"]["usage"] + delta_usage = next( + data["usage"] for ev, data in events if ev == "message_delta" + ) + assert start_usage["input_tokens"] == 50 + assert "cache_read_input_tokens" not in start_usage + assert "cache_creation_input_tokens" not in start_usage + assert delta_usage["input_tokens"] == 50 # 50 - 0 + assert delta_usage["cache_read_input_tokens"] == 0 + assert delta_usage["cache_creation_input_tokens"] == 0 + + @pytest.mark.asyncio + async def test_streaming_no_prompt_tokens_details_at_all(self): + """If --enable-prompt-tokens-details is off, no chunk carries cache + info; both message_start and message_delta omit cache fields.""" + + async def sse_input(): + yield _make_stream_chunk( + delta=DeltaMessage(role="assistant"), + usage=UsageInfo(prompt_tokens=30, total_tokens=30), + ) + yield _make_stream_chunk(finish_reason="stop") + yield _make_stream_chunk( + choices=[], + usage=UsageInfo(prompt_tokens=30, completion_tokens=2, total_tokens=32), + ) + yield "data: [DONE]" + + converter = _make_stream_converter() + output = [] + async for event in converter.message_stream_converter(sse_input()): + output.append(event) + events = _parse_sse_events(output) + + start_usage = events[0][1]["message"]["usage"] + delta_usage = next( + data["usage"] for ev, data in events if ev == "message_delta" + ) + assert "cache_read_input_tokens" not in start_usage + assert "cache_creation_input_tokens" not in start_usage + assert "cache_read_input_tokens" not in delta_usage + assert "cache_creation_input_tokens" not in delta_usage + + # ====================================================================== # Auto-detection of system-first template requirement # ====================================================================== diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 9d5852428d..3d0151aefa 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -43,6 +43,7 @@ from vllm.entrypoints.openai.engine.protocol import ( JsonSchemaResponseFormat, ResponseFormat, StreamOptions, + UsageInfo, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.utils.api_utils import sanitize_message @@ -54,6 +55,49 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +def _get_cached_tokens(usage: UsageInfo | None) -> int | None: + """Extract cached token count from OpenAI UsageInfo.""" + if usage is None or usage.prompt_tokens_details is None: + return None + return usage.prompt_tokens_details.cached_tokens + + +def _build_anthropic_usage( + prompt_tokens: int, + completion_tokens: int | None, + usage: UsageInfo | None, +) -> AnthropicUsage: + """Build an AnthropicUsage from OpenAI-style token counts. + + Anthropic defines ``total_input == input_tokens + cache_read + + cache_creation``. vLLM's ``prompt_tokens`` is the total, so + ``input_tokens = prompt_tokens - cached_tokens``. + + OpenAI usage only exposes ``cached_tokens`` (hits); there is no + cache-creation analog, so ``cache_creation_input_tokens`` is ``0`` + when cache info is present. When cache info is absent (e.g. + ``--enable-prompt-tokens-details`` off, or a streaming chunk that + hasn't carried it yet), cache fields are left **unset** so + ``exclude_unset=True`` serialization omits them entirely. + + ``completion_tokens`` follows ``UsageInfo`` and may be ``None`` on + intermediate stream chunks; we coerce to ``0`` for the wire format. + """ + output_tokens = completion_tokens or 0 + cached = _get_cached_tokens(usage) + if cached is not None: + return AnthropicUsage( + input_tokens=prompt_tokens - cached, + output_tokens=output_tokens, + cache_read_input_tokens=cached, + cache_creation_input_tokens=0, + ) + return AnthropicUsage( + input_tokens=prompt_tokens, + output_tokens=output_tokens, + ) + + def wrap_data_with_event(data: str, event: str): return f"event: {event}\ndata: {data}\n\n" @@ -582,9 +626,10 @@ class AnthropicServingMessages(OpenAIServingChat): id=generator.id, content=[], model=generator.model, - usage=AnthropicUsage( - input_tokens=generator.usage.prompt_tokens, - output_tokens=generator.usage.completion_tokens, + usage=_build_anthropic_usage( + generator.usage.prompt_tokens, + generator.usage.completion_tokens, + generator.usage, ), kv_transfer_params=generator.kv_transfer_params, ) @@ -765,11 +810,12 @@ class AnthropicServingMessages(OpenAIServingChat): model=origin_chunk.model, stop_reason=None, stop_sequence=None, - usage=AnthropicUsage( - input_tokens=origin_chunk.usage.prompt_tokens + usage=_build_anthropic_usage( + origin_chunk.usage.prompt_tokens if origin_chunk.usage else 0, - output_tokens=0, + 0, + origin_chunk.usage, ), ), ) @@ -788,13 +834,14 @@ class AnthropicServingMessages(OpenAIServingChat): chunk = AnthropicStreamEvent( type="message_delta", delta=AnthropicDelta(stop_reason=stop_reason), - usage=AnthropicUsage( - input_tokens=origin_chunk.usage.prompt_tokens + usage=_build_anthropic_usage( + origin_chunk.usage.prompt_tokens if origin_chunk.usage else 0, - output_tokens=origin_chunk.usage.completion_tokens + origin_chunk.usage.completion_tokens if origin_chunk.usage else 0, + origin_chunk.usage, ), ) data = chunk.model_dump_json(exclude_unset=True)