[Front-end] [Messages] Populate num_cache_creation_tokens (#48535)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-07-18 13:04:35 -04:00
committed by GitHub
parent e94243893d
commit a287eb163f
15 changed files with 201 additions and 127 deletions
+29 -51
View File
@@ -53,47 +53,43 @@ from vllm.renderers.online_renderer import OnlineRenderer
logger = logging.getLogger(__name__)
def _get_cached_tokens(usage: UsageInfo | None) -> int | None:
"""Extract cached token count from OpenAI UsageInfo."""
if usage is None or usage.prompt_tokens_details is None:
return None
return usage.prompt_tokens_details.cached_tokens
def _build_anthropic_usage(
prompt_tokens: int,
completion_tokens: int | None,
usage: UsageInfo | None,
) -> AnthropicUsage:
"""Build an AnthropicUsage from OpenAI-style token counts.
"""Build an AnthropicUsage from UsageInfo.
Anthropic defines ``total_input == input_tokens + cache_read +
cache_creation``. vLLM's ``prompt_tokens`` is the total, so
``input_tokens = prompt_tokens - cached_tokens``.
``input_tokens = prompt_tokens - cache_read - cache_creation``.
OpenAI usage only exposes ``cached_tokens`` (hits); there is no
cache-creation analog, so ``cache_creation_input_tokens`` is ``0``
when cache info is present. When cache info is absent (e.g.
``--enable-prompt-tokens-details`` off, or a streaming chunk that
hasn't carried it yet), cache fields are left **unset** so
``exclude_unset=True`` serialization omits them entirely.
Cache fields are taken from ``UsageInfo.prompt_tokens_details``.
When cache info is absent (e.g. ``--enable-prompt-tokens-details``
off, or a streaming chunk that hasn't carried it yet), cache fields
are left **unset** so ``exclude_unset=True`` serialization omits them
entirely.
``completion_tokens`` follows ``UsageInfo`` and may be ``None`` on
intermediate stream chunks; we coerce to ``0`` for the wire format.
"""
output_tokens = completion_tokens or 0
cached = _get_cached_tokens(usage)
if cached is not None:
return AnthropicUsage(
input_tokens=prompt_tokens - cached,
output_tokens=output_tokens,
cache_read_input_tokens=cached,
cache_creation_input_tokens=0,
)
return AnthropicUsage(
input_tokens=prompt_tokens,
output_tokens=output_tokens,
)
kwargs = {}
if usage is None:
kwargs["input_tokens"] = 0
kwargs["output_tokens"] = 0
else:
kwargs["output_tokens"] = usage.completion_tokens or 0
input_tokens = usage.prompt_tokens
if (details := usage.prompt_tokens_details) is not None:
if (cache_read := details.cached_tokens) is not None:
input_tokens -= cache_read
kwargs["cache_read_input_tokens"] = cache_read
if (cache_creation := details.created_cache_tokens) is not None:
input_tokens -= cache_creation
kwargs["cache_creation_input_tokens"] = cache_creation
kwargs["input_tokens"] = max(0, input_tokens)
return AnthropicUsage(**kwargs)
def wrap_data_with_event(data: str, event: str):
@@ -625,11 +621,7 @@ class AnthropicServingMessages(OpenAIServingChat):
id=generator.id,
content=[],
model=generator.model,
usage=_build_anthropic_usage(
generator.usage.prompt_tokens,
generator.usage.completion_tokens,
generator.usage,
),
usage=_build_anthropic_usage(generator.usage),
kv_transfer_params=generator.kv_transfer_params,
ec_transfer_params=generator.ec_transfer_params,
)
@@ -816,13 +808,7 @@ class AnthropicServingMessages(OpenAIServingChat):
model=origin_chunk.model,
stop_reason=None,
stop_sequence=None,
usage=_build_anthropic_usage(
origin_chunk.usage.prompt_tokens
if origin_chunk.usage
else 0,
0,
origin_chunk.usage,
),
usage=_build_anthropic_usage(origin_chunk.usage),
),
)
first_item = False
@@ -840,15 +826,7 @@ class AnthropicServingMessages(OpenAIServingChat):
chunk = AnthropicStreamEvent(
type="message_delta",
delta=AnthropicDelta(stop_reason=stop_reason),
usage=_build_anthropic_usage(
origin_chunk.usage.prompt_tokens
if origin_chunk.usage
else 0,
origin_chunk.usage.completion_tokens
if origin_chunk.usage
else 0,
origin_chunk.usage,
),
usage=_build_anthropic_usage(origin_chunk.usage),
)
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(data, "message_delta")
@@ -90,15 +90,21 @@ def _get_mm_token_counts(engine_input: EngineInput) -> dict[str, int]:
def _make_prompt_tokens_details(
enable_prompt_tokens_details: bool,
num_cached_tokens: int | None,
num_cache_creation_tokens: int | None,
mm_token_counts: dict[str, int] | None,
) -> PromptTokenUsageInfo | None:
"""Build ``prompt_tokens_details`` from cached + multimodal token counts."""
if not enable_prompt_tokens_details:
return None
if num_cached_tokens is None and not mm_token_counts:
if (
num_cached_tokens is None
and num_cache_creation_tokens is None
and not mm_token_counts
):
return None
return PromptTokenUsageInfo(
cached_tokens=num_cached_tokens,
created_cache_tokens=num_cache_creation_tokens,
multimodal_tokens=mm_token_counts or None,
)
@@ -427,6 +433,7 @@ class OpenAIServingChat(GenerateBaseServing):
finish_reason_sent = [False] * num_choices
num_prompt_tokens = 0
num_cached_tokens = None
num_cache_creation_tokens = None
tools_streamed = [False] * num_choices
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
@@ -479,6 +486,7 @@ class OpenAIServingChat(GenerateBaseServing):
# response (by the try...catch).
if first_iteration:
num_cached_tokens = res.num_cached_tokens
num_cache_creation_tokens = res.num_cache_creation_tokens
# Send first response for each request.n (index) with
# the role
role = self.get_chat_request_role(request)
@@ -756,6 +764,7 @@ class OpenAIServingChat(GenerateBaseServing):
final_usage.prompt_tokens_details = _make_prompt_tokens_details(
self.enable_prompt_tokens_details,
num_cached_tokens,
num_cache_creation_tokens,
mm_token_counts,
)
@@ -1030,6 +1039,7 @@ class OpenAIServingChat(GenerateBaseServing):
usage.prompt_tokens_details = _make_prompt_tokens_details(
self.enable_prompt_tokens_details,
final_res.num_cached_tokens,
final_res.num_cache_creation_tokens,
mm_token_counts,
)
@@ -101,6 +101,7 @@ class ModelList(OpenAIBaseModel):
class PromptTokenUsageInfo(OpenAIBaseModel):
cached_tokens: int | None = None
created_cache_tokens: int | None = None
multimodal_tokens: dict[str, int] | None = None
"""Prompt tokens contributed by each input modality, keyed by modality name
(e.g. `image`, `audio`, `video`). A breakdown of the multimodal