[Front-end] [Messages] Populate num_cache_creation_tokens (#48535)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
@@ -53,47 +53,43 @@ from vllm.renderers.online_renderer import OnlineRenderer
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_cached_tokens(usage: UsageInfo | None) -> int | None:
|
||||
"""Extract cached token count from OpenAI UsageInfo."""
|
||||
if usage is None or usage.prompt_tokens_details is None:
|
||||
return None
|
||||
return usage.prompt_tokens_details.cached_tokens
|
||||
|
||||
|
||||
def _build_anthropic_usage(
|
||||
prompt_tokens: int,
|
||||
completion_tokens: int | None,
|
||||
usage: UsageInfo | None,
|
||||
) -> AnthropicUsage:
|
||||
"""Build an AnthropicUsage from OpenAI-style token counts.
|
||||
"""Build an AnthropicUsage from UsageInfo.
|
||||
|
||||
Anthropic defines ``total_input == input_tokens + cache_read +
|
||||
cache_creation``. vLLM's ``prompt_tokens`` is the total, so
|
||||
``input_tokens = prompt_tokens - cached_tokens``.
|
||||
``input_tokens = prompt_tokens - cache_read - cache_creation``.
|
||||
|
||||
OpenAI usage only exposes ``cached_tokens`` (hits); there is no
|
||||
cache-creation analog, so ``cache_creation_input_tokens`` is ``0``
|
||||
when cache info is present. When cache info is absent (e.g.
|
||||
``--enable-prompt-tokens-details`` off, or a streaming chunk that
|
||||
hasn't carried it yet), cache fields are left **unset** so
|
||||
``exclude_unset=True`` serialization omits them entirely.
|
||||
Cache fields are taken from ``UsageInfo.prompt_tokens_details``.
|
||||
When cache info is absent (e.g. ``--enable-prompt-tokens-details``
|
||||
off, or a streaming chunk that hasn't carried it yet), cache fields
|
||||
are left **unset** so ``exclude_unset=True`` serialization omits them
|
||||
entirely.
|
||||
|
||||
``completion_tokens`` follows ``UsageInfo`` and may be ``None`` on
|
||||
intermediate stream chunks; we coerce to ``0`` for the wire format.
|
||||
"""
|
||||
output_tokens = completion_tokens or 0
|
||||
cached = _get_cached_tokens(usage)
|
||||
if cached is not None:
|
||||
return AnthropicUsage(
|
||||
input_tokens=prompt_tokens - cached,
|
||||
output_tokens=output_tokens,
|
||||
cache_read_input_tokens=cached,
|
||||
cache_creation_input_tokens=0,
|
||||
)
|
||||
return AnthropicUsage(
|
||||
input_tokens=prompt_tokens,
|
||||
output_tokens=output_tokens,
|
||||
)
|
||||
kwargs = {}
|
||||
if usage is None:
|
||||
kwargs["input_tokens"] = 0
|
||||
kwargs["output_tokens"] = 0
|
||||
else:
|
||||
kwargs["output_tokens"] = usage.completion_tokens or 0
|
||||
input_tokens = usage.prompt_tokens
|
||||
|
||||
if (details := usage.prompt_tokens_details) is not None:
|
||||
if (cache_read := details.cached_tokens) is not None:
|
||||
input_tokens -= cache_read
|
||||
kwargs["cache_read_input_tokens"] = cache_read
|
||||
|
||||
if (cache_creation := details.created_cache_tokens) is not None:
|
||||
input_tokens -= cache_creation
|
||||
kwargs["cache_creation_input_tokens"] = cache_creation
|
||||
|
||||
kwargs["input_tokens"] = max(0, input_tokens)
|
||||
return AnthropicUsage(**kwargs)
|
||||
|
||||
|
||||
def wrap_data_with_event(data: str, event: str):
|
||||
@@ -625,11 +621,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
id=generator.id,
|
||||
content=[],
|
||||
model=generator.model,
|
||||
usage=_build_anthropic_usage(
|
||||
generator.usage.prompt_tokens,
|
||||
generator.usage.completion_tokens,
|
||||
generator.usage,
|
||||
),
|
||||
usage=_build_anthropic_usage(generator.usage),
|
||||
kv_transfer_params=generator.kv_transfer_params,
|
||||
ec_transfer_params=generator.ec_transfer_params,
|
||||
)
|
||||
@@ -816,13 +808,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
model=origin_chunk.model,
|
||||
stop_reason=None,
|
||||
stop_sequence=None,
|
||||
usage=_build_anthropic_usage(
|
||||
origin_chunk.usage.prompt_tokens
|
||||
if origin_chunk.usage
|
||||
else 0,
|
||||
0,
|
||||
origin_chunk.usage,
|
||||
),
|
||||
usage=_build_anthropic_usage(origin_chunk.usage),
|
||||
),
|
||||
)
|
||||
first_item = False
|
||||
@@ -840,15 +826,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
chunk = AnthropicStreamEvent(
|
||||
type="message_delta",
|
||||
delta=AnthropicDelta(stop_reason=stop_reason),
|
||||
usage=_build_anthropic_usage(
|
||||
origin_chunk.usage.prompt_tokens
|
||||
if origin_chunk.usage
|
||||
else 0,
|
||||
origin_chunk.usage.completion_tokens
|
||||
if origin_chunk.usage
|
||||
else 0,
|
||||
origin_chunk.usage,
|
||||
),
|
||||
usage=_build_anthropic_usage(origin_chunk.usage),
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(data, "message_delta")
|
||||
|
||||
@@ -90,15 +90,21 @@ def _get_mm_token_counts(engine_input: EngineInput) -> dict[str, int]:
|
||||
def _make_prompt_tokens_details(
|
||||
enable_prompt_tokens_details: bool,
|
||||
num_cached_tokens: int | None,
|
||||
num_cache_creation_tokens: int | None,
|
||||
mm_token_counts: dict[str, int] | None,
|
||||
) -> PromptTokenUsageInfo | None:
|
||||
"""Build ``prompt_tokens_details`` from cached + multimodal token counts."""
|
||||
if not enable_prompt_tokens_details:
|
||||
return None
|
||||
if num_cached_tokens is None and not mm_token_counts:
|
||||
if (
|
||||
num_cached_tokens is None
|
||||
and num_cache_creation_tokens is None
|
||||
and not mm_token_counts
|
||||
):
|
||||
return None
|
||||
return PromptTokenUsageInfo(
|
||||
cached_tokens=num_cached_tokens,
|
||||
created_cache_tokens=num_cache_creation_tokens,
|
||||
multimodal_tokens=mm_token_counts or None,
|
||||
)
|
||||
|
||||
@@ -427,6 +433,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
finish_reason_sent = [False] * num_choices
|
||||
num_prompt_tokens = 0
|
||||
num_cached_tokens = None
|
||||
num_cache_creation_tokens = None
|
||||
tools_streamed = [False] * num_choices
|
||||
|
||||
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
|
||||
@@ -479,6 +486,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
# response (by the try...catch).
|
||||
if first_iteration:
|
||||
num_cached_tokens = res.num_cached_tokens
|
||||
num_cache_creation_tokens = res.num_cache_creation_tokens
|
||||
# Send first response for each request.n (index) with
|
||||
# the role
|
||||
role = self.get_chat_request_role(request)
|
||||
@@ -756,6 +764,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
final_usage.prompt_tokens_details = _make_prompt_tokens_details(
|
||||
self.enable_prompt_tokens_details,
|
||||
num_cached_tokens,
|
||||
num_cache_creation_tokens,
|
||||
mm_token_counts,
|
||||
)
|
||||
|
||||
@@ -1030,6 +1039,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
usage.prompt_tokens_details = _make_prompt_tokens_details(
|
||||
self.enable_prompt_tokens_details,
|
||||
final_res.num_cached_tokens,
|
||||
final_res.num_cache_creation_tokens,
|
||||
mm_token_counts,
|
||||
)
|
||||
|
||||
|
||||
@@ -101,6 +101,7 @@ class ModelList(OpenAIBaseModel):
|
||||
|
||||
class PromptTokenUsageInfo(OpenAIBaseModel):
|
||||
cached_tokens: int | None = None
|
||||
created_cache_tokens: int | None = None
|
||||
multimodal_tokens: dict[str, int] | None = None
|
||||
"""Prompt tokens contributed by each input modality, keyed by modality name
|
||||
(e.g. `image`, `audio`, `video`). A breakdown of the multimodal
|
||||
|
||||
Reference in New Issue
Block a user