[Feature][Frontend] Report multimodal token counts in usage.prompt_tokens_details (#45458)

Signed-off-by: Ting Sun <suntcrick@gmail.com>
This commit is contained in:
Ting SUN
2026-06-15 05:20:58 +00:00
committed by GitHub
parent 7df4fe1bd7
commit 48df95c43e
3 changed files with 93 additions and 14 deletions
@@ -23,7 +23,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
)
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.chat_completion.serving import (
OpenAIServingChat,
_get_mm_token_counts,
_make_prompt_tokens_details,
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
RequestResponseMetadata,
@@ -37,6 +41,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.exceptions import VLLMValidationError
from vllm.inputs import TokensPrompt
from vllm.multimodal.inputs import PlaceholderRange
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser import HarmonyParser
from vllm.renderers.hf import HfRenderer
@@ -635,6 +640,37 @@ def test_async_serving_chat_init():
assert serving_completion.chat_template == CHAT_TEMPLATE
def test_mm_prompt_tokens_details():
# Text-only input has no multimodal placeholders.
assert _get_mm_token_counts({"type": "tokens"}) == {}
# Per-modality counts sum each modality's placeholder ranges.
counts = _get_mm_token_counts(
{
"mm_placeholders": {
"image": [
PlaceholderRange(offset=0, length=576),
PlaceholderRange(offset=600, length=24),
],
"video": [PlaceholderRange(offset=700, length=1200)],
}
}
)
assert counts == {"image": 600, "video": 1200}
# Gated off, or nothing to report -> no details.
assert _make_prompt_tokens_details(False, 5, counts) is None
assert _make_prompt_tokens_details(True, None, None) is None
# Zero cached_tokens is still reported (not None), matching the cached-only
# behavior; multimodal counts ride alongside even when cached_tokens is None.
assert _make_prompt_tokens_details(True, 0, None).cached_tokens == 0
details = _make_prompt_tokens_details(True, None, counts)
assert details.cached_tokens is None
assert details.multimodal_tokens == {"image": 600, "video": 1200}
assert _make_prompt_tokens_details(True, 3, counts).cached_tokens == 3
@pytest.mark.asyncio
async def test_serving_chat_returns_correct_model_name():
mock_engine = MagicMock(spec=AsyncLLM)
@@ -7,7 +7,7 @@ import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, cast
import numpy as np
import pybase64 as base64
@@ -54,7 +54,7 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.entrypoints.serve.utils.tool_calls_utils import (
maybe_filter_parallel_tool_calls,
)
from vllm.inputs import EngineInput
from vllm.inputs import EngineInput, MultiModalPlaceholders
from vllm.logger import init_logger
from vllm.logprobs import Logprob
from vllm.outputs import RequestOutput
@@ -73,6 +73,39 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _get_mm_token_counts(engine_input: EngineInput) -> dict[str, int]:
"""Sum per-modality placeholder tokens from ``mm_placeholders``.
Keyed by modality name; ``PlaceholderRange.length`` is the placeholder's
prompt token span, so each sum matches the placeholder tokens already
counted in ``usage.prompt_tokens``.
"""
mm_placeholders = cast(
"MultiModalPlaceholders | None", engine_input.get("mm_placeholders")
)
return {
modality: sum(p.length for p in ranges)
for modality, ranges in (mm_placeholders or {}).items()
if ranges
}
def _make_prompt_tokens_details(
enable_prompt_tokens_details: bool,
num_cached_tokens: int | None,
mm_token_counts: dict[str, int] | None,
) -> PromptTokenUsageInfo | None:
"""Build ``prompt_tokens_details`` from cached + multimodal token counts."""
if not enable_prompt_tokens_details:
return None
if num_cached_tokens is None and not mm_token_counts:
return None
return PromptTokenUsageInfo(
cached_tokens=num_cached_tokens,
multimodal_tokens=mm_token_counts or None,
)
class OpenAIServingChat(OpenAIServing):
def __init__(
self,
@@ -264,8 +297,10 @@ class OpenAIServingChat(OpenAIServing):
# Schedule the request and get the result generator.
max_model_len = self.model_config.max_model_len
generators: list[AsyncGenerator[RequestOutput, None]] = []
mm_token_counts: dict[str, int] | None = None
for i, engine_input in enumerate(engine_inputs):
prompt_token_ids = self._extract_prompt_components(engine_input).token_ids
mm_token_counts = _get_mm_token_counts(engine_input)
# If we are creating sub requests for multiple prompts, ensure that they
# have unique request ids.
@@ -362,6 +397,7 @@ class OpenAIServingChat(OpenAIServing):
tokenizer,
request_metadata,
chat_template_kwargs=chat_template_kwargs,
mm_token_counts=mm_token_counts,
)
return await self.chat_completion_full_generator(
@@ -373,6 +409,7 @@ class OpenAIServingChat(OpenAIServing):
tokenizer,
request_metadata,
chat_template_kwargs=chat_template_kwargs,
mm_token_counts=mm_token_counts,
)
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
@@ -390,6 +427,7 @@ class OpenAIServingChat(OpenAIServing):
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
chat_template_kwargs: dict[str, Any] | None = None,
mm_token_counts: dict[str, int] | None = None,
) -> AsyncGenerator[str, None]:
created_time = int(time.time())
chunk_object_type: Final = "chat.completion.chunk"
@@ -733,10 +771,11 @@ class OpenAIServingChat(OpenAIServing):
completion_tokens=completion_tokens,
total_tokens=num_prompt_tokens + completion_tokens,
)
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
final_usage.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=num_cached_tokens
)
final_usage.prompt_tokens_details = _make_prompt_tokens_details(
self.enable_prompt_tokens_details,
num_cached_tokens,
mm_token_counts,
)
final_usage_chunk = ChatCompletionStreamResponse(
id=request_id,
@@ -797,6 +836,7 @@ class OpenAIServingChat(OpenAIServing):
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
chat_template_kwargs: dict[str, Any] | None = None,
mm_token_counts: dict[str, int] | None = None,
) -> ErrorResponse | ChatCompletionResponse:
created_time = int(time.time())
final_res: RequestOutput | None = None
@@ -1024,13 +1064,11 @@ class OpenAIServingChat(OpenAIServing):
completion_tokens=num_generated_tokens,
total_tokens=num_prompt_tokens + num_generated_tokens,
)
if (
self.enable_prompt_tokens_details
and final_res.num_cached_tokens is not None
):
usage.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=final_res.num_cached_tokens
)
usage.prompt_tokens_details = _make_prompt_tokens_details(
self.enable_prompt_tokens_details,
final_res.num_cached_tokens,
mm_token_counts,
)
request_metadata.final_usage_info = usage
@@ -101,6 +101,11 @@ class ModelList(OpenAIBaseModel):
class PromptTokenUsageInfo(OpenAIBaseModel):
cached_tokens: int | None = None
multimodal_tokens: dict[str, int] | None = None
"""Prompt tokens contributed by each input modality, keyed by modality name
(e.g. `image`, `audio`, `video`). A breakdown of the multimodal
placeholder tokens already counted in `prompt_tokens`; `None` when the
request has no multimodal input."""
class UsageInfo(OpenAIBaseModel):