From 48df95c43e05347070b6a99d39c053acfdeb8900 Mon Sep 17 00:00:00 2001 From: Ting SUN Date: Mon, 15 Jun 2026 13:20:58 +0800 Subject: [PATCH] [Feature][Frontend] Report multimodal token counts in usage.prompt_tokens_details (#45458) Signed-off-by: Ting Sun --- .../chat_completion/test_serving_chat.py | 38 ++++++++++- .../openai/chat_completion/serving.py | 64 +++++++++++++++---- vllm/entrypoints/openai/engine/protocol.py | 5 ++ 3 files changed, 93 insertions(+), 14 deletions(-) diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index e523cc2d4a..27503ae56f 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -23,7 +23,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionResponse, ) -from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat +from vllm.entrypoints.openai.chat_completion.serving import ( + OpenAIServingChat, + _get_mm_token_counts, + _make_prompt_tokens_details, +) from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, RequestResponseMetadata, @@ -37,6 +41,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import get_encoding from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.exceptions import VLLMValidationError from vllm.inputs import TokensPrompt +from vllm.multimodal.inputs import PlaceholderRange from vllm.outputs import CompletionOutput, RequestOutput from vllm.parser import HarmonyParser from vllm.renderers.hf import HfRenderer @@ -635,6 +640,37 @@ def test_async_serving_chat_init(): assert serving_completion.chat_template == CHAT_TEMPLATE +def test_mm_prompt_tokens_details(): + # Text-only input has no multimodal placeholders. + assert _get_mm_token_counts({"type": "tokens"}) == {} + + # Per-modality counts sum each modality's placeholder ranges. + counts = _get_mm_token_counts( + { + "mm_placeholders": { + "image": [ + PlaceholderRange(offset=0, length=576), + PlaceholderRange(offset=600, length=24), + ], + "video": [PlaceholderRange(offset=700, length=1200)], + } + } + ) + assert counts == {"image": 600, "video": 1200} + + # Gated off, or nothing to report -> no details. + assert _make_prompt_tokens_details(False, 5, counts) is None + assert _make_prompt_tokens_details(True, None, None) is None + + # Zero cached_tokens is still reported (not None), matching the cached-only + # behavior; multimodal counts ride alongside even when cached_tokens is None. + assert _make_prompt_tokens_details(True, 0, None).cached_tokens == 0 + details = _make_prompt_tokens_details(True, None, counts) + assert details.cached_tokens is None + assert details.multimodal_tokens == {"image": 600, "video": 1200} + assert _make_prompt_tokens_details(True, 3, counts).cached_tokens == 3 + + @pytest.mark.asyncio async def test_serving_chat_returns_correct_model_name(): mock_engine = MagicMock(spec=AsyncLLM) diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index b570b0c987..ed1820f4c4 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -7,7 +7,7 @@ import time from collections.abc import AsyncGenerator, AsyncIterator from collections.abc import Sequence as GenericSequence from http import HTTPStatus -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast import numpy as np import pybase64 as base64 @@ -54,7 +54,7 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.entrypoints.serve.utils.tool_calls_utils import ( maybe_filter_parallel_tool_calls, ) -from vllm.inputs import EngineInput +from vllm.inputs import EngineInput, MultiModalPlaceholders from vllm.logger import init_logger from vllm.logprobs import Logprob from vllm.outputs import RequestOutput @@ -73,6 +73,39 @@ if TYPE_CHECKING: logger = init_logger(__name__) +def _get_mm_token_counts(engine_input: EngineInput) -> dict[str, int]: + """Sum per-modality placeholder tokens from ``mm_placeholders``. + + Keyed by modality name; ``PlaceholderRange.length`` is the placeholder's + prompt token span, so each sum matches the placeholder tokens already + counted in ``usage.prompt_tokens``. + """ + mm_placeholders = cast( + "MultiModalPlaceholders | None", engine_input.get("mm_placeholders") + ) + return { + modality: sum(p.length for p in ranges) + for modality, ranges in (mm_placeholders or {}).items() + if ranges + } + + +def _make_prompt_tokens_details( + enable_prompt_tokens_details: bool, + num_cached_tokens: int | None, + mm_token_counts: dict[str, int] | None, +) -> PromptTokenUsageInfo | None: + """Build ``prompt_tokens_details`` from cached + multimodal token counts.""" + if not enable_prompt_tokens_details: + return None + if num_cached_tokens is None and not mm_token_counts: + return None + return PromptTokenUsageInfo( + cached_tokens=num_cached_tokens, + multimodal_tokens=mm_token_counts or None, + ) + + class OpenAIServingChat(OpenAIServing): def __init__( self, @@ -264,8 +297,10 @@ class OpenAIServingChat(OpenAIServing): # Schedule the request and get the result generator. max_model_len = self.model_config.max_model_len generators: list[AsyncGenerator[RequestOutput, None]] = [] + mm_token_counts: dict[str, int] | None = None for i, engine_input in enumerate(engine_inputs): prompt_token_ids = self._extract_prompt_components(engine_input).token_ids + mm_token_counts = _get_mm_token_counts(engine_input) # If we are creating sub requests for multiple prompts, ensure that they # have unique request ids. @@ -362,6 +397,7 @@ class OpenAIServingChat(OpenAIServing): tokenizer, request_metadata, chat_template_kwargs=chat_template_kwargs, + mm_token_counts=mm_token_counts, ) return await self.chat_completion_full_generator( @@ -373,6 +409,7 @@ class OpenAIServingChat(OpenAIServing): tokenizer, request_metadata, chat_template_kwargs=chat_template_kwargs, + mm_token_counts=mm_token_counts, ) def get_chat_request_role(self, request: ChatCompletionRequest) -> str: @@ -390,6 +427,7 @@ class OpenAIServingChat(OpenAIServing): tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, chat_template_kwargs: dict[str, Any] | None = None, + mm_token_counts: dict[str, int] | None = None, ) -> AsyncGenerator[str, None]: created_time = int(time.time()) chunk_object_type: Final = "chat.completion.chunk" @@ -733,10 +771,11 @@ class OpenAIServingChat(OpenAIServing): completion_tokens=completion_tokens, total_tokens=num_prompt_tokens + completion_tokens, ) - if self.enable_prompt_tokens_details and num_cached_tokens is not None: - final_usage.prompt_tokens_details = PromptTokenUsageInfo( - cached_tokens=num_cached_tokens - ) + final_usage.prompt_tokens_details = _make_prompt_tokens_details( + self.enable_prompt_tokens_details, + num_cached_tokens, + mm_token_counts, + ) final_usage_chunk = ChatCompletionStreamResponse( id=request_id, @@ -797,6 +836,7 @@ class OpenAIServingChat(OpenAIServing): tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, chat_template_kwargs: dict[str, Any] | None = None, + mm_token_counts: dict[str, int] | None = None, ) -> ErrorResponse | ChatCompletionResponse: created_time = int(time.time()) final_res: RequestOutput | None = None @@ -1024,13 +1064,11 @@ class OpenAIServingChat(OpenAIServing): completion_tokens=num_generated_tokens, total_tokens=num_prompt_tokens + num_generated_tokens, ) - if ( - self.enable_prompt_tokens_details - and final_res.num_cached_tokens is not None - ): - usage.prompt_tokens_details = PromptTokenUsageInfo( - cached_tokens=final_res.num_cached_tokens - ) + usage.prompt_tokens_details = _make_prompt_tokens_details( + self.enable_prompt_tokens_details, + final_res.num_cached_tokens, + mm_token_counts, + ) request_metadata.final_usage_info = usage diff --git a/vllm/entrypoints/openai/engine/protocol.py b/vllm/entrypoints/openai/engine/protocol.py index 434888df9e..3cd998780f 100644 --- a/vllm/entrypoints/openai/engine/protocol.py +++ b/vllm/entrypoints/openai/engine/protocol.py @@ -101,6 +101,11 @@ class ModelList(OpenAIBaseModel): class PromptTokenUsageInfo(OpenAIBaseModel): cached_tokens: int | None = None + multimodal_tokens: dict[str, int] | None = None + """Prompt tokens contributed by each input modality, keyed by modality name + (e.g. `image`, `audio`, `video`). A breakdown of the multimodal + placeholder tokens already counted in `prompt_tokens`; `None` when the + request has no multimodal input.""" class UsageInfo(OpenAIBaseModel):