[chore] Refactor pooling metadata token ID accessors (#42368)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-05-13 06:08:01 +00:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Wentao Ye
parent 71bcd02ef3
commit 503697c9ce
4 changed files with 142 additions and 142 deletions
+92 -11
View File
@@ -2,16 +2,19 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
import json
import math
from collections.abc import Callable, Sequence
from dataclasses import dataclass
from functools import lru_cache
from typing import Any
from functools import lru_cache, partial
from typing import Any, Literal, cast
import pybase64
import torch
from fastapi.responses import JSONResponse
from fastapi.responses import JSONResponse, StreamingResponse
from vllm.config import ModelConfig
from vllm.entrypoints.openai.engine.protocol import UsageInfo
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput
from vllm.tasks import SupportedTask
@@ -25,6 +28,11 @@ from vllm.utils.serial_utils import (
logger = init_logger(__name__)
JsonEncodingFormat = Literal["float", "base64"]
BytesEncodingFormat = Literal["bytes", "bytes_only"]
FloatEncodedPoolingOutput = list[float] | list[list[float]]
JsonEncodedPoolingOutput = FloatEncodedPoolingOutput | str
@dataclass
class MetadataItem:
@@ -58,7 +66,9 @@ def build_metadata_items(
]
def encode_pooling_output_float(output: PoolingRequestOutput) -> list[float]:
def encode_pooling_output_float(
output: PoolingRequestOutput,
) -> FloatEncodedPoolingOutput:
return output.outputs.data.tolist()
@@ -86,8 +96,7 @@ def encode_pooling_bytes(
pooling_outputs: list[PoolingRequestOutput],
embed_dtype: EmbedDType,
endianness: Endianness,
) -> tuple[list[bytes], list[dict[str, Any]], dict[str, Any]]:
num_prompt_tokens = 0
) -> tuple[list[bytes], list[dict[str, Any]]]:
items: list[dict[str, Any]] = []
body: list[bytes] = []
offset = 0
@@ -111,17 +120,89 @@ def encode_pooling_bytes(
body.append(binary)
items.append(item)
prompt_token_ids = output.prompt_token_ids
num_prompt_tokens += len(prompt_token_ids)
offset += size
# Dictionary form of UsageInfo
usage = dict(
return body, items
def get_pooling_output_encoder(
encoding_format: JsonEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> Callable[[PoolingRequestOutput], JsonEncodedPoolingOutput]:
return cast(
Callable[[PoolingRequestOutput], JsonEncodedPoolingOutput],
(
encode_pooling_output_float
if encoding_format == "float"
else partial(
encode_pooling_output_base64,
embed_dtype=embed_dtype,
endianness=endianness,
)
),
)
def get_pooling_usage(
pooling_outputs: Sequence[PoolingRequestOutput],
) -> UsageInfo:
num_prompt_tokens = sum(
len(output.prompt_token_ids) if output.prompt_token_ids is not None else 0
for output in pooling_outputs
)
return UsageInfo(
prompt_tokens=num_prompt_tokens,
total_tokens=num_prompt_tokens,
)
return body, items, usage
def get_pooling_usage_payload(
pooling_outputs: Sequence[PoolingRequestOutput],
) -> dict[str, int]:
usage = get_pooling_usage(pooling_outputs)
return {
"prompt_tokens": usage.prompt_tokens,
"total_tokens": usage.total_tokens,
}
def build_pooling_bytes_streaming_response(
pooling_outputs: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
encoding_format: BytesEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> StreamingResponse:
content, items = encode_pooling_bytes(
pooling_outputs=pooling_outputs,
embed_dtype=embed_dtype,
endianness=endianness,
)
headers = (
None
if encoding_format == "bytes_only"
else {
"metadata": json.dumps(
{
"id": request_id,
"created": created_time,
"model": model_name,
"data": items,
"usage": get_pooling_usage_payload(pooling_outputs),
}
)
}
)
return StreamingResponse(
content=content,
headers=headers,
media_type="application/octet-stream",
)
def decode_pooling_output(items: list[MetadataItem], body: bytes) -> list[torch.Tensor]: