[chore] Refactor pooling metadata token ID accessors (#42368)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Wentao Ye
parent
71bcd02ef3
commit
503697c9ce
@@ -2,16 +2,19 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
from functools import lru_cache, partial
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
import pybase64
|
||||
import torch
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.tasks import SupportedTask
|
||||
@@ -25,6 +28,11 @@ from vllm.utils.serial_utils import (
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
JsonEncodingFormat = Literal["float", "base64"]
|
||||
BytesEncodingFormat = Literal["bytes", "bytes_only"]
|
||||
FloatEncodedPoolingOutput = list[float] | list[list[float]]
|
||||
JsonEncodedPoolingOutput = FloatEncodedPoolingOutput | str
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetadataItem:
|
||||
@@ -58,7 +66,9 @@ def build_metadata_items(
|
||||
]
|
||||
|
||||
|
||||
def encode_pooling_output_float(output: PoolingRequestOutput) -> list[float]:
|
||||
def encode_pooling_output_float(
|
||||
output: PoolingRequestOutput,
|
||||
) -> FloatEncodedPoolingOutput:
|
||||
return output.outputs.data.tolist()
|
||||
|
||||
|
||||
@@ -86,8 +96,7 @@ def encode_pooling_bytes(
|
||||
pooling_outputs: list[PoolingRequestOutput],
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> tuple[list[bytes], list[dict[str, Any]], dict[str, Any]]:
|
||||
num_prompt_tokens = 0
|
||||
) -> tuple[list[bytes], list[dict[str, Any]]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
body: list[bytes] = []
|
||||
offset = 0
|
||||
@@ -111,17 +120,89 @@ def encode_pooling_bytes(
|
||||
|
||||
body.append(binary)
|
||||
items.append(item)
|
||||
prompt_token_ids = output.prompt_token_ids
|
||||
num_prompt_tokens += len(prompt_token_ids)
|
||||
offset += size
|
||||
|
||||
# Dictionary form of UsageInfo
|
||||
usage = dict(
|
||||
return body, items
|
||||
|
||||
|
||||
def get_pooling_output_encoder(
|
||||
encoding_format: JsonEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> Callable[[PoolingRequestOutput], JsonEncodedPoolingOutput]:
|
||||
return cast(
|
||||
Callable[[PoolingRequestOutput], JsonEncodedPoolingOutput],
|
||||
(
|
||||
encode_pooling_output_float
|
||||
if encoding_format == "float"
|
||||
else partial(
|
||||
encode_pooling_output_base64,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_pooling_usage(
|
||||
pooling_outputs: Sequence[PoolingRequestOutput],
|
||||
) -> UsageInfo:
|
||||
num_prompt_tokens = sum(
|
||||
len(output.prompt_token_ids) if output.prompt_token_ids is not None else 0
|
||||
for output in pooling_outputs
|
||||
)
|
||||
return UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
total_tokens=num_prompt_tokens,
|
||||
)
|
||||
|
||||
return body, items, usage
|
||||
|
||||
def get_pooling_usage_payload(
|
||||
pooling_outputs: Sequence[PoolingRequestOutput],
|
||||
) -> dict[str, int]:
|
||||
usage = get_pooling_usage(pooling_outputs)
|
||||
return {
|
||||
"prompt_tokens": usage.prompt_tokens,
|
||||
"total_tokens": usage.total_tokens,
|
||||
}
|
||||
|
||||
|
||||
def build_pooling_bytes_streaming_response(
|
||||
pooling_outputs: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: BytesEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> StreamingResponse:
|
||||
content, items = encode_pooling_bytes(
|
||||
pooling_outputs=pooling_outputs,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
|
||||
headers = (
|
||||
None
|
||||
if encoding_format == "bytes_only"
|
||||
else {
|
||||
"metadata": json.dumps(
|
||||
{
|
||||
"id": request_id,
|
||||
"created": created_time,
|
||||
"model": model_name,
|
||||
"data": items,
|
||||
"usage": get_pooling_usage_payload(pooling_outputs),
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
return StreamingResponse(
|
||||
content=content,
|
||||
headers=headers,
|
||||
media_type="application/octet-stream",
|
||||
)
|
||||
|
||||
|
||||
def decode_pooling_output(items: list[MetadataItem], body: bytes) -> list[torch.Tensor]:
|
||||
|
||||
Reference in New Issue
Block a user