[chore] Refactor pooling metadata token ID accessors (#42368)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Wentao Ye
parent
71bcd02ef3
commit
503697c9ce
@@ -1,14 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from functools import partial
|
||||
from typing import Literal, TypeAlias, cast
|
||||
from typing import TypeAlias, cast
|
||||
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.utils.serial_utils import EmbedDType, Endianness
|
||||
@@ -16,11 +12,14 @@ from vllm.utils.serial_utils import EmbedDType, Endianness
|
||||
from ..base.serving import PoolingServing
|
||||
from ..typing import PoolingServeContext
|
||||
from ..utils import (
|
||||
encode_pooling_bytes,
|
||||
encode_pooling_output_base64,
|
||||
BytesEncodingFormat,
|
||||
JsonEncodingFormat,
|
||||
build_pooling_bytes_streaming_response,
|
||||
encode_pooling_output_float,
|
||||
encode_pooling_output_float_or_ndarray,
|
||||
get_json_response_cls,
|
||||
get_pooling_output_encoder,
|
||||
get_pooling_usage,
|
||||
)
|
||||
from .io_processor import EmbedIOProcessor
|
||||
from .protocol import (
|
||||
@@ -28,7 +27,6 @@ from .protocol import (
|
||||
CohereEmbedRequest,
|
||||
CohereEmbedResponse,
|
||||
CohereMeta,
|
||||
EmbeddingBytesResponse,
|
||||
EmbeddingRequest,
|
||||
EmbeddingResponse,
|
||||
EmbeddingResponseData,
|
||||
@@ -101,7 +99,7 @@ class ServingEmbedding(PoolingServing):
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: Literal["float", "base64"],
|
||||
encoding_format: JsonEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> JSONResponse:
|
||||
@@ -111,7 +109,6 @@ class ServingEmbedding(PoolingServing):
|
||||
)
|
||||
if use_ndarray_response:
|
||||
ndarray_items: list[dict[str, object]] = []
|
||||
ndarray_num_tokens = 0
|
||||
|
||||
for idx, final_res in enumerate(final_res_batch):
|
||||
item_dict = EmbeddingResponseData(
|
||||
@@ -122,60 +119,39 @@ class ServingEmbedding(PoolingServing):
|
||||
final_res
|
||||
)
|
||||
ndarray_items.append(item_dict)
|
||||
ndarray_num_tokens += len(final_res.prompt_token_ids)
|
||||
|
||||
ndarray_usage = UsageInfo(
|
||||
prompt_tokens=ndarray_num_tokens,
|
||||
total_tokens=ndarray_num_tokens,
|
||||
)
|
||||
ndarray_response = EmbeddingResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
data=[], # type: ignore[arg-type]
|
||||
usage=ndarray_usage,
|
||||
usage=get_pooling_usage(final_res_batch),
|
||||
).model_dump()
|
||||
ndarray_response["data"] = ndarray_items
|
||||
|
||||
return self.json_response_cls(content=ndarray_response)
|
||||
|
||||
encode_fn = cast(
|
||||
Callable[[PoolingRequestOutput], list[float] | str],
|
||||
(
|
||||
encode_pooling_output_float
|
||||
if encoding_format == "float"
|
||||
else partial(
|
||||
encode_pooling_output_base64,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
),
|
||||
encode_fn = get_pooling_output_encoder(
|
||||
encoding_format=encoding_format,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
|
||||
items: list[EmbeddingResponseData] = []
|
||||
num_prompt_tokens = 0
|
||||
|
||||
for idx, final_res in enumerate(final_res_batch):
|
||||
item = EmbeddingResponseData(
|
||||
index=idx,
|
||||
embedding=encode_fn(final_res),
|
||||
embedding=cast(list[float] | str, encode_fn(final_res)),
|
||||
)
|
||||
prompt_token_ids = final_res.prompt_token_ids
|
||||
|
||||
items.append(item)
|
||||
num_prompt_tokens += len(prompt_token_ids)
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
total_tokens=num_prompt_tokens,
|
||||
)
|
||||
|
||||
response = EmbeddingResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
data=items,
|
||||
usage=usage,
|
||||
usage=get_pooling_usage(final_res_batch),
|
||||
)
|
||||
return self.json_response_cls(content=response.model_dump())
|
||||
|
||||
@@ -185,39 +161,20 @@ class ServingEmbedding(PoolingServing):
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: Literal["bytes", "bytes_only"],
|
||||
encoding_format: BytesEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> StreamingResponse:
|
||||
content, items, usage = encode_pooling_bytes(
|
||||
return build_pooling_bytes_streaming_response(
|
||||
pooling_outputs=final_res_batch,
|
||||
request_id=request_id,
|
||||
created_time=created_time,
|
||||
model_name=model_name,
|
||||
encoding_format=encoding_format,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
|
||||
headers = (
|
||||
None
|
||||
if encoding_format == "bytes_only"
|
||||
else {
|
||||
"metadata": json.dumps(
|
||||
{
|
||||
"id": request_id,
|
||||
"created": created_time,
|
||||
"model": model_name,
|
||||
"data": items,
|
||||
"usage": usage,
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
response = EmbeddingBytesResponse(content=content, headers=headers)
|
||||
return StreamingResponse(
|
||||
content=response.content,
|
||||
headers=response.headers,
|
||||
media_type=response.media_type,
|
||||
)
|
||||
|
||||
def _build_cohere_response_from_ctx(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
@@ -225,8 +182,11 @@ class ServingEmbedding(PoolingServing):
|
||||
request = ctx.request
|
||||
assert isinstance(request, CohereEmbedRequest)
|
||||
|
||||
all_floats = [encode_pooling_output_float(out) for out in ctx.final_res_batch]
|
||||
total_tokens = sum(len(out.prompt_token_ids) for out in ctx.final_res_batch)
|
||||
all_floats = [
|
||||
cast(list[float], encode_pooling_output_float(out))
|
||||
for out in ctx.final_res_batch
|
||||
]
|
||||
total_tokens = get_pooling_usage(ctx.final_res_batch).prompt_tokens
|
||||
|
||||
image_tokens = total_tokens if request.images is not None else 0
|
||||
texts_echo = request.texts
|
||||
|
||||
Reference in New Issue
Block a user