[Frontend][2/n] Improve pooling entrypoints | embed. (#36110)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
This commit is contained in:
wang.yuqi
2026-03-09 11:42:19 +08:00
committed by GitHub
parent c4d859c274
commit fff3711a24
20 changed files with 508 additions and 755 deletions
+19
View File
@@ -1,12 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
import math
from dataclasses import dataclass
from functools import lru_cache
from typing import Any
import pybase64
import torch
from fastapi.responses import JSONResponse
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput
from vllm.utils.serial_utils import (
EMBED_DTYPES,
@@ -16,6 +21,8 @@ from vllm.utils.serial_utils import (
tensor2binary,
)
logger = init_logger(__name__)
@dataclass
class MetadataItem:
@@ -122,3 +129,15 @@ def decode_pooling_output(items: list[MetadataItem], body: bytes) -> list[torch.
)
for item in sorted(items, key=lambda x: x.index)
]
@lru_cache(maxsize=1)
def get_json_response_cls() -> type[JSONResponse]:
if importlib.util.find_spec("orjson") is not None:
from fastapi.responses import ORJSONResponse
return ORJSONResponse
logger.warning_once(
"To make v1/embeddings API fast, please install orjson by `pip install orjson`"
)
return JSONResponse