[Pooling] Honor request_id from request bodies (#55665)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -54,6 +54,7 @@ def server():
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_pooling_token_embed(server: RemoteOpenAIServer, model_name: str):
|
||||
task = "token_embed"
|
||||
request_id = "client-request-id"
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
@@ -61,11 +62,13 @@ async def test_pooling_token_embed(server: RemoteOpenAIServer, model_name: str):
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
"task": task,
|
||||
"request_id": request_id,
|
||||
},
|
||||
)
|
||||
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
|
||||
assert poolings.id == f"pooling-{request_id}"
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == len(input_tokens)
|
||||
assert len(poolings.data[0].data[0]) == 384
|
||||
|
||||
@@ -109,7 +109,10 @@ class PoolingBaseServing(ABC, BaseServing):
|
||||
raw_request: Request | None = None,
|
||||
):
|
||||
model_name = self.models.model_name()
|
||||
request_id = f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
|
||||
base_request_id = self._base_request_id(
|
||||
raw_request, getattr(request, "request_id", None)
|
||||
)
|
||||
request_id = f"{self.request_id_prefix}-{base_request_id}"
|
||||
await self._check_model(request)
|
||||
|
||||
pooling_params = io_processor.create_pooling_params(request)
|
||||
|
||||
Reference in New Issue
Block a user