[Model] Add num_cached_tokens for PoolingRequestOutput (#27378)

Signed-off-by: wang.yuqi <noooop@126.com>
This commit is contained in:
wang.yuqi
2025-10-23 14:03:42 +08:00
committed by GitHub
parent 6644796bf4
commit 3729ed00ba
7 changed files with 75 additions and 5 deletions
+3
View File
@@ -1078,6 +1078,9 @@ class LLM:
PoolingRequestOutput[Any](
request_id="",
outputs=processed_outputs,
num_cached_tokens=getattr(
processed_outputs, "num_cached_tokens", 0
),
prompt_token_ids=[],
finished=True,
)
@@ -583,6 +583,7 @@ class EmbeddingMixin(OpenAIServing):
request_id=aggregator["request_id"],
prompt_token_ids=original_token_ids,
outputs=pooling_output_data,
num_cached_tokens=0,
finished=True,
)
+1
View File
@@ -66,6 +66,7 @@ def _cosine_similarity(
request_id=f"{emb_1.request_id}_{emb_2.request_id}",
outputs=pair_score,
prompt_token_ids=tokens,
num_cached_tokens=emb_1.num_cached_tokens + emb_2.num_cached_tokens,
finished=True,
)
)