feat: add max_tokens_per_doc in rerank request. (#38827)
Signed-off-by: Jesus Federico <jefp@amazon.com> Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
co-authored by
Claude Opus 4.6
wang.yuqi
parent
0e39202ca9
commit
edbc1abd1c
@@ -237,6 +237,7 @@ class ServingScores(PoolingServing):
|
||||
|
||||
await self._prepare_generators(query_ctx)
|
||||
await self._collect_batch(query_ctx)
|
||||
ctx.query_final_res_batch = query_ctx.final_res_batch
|
||||
|
||||
async def _flash_late_interaction_encode_docs(self, ctx: ScoringServeContext):
|
||||
assert ctx.n_queries is not None
|
||||
|
||||
Reference in New Issue
Block a user