feat: add max_tokens_per_doc in rerank request. (#38827)

Signed-off-by: Jesus Federico <jefp@amazon.com>
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
Jesus Federico
2026-04-13 01:24:09 -07:00
committed by GitHub
co-authored by Claude Opus 4.6 wang.yuqi
parent 0e39202ca9
commit edbc1abd1c
9 changed files with 544 additions and 27 deletions
@@ -237,6 +237,7 @@ class ServingScores(PoolingServing):
await self._prepare_generators(query_ctx)
await self._collect_batch(query_ctx)
ctx.query_final_res_batch = query_ctx.final_res_batch
async def _flash_late_interaction_encode_docs(self, ctx: ScoringServeContext):
assert ctx.n_queries is not None