[Bugfix] Fix pooling non-determinism from pinned prompt_lens aliasing (#37775)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-03-22 03:22:24 +00:00
committed by GitHub
parent e78bc74268
commit 66f927f205
2 changed files with 63 additions and 1 deletions
+1 -1
View File
@@ -892,7 +892,7 @@ class InputBatch:
pooling_states = self.get_pooling_states()
return PoolingMetadata(
prompt_lens=torch.from_numpy(self.num_prompt_tokens[: self.num_reqs]),
prompt_lens=self.num_prompt_tokens_cpu_tensor[: self.num_reqs].clone(),
prompt_token_ids=self.sampling_metadata.prompt_token_ids,
pooling_params=pooling_params,
pooling_states=pooling_states,