[Perf][3/n] Eliminate GPU<->CPU syncs in attention impls (#41434)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-05-07 19:44:24 -07:00
committed by GitHub
parent cd58e30872
commit 989c176c0a
10 changed files with 169 additions and 68 deletions
+1 -2
View File
@@ -58,8 +58,7 @@ class PenaltiesState:
idx_mapping = async_tensor_h2d(
self._new_penalties_reqs,
dtype=torch.int32,
target_device=self.device,
pin_memory=True,
device=self.device,
)
prefill_lens = self.req_states.prefill_len.np[self._new_penalties_reqs]