[Perf][4/n] Eliminate various GPU<->CPU syncs (#42347)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-05-19 10:35:54 -04:00
committed by GitHub
parent a78b842d0e
commit b82e908b4c
23 changed files with 129 additions and 108 deletions
+3 -6
View File
@@ -559,13 +559,10 @@ def _encode_token_type_ids(
def _decode_token_type_ids(input_ids: torch.Tensor) -> torch.Tensor:
ids_mask = (
torch.ones_like(input_ids, dtype=torch.int32, device=input_ids.device)
<< TOKEN_TYPE_SHIFT
)
tokens_mask = ids_mask.bitwise_not()
ids_mask = 1 << TOKEN_TYPE_SHIFT
tokens_mask = ~ids_mask
token_type_ids = input_ids.bitwise_and(ids_mask) >> TOKEN_TYPE_SHIFT
token_type_ids = (input_ids & ids_mask) >> TOKEN_TYPE_SHIFT
input_ids.bitwise_and_(tokens_mask)