[Perf][1/n] Eliminate various GPU<->CPU syncs (#41429)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-05-11 20:36:03 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 53181384e0
commit bbee532988
7 changed files with 49 additions and 33 deletions
+17 -14
View File
@@ -175,27 +175,30 @@ class Sampler(nn.Module):
# Find max number of tokens across all requests
max_num_tokens = max(len(tids) for tids in logprob_token_ids.values())
pin = self.pin_memory
# Create padded token_ids tensor: [batch_size, max_num_tokens + 1]
# +1 for sampled token in first position
token_ids_tensor = torch.zeros(
batch_size, max_num_tokens + 1, dtype=torch.int64, device=device
# Build the padded token_ids and valid_mask matrices on pinned CPU,
# then upload non-blocking.
token_ids_cpu = torch.zeros(
batch_size, max_num_tokens + 1, dtype=torch.int64, pin_memory=pin
)
token_ids_tensor[:, 0] = sampled # First column is sampled token
# Create mask for valid positions (True = valid, False = padded)
valid_mask = torch.zeros(
batch_size, max_num_tokens + 1, dtype=torch.bool, device=device
valid_mask_cpu = torch.zeros(
batch_size, max_num_tokens + 1, dtype=torch.bool, pin_memory=pin
)
valid_mask[:, 0] = True # Sampled token is always valid
# Fill in token IDs for each request
valid_mask_cpu[:, 0] = True # Sampled token is always valid
for req_idx, token_ids in logprob_token_ids.items():
num_tokens = len(token_ids)
token_ids_tensor[req_idx, 1 : num_tokens + 1] = torch.tensor(
token_ids, dtype=torch.int64, device=device
token_ids_cpu[req_idx, 1 : num_tokens + 1] = torch.as_tensor(
token_ids, dtype=torch.int64
)
valid_mask[req_idx, 1 : num_tokens + 1] = True
valid_mask_cpu[req_idx, 1 : num_tokens + 1] = True
token_ids_tensor = token_ids_cpu.to(device, non_blocking=True)
valid_mask = valid_mask_cpu.to(device, non_blocking=True)
# Sampled token in column 0 — fill on-device from the sampled GPU
# tensor so we don't need to D2H + re-upload.
token_ids_tensor[:, 0] = sampled
# Compute logprobs using the fused Triton kernel (log_softmax + gather)
logprobs = compute_token_logprobs(logits, token_ids_tensor)