[Perf][1/n] Eliminate various GPU<->CPU syncs (#41429)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
53181384e0
commit
bbee532988
+17
-14
@@ -175,27 +175,30 @@ class Sampler(nn.Module):
|
||||
|
||||
# Find max number of tokens across all requests
|
||||
max_num_tokens = max(len(tids) for tids in logprob_token_ids.values())
|
||||
pin = self.pin_memory
|
||||
|
||||
# Create padded token_ids tensor: [batch_size, max_num_tokens + 1]
|
||||
# +1 for sampled token in first position
|
||||
token_ids_tensor = torch.zeros(
|
||||
batch_size, max_num_tokens + 1, dtype=torch.int64, device=device
|
||||
# Build the padded token_ids and valid_mask matrices on pinned CPU,
|
||||
# then upload non-blocking.
|
||||
token_ids_cpu = torch.zeros(
|
||||
batch_size, max_num_tokens + 1, dtype=torch.int64, pin_memory=pin
|
||||
)
|
||||
token_ids_tensor[:, 0] = sampled # First column is sampled token
|
||||
|
||||
# Create mask for valid positions (True = valid, False = padded)
|
||||
valid_mask = torch.zeros(
|
||||
batch_size, max_num_tokens + 1, dtype=torch.bool, device=device
|
||||
valid_mask_cpu = torch.zeros(
|
||||
batch_size, max_num_tokens + 1, dtype=torch.bool, pin_memory=pin
|
||||
)
|
||||
valid_mask[:, 0] = True # Sampled token is always valid
|
||||
|
||||
# Fill in token IDs for each request
|
||||
valid_mask_cpu[:, 0] = True # Sampled token is always valid
|
||||
for req_idx, token_ids in logprob_token_ids.items():
|
||||
num_tokens = len(token_ids)
|
||||
token_ids_tensor[req_idx, 1 : num_tokens + 1] = torch.tensor(
|
||||
token_ids, dtype=torch.int64, device=device
|
||||
token_ids_cpu[req_idx, 1 : num_tokens + 1] = torch.as_tensor(
|
||||
token_ids, dtype=torch.int64
|
||||
)
|
||||
valid_mask[req_idx, 1 : num_tokens + 1] = True
|
||||
valid_mask_cpu[req_idx, 1 : num_tokens + 1] = True
|
||||
|
||||
token_ids_tensor = token_ids_cpu.to(device, non_blocking=True)
|
||||
valid_mask = valid_mask_cpu.to(device, non_blocking=True)
|
||||
# Sampled token in column 0 — fill on-device from the sampled GPU
|
||||
# tensor so we don't need to D2H + re-upload.
|
||||
token_ids_tensor[:, 0] = sampled
|
||||
|
||||
# Compute logprobs using the fused Triton kernel (log_softmax + gather)
|
||||
logprobs = compute_token_logprobs(logits, token_ids_tensor)
|
||||
|
||||
Reference in New Issue
Block a user