[Perf][1/n] Eliminate various GPU<->CPU syncs (#41429)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-05-11 20:36:03 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 53181384e0
commit bbee532988
7 changed files with 49 additions and 33 deletions
@@ -362,6 +362,7 @@ class MambaMixer(MambaBase, PluggableLayer):
initial_state_idx=block_idx_last_computed_token_p,
num_computed_tokens=num_computed_tokens_p,
block_size_to_align=mamba_block_size,
metadata=attn_metadata,
)
# 3. State Space Model sequence transformations.
discrete_time_step_p, B_p, C_p = self._ssm_transform(
+2 -1
View File
@@ -23,7 +23,8 @@ def _apply_bad_words_single_batch(
assert len(actual_prefix) == len(expected_prefix)
if actual_prefix == expected_prefix:
logits[last_token_id] = _SMALLEST_LOGIT
# Assign to slice to avoid cpu->gpu sync.
logits[last_token_id : last_token_id + 1] = _SMALLEST_LOGIT
def apply_bad_words(
+17 -14
View File
@@ -175,27 +175,30 @@ class Sampler(nn.Module):
# Find max number of tokens across all requests
max_num_tokens = max(len(tids) for tids in logprob_token_ids.values())
pin = self.pin_memory
# Create padded token_ids tensor: [batch_size, max_num_tokens + 1]
# +1 for sampled token in first position
token_ids_tensor = torch.zeros(
batch_size, max_num_tokens + 1, dtype=torch.int64, device=device
# Build the padded token_ids and valid_mask matrices on pinned CPU,
# then upload non-blocking.
token_ids_cpu = torch.zeros(
batch_size, max_num_tokens + 1, dtype=torch.int64, pin_memory=pin
)
token_ids_tensor[:, 0] = sampled # First column is sampled token
# Create mask for valid positions (True = valid, False = padded)
valid_mask = torch.zeros(
batch_size, max_num_tokens + 1, dtype=torch.bool, device=device
valid_mask_cpu = torch.zeros(
batch_size, max_num_tokens + 1, dtype=torch.bool, pin_memory=pin
)
valid_mask[:, 0] = True # Sampled token is always valid
# Fill in token IDs for each request
valid_mask_cpu[:, 0] = True # Sampled token is always valid
for req_idx, token_ids in logprob_token_ids.items():
num_tokens = len(token_ids)
token_ids_tensor[req_idx, 1 : num_tokens + 1] = torch.tensor(
token_ids, dtype=torch.int64, device=device
token_ids_cpu[req_idx, 1 : num_tokens + 1] = torch.as_tensor(
token_ids, dtype=torch.int64
)
valid_mask[req_idx, 1 : num_tokens + 1] = True
valid_mask_cpu[req_idx, 1 : num_tokens + 1] = True
token_ids_tensor = token_ids_cpu.to(device, non_blocking=True)
valid_mask = valid_mask_cpu.to(device, non_blocking=True)
# Sampled token in column 0 — fill on-device from the sampled GPU
# tensor so we don't need to D2H + re-upload.
token_ids_tensor[:, 0] = sampled
# Compute logprobs using the fused Triton kernel (log_softmax + gather)
logprobs = compute_token_logprobs(logits, token_ids_tensor)
+3 -2
View File
@@ -18,6 +18,7 @@ from vllm.config import (
VllmConfig,
)
from vllm.forward_context import set_forward_context
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.utils import record_function_or_nullcontext
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
@@ -569,8 +570,8 @@ def update_ngram_gpu_tensors_incremental(
reorder_dst.append(curr_idx)
if reorder_src:
src_tensor = torch.tensor(reorder_src, dtype=torch.long, device=device)
dst_tensor = torch.tensor(reorder_dst, dtype=torch.long, device=device)
src_tensor = async_tensor_h2d(reorder_src, dtype=torch.long, device=device)
dst_tensor = async_tensor_h2d(reorder_dst, dtype=torch.long, device=device)
temp_token_ids = token_ids_gpu_tensor[src_tensor].clone()
temp_num_tokens = num_tokens_no_spec_gpu[src_tensor].clone()
+7 -5
View File
@@ -43,11 +43,13 @@ def _run_ar(
dp_size = parallel_config.data_parallel_size
dp_rank = parallel_config.data_parallel_rank
device, group = _get_device_and_group(parallel_config)
tensor = torch.zeros(4, dp_size, device=device, dtype=torch.int32)
tensor[0][dp_rank] = orig_num_tokens_per_ubatch
tensor[1][dp_rank] = padded_num_tokens_per_ubatch
tensor[2][dp_rank] = 1 if should_ubatch else 0
tensor[3][dp_rank] = cudagraph_mode
# Populate this rank's contribution on CPU to reduce GPU syncs.
tensor_cpu = torch.zeros(4, dp_size, dtype=torch.int32)
tensor_cpu[0][dp_rank] = orig_num_tokens_per_ubatch
tensor_cpu[1][dp_rank] = padded_num_tokens_per_ubatch
tensor_cpu[2][dp_rank] = 1 if should_ubatch else 0
tensor_cpu[3][dp_rank] = cudagraph_mode
tensor = tensor_cpu.to(device, non_blocking=True)
dist.all_reduce(tensor, group=group)
return tensor
+4 -2
View File
@@ -283,8 +283,10 @@ def bincount(
output_bin_counts: torch.Tensor,
max_prefill_len: int,
) -> None:
prompt_bin_mask[expanded_idx_mapping] = 0
output_bin_counts[expanded_idx_mapping] = 0
# Use index_fill_ instead of `tensor[idx] = 0` to avoid sync.
idx_long = expanded_idx_mapping.long()
prompt_bin_mask.index_fill_(0, idx_long, 0)
output_bin_counts.index_fill_(0, idx_long, 0)
num_tokens = expanded_idx_mapping.shape[0]
BLOCK_SIZE = 1024
num_blocks = triton.cdiv(max_prefill_len, BLOCK_SIZE)
+15 -9
View File
@@ -995,16 +995,23 @@ class GPUModelRunner(
if len(token_type_id_requests) == 0:
return model_kwargs
seq_lens = self.seq_lens[:num_reqs]
# Build ids on CPU using the CPU-resident upper bound for seq_lens;
# `torch.arange(seq_lens[i])` with a GPU scalar would force a sync.
seq_lens_cpu = self.optimistic_seq_lens_cpu[:num_reqs].tolist()
token_type_ids = []
for i in range(num_reqs):
pos = token_type_id_requests.get(i, seq_lens[i])
ids = (torch.arange(seq_lens[i]) >= pos).int()
seq_len_i = seq_lens_cpu[i]
pos = token_type_id_requests.get(i, seq_len_i)
ids = (torch.arange(seq_len_i) >= pos).int()
token_type_ids.append(ids)
model_kwargs["token_type_ids"] = torch.concat(token_type_ids).to(
device=self.device
token_type_ids_cpu = torch.empty(
sum(seq_lens_cpu), dtype=torch.int32, pin_memory=self.pin_memory
)
torch.cat(token_type_ids, out=token_type_ids_cpu)
model_kwargs["token_type_ids"] = token_type_ids_cpu.to(
device=self.device, non_blocking=True
)
return model_kwargs
@@ -2715,10 +2722,9 @@ class GPUModelRunner(
# There might have leftover indices in logits_indices[num_logits:]
# from previous iterations, whose values may be greater than the
# batch size in the current iteration. To ensure indices are always
# valid, we fill the padded indices with the last index.
self.kv_sharing_fast_prefill_logits_indices[num_logits:].fill_(
logits_indices[-1].item()
)
# valid, fill the padded indices with the last index. Broadcast the
# scalar GPU-side to avoid a D2H sync on `.item()`.
self.kv_sharing_fast_prefill_logits_indices[num_logits:] = logits_indices[-1]
# Dispatch for the decoder portion of the model.
_, batch_desc = self.cudagraph_dispatcher.dispatch(
num_logits, invalid_modes={CUDAGraphMode.FULL}