[Perf][1/n] Eliminate various GPU<->CPU syncs (#41429)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
53181384e0
commit
bbee532988
@@ -362,6 +362,7 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
initial_state_idx=block_idx_last_computed_token_p,
|
||||
num_computed_tokens=num_computed_tokens_p,
|
||||
block_size_to_align=mamba_block_size,
|
||||
metadata=attn_metadata,
|
||||
)
|
||||
# 3. State Space Model sequence transformations.
|
||||
discrete_time_step_p, B_p, C_p = self._ssm_transform(
|
||||
|
||||
@@ -23,7 +23,8 @@ def _apply_bad_words_single_batch(
|
||||
assert len(actual_prefix) == len(expected_prefix)
|
||||
|
||||
if actual_prefix == expected_prefix:
|
||||
logits[last_token_id] = _SMALLEST_LOGIT
|
||||
# Assign to slice to avoid cpu->gpu sync.
|
||||
logits[last_token_id : last_token_id + 1] = _SMALLEST_LOGIT
|
||||
|
||||
|
||||
def apply_bad_words(
|
||||
|
||||
+17
-14
@@ -175,27 +175,30 @@ class Sampler(nn.Module):
|
||||
|
||||
# Find max number of tokens across all requests
|
||||
max_num_tokens = max(len(tids) for tids in logprob_token_ids.values())
|
||||
pin = self.pin_memory
|
||||
|
||||
# Create padded token_ids tensor: [batch_size, max_num_tokens + 1]
|
||||
# +1 for sampled token in first position
|
||||
token_ids_tensor = torch.zeros(
|
||||
batch_size, max_num_tokens + 1, dtype=torch.int64, device=device
|
||||
# Build the padded token_ids and valid_mask matrices on pinned CPU,
|
||||
# then upload non-blocking.
|
||||
token_ids_cpu = torch.zeros(
|
||||
batch_size, max_num_tokens + 1, dtype=torch.int64, pin_memory=pin
|
||||
)
|
||||
token_ids_tensor[:, 0] = sampled # First column is sampled token
|
||||
|
||||
# Create mask for valid positions (True = valid, False = padded)
|
||||
valid_mask = torch.zeros(
|
||||
batch_size, max_num_tokens + 1, dtype=torch.bool, device=device
|
||||
valid_mask_cpu = torch.zeros(
|
||||
batch_size, max_num_tokens + 1, dtype=torch.bool, pin_memory=pin
|
||||
)
|
||||
valid_mask[:, 0] = True # Sampled token is always valid
|
||||
|
||||
# Fill in token IDs for each request
|
||||
valid_mask_cpu[:, 0] = True # Sampled token is always valid
|
||||
for req_idx, token_ids in logprob_token_ids.items():
|
||||
num_tokens = len(token_ids)
|
||||
token_ids_tensor[req_idx, 1 : num_tokens + 1] = torch.tensor(
|
||||
token_ids, dtype=torch.int64, device=device
|
||||
token_ids_cpu[req_idx, 1 : num_tokens + 1] = torch.as_tensor(
|
||||
token_ids, dtype=torch.int64
|
||||
)
|
||||
valid_mask[req_idx, 1 : num_tokens + 1] = True
|
||||
valid_mask_cpu[req_idx, 1 : num_tokens + 1] = True
|
||||
|
||||
token_ids_tensor = token_ids_cpu.to(device, non_blocking=True)
|
||||
valid_mask = valid_mask_cpu.to(device, non_blocking=True)
|
||||
# Sampled token in column 0 — fill on-device from the sampled GPU
|
||||
# tensor so we don't need to D2H + re-upload.
|
||||
token_ids_tensor[:, 0] = sampled
|
||||
|
||||
# Compute logprobs using the fused Triton kernel (log_softmax + gather)
|
||||
logprobs = compute_token_logprobs(logits, token_ids_tensor)
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.config import (
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
@@ -569,8 +570,8 @@ def update_ngram_gpu_tensors_incremental(
|
||||
reorder_dst.append(curr_idx)
|
||||
|
||||
if reorder_src:
|
||||
src_tensor = torch.tensor(reorder_src, dtype=torch.long, device=device)
|
||||
dst_tensor = torch.tensor(reorder_dst, dtype=torch.long, device=device)
|
||||
src_tensor = async_tensor_h2d(reorder_src, dtype=torch.long, device=device)
|
||||
dst_tensor = async_tensor_h2d(reorder_dst, dtype=torch.long, device=device)
|
||||
|
||||
temp_token_ids = token_ids_gpu_tensor[src_tensor].clone()
|
||||
temp_num_tokens = num_tokens_no_spec_gpu[src_tensor].clone()
|
||||
|
||||
@@ -43,11 +43,13 @@ def _run_ar(
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
device, group = _get_device_and_group(parallel_config)
|
||||
tensor = torch.zeros(4, dp_size, device=device, dtype=torch.int32)
|
||||
tensor[0][dp_rank] = orig_num_tokens_per_ubatch
|
||||
tensor[1][dp_rank] = padded_num_tokens_per_ubatch
|
||||
tensor[2][dp_rank] = 1 if should_ubatch else 0
|
||||
tensor[3][dp_rank] = cudagraph_mode
|
||||
# Populate this rank's contribution on CPU to reduce GPU syncs.
|
||||
tensor_cpu = torch.zeros(4, dp_size, dtype=torch.int32)
|
||||
tensor_cpu[0][dp_rank] = orig_num_tokens_per_ubatch
|
||||
tensor_cpu[1][dp_rank] = padded_num_tokens_per_ubatch
|
||||
tensor_cpu[2][dp_rank] = 1 if should_ubatch else 0
|
||||
tensor_cpu[3][dp_rank] = cudagraph_mode
|
||||
tensor = tensor_cpu.to(device, non_blocking=True)
|
||||
dist.all_reduce(tensor, group=group)
|
||||
return tensor
|
||||
|
||||
|
||||
@@ -283,8 +283,10 @@ def bincount(
|
||||
output_bin_counts: torch.Tensor,
|
||||
max_prefill_len: int,
|
||||
) -> None:
|
||||
prompt_bin_mask[expanded_idx_mapping] = 0
|
||||
output_bin_counts[expanded_idx_mapping] = 0
|
||||
# Use index_fill_ instead of `tensor[idx] = 0` to avoid sync.
|
||||
idx_long = expanded_idx_mapping.long()
|
||||
prompt_bin_mask.index_fill_(0, idx_long, 0)
|
||||
output_bin_counts.index_fill_(0, idx_long, 0)
|
||||
num_tokens = expanded_idx_mapping.shape[0]
|
||||
BLOCK_SIZE = 1024
|
||||
num_blocks = triton.cdiv(max_prefill_len, BLOCK_SIZE)
|
||||
|
||||
@@ -995,16 +995,23 @@ class GPUModelRunner(
|
||||
if len(token_type_id_requests) == 0:
|
||||
return model_kwargs
|
||||
|
||||
seq_lens = self.seq_lens[:num_reqs]
|
||||
# Build ids on CPU using the CPU-resident upper bound for seq_lens;
|
||||
# `torch.arange(seq_lens[i])` with a GPU scalar would force a sync.
|
||||
seq_lens_cpu = self.optimistic_seq_lens_cpu[:num_reqs].tolist()
|
||||
token_type_ids = []
|
||||
|
||||
for i in range(num_reqs):
|
||||
pos = token_type_id_requests.get(i, seq_lens[i])
|
||||
ids = (torch.arange(seq_lens[i]) >= pos).int()
|
||||
seq_len_i = seq_lens_cpu[i]
|
||||
pos = token_type_id_requests.get(i, seq_len_i)
|
||||
ids = (torch.arange(seq_len_i) >= pos).int()
|
||||
token_type_ids.append(ids)
|
||||
|
||||
model_kwargs["token_type_ids"] = torch.concat(token_type_ids).to(
|
||||
device=self.device
|
||||
token_type_ids_cpu = torch.empty(
|
||||
sum(seq_lens_cpu), dtype=torch.int32, pin_memory=self.pin_memory
|
||||
)
|
||||
torch.cat(token_type_ids, out=token_type_ids_cpu)
|
||||
model_kwargs["token_type_ids"] = token_type_ids_cpu.to(
|
||||
device=self.device, non_blocking=True
|
||||
)
|
||||
return model_kwargs
|
||||
|
||||
@@ -2715,10 +2722,9 @@ class GPUModelRunner(
|
||||
# There might have leftover indices in logits_indices[num_logits:]
|
||||
# from previous iterations, whose values may be greater than the
|
||||
# batch size in the current iteration. To ensure indices are always
|
||||
# valid, we fill the padded indices with the last index.
|
||||
self.kv_sharing_fast_prefill_logits_indices[num_logits:].fill_(
|
||||
logits_indices[-1].item()
|
||||
)
|
||||
# valid, fill the padded indices with the last index. Broadcast the
|
||||
# scalar GPU-side to avoid a D2H sync on `.item()`.
|
||||
self.kv_sharing_fast_prefill_logits_indices[num_logits:] = logits_indices[-1]
|
||||
# Dispatch for the decoder portion of the model.
|
||||
_, batch_desc = self.cudagraph_dispatcher.dispatch(
|
||||
num_logits, invalid_modes={CUDAGraphMode.FULL}
|
||||
|
||||
Reference in New Issue
Block a user