diff --git a/vllm/model_executor/layers/mamba/mamba_mixer.py b/vllm/model_executor/layers/mamba/mamba_mixer.py index 46f02c8b12..1d3159d1e7 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer.py @@ -362,6 +362,7 @@ class MambaMixer(MambaBase, PluggableLayer): initial_state_idx=block_idx_last_computed_token_p, num_computed_tokens=num_computed_tokens_p, block_size_to_align=mamba_block_size, + metadata=attn_metadata, ) # 3. State Space Model sequence transformations. discrete_time_step_p, B_p, C_p = self._ssm_transform( diff --git a/vllm/v1/sample/ops/bad_words.py b/vllm/v1/sample/ops/bad_words.py index 56972e5179..62ea430ac6 100644 --- a/vllm/v1/sample/ops/bad_words.py +++ b/vllm/v1/sample/ops/bad_words.py @@ -23,7 +23,8 @@ def _apply_bad_words_single_batch( assert len(actual_prefix) == len(expected_prefix) if actual_prefix == expected_prefix: - logits[last_token_id] = _SMALLEST_LOGIT + # Assign to slice to avoid cpu->gpu sync. + logits[last_token_id : last_token_id + 1] = _SMALLEST_LOGIT def apply_bad_words( diff --git a/vllm/v1/sample/sampler.py b/vllm/v1/sample/sampler.py index a77eafba25..d4ec2a2ddc 100644 --- a/vllm/v1/sample/sampler.py +++ b/vllm/v1/sample/sampler.py @@ -175,27 +175,30 @@ class Sampler(nn.Module): # Find max number of tokens across all requests max_num_tokens = max(len(tids) for tids in logprob_token_ids.values()) + pin = self.pin_memory - # Create padded token_ids tensor: [batch_size, max_num_tokens + 1] - # +1 for sampled token in first position - token_ids_tensor = torch.zeros( - batch_size, max_num_tokens + 1, dtype=torch.int64, device=device + # Build the padded token_ids and valid_mask matrices on pinned CPU, + # then upload non-blocking. + token_ids_cpu = torch.zeros( + batch_size, max_num_tokens + 1, dtype=torch.int64, pin_memory=pin ) - token_ids_tensor[:, 0] = sampled # First column is sampled token - # Create mask for valid positions (True = valid, False = padded) - valid_mask = torch.zeros( - batch_size, max_num_tokens + 1, dtype=torch.bool, device=device + valid_mask_cpu = torch.zeros( + batch_size, max_num_tokens + 1, dtype=torch.bool, pin_memory=pin ) - valid_mask[:, 0] = True # Sampled token is always valid - - # Fill in token IDs for each request + valid_mask_cpu[:, 0] = True # Sampled token is always valid for req_idx, token_ids in logprob_token_ids.items(): num_tokens = len(token_ids) - token_ids_tensor[req_idx, 1 : num_tokens + 1] = torch.tensor( - token_ids, dtype=torch.int64, device=device + token_ids_cpu[req_idx, 1 : num_tokens + 1] = torch.as_tensor( + token_ids, dtype=torch.int64 ) - valid_mask[req_idx, 1 : num_tokens + 1] = True + valid_mask_cpu[req_idx, 1 : num_tokens + 1] = True + + token_ids_tensor = token_ids_cpu.to(device, non_blocking=True) + valid_mask = valid_mask_cpu.to(device, non_blocking=True) + # Sampled token in column 0 — fill on-device from the sampled GPU + # tensor so we don't need to D2H + re-upload. + token_ids_tensor[:, 0] = sampled # Compute logprobs using the fused Triton kernel (log_softmax + gather) logprobs = compute_token_logprobs(logits, token_ids_tensor) diff --git a/vllm/v1/spec_decode/ngram_proposer_gpu.py b/vllm/v1/spec_decode/ngram_proposer_gpu.py index eb24a9c933..7759d5c32f 100644 --- a/vllm/v1/spec_decode/ngram_proposer_gpu.py +++ b/vllm/v1/spec_decode/ngram_proposer_gpu.py @@ -18,6 +18,7 @@ from vllm.config import ( VllmConfig, ) from vllm.forward_context import set_forward_context +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.utils import record_function_or_nullcontext from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch @@ -569,8 +570,8 @@ def update_ngram_gpu_tensors_incremental( reorder_dst.append(curr_idx) if reorder_src: - src_tensor = torch.tensor(reorder_src, dtype=torch.long, device=device) - dst_tensor = torch.tensor(reorder_dst, dtype=torch.long, device=device) + src_tensor = async_tensor_h2d(reorder_src, dtype=torch.long, device=device) + dst_tensor = async_tensor_h2d(reorder_dst, dtype=torch.long, device=device) temp_token_ids = token_ids_gpu_tensor[src_tensor].clone() temp_num_tokens = num_tokens_no_spec_gpu[src_tensor].clone() diff --git a/vllm/v1/worker/dp_utils.py b/vllm/v1/worker/dp_utils.py index 64f4e59031..e7c6d81a99 100644 --- a/vllm/v1/worker/dp_utils.py +++ b/vllm/v1/worker/dp_utils.py @@ -43,11 +43,13 @@ def _run_ar( dp_size = parallel_config.data_parallel_size dp_rank = parallel_config.data_parallel_rank device, group = _get_device_and_group(parallel_config) - tensor = torch.zeros(4, dp_size, device=device, dtype=torch.int32) - tensor[0][dp_rank] = orig_num_tokens_per_ubatch - tensor[1][dp_rank] = padded_num_tokens_per_ubatch - tensor[2][dp_rank] = 1 if should_ubatch else 0 - tensor[3][dp_rank] = cudagraph_mode + # Populate this rank's contribution on CPU to reduce GPU syncs. + tensor_cpu = torch.zeros(4, dp_size, dtype=torch.int32) + tensor_cpu[0][dp_rank] = orig_num_tokens_per_ubatch + tensor_cpu[1][dp_rank] = padded_num_tokens_per_ubatch + tensor_cpu[2][dp_rank] = 1 if should_ubatch else 0 + tensor_cpu[3][dp_rank] = cudagraph_mode + tensor = tensor_cpu.to(device, non_blocking=True) dist.all_reduce(tensor, group=group) return tensor diff --git a/vllm/v1/worker/gpu/sample/penalties.py b/vllm/v1/worker/gpu/sample/penalties.py index 057517479b..2cfa8b0aec 100644 --- a/vllm/v1/worker/gpu/sample/penalties.py +++ b/vllm/v1/worker/gpu/sample/penalties.py @@ -283,8 +283,10 @@ def bincount( output_bin_counts: torch.Tensor, max_prefill_len: int, ) -> None: - prompt_bin_mask[expanded_idx_mapping] = 0 - output_bin_counts[expanded_idx_mapping] = 0 + # Use index_fill_ instead of `tensor[idx] = 0` to avoid sync. + idx_long = expanded_idx_mapping.long() + prompt_bin_mask.index_fill_(0, idx_long, 0) + output_bin_counts.index_fill_(0, idx_long, 0) num_tokens = expanded_idx_mapping.shape[0] BLOCK_SIZE = 1024 num_blocks = triton.cdiv(max_prefill_len, BLOCK_SIZE) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 98b2fbf120..df061255b3 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -995,16 +995,23 @@ class GPUModelRunner( if len(token_type_id_requests) == 0: return model_kwargs - seq_lens = self.seq_lens[:num_reqs] + # Build ids on CPU using the CPU-resident upper bound for seq_lens; + # `torch.arange(seq_lens[i])` with a GPU scalar would force a sync. + seq_lens_cpu = self.optimistic_seq_lens_cpu[:num_reqs].tolist() token_type_ids = [] for i in range(num_reqs): - pos = token_type_id_requests.get(i, seq_lens[i]) - ids = (torch.arange(seq_lens[i]) >= pos).int() + seq_len_i = seq_lens_cpu[i] + pos = token_type_id_requests.get(i, seq_len_i) + ids = (torch.arange(seq_len_i) >= pos).int() token_type_ids.append(ids) - model_kwargs["token_type_ids"] = torch.concat(token_type_ids).to( - device=self.device + token_type_ids_cpu = torch.empty( + sum(seq_lens_cpu), dtype=torch.int32, pin_memory=self.pin_memory + ) + torch.cat(token_type_ids, out=token_type_ids_cpu) + model_kwargs["token_type_ids"] = token_type_ids_cpu.to( + device=self.device, non_blocking=True ) return model_kwargs @@ -2715,10 +2722,9 @@ class GPUModelRunner( # There might have leftover indices in logits_indices[num_logits:] # from previous iterations, whose values may be greater than the # batch size in the current iteration. To ensure indices are always - # valid, we fill the padded indices with the last index. - self.kv_sharing_fast_prefill_logits_indices[num_logits:].fill_( - logits_indices[-1].item() - ) + # valid, fill the padded indices with the last index. Broadcast the + # scalar GPU-side to avoid a D2H sync on `.item()`. + self.kv_sharing_fast_prefill_logits_indices[num_logits:] = logits_indices[-1] # Dispatch for the decoder portion of the model. _, batch_desc = self.cudagraph_dispatcher.dispatch( num_logits, invalid_modes={CUDAGraphMode.FULL}