[Perf] Avoid some more unnecessary GPU<->CPU syncs (#51458)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-08 17:13:26 -07:00
committed by GitHub
parent d608dfabfd
commit 9b0afeb4f6
13 changed files with 99 additions and 49 deletions
@@ -307,7 +307,8 @@ def test_raise_on_logit_nans(
def compute_nan_logits(self, *args, **kwargs):
logits = original_compute_logits(self, *args, **kwargs)
logits[0, 0] = float("nan")
# `logits[0, 0] = ...` copies the scalar H2D and blocks.
logits[0, 0].fill_(float("nan"))
return logits
monkeypatch.setattr(model_cls, "compute_logits", compute_nan_logits)
@@ -19,6 +19,7 @@ from vllm.distributed import cleanup_dist_env_and_memory
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
from vllm.v1.core.sched.output import SchedulerOutput
@@ -77,7 +78,7 @@ def get_fake_sample_fn() -> SamplerOutput:
first_token_id_index = num_computed_tokens + 1
if spec_decode_metadata is None:
return SamplerOutput(
sampled_token_ids=torch.tensor(
sampled_token_ids=async_tensor_h2d(
[[prompt_token_ids[first_token_id_index]]],
device=DEVICE_TYPE,
dtype=torch.int32,
@@ -90,7 +91,7 @@ def get_fake_sample_fn() -> SamplerOutput:
]
sampled_token_ids = accepted_tokens
return SamplerOutput(
sampled_token_ids=torch.tensor(
sampled_token_ids=async_tensor_h2d(
[sampled_token_ids],
device=DEVICE_TYPE,
dtype=torch.int32,
@@ -132,7 +133,7 @@ def get_fake_propose_draft_token_ids_fn():
]
]
next_token_ids = torch.tensor(
next_token_ids = async_tensor_h2d(
prompt_token_ids[
first_token_id_index - 1 : first_token_id_index
- 1
@@ -142,7 +143,7 @@ def get_fake_propose_draft_token_ids_fn():
dtype=torch.int32,
)
valid_sampled_tokens_count = torch.tensor(
valid_sampled_tokens_count = async_tensor_h2d(
[num_accepted_tokens],
device=DEVICE_TYPE,
dtype=torch.int32,
@@ -150,7 +151,7 @@ def get_fake_propose_draft_token_ids_fn():
self._copy_valid_sampled_token_count(next_token_ids, valid_sampled_tokens_count)
return torch.tensor(
return async_tensor_h2d(
proposed_draft_token_ids,
device=DEVICE_TYPE,
dtype=torch.int32,
@@ -1088,11 +1089,10 @@ def _run_mamba_prefix_cache_mrv2(
device=hidden_states.device,
dtype=torch.int64,
)
num_logits = torch.tensor(
num_logits = async_tensor_h2d(
input_batch.cu_num_logits_np[1 : num_reqs + 1]
- input_batch.cu_num_logits_np[:num_reqs],
device=hidden_states.device,
dtype=torch.int32,
)
accepted = torch.full_like(num_logits, num_accepted_tokens)
num_sampled = torch.minimum(accepted, num_logits)
+14 -6
View File
@@ -14,6 +14,7 @@ from vllm.config import VllmConfig
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.sampling_params import SamplingParams
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.sample.logits_processor import (
LOGITSPROCS_GROUP,
AdapterLogitsProcessor,
@@ -90,17 +91,24 @@ class DummyLogitsProcessor(LogitsProcessor):
if not self.req_info:
return logits
# Save target values before modification
cols = torch.tensor(
# Save target values before modification.
cols = async_tensor_h2d(
list(self.req_info.values()), dtype=torch.long, device=logits.device
)
rows = torch.tensor(
rows = async_tensor_h2d(
list(self.req_info.keys()), dtype=torch.long, device=logits.device
)
values_to_keep = logits[rows, cols].clone()
# Mask all but target tokens
logits[rows] = float("-inf")
# Mask all but target tokens. Use an on-device fill tensor so the
# scatter doesn't force a synchronizing scalar H2D.
fill = torch.full(
(rows.numel(), logits.size(-1)),
float("-inf"),
dtype=logits.dtype,
device=logits.device,
)
logits[rows] = fill
logits[rows, cols] = values_to_keep
return logits
@@ -141,7 +149,7 @@ class DummyPerReqLogitsProcessor:
output_ids: list[int],
logits: torch.Tensor,
) -> torch.Tensor:
val_to_keep = logits[self.target_token].item()
val_to_keep = logits[self.target_token].clone()
logits[:] = float("-inf")
logits[self.target_token] = val_to_keep
return logits