[Perf] Avoid some more unnecessary GPU<->CPU syncs (#51458)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-08 17:13:26 -07:00
committed by GitHub
parent d608dfabfd
commit 9b0afeb4f6
13 changed files with 99 additions and 49 deletions
@@ -19,6 +19,7 @@ from vllm.distributed import cleanup_dist_env_and_memory
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
from vllm.v1.core.sched.output import SchedulerOutput
@@ -77,7 +78,7 @@ def get_fake_sample_fn() -> SamplerOutput:
first_token_id_index = num_computed_tokens + 1
if spec_decode_metadata is None:
return SamplerOutput(
sampled_token_ids=torch.tensor(
sampled_token_ids=async_tensor_h2d(
[[prompt_token_ids[first_token_id_index]]],
device=DEVICE_TYPE,
dtype=torch.int32,
@@ -90,7 +91,7 @@ def get_fake_sample_fn() -> SamplerOutput:
]
sampled_token_ids = accepted_tokens
return SamplerOutput(
sampled_token_ids=torch.tensor(
sampled_token_ids=async_tensor_h2d(
[sampled_token_ids],
device=DEVICE_TYPE,
dtype=torch.int32,
@@ -132,7 +133,7 @@ def get_fake_propose_draft_token_ids_fn():
]
]
next_token_ids = torch.tensor(
next_token_ids = async_tensor_h2d(
prompt_token_ids[
first_token_id_index - 1 : first_token_id_index
- 1
@@ -142,7 +143,7 @@ def get_fake_propose_draft_token_ids_fn():
dtype=torch.int32,
)
valid_sampled_tokens_count = torch.tensor(
valid_sampled_tokens_count = async_tensor_h2d(
[num_accepted_tokens],
device=DEVICE_TYPE,
dtype=torch.int32,
@@ -150,7 +151,7 @@ def get_fake_propose_draft_token_ids_fn():
self._copy_valid_sampled_token_count(next_token_ids, valid_sampled_tokens_count)
return torch.tensor(
return async_tensor_h2d(
proposed_draft_token_ids,
device=DEVICE_TYPE,
dtype=torch.int32,
@@ -1088,11 +1089,10 @@ def _run_mamba_prefix_cache_mrv2(
device=hidden_states.device,
dtype=torch.int64,
)
num_logits = torch.tensor(
num_logits = async_tensor_h2d(
input_batch.cu_num_logits_np[1 : num_reqs + 1]
- input_batch.cu_num_logits_np[:num_reqs],
device=hidden_states.device,
dtype=torch.int32,
)
accepted = torch.full_like(num_logits, num_accepted_tokens)
num_sampled = torch.minimum(accepted, num_logits)