[Perf] Avoid some more unnecessary GPU<->CPU syncs (#51458)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -19,6 +19,7 @@ from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
@@ -77,7 +78,7 @@ def get_fake_sample_fn() -> SamplerOutput:
|
||||
first_token_id_index = num_computed_tokens + 1
|
||||
if spec_decode_metadata is None:
|
||||
return SamplerOutput(
|
||||
sampled_token_ids=torch.tensor(
|
||||
sampled_token_ids=async_tensor_h2d(
|
||||
[[prompt_token_ids[first_token_id_index]]],
|
||||
device=DEVICE_TYPE,
|
||||
dtype=torch.int32,
|
||||
@@ -90,7 +91,7 @@ def get_fake_sample_fn() -> SamplerOutput:
|
||||
]
|
||||
sampled_token_ids = accepted_tokens
|
||||
return SamplerOutput(
|
||||
sampled_token_ids=torch.tensor(
|
||||
sampled_token_ids=async_tensor_h2d(
|
||||
[sampled_token_ids],
|
||||
device=DEVICE_TYPE,
|
||||
dtype=torch.int32,
|
||||
@@ -132,7 +133,7 @@ def get_fake_propose_draft_token_ids_fn():
|
||||
]
|
||||
]
|
||||
|
||||
next_token_ids = torch.tensor(
|
||||
next_token_ids = async_tensor_h2d(
|
||||
prompt_token_ids[
|
||||
first_token_id_index - 1 : first_token_id_index
|
||||
- 1
|
||||
@@ -142,7 +143,7 @@ def get_fake_propose_draft_token_ids_fn():
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
valid_sampled_tokens_count = torch.tensor(
|
||||
valid_sampled_tokens_count = async_tensor_h2d(
|
||||
[num_accepted_tokens],
|
||||
device=DEVICE_TYPE,
|
||||
dtype=torch.int32,
|
||||
@@ -150,7 +151,7 @@ def get_fake_propose_draft_token_ids_fn():
|
||||
|
||||
self._copy_valid_sampled_token_count(next_token_ids, valid_sampled_tokens_count)
|
||||
|
||||
return torch.tensor(
|
||||
return async_tensor_h2d(
|
||||
proposed_draft_token_ids,
|
||||
device=DEVICE_TYPE,
|
||||
dtype=torch.int32,
|
||||
@@ -1088,11 +1089,10 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
device=hidden_states.device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
num_logits = torch.tensor(
|
||||
num_logits = async_tensor_h2d(
|
||||
input_batch.cu_num_logits_np[1 : num_reqs + 1]
|
||||
- input_batch.cu_num_logits_np[:num_reqs],
|
||||
device=hidden_states.device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
accepted = torch.full_like(num_logits, num_accepted_tokens)
|
||||
num_sampled = torch.minimum(accepted, num_logits)
|
||||
|
||||
Reference in New Issue
Block a user