[Perf] Avoid some more unnecessary GPU<->CPU syncs (#51458)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -307,7 +307,8 @@ def test_raise_on_logit_nans(
|
||||
|
||||
def compute_nan_logits(self, *args, **kwargs):
|
||||
logits = original_compute_logits(self, *args, **kwargs)
|
||||
logits[0, 0] = float("nan")
|
||||
# `logits[0, 0] = ...` copies the scalar H2D and blocks.
|
||||
logits[0, 0].fill_(float("nan"))
|
||||
return logits
|
||||
|
||||
monkeypatch.setattr(model_cls, "compute_logits", compute_nan_logits)
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
@@ -77,7 +78,7 @@ def get_fake_sample_fn() -> SamplerOutput:
|
||||
first_token_id_index = num_computed_tokens + 1
|
||||
if spec_decode_metadata is None:
|
||||
return SamplerOutput(
|
||||
sampled_token_ids=torch.tensor(
|
||||
sampled_token_ids=async_tensor_h2d(
|
||||
[[prompt_token_ids[first_token_id_index]]],
|
||||
device=DEVICE_TYPE,
|
||||
dtype=torch.int32,
|
||||
@@ -90,7 +91,7 @@ def get_fake_sample_fn() -> SamplerOutput:
|
||||
]
|
||||
sampled_token_ids = accepted_tokens
|
||||
return SamplerOutput(
|
||||
sampled_token_ids=torch.tensor(
|
||||
sampled_token_ids=async_tensor_h2d(
|
||||
[sampled_token_ids],
|
||||
device=DEVICE_TYPE,
|
||||
dtype=torch.int32,
|
||||
@@ -132,7 +133,7 @@ def get_fake_propose_draft_token_ids_fn():
|
||||
]
|
||||
]
|
||||
|
||||
next_token_ids = torch.tensor(
|
||||
next_token_ids = async_tensor_h2d(
|
||||
prompt_token_ids[
|
||||
first_token_id_index - 1 : first_token_id_index
|
||||
- 1
|
||||
@@ -142,7 +143,7 @@ def get_fake_propose_draft_token_ids_fn():
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
valid_sampled_tokens_count = torch.tensor(
|
||||
valid_sampled_tokens_count = async_tensor_h2d(
|
||||
[num_accepted_tokens],
|
||||
device=DEVICE_TYPE,
|
||||
dtype=torch.int32,
|
||||
@@ -150,7 +151,7 @@ def get_fake_propose_draft_token_ids_fn():
|
||||
|
||||
self._copy_valid_sampled_token_count(next_token_ids, valid_sampled_tokens_count)
|
||||
|
||||
return torch.tensor(
|
||||
return async_tensor_h2d(
|
||||
proposed_draft_token_ids,
|
||||
device=DEVICE_TYPE,
|
||||
dtype=torch.int32,
|
||||
@@ -1088,11 +1089,10 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
device=hidden_states.device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
num_logits = torch.tensor(
|
||||
num_logits = async_tensor_h2d(
|
||||
input_batch.cu_num_logits_np[1 : num_reqs + 1]
|
||||
- input_batch.cu_num_logits_np[:num_reqs],
|
||||
device=hidden_states.device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
accepted = torch.full_like(num_logits, num_accepted_tokens)
|
||||
num_sampled = torch.minimum(accepted, num_logits)
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.sample.logits_processor import (
|
||||
LOGITSPROCS_GROUP,
|
||||
AdapterLogitsProcessor,
|
||||
@@ -90,17 +91,24 @@ class DummyLogitsProcessor(LogitsProcessor):
|
||||
if not self.req_info:
|
||||
return logits
|
||||
|
||||
# Save target values before modification
|
||||
cols = torch.tensor(
|
||||
# Save target values before modification.
|
||||
cols = async_tensor_h2d(
|
||||
list(self.req_info.values()), dtype=torch.long, device=logits.device
|
||||
)
|
||||
rows = torch.tensor(
|
||||
rows = async_tensor_h2d(
|
||||
list(self.req_info.keys()), dtype=torch.long, device=logits.device
|
||||
)
|
||||
values_to_keep = logits[rows, cols].clone()
|
||||
|
||||
# Mask all but target tokens
|
||||
logits[rows] = float("-inf")
|
||||
# Mask all but target tokens. Use an on-device fill tensor so the
|
||||
# scatter doesn't force a synchronizing scalar H2D.
|
||||
fill = torch.full(
|
||||
(rows.numel(), logits.size(-1)),
|
||||
float("-inf"),
|
||||
dtype=logits.dtype,
|
||||
device=logits.device,
|
||||
)
|
||||
logits[rows] = fill
|
||||
logits[rows, cols] = values_to_keep
|
||||
|
||||
return logits
|
||||
@@ -141,7 +149,7 @@ class DummyPerReqLogitsProcessor:
|
||||
output_ids: list[int],
|
||||
logits: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
val_to_keep = logits[self.target_token].item()
|
||||
val_to_keep = logits[self.target_token].clone()
|
||||
logits[:] = float("-inf")
|
||||
logits[self.target_token] = val_to_keep
|
||||
return logits
|
||||
|
||||
Reference in New Issue
Block a user