[Core] Check for GPU<->CPU syncs during CI (#43107)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
7b544ecb52
commit
615d4cfade
@@ -120,6 +120,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.nvtx_pytorch_hooks import PytHooks
|
||||
@@ -3848,26 +3849,28 @@ class GPUModelRunner(
|
||||
self.routed_experts_slot_mapping_device[:total],
|
||||
non_blocking=True,
|
||||
)
|
||||
with gpu_sync_allowed():
|
||||
# Get the valid generated tokens.
|
||||
max_gen_len = sampled_token_ids.shape[-1]
|
||||
if max_gen_len == 1:
|
||||
# No spec decode tokens.
|
||||
valid_sampled_token_ids = self._to_list(sampled_token_ids)
|
||||
# Mask out the sampled tokens that should not be sampled.
|
||||
for i in discard_sampled_tokens_req_indices:
|
||||
valid_sampled_token_ids[int(i)].clear()
|
||||
|
||||
# Get the valid generated tokens.
|
||||
max_gen_len = sampled_token_ids.shape[-1]
|
||||
if max_gen_len == 1:
|
||||
# No spec decode tokens.
|
||||
valid_sampled_token_ids = self._to_list(sampled_token_ids)
|
||||
# Mask out the sampled tokens that should not be sampled.
|
||||
for i in discard_sampled_tokens_req_indices:
|
||||
valid_sampled_token_ids[int(i)].clear()
|
||||
|
||||
if logprobs_tensors is not None:
|
||||
logprobs_lists = logprobs_tensors.tolists()
|
||||
else:
|
||||
# Includes spec decode tokens.
|
||||
valid_sampled_token_ids, logprobs_lists = RejectionSampler.parse_output(
|
||||
sampled_token_ids,
|
||||
self.input_batch.vocab_size,
|
||||
discard_sampled_tokens_req_indices,
|
||||
logprobs_tensors=logprobs_tensors,
|
||||
)
|
||||
if logprobs_tensors is not None:
|
||||
logprobs_lists = logprobs_tensors.tolists()
|
||||
else:
|
||||
# Includes spec decode tokens.
|
||||
valid_sampled_token_ids, logprobs_lists = (
|
||||
RejectionSampler.parse_output(
|
||||
sampled_token_ids,
|
||||
self.input_batch.vocab_size,
|
||||
discard_sampled_tokens_req_indices,
|
||||
logprobs_tensors=logprobs_tensors,
|
||||
)
|
||||
)
|
||||
else:
|
||||
valid_sampled_token_ids = []
|
||||
invalid_req_indices = discard_sampled_tokens_req_indices.tolist()
|
||||
@@ -5843,7 +5846,10 @@ class GPUModelRunner(
|
||||
on device instead; see`AsyncGPUModelRunnerOutput`.
|
||||
"""
|
||||
try:
|
||||
counts = [] if logits is None else count_nans_per_row(logits).tolist()
|
||||
# Reporting per-request NaN counts requires them on the host; this
|
||||
# path is opt-in diagnostics, so the D2H is intended.
|
||||
with gpu_sync_allowed():
|
||||
counts = [] if logits is None else count_nans_per_row(logits).tolist()
|
||||
num_nans_in_logits = nans_to_dict(counts, self.input_batch.req_id_to_index)
|
||||
if envs.VLLM_RAISE_ON_LOGIT_NANS:
|
||||
raise_if_nan_logits(num_nans_in_logits)
|
||||
|
||||
Reference in New Issue
Block a user