From cec2ec11760f9f3beabd4c90451936078bf91533 Mon Sep 17 00:00:00 2001 From: Weiwei Sun <68775773+sunnweiwei@users.noreply.github.com> Date: Mon, 22 Jun 2026 01:53:16 -0700 Subject: [PATCH] [Bugfix] Avoid racy accepted counts in async spec decode (#45100) Signed-off-by: Weiwei Sun <68775773+sunnweiwei@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Jiangyun Zhu --- .../v1/attention/test_gdn_metadata_builder.py | 34 ++++++++++++++++++- vllm/v1/attention/backends/gdn_attn.py | 7 ++-- vllm/v1/worker/gpu_model_runner.py | 10 +++++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/tests/v1/attention/test_gdn_metadata_builder.py b/tests/v1/attention/test_gdn_metadata_builder.py index 6576a9bf33..221f933d89 100644 --- a/tests/v1/attention/test_gdn_metadata_builder.py +++ b/tests/v1/attention/test_gdn_metadata_builder.py @@ -16,6 +16,7 @@ from tests.v1.attention.utils import ( create_vllm_config, ) from vllm.config import SpeculativeConfig +from vllm.config.compilation import CUDAGraphMode from vllm.v1.attention.backends.gdn_attn import ( GDNAttentionMetadata, GDNAttentionMetadataBuilder, @@ -123,9 +124,15 @@ GDN_BUILD_TEST_CASES = { def _create_gdn_builder( num_speculative_tokens: int = 0, + full_cuda_graph: bool = False, ) -> GDNAttentionMetadataBuilder: """Create a GDNAttentionMetadataBuilder with minimal config.""" - vllm_config = create_vllm_config(block_size=BLOCK_SIZE) + vllm_config = create_vllm_config( + model_name="Qwen/Qwen3.5-0.8B", + block_size=BLOCK_SIZE, + ) + if full_cuda_graph: + vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE if num_speculative_tokens > 0: vllm_config.speculative_config = SpeculativeConfig( method="ngram", @@ -189,3 +196,28 @@ def test_has_initial_state_after_reclassification(): assert meta.has_initial_state is not None # req0 has context_lens = 65 - 1 = 64 > 0, so has_initial_state[0] = True assert meta.has_initial_state[0].item() is True + + +def test_full_cudagraph_spec_metadata_uses_request_count(): + """FULL cudagraph token padding must not pad request-indexed metadata.""" + num_speculative_tokens = 3 + builder = _create_gdn_builder( + num_speculative_tokens=num_speculative_tokens, + full_cuda_graph=True, + ) + batch = BatchSpec(seq_lens=[80, 96], query_lens=[4, 4]) + meta = _build(builder, batch, num_decode_draft_tokens=[3, 3]) + + assert meta.num_spec_decodes == batch.batch_size + assert meta.num_spec_decode_tokens == batch.compute_num_tokens() + assert meta.spec_state_indices_tensor is not None + assert meta.spec_state_indices_tensor.shape == ( + batch.batch_size, + num_speculative_tokens + 1, + ) + assert meta.spec_sequence_masks is not None + assert meta.spec_sequence_masks.shape == (batch.batch_size,) + assert meta.spec_query_start_loc is not None + assert meta.spec_query_start_loc.shape == (batch.batch_size + 1,) + assert meta.num_accepted_tokens is not None + assert meta.num_accepted_tokens.shape == (batch.batch_size,) diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index c615ab62c1..340a304030 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -410,9 +410,10 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] f"num_decodes: {num_decodes}, num_spec_decodes: {num_spec_decodes}" ) - # Prepare tensors for cudagraph - # Note: m.num_actual_tokens is already padded by the model runner for CUDAGraph - batch_size = m.num_actual_tokens + # Prepare per-request tensors for cudagraph. m.num_actual_tokens is + # token-padded for FULL graph replay, but the GDN state/query/accepted + # metadata below is indexed by request. + batch_size = m.num_reqs if ( self.use_full_cuda_graph diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 0b72870fc4..173e285ca8 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2035,7 +2035,13 @@ class GPUModelRunner( # Sync num_accepted_tokens from CPU (set by # _update_states_after_model_execute for hybrid models). - if self.num_accepted_tokens_event is not None: + # Skipped under async scheduling (non-align): the CPU copy races with + # the in-flight D2H copy and with input-batch row moves. + needs_cpu_accepted_counts = self.num_accepted_tokens_event is not None and not ( + self.use_async_scheduling and self.cache_config.mamba_cache_mode != "align" + ) + if needs_cpu_accepted_counts: + assert self.num_accepted_tokens_event is not None self.num_accepted_tokens_event.synchronize() # Async mode: condense() reordered indices, use prev_positions mapping if self.use_async_scheduling and prev_req_id_to_index: @@ -2058,6 +2064,8 @@ class GPUModelRunner( self.num_accepted_tokens.np[num_reqs:].fill(1) self.num_accepted_tokens.copy_to_gpu() else: + # Default to 1; update_num_computed_tokens_for_batch_change below + # corrects rows that had drafts from valid_sampled_token_count. self.num_accepted_tokens.np.fill(1) self.num_accepted_tokens.gpu.fill_(1)