From 1117140edbaf3d1e1f73b01b09cde3cf98ec24ff Mon Sep 17 00:00:00 2001 From: Fabian Joswig Date: Wed, 30 Sep 2026 07:34:26 +0200 Subject: [PATCH] [Bugfix][Model] Fix DiffusionGemma silently freezing attention mask under CUDA graph replay (#51994) Signed-off-by: Fabian Joswig Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/model_executor/models/diffusion_gemma.py | 15 +++++++++------ vllm/v1/attention/backends/flash_attn.py | 6 +++++- vllm/v1/attention/ops/triton_unified_attention.py | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index 8f26edf9e4..404c154b54 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -909,9 +909,12 @@ class DiffusionGemmaModelState(ModelState): self._req_id_to_index: dict[str, int] = {} # Persistent buffer for per-request causal flags, updated in-place - # so FULL CUDA graph replay sees the latest values. + # so FULL CUDA graph replay sees the latest values. Must be int32: + # FlashAttentionMetadataBuilder.build() rejects other dtypes, since + # an out-of-place cast there would detach the captured graph from + # this buffer and freeze replay at the capture-time snapshot. self._causal_buf = torch.zeros( - self.max_num_reqs, dtype=torch.bool, device=device + self.max_num_reqs, dtype=torch.int32, device=device ) # Persistent inputs_embeds buffer — required so FULL CUDA graph @@ -1116,11 +1119,11 @@ class DiffusionGemmaModelState(ModelState): # Invariant: the sampler flips is_encoder_phase to False only after a # request's FINAL prompt chunk, so a prompt spanning multiple chunks # (longer than the token budget) stays causal for every chunk. - self._causal_buf[:actual_num_reqs] = self.diffusion_states.is_encoder_phase[ - slots - ] + self._causal_buf[:actual_num_reqs].copy_( + self.diffusion_states.is_encoder_phase[slots] + ) if actual_num_reqs < num_reqs: - self._causal_buf[actual_num_reqs:num_reqs] = False + self._causal_buf[actual_num_reqs:num_reqs] = 0 causal: bool | torch.Tensor = self._causal_buf[:num_reqs] return build_attn_metadata( diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 6035678394..6179b27bb6 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -987,7 +987,11 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad scheduler_metadata = self._store_scheduler_metadata(scheduler_metadata) if isinstance(causal, torch.Tensor) and causal.dtype != torch.int32: - causal = causal.to(torch.int32) + raise ValueError( + f"Per-request causal tensor must be int32, got {causal.dtype}. " + "Casting here would allocate a fresh tensor each build() and " + "break FULL CUDA graph replay; allocate the buffer as int32." + ) attn_metadata = FlashAttentionMetadata( num_actual_tokens=num_actual_tokens, diff --git a/vllm/v1/attention/ops/triton_unified_attention.py b/vllm/v1/attention/ops/triton_unified_attention.py index b0231dd0c0..4d73e49e91 100644 --- a/vllm/v1/attention/ops/triton_unified_attention.py +++ b/vllm/v1/attention/ops/triton_unified_attention.py @@ -217,7 +217,7 @@ def kernel_unified_attention( SLIDING_WINDOW: tl.constexpr, # int USE_CAUSAL: tl.constexpr, # bool USE_PER_SEQ_CAUSAL: tl.constexpr, # bool - per_seq_causal_ptr, # [num_seqs] bool, or None + per_seq_causal_ptr, # [num_seqs] int32, or None USE_MM_PREFIX: tl.constexpr, # bool MAX_MM_RANGES: tl.constexpr, # int mm_prefix_range_ptr,