[Bugfix][Model] Fix DiffusionGemma silently freezing attention mask under CUDA graph replay (#51994)

Signed-off-by: Fabian Joswig <fjosw@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Fabian Joswig
2026-09-30 05:34:26 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 329e6eee3f
commit 1117140edb
3 changed files with 15 additions and 8 deletions
@@ -909,9 +909,12 @@ class DiffusionGemmaModelState(ModelState):
self._req_id_to_index: dict[str, int] = {}
# Persistent buffer for per-request causal flags, updated in-place
# so FULL CUDA graph replay sees the latest values.
# so FULL CUDA graph replay sees the latest values. Must be int32:
# FlashAttentionMetadataBuilder.build() rejects other dtypes, since
# an out-of-place cast there would detach the captured graph from
# this buffer and freeze replay at the capture-time snapshot.
self._causal_buf = torch.zeros(
self.max_num_reqs, dtype=torch.bool, device=device
self.max_num_reqs, dtype=torch.int32, device=device
)
# Persistent inputs_embeds buffer — required so FULL CUDA graph
@@ -1116,11 +1119,11 @@ class DiffusionGemmaModelState(ModelState):
# Invariant: the sampler flips is_encoder_phase to False only after a
# request's FINAL prompt chunk, so a prompt spanning multiple chunks
# (longer than the token budget) stays causal for every chunk.
self._causal_buf[:actual_num_reqs] = self.diffusion_states.is_encoder_phase[
slots
]
self._causal_buf[:actual_num_reqs].copy_(
self.diffusion_states.is_encoder_phase[slots]
)
if actual_num_reqs < num_reqs:
self._causal_buf[actual_num_reqs:num_reqs] = False
self._causal_buf[actual_num_reqs:num_reqs] = 0
causal: bool | torch.Tensor = self._causal_buf[:num_reqs]
return build_attn_metadata(