[Bugfix][Model] Fix DiffusionGemma silently freezing attention mask under CUDA graph replay (#51994)
Signed-off-by: Fabian Joswig <fjosw@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
329e6eee3f
commit
1117140edb
@@ -909,9 +909,12 @@ class DiffusionGemmaModelState(ModelState):
|
|||||||
self._req_id_to_index: dict[str, int] = {}
|
self._req_id_to_index: dict[str, int] = {}
|
||||||
|
|
||||||
# Persistent buffer for per-request causal flags, updated in-place
|
# Persistent buffer for per-request causal flags, updated in-place
|
||||||
# so FULL CUDA graph replay sees the latest values.
|
# so FULL CUDA graph replay sees the latest values. Must be int32:
|
||||||
|
# FlashAttentionMetadataBuilder.build() rejects other dtypes, since
|
||||||
|
# an out-of-place cast there would detach the captured graph from
|
||||||
|
# this buffer and freeze replay at the capture-time snapshot.
|
||||||
self._causal_buf = torch.zeros(
|
self._causal_buf = torch.zeros(
|
||||||
self.max_num_reqs, dtype=torch.bool, device=device
|
self.max_num_reqs, dtype=torch.int32, device=device
|
||||||
)
|
)
|
||||||
|
|
||||||
# Persistent inputs_embeds buffer — required so FULL CUDA graph
|
# Persistent inputs_embeds buffer — required so FULL CUDA graph
|
||||||
@@ -1116,11 +1119,11 @@ class DiffusionGemmaModelState(ModelState):
|
|||||||
# Invariant: the sampler flips is_encoder_phase to False only after a
|
# Invariant: the sampler flips is_encoder_phase to False only after a
|
||||||
# request's FINAL prompt chunk, so a prompt spanning multiple chunks
|
# request's FINAL prompt chunk, so a prompt spanning multiple chunks
|
||||||
# (longer than the token budget) stays causal for every chunk.
|
# (longer than the token budget) stays causal for every chunk.
|
||||||
self._causal_buf[:actual_num_reqs] = self.diffusion_states.is_encoder_phase[
|
self._causal_buf[:actual_num_reqs].copy_(
|
||||||
slots
|
self.diffusion_states.is_encoder_phase[slots]
|
||||||
]
|
)
|
||||||
if actual_num_reqs < num_reqs:
|
if actual_num_reqs < num_reqs:
|
||||||
self._causal_buf[actual_num_reqs:num_reqs] = False
|
self._causal_buf[actual_num_reqs:num_reqs] = 0
|
||||||
causal: bool | torch.Tensor = self._causal_buf[:num_reqs]
|
causal: bool | torch.Tensor = self._causal_buf[:num_reqs]
|
||||||
|
|
||||||
return build_attn_metadata(
|
return build_attn_metadata(
|
||||||
|
|||||||
@@ -987,7 +987,11 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
|||||||
scheduler_metadata = self._store_scheduler_metadata(scheduler_metadata)
|
scheduler_metadata = self._store_scheduler_metadata(scheduler_metadata)
|
||||||
|
|
||||||
if isinstance(causal, torch.Tensor) and causal.dtype != torch.int32:
|
if isinstance(causal, torch.Tensor) and causal.dtype != torch.int32:
|
||||||
causal = causal.to(torch.int32)
|
raise ValueError(
|
||||||
|
f"Per-request causal tensor must be int32, got {causal.dtype}. "
|
||||||
|
"Casting here would allocate a fresh tensor each build() and "
|
||||||
|
"break FULL CUDA graph replay; allocate the buffer as int32."
|
||||||
|
)
|
||||||
|
|
||||||
attn_metadata = FlashAttentionMetadata(
|
attn_metadata = FlashAttentionMetadata(
|
||||||
num_actual_tokens=num_actual_tokens,
|
num_actual_tokens=num_actual_tokens,
|
||||||
|
|||||||
@@ -217,7 +217,7 @@ def kernel_unified_attention(
|
|||||||
SLIDING_WINDOW: tl.constexpr, # int
|
SLIDING_WINDOW: tl.constexpr, # int
|
||||||
USE_CAUSAL: tl.constexpr, # bool
|
USE_CAUSAL: tl.constexpr, # bool
|
||||||
USE_PER_SEQ_CAUSAL: tl.constexpr, # bool
|
USE_PER_SEQ_CAUSAL: tl.constexpr, # bool
|
||||||
per_seq_causal_ptr, # [num_seqs] bool, or None
|
per_seq_causal_ptr, # [num_seqs] int32, or None
|
||||||
USE_MM_PREFIX: tl.constexpr, # bool
|
USE_MM_PREFIX: tl.constexpr, # bool
|
||||||
MAX_MM_RANGES: tl.constexpr, # int
|
MAX_MM_RANGES: tl.constexpr, # int
|
||||||
mm_prefix_range_ptr,
|
mm_prefix_range_ptr,
|
||||||
|
|||||||
Reference in New Issue
Block a user