[Model Runner V2] Reserve CUDA graph memory (#53306)

Signed-off-by: Anh Tran <anh.tran.3889@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Anh Tran <anh.tran.3889@gmail.com>
Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
Nick Hill
2026-08-23 23:11:19 -07:00
committed by GitHub
co-authored by Anh Tran anhtra3889
parent 8c2bbe00d5
commit a4d70bef37
13 changed files with 573 additions and 31 deletions
@@ -416,7 +416,10 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
)
assert isinstance(attn_metadata_raw, dict)
attn_metadata_narrowed = attn_metadata_raw[self.prefix]
attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
if attn_metadata_narrowed is None:
# Profile/warmup dummy runs skip mamba-family metadata.
return
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
m = attn_metadata_narrowed
has_initial_state = m.has_initial_state
@@ -305,7 +305,10 @@ class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
return
assert isinstance(attn_metadata, dict)
attn_metadata = attn_metadata[self.prefix] # type: ignore[assignment]
attn_metadata = attn_metadata.get(self.prefix) # type: ignore[assignment]
if attn_metadata is None:
# Profile/warmup dummy runs skip mamba-family metadata.
return
assert isinstance(attn_metadata, GDNAttentionMetadata)
has_initial_state = attn_metadata.has_initial_state
spec_query_start_loc = attn_metadata.spec_query_start_loc
@@ -1202,13 +1202,14 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
if attn_metadata_raw is None:
attn_metadata = None
if isinstance(attn_metadata_raw, dict):
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is None:
v_dim = core_attn_out.shape[-1] * core_attn_out.shape[-2]
self._warmup_prefill_kernels(qkvz, v_dim)
return
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
assert isinstance(attn_metadata, GDNAttentionMetadata)
# The AITER fused reshape/conv kernel expects Qwen3-Next's interleaved
@@ -1259,12 +1260,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
if attn_metadata_raw is None:
attn_metadata = None
if isinstance(attn_metadata_raw, dict):
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is None:
self._warmup_prefill_kernels(mixed_qkv, 0)
return
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
assert isinstance(attn_metadata, GDNAttentionMetadata)
if (
@@ -1773,12 +1775,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
if attn_metadata_raw is None:
attn_metadata = None
if isinstance(attn_metadata_raw, dict):
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is None:
self._warmup_prefill_kernels(mixed_qkvz[:, :qkv_size], 0)
return
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
assert isinstance(attn_metadata, GDNAttentionMetadata)
mixed_qkv, output_gate_flat = mixed_qkvz.split(
[qkv_size, self.value_dim // self.tp_size], dim=-1
@@ -1856,12 +1859,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
if attn_metadata_raw is None:
attn_metadata = None
if isinstance(attn_metadata_raw, dict):
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is None:
self._warmup_prefill_kernels(mixed_qkv, 0)
return
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
assert isinstance(attn_metadata, GDNAttentionMetadata)
if (
self._can_use_fused_gdn_mtp_decode(attn_metadata)
@@ -542,7 +542,8 @@ class BailingMoELinearAttention(LinearAttention):
attn_metadata = forward_context.attn_metadata
if attn_metadata is not None:
assert isinstance(attn_metadata, dict)
attn_metadata = attn_metadata[self.prefix] # type: ignore
attn_metadata = attn_metadata.get(self.prefix) # type: ignore
if attn_metadata is not None:
assert isinstance(attn_metadata, LinearAttentionMetadata)
num_actual_tokens = (
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
@@ -282,7 +282,8 @@ class MiniMaxText01LinearAttention(LinearAttention):
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix]
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, LinearAttentionMetadata)
num_actual_tokens = (
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
@@ -269,7 +269,8 @@ class MambaMixer(MambaBase, PluggableLayer):
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix]
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, Mamba1AttentionMetadata)
query_start_loc_p = attn_metadata.query_start_loc_p
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
@@ -710,7 +710,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix]
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, Mamba2AttentionMetadata)
# conv_state must be (..., dim, width-1) for the conv kernels.
# DS layout stores it that way directly; SD layout needs a
@@ -113,8 +113,10 @@ class ShortConv(MambaBase, PluggableLayer):
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix]
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
attn_metadata = attn_metadata_raw.get(self.prefix)
assert attn_metadata is None or isinstance(
attn_metadata, ShortConvAttentionMetadata
)
BCx, _ = self.in_proj(hidden_states)
B, C, x = BCx.chunk(3, dim=-1)
@@ -225,7 +227,8 @@ class ShortConv(MambaBase, PluggableLayer):
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix]
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
conv_state = (
self.kv_cache[0]