[Model Runner V2] Reserve CUDA graph memory (#53306)
Signed-off-by: Anh Tran <anh.tran.3889@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Anh Tran <anh.tran.3889@gmail.com> Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
co-authored by
Anh Tran
anhtra3889
parent
8c2bbe00d5
commit
a4d70bef37
@@ -416,7 +416,10 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
)
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata_narrowed = attn_metadata_raw[self.prefix]
|
||||
attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata_narrowed is None:
|
||||
# Profile/warmup dummy runs skip mamba-family metadata.
|
||||
return
|
||||
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
|
||||
m = attn_metadata_narrowed
|
||||
has_initial_state = m.has_initial_state
|
||||
|
||||
@@ -305,7 +305,10 @@ class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
attn_metadata = attn_metadata[self.prefix] # type: ignore[assignment]
|
||||
attn_metadata = attn_metadata.get(self.prefix) # type: ignore[assignment]
|
||||
if attn_metadata is None:
|
||||
# Profile/warmup dummy runs skip mamba-family metadata.
|
||||
return
|
||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||
has_initial_state = attn_metadata.has_initial_state
|
||||
spec_query_start_loc = attn_metadata.spec_query_start_loc
|
||||
|
||||
@@ -1202,13 +1202,14 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata_raw is None:
|
||||
attn_metadata = None
|
||||
if isinstance(attn_metadata_raw, dict):
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is None:
|
||||
v_dim = core_attn_out.shape[-1] * core_attn_out.shape[-2]
|
||||
self._warmup_prefill_kernels(qkvz, v_dim)
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
|
||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||
|
||||
# The AITER fused reshape/conv kernel expects Qwen3-Next's interleaved
|
||||
@@ -1259,12 +1260,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata_raw is None:
|
||||
attn_metadata = None
|
||||
if isinstance(attn_metadata_raw, dict):
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is None:
|
||||
self._warmup_prefill_kernels(mixed_qkv, 0)
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
|
||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||
|
||||
if (
|
||||
@@ -1773,12 +1775,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
|
||||
if attn_metadata_raw is None:
|
||||
attn_metadata = None
|
||||
if isinstance(attn_metadata_raw, dict):
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is None:
|
||||
self._warmup_prefill_kernels(mixed_qkvz[:, :qkv_size], 0)
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
|
||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||
mixed_qkv, output_gate_flat = mixed_qkvz.split(
|
||||
[qkv_size, self.value_dim // self.tp_size], dim=-1
|
||||
@@ -1856,12 +1859,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
if attn_metadata_raw is None:
|
||||
attn_metadata = None
|
||||
if isinstance(attn_metadata_raw, dict):
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is None:
|
||||
self._warmup_prefill_kernels(mixed_qkv, 0)
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
|
||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||
if (
|
||||
self._can_use_fused_gdn_mtp_decode(attn_metadata)
|
||||
|
||||
@@ -542,7 +542,8 @@ class BailingMoELinearAttention(LinearAttention):
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, dict)
|
||||
attn_metadata = attn_metadata[self.prefix] # type: ignore
|
||||
attn_metadata = attn_metadata.get(self.prefix) # type: ignore
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, LinearAttentionMetadata)
|
||||
num_actual_tokens = (
|
||||
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
||||
|
||||
@@ -282,7 +282,8 @@ class MiniMaxText01LinearAttention(LinearAttention):
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix]
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, LinearAttentionMetadata)
|
||||
num_actual_tokens = (
|
||||
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
||||
|
||||
@@ -269,7 +269,8 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix]
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, Mamba1AttentionMetadata)
|
||||
query_start_loc_p = attn_metadata.query_start_loc_p
|
||||
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
|
||||
|
||||
@@ -710,7 +710,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix]
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, Mamba2AttentionMetadata)
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
# DS layout stores it that way directly; SD layout needs a
|
||||
|
||||
@@ -113,8 +113,10 @@ class ShortConv(MambaBase, PluggableLayer):
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix]
|
||||
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
assert attn_metadata is None or isinstance(
|
||||
attn_metadata, ShortConvAttentionMetadata
|
||||
)
|
||||
|
||||
BCx, _ = self.in_proj(hidden_states)
|
||||
B, C, x = BCx.chunk(3, dim=-1)
|
||||
@@ -225,7 +227,8 @@ class ShortConv(MambaBase, PluggableLayer):
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix]
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
|
||||
conv_state = (
|
||||
self.kv_cache[0]
|
||||
|
||||
Reference in New Issue
Block a user