[Model Runner V2] Reserve CUDA graph memory (#53306)

Signed-off-by: Anh Tran <anh.tran.3889@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Anh Tran <anh.tran.3889@gmail.com>
Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
Nick Hill
2026-08-23 23:11:19 -07:00
committed by GitHub
co-authored by Anh Tran anhtra3889
parent 8c2bbe00d5
commit a4d70bef37
13 changed files with 573 additions and 31 deletions
@@ -710,7 +710,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix]
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, Mamba2AttentionMetadata)
# conv_state must be (..., dim, width-1) for the conv kernels.
# DS layout stores it that way directly; SD layout needs a