From ff440c780544abd63bc82d57557adedf15647bfd Mon Sep 17 00:00:00 2001 From: liuzhenwei Date: Sat, 10 Oct 2026 14:06:57 +0800 Subject: [PATCH] [BreakableCG] Add eager break for Mamba-family ops (#57881) Signed-off-by: zhenwei-intel Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/model_executor/layers/mamba/gdn/olmo_gdn_linear_attn.py | 2 ++ vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py | 2 ++ vllm/model_executor/layers/mamba/mamba_mixer.py | 2 ++ vllm/model_executor/layers/mamba/mamba_mixer2.py | 2 ++ vllm/model_executor/layers/mamba/short_conv.py | 2 ++ 5 files changed, 10 insertions(+) diff --git a/vllm/model_executor/layers/mamba/gdn/olmo_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/olmo_gdn_linear_attn.py index c8f5a9ba64..a7bbac74f6 100644 --- a/vllm/model_executor/layers/mamba/gdn/olmo_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/olmo_gdn_linear_attn.py @@ -4,6 +4,7 @@ import torch from einops import rearrange from torch import nn +from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import ( VllmConfig, get_current_vllm_config, @@ -524,6 +525,7 @@ def _make_fused_conv1d_weight_loader(dims, tp_size, tp_rank): return weight_loader +@eager_break_during_capture def olmo_hybrid_gdn_full_forward( hidden_states: torch.Tensor, output: torch.Tensor, diff --git a/vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py b/vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py index e7618b29b7..2d88f17e6a 100644 --- a/vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py +++ b/vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py @@ -8,6 +8,7 @@ import torch import torch.nn.functional as F from einops import rearrange +from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import get_current_vllm_config from vllm.forward_context import ForwardContext, get_forward_context from vllm.model_executor.custom_op import PluggableLayer @@ -325,6 +326,7 @@ class MiniMaxText01LinearAttention(LinearAttention): output[:num_actual_tokens], _ = self.out_proj(hidden) +@eager_break_during_capture def linear_attention( hidden_states: torch.Tensor, output: torch.Tensor, diff --git a/vllm/model_executor/layers/mamba/mamba_mixer.py b/vllm/model_executor/layers/mamba/mamba_mixer.py index 8f3b8a2638..74cac8b630 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer.py @@ -7,6 +7,7 @@ import torch from torch import nn from torch.nn.parameter import Parameter +from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config from vllm.distributed.parallel_state import ( get_tensor_model_parallel_rank, @@ -465,6 +466,7 @@ def split_batch_to_prefill_and_decode( ) +@eager_break_during_capture def mamba_mixer( hidden_states: torch.Tensor, output: torch.Tensor, diff --git a/vllm/model_executor/layers/mamba/mamba_mixer2.py b/vllm/model_executor/layers/mamba/mamba_mixer2.py index 3753153fa3..bfa1103bb3 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer2.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer2.py @@ -8,6 +8,7 @@ from dataclasses import replace import torch from torch import nn +from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config from vllm.config.mamba import MambaBackendEnum from vllm.distributed import ( @@ -1214,6 +1215,7 @@ def share_replayssm_ring_trackers( replayssm_mixers[last_layer_name]._updates_replayssm_trackers = True +@eager_break_during_capture def mamba_mixer2( projected_states: torch.Tensor, output: torch.Tensor, diff --git a/vllm/model_executor/layers/mamba/short_conv.py b/vllm/model_executor/layers/mamba/short_conv.py index 37aef2b5bd..7a695dfd31 100644 --- a/vllm/model_executor/layers/mamba/short_conv.py +++ b/vllm/model_executor/layers/mamba/short_conv.py @@ -4,6 +4,7 @@ import torch +from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config from vllm.distributed import get_tensor_model_parallel_world_size from vllm.forward_context import ForwardContext, get_forward_context @@ -360,6 +361,7 @@ class ShortConv(MambaBase, PluggableLayer): return MambaAttentionBackendEnum.SHORT_CONV +@eager_break_during_capture def short_conv( hidden_states: torch.Tensor, output: torch.Tensor,