[BreakableCG] Add eager break for Mamba-family ops (#57881)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
a205fb3177
commit
ff440c7805
@@ -4,6 +4,7 @@ import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
@@ -524,6 +525,7 @@ def _make_fused_conv1d_weight_loader(dims, tp_size, tp_rank):
|
||||
return weight_loader
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def olmo_hybrid_gdn_full_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
@@ -325,6 +326,7 @@ class MiniMaxText01LinearAttention(LinearAttention):
|
||||
output[:num_actual_tokens], _ = self.out_proj(hidden)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def linear_attention(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
@@ -465,6 +466,7 @@ def split_batch_to_prefill_and_decode(
|
||||
)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def mamba_mixer(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -8,6 +8,7 @@ from dataclasses import replace
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.distributed import (
|
||||
@@ -1214,6 +1215,7 @@ def share_replayssm_ring_trackers(
|
||||
replayssm_mixers[last_layer_name]._updates_replayssm_trackers = True
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def mamba_mixer2(
|
||||
projected_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
@@ -360,6 +361,7 @@ class ShortConv(MambaBase, PluggableLayer):
|
||||
return MambaAttentionBackendEnum.SHORT_CONV
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def short_conv(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user