[compile] Allow strings in custom ops without regressing compilation times (#38123)

Signed-off-by: Richard Zou <zou3519@gmail.com>
This commit is contained in:
Richard Zou
2026-04-10 07:26:37 +00:00
committed by GitHub
parent 447ce22212
commit f44afef6d6
16 changed files with 564 additions and 167 deletions
@@ -56,7 +56,12 @@ from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
@@ -568,7 +573,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
b,
a,
core_attn_out,
self.prefix,
_encode_layer_name(self.prefix),
)
# ============================================================
@@ -1084,13 +1089,14 @@ def gdn_attention_core(
b: torch.Tensor,
a: torch.Tensor,
core_attn_out: torch.Tensor,
layer_name: str,
layer_name: LayerNameType,
) -> None:
"""
Custom op for the core attention computation.
Only handles the convolution + recurrent attention part.
Input/output projections are handled outside this op.
"""
layer_name = _resolve_layer_name(layer_name)
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._forward_core(
@@ -1106,7 +1112,7 @@ def gdn_attention_core_fake(
b: torch.Tensor,
a: torch.Tensor,
core_attn_out: torch.Tensor,
layer_name: str,
layer_name: LayerNameType,
) -> None:
"""Fake implementation for torch.compile."""
return
@@ -36,7 +36,12 @@ from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionMetadata
@@ -228,7 +233,7 @@ class MambaMixer(MambaBase, PluggableLayer):
torch.ops.vllm.mamba_mixer(
hidden_states,
output,
self.prefix,
_encode_layer_name(self.prefix),
)
def forward_impl(self, hidden_states: torch.Tensor, output: torch.Tensor):
@@ -515,8 +520,9 @@ def split_batch_to_prefill_and_decode(
def mamba_mixer(
hidden_states: torch.Tensor,
output: torch.Tensor,
layer_name: str,
layer_name: LayerNameType,
) -> None:
layer_name = _resolve_layer_name(layer_name)
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self.forward_impl(hidden_states=hidden_states, output=output)
@@ -525,7 +531,7 @@ def mamba_mixer(
def mamba_mixer_fake(
hidden_states: torch.Tensor,
output: torch.Tensor,
layer_name: str,
layer_name: LayerNameType,
) -> None:
return
@@ -44,7 +44,12 @@ from vllm.model_executor.model_loader.weight_utils import (
from vllm.model_executor.parameter import BasevLLMParameter
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
@@ -536,7 +541,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
torch.ops.vllm.mamba_mixer2(
projected_states,
ssm_output,
self.prefix,
_encode_layer_name(self.prefix),
)
# 4. gated MLP
@@ -927,8 +932,9 @@ class MambaMixer2(MambaBase, PluggableLayer):
def mamba_mixer2(
projected_states: torch.Tensor,
output: torch.Tensor,
layer_name: str,
layer_name: LayerNameType,
) -> None:
layer_name = _resolve_layer_name(layer_name)
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self.conv_ssm_forward(projected_states=projected_states, output=output)
@@ -937,7 +943,7 @@ def mamba_mixer2(
def mamba_mixer2_fake(
projected_states: torch.Tensor,
output: torch.Tensor,
layer_name: str,
layer_name: LayerNameType,
) -> None:
return