[compile] Allow strings in custom ops without regressing compilation times (#38123)
Signed-off-by: Richard Zou <zou3519@gmail.com>
This commit is contained in:
@@ -36,7 +36,12 @@ from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionMetadata
|
||||
|
||||
|
||||
@@ -228,7 +233,7 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
torch.ops.vllm.mamba_mixer(
|
||||
hidden_states,
|
||||
output,
|
||||
self.prefix,
|
||||
_encode_layer_name(self.prefix),
|
||||
)
|
||||
|
||||
def forward_impl(self, hidden_states: torch.Tensor, output: torch.Tensor):
|
||||
@@ -515,8 +520,9 @@ def split_batch_to_prefill_and_decode(
|
||||
def mamba_mixer(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self.forward_impl(hidden_states=hidden_states, output=output)
|
||||
@@ -525,7 +531,7 @@ def mamba_mixer(
|
||||
def mamba_mixer_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user