Gate SSU dispatch setup (#40039)

Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
roikoren755
2026-04-16 13:06:01 -07:00
committed by GitHub
parent adf9bb3c57
commit b897f00c9c
4 changed files with 73 additions and 10 deletions
+48 -4
View File
@@ -13,6 +13,11 @@ from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
selective_state_update,
)
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
)
try:
import flashinfer.mamba # noqa: F401
@@ -22,22 +27,40 @@ except ImportError:
HAS_FLASHINFER = False
def _kv_cache_config_with_ssu(mamba_type: str = "mamba2") -> KVCacheConfig:
spec = MambaSpec(
block_size=16,
shapes=((16, 64),),
dtypes=(torch.float16,),
mamba_type=mamba_type,
)
return KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(layer_names=["l0"], kv_cache_spec=spec)],
)
def test_default_backend_is_triton():
initialize_mamba_ssu_backend(MambaConfig())
initialize_mamba_ssu_backend(MambaConfig(), _kv_cache_config_with_ssu())
backend = get_mamba_ssu_backend()
assert isinstance(backend, TritonSSUBackend)
assert backend.name == "triton"
def test_explicit_triton_backend():
initialize_mamba_ssu_backend(MambaConfig(backend=MambaBackendEnum.TRITON))
initialize_mamba_ssu_backend(
MambaConfig(backend=MambaBackendEnum.TRITON), _kv_cache_config_with_ssu()
)
backend = get_mamba_ssu_backend()
assert isinstance(backend, TritonSSUBackend)
@pytest.mark.skipif(not HAS_FLASHINFER, reason="flashinfer not installed")
def test_flashinfer_backend_init():
initialize_mamba_ssu_backend(MambaConfig(backend=MambaBackendEnum.FLASHINFER))
initialize_mamba_ssu_backend(
MambaConfig(backend=MambaBackendEnum.FLASHINFER), _kv_cache_config_with_ssu()
)
backend = get_mamba_ssu_backend()
assert isinstance(backend, FlashInferSSUBackend)
assert backend.name == "flashinfer"
@@ -53,6 +76,25 @@ def test_uninitialized_backend_raises():
mod._mamba_ssu_backend = old
@pytest.mark.parametrize(
"mamba_type", ["linear_attention", "gdn_attention", "short_conv"]
)
def test_init_is_noop_for_non_ssu_mamba_type(mamba_type):
import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod
old = mod._mamba_ssu_backend
mod._mamba_ssu_backend = None
try:
initialize_mamba_ssu_backend(
MambaConfig(), _kv_cache_config_with_ssu(mamba_type)
)
assert mod._mamba_ssu_backend is None
with pytest.raises(RuntimeError, match="not been initialized"):
get_mamba_ssu_backend()
finally:
mod._mamba_ssu_backend = old
@pytest.mark.skipif(HAS_FLASHINFER, reason="flashinfer is installed")
def test_flashinfer_import_error():
with pytest.raises(ImportError, match="FlashInfer is required"):
@@ -61,7 +103,9 @@ def test_flashinfer_import_error():
def test_triton_basic_call():
set_random_seed(0)
initialize_mamba_ssu_backend(MambaConfig(backend=MambaBackendEnum.TRITON))
initialize_mamba_ssu_backend(
MambaConfig(backend=MambaBackendEnum.TRITON), _kv_cache_config_with_ssu()
)
device = "cuda"
batch_size = 2
dim = 64