Tuning script and configs for Triton Mamba SSU kernel (#43083)

Signed-off-by: Banani Ghosh <bg2502@nyu.edu>
Signed-off-by: Daniel Serebrenik <daserebrenik@nvidia.com>
Co-authored-by: Banani Ghosh <bg2502@nyu.edu>
This commit is contained in:
danisereb
2026-05-24 20:12:44 +03:00
committed by GitHub
co-authored by Banani Ghosh
parent 1806d1adfc
commit d56285c747
13 changed files with 1770 additions and 91 deletions
@@ -0,0 +1,87 @@
{
"triton_version": "3.6.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 4
},
"16": {
"BLOCK_SIZE_M": 8,
"num_warps": 8
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"128": {
"BLOCK_SIZE_M": 32,
"num_warps": 4
},
"256": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"512": {
"BLOCK_SIZE_M": 32,
"num_warps": 4
},
"1024": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"2048": {
"BLOCK_SIZE_M": 32,
"num_warps": 2
},
"4096": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"8192": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"12288": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"16384": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"24576": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"32768": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"49152": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"65536": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"98304": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"131072": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"196608": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"262144": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
}
}
@@ -0,0 +1,87 @@
{
"triton_version": "3.6.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 4
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"12288": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"24576": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"49152": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"98304": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"196608": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"262144": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,87 @@
{
"triton_version": "3.6.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 4
},
"16": {
"BLOCK_SIZE_M": 8,
"num_warps": 8
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"128": {
"BLOCK_SIZE_M": 32,
"num_warps": 4
},
"256": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"512": {
"BLOCK_SIZE_M": 32,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"2048": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"4096": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"8192": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"12288": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"16384": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"24576": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"32768": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"49152": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"65536": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"98304": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"131072": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"196608": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"262144": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
}
}
@@ -0,0 +1,87 @@
{
"triton_version": "3.6.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"12288": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"24576": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"49152": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"98304": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"196608": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"262144": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,87 @@
{
"triton_version": "3.6.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"128": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"256": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"512": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"1024": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"4096": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"8192": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"12288": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"16384": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"24576": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"32768": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"49152": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"65536": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"98304": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"131072": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"196608": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"262144": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
}
}
@@ -0,0 +1,87 @@
{
"triton_version": "3.6.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"12288": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"16384": {
"BLOCK_SIZE_M": 16,
"num_warps": 1
},
"24576": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"32768": {
"BLOCK_SIZE_M": 16,
"num_warps": 1
},
"49152": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"65536": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"98304": {
"BLOCK_SIZE_M": 16,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"196608": {
"BLOCK_SIZE_M": 16,
"num_warps": 1
},
"262144": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
}
}
+179 -18
View File
@@ -4,16 +4,190 @@
# Copyright (c) 2024, Tri Dao, Albert Gu.
# Adapted from https://github.com/state-spaces/mamba/blob/v2.2.4/mamba_ssm/ops/triton/selective_state_update.py
import functools
import json
import os
from contextlib import contextmanager
from typing import Any
import torch
from packaging import version
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.ops.triton_helpers import fast_exp
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON, tl, triton
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
logger = init_logger(__name__)
TRITON3 = HAS_TRITON and (version.parse(triton.__version__) >= version.parse("3.0.0"))
# ---------------------------------------------------------------------------
# JSON config loading
# ---------------------------------------------------------------------------
_CONFIGS_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "configs", "selective_state_update"
)
def get_ssm_config_file_name(
headdim: int, dstate: int, cache_dtype: str, device_name: str
) -> str:
"""Return the JSON filename for the given kernel shape.
Layout: ``configs/selective_state_update/
headdim=<H>,dstate=<D>,device_name=<dev>,cache_dtype=<dt>.json``.
"""
return (
f"headdim={headdim},dstate={dstate},"
f"device_name={device_name},cache_dtype={cache_dtype}.json"
)
def get_ssm_device_name() -> str:
return current_platform.get_device_name().replace(" ", "_")
def _canonical_cache_dtype(cache_dtype: str) -> str:
"""Canonical key for config lookup. bf16 and fp16 share the same tuned
configs because the kernel only sees bit width when accessing state."""
return "float16" if cache_dtype == "bfloat16" else cache_dtype
@functools.cache
def get_ssm_configs(
headdim: int, dstate: int, cache_dtype: str
) -> dict[int, Any] | None:
"""
Return tuned (BLOCK_SIZE_M, num_warps) configs for *selective_state_update*
keyed by ``effective_batch = batch * nheads``, or ``None`` if no config
file is found for the (headdim, dstate, cache_dtype, device) combination.
They can be generated with:
benchmarks/kernels/benchmark_selective_state_update.py --save-configs
"""
cache_dtype = _canonical_cache_dtype(cache_dtype)
device_name = get_ssm_device_name()
json_file_name = get_ssm_config_file_name(headdim, dstate, cache_dtype, device_name)
config_file_paths: list[str] = []
# User-supplied override
user_defined_config_folder = envs.VLLM_TUNED_CONFIG_FOLDER
if user_defined_config_folder is not None:
config_file_paths.append(
os.path.join(user_defined_config_folder, json_file_name)
)
# Bundled default
config_file_paths.append(os.path.join(_CONFIGS_DIR, json_file_name))
for path in config_file_paths:
if os.path.exists(path):
with open(path) as f:
logger.info_once(
"Using SSM config from %s for selective_state_update.",
path,
scope="global",
)
raw = json.load(f)
if isinstance(raw, dict):
# triton_version included in the config file only for reference
raw.pop("triton_version", None)
return {int(k): v for k, v in raw.items() if k.isdigit()}
logger.warning_once(
"Using default Mamba SSU config. Performance might be sub-optimal! "
"Config file not found at %s",
", ".join(config_file_paths),
)
return None
def _get_default_ssm_launch_config(
dstate: int,
is_blackwell: bool,
) -> tuple[int, int]:
"""Hard-coded fallback heuristic used when no tuned config is available."""
BLOCK_SIZE_M, num_warps = 4, 8
if dstate <= 16:
BLOCK_SIZE_M, num_warps = 32, 4
elif dstate <= 32:
BLOCK_SIZE_M, num_warps = 16, 4
elif dstate <= 64:
BLOCK_SIZE_M, num_warps = 8, 4
else:
if is_blackwell:
BLOCK_SIZE_M, num_warps = 32, 8
elif dstate <= 128:
BLOCK_SIZE_M, num_warps = 4, 4
return BLOCK_SIZE_M, num_warps
@functools.cache
def _try_get_optimal_ssm_config_cached(
headdim: int,
dstate: int,
batch: int,
nheads: int,
cache_dtype: str,
is_blackwell: bool,
) -> tuple[int, int]:
"""Cached resolution. See :func:`try_get_optimal_ssm_config`."""
effective_batch = batch * nheads
configs = get_ssm_configs(headdim, dstate, cache_dtype)
if configs:
# Pick the closest effective_batch in the tuned grid (MoE strategy).
closest = min(configs.keys(), key=lambda x: abs(x - effective_batch))
cfg = configs[closest]
return cfg["BLOCK_SIZE_M"], cfg["num_warps"]
return _get_default_ssm_launch_config(dstate, is_blackwell)
# Override hook for benchmarks/tests, see `override_ssm_config`.
_ssm_config_override: tuple[int, int] | None = None
@contextmanager
def override_ssm_config(config: tuple[int, int]):
"""Pin ``try_get_optimal_ssm_config`` to ``config`` for the duration of
the context. Used by the tuning benchmark to time specific configs."""
global _ssm_config_override
prev = _ssm_config_override
_ssm_config_override = config
try:
yield
finally:
_ssm_config_override = prev
def try_get_optimal_ssm_config(
headdim: int,
dstate: int,
batch: int,
nheads: int,
cache_dtype: str,
is_blackwell: bool,
) -> tuple[int, int]:
"""Return (BLOCK_SIZE_M, num_warps) for the given kernel shape.
Tuning is keyed on ``effective_batch = batch * nheads`` (the kernel grid
scales with the product), so configs transfer across (model, TP) combos
sharing ``(headdim, dstate, cache_dtype)``.
"""
if _ssm_config_override is not None:
return _ssm_config_override
return _try_get_optimal_ssm_config_cached(
headdim, dstate, batch, nheads, cache_dtype, is_blackwell
)
if TRITON3:
@triton.jit
@@ -440,24 +614,11 @@ def selective_state_update(
else (0, 0)
)
# We don't want autotune since it will overwrite the state.
# We instead tune by hand based on dstate.
# Default
BLOCK_SIZE_M, num_warps = 4, 8
if dstate <= 16:
BLOCK_SIZE_M, num_warps = 32, 4
elif dstate <= 32:
BLOCK_SIZE_M, num_warps = 16, 4
elif dstate <= 64:
BLOCK_SIZE_M, num_warps = 8, 4
else:
# dstate > 64
if is_blackwell:
# Optimized for B200 with dstate>64
BLOCK_SIZE_M, num_warps = 32, 8
elif dstate <= 128:
BLOCK_SIZE_M, num_warps = 4, 4
# Load from JSON config if available, otherwise fall back to heuristic.
cache_dtype = str(state.dtype).removeprefix("torch.")
BLOCK_SIZE_M, num_warps = try_get_optimal_ssm_config(
dim, dstate, N, nheads, cache_dtype, is_blackwell
)
tie_hdim = (
A.stride(-1) == 0