Tuning script and configs for Triton Mamba SSU kernel (#43083)
Signed-off-by: Banani Ghosh <bg2502@nyu.edu> Signed-off-by: Daniel Serebrenik <daserebrenik@nvidia.com> Co-authored-by: Banani Ghosh <bg2502@nyu.edu>
This commit is contained in:
co-authored by
Banani Ghosh
parent
1806d1adfc
commit
d56285c747
+87
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"triton_version": "3.6.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 4
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"12288": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"24576": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"49152": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"98304": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"196608": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"262144": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
}
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"triton_version": "3.6.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"12288": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"24576": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"49152": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"98304": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"196608": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"262144": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"triton_version": "3.6.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"12288": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"24576": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"49152": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"98304": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"196608": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"262144": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
}
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"triton_version": "3.6.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"12288": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"24576": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"49152": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"98304": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"196608": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"262144": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"triton_version": "3.6.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"12288": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"24576": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"49152": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"98304": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"196608": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"262144": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
}
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"triton_version": "3.6.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"12288": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 1
|
||||
},
|
||||
"24576": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 1
|
||||
},
|
||||
"49152": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"98304": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"196608": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 1
|
||||
},
|
||||
"262144": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
}
|
||||
}
|
||||
@@ -4,16 +4,190 @@
|
||||
# Copyright (c) 2024, Tri Dao, Albert Gu.
|
||||
# Adapted from https://github.com/state-spaces/mamba/blob/v2.2.4/mamba_ssm/ops/triton/selective_state_update.py
|
||||
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from packaging import version
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.mamba.ops.triton_helpers import fast_exp
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
TRITON3 = HAS_TRITON and (version.parse(triton.__version__) >= version.parse("3.0.0"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON config loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_CONFIGS_DIR = os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)), "configs", "selective_state_update"
|
||||
)
|
||||
|
||||
|
||||
def get_ssm_config_file_name(
|
||||
headdim: int, dstate: int, cache_dtype: str, device_name: str
|
||||
) -> str:
|
||||
"""Return the JSON filename for the given kernel shape.
|
||||
|
||||
Layout: ``configs/selective_state_update/
|
||||
headdim=<H>,dstate=<D>,device_name=<dev>,cache_dtype=<dt>.json``.
|
||||
"""
|
||||
return (
|
||||
f"headdim={headdim},dstate={dstate},"
|
||||
f"device_name={device_name},cache_dtype={cache_dtype}.json"
|
||||
)
|
||||
|
||||
|
||||
def get_ssm_device_name() -> str:
|
||||
return current_platform.get_device_name().replace(" ", "_")
|
||||
|
||||
|
||||
def _canonical_cache_dtype(cache_dtype: str) -> str:
|
||||
"""Canonical key for config lookup. bf16 and fp16 share the same tuned
|
||||
configs because the kernel only sees bit width when accessing state."""
|
||||
return "float16" if cache_dtype == "bfloat16" else cache_dtype
|
||||
|
||||
|
||||
@functools.cache
|
||||
def get_ssm_configs(
|
||||
headdim: int, dstate: int, cache_dtype: str
|
||||
) -> dict[int, Any] | None:
|
||||
"""
|
||||
Return tuned (BLOCK_SIZE_M, num_warps) configs for *selective_state_update*
|
||||
keyed by ``effective_batch = batch * nheads``, or ``None`` if no config
|
||||
file is found for the (headdim, dstate, cache_dtype, device) combination.
|
||||
|
||||
They can be generated with:
|
||||
benchmarks/kernels/benchmark_selective_state_update.py --save-configs
|
||||
"""
|
||||
cache_dtype = _canonical_cache_dtype(cache_dtype)
|
||||
device_name = get_ssm_device_name()
|
||||
json_file_name = get_ssm_config_file_name(headdim, dstate, cache_dtype, device_name)
|
||||
|
||||
config_file_paths: list[str] = []
|
||||
|
||||
# User-supplied override
|
||||
user_defined_config_folder = envs.VLLM_TUNED_CONFIG_FOLDER
|
||||
if user_defined_config_folder is not None:
|
||||
config_file_paths.append(
|
||||
os.path.join(user_defined_config_folder, json_file_name)
|
||||
)
|
||||
|
||||
# Bundled default
|
||||
config_file_paths.append(os.path.join(_CONFIGS_DIR, json_file_name))
|
||||
|
||||
for path in config_file_paths:
|
||||
if os.path.exists(path):
|
||||
with open(path) as f:
|
||||
logger.info_once(
|
||||
"Using SSM config from %s for selective_state_update.",
|
||||
path,
|
||||
scope="global",
|
||||
)
|
||||
raw = json.load(f)
|
||||
if isinstance(raw, dict):
|
||||
# triton_version included in the config file only for reference
|
||||
raw.pop("triton_version", None)
|
||||
return {int(k): v for k, v in raw.items() if k.isdigit()}
|
||||
|
||||
logger.warning_once(
|
||||
"Using default Mamba SSU config. Performance might be sub-optimal! "
|
||||
"Config file not found at %s",
|
||||
", ".join(config_file_paths),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _get_default_ssm_launch_config(
|
||||
dstate: int,
|
||||
is_blackwell: bool,
|
||||
) -> tuple[int, int]:
|
||||
"""Hard-coded fallback heuristic used when no tuned config is available."""
|
||||
BLOCK_SIZE_M, num_warps = 4, 8
|
||||
if dstate <= 16:
|
||||
BLOCK_SIZE_M, num_warps = 32, 4
|
||||
elif dstate <= 32:
|
||||
BLOCK_SIZE_M, num_warps = 16, 4
|
||||
elif dstate <= 64:
|
||||
BLOCK_SIZE_M, num_warps = 8, 4
|
||||
else:
|
||||
if is_blackwell:
|
||||
BLOCK_SIZE_M, num_warps = 32, 8
|
||||
elif dstate <= 128:
|
||||
BLOCK_SIZE_M, num_warps = 4, 4
|
||||
return BLOCK_SIZE_M, num_warps
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _try_get_optimal_ssm_config_cached(
|
||||
headdim: int,
|
||||
dstate: int,
|
||||
batch: int,
|
||||
nheads: int,
|
||||
cache_dtype: str,
|
||||
is_blackwell: bool,
|
||||
) -> tuple[int, int]:
|
||||
"""Cached resolution. See :func:`try_get_optimal_ssm_config`."""
|
||||
effective_batch = batch * nheads
|
||||
configs = get_ssm_configs(headdim, dstate, cache_dtype)
|
||||
if configs:
|
||||
# Pick the closest effective_batch in the tuned grid (MoE strategy).
|
||||
closest = min(configs.keys(), key=lambda x: abs(x - effective_batch))
|
||||
cfg = configs[closest]
|
||||
return cfg["BLOCK_SIZE_M"], cfg["num_warps"]
|
||||
|
||||
return _get_default_ssm_launch_config(dstate, is_blackwell)
|
||||
|
||||
|
||||
# Override hook for benchmarks/tests, see `override_ssm_config`.
|
||||
_ssm_config_override: tuple[int, int] | None = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def override_ssm_config(config: tuple[int, int]):
|
||||
"""Pin ``try_get_optimal_ssm_config`` to ``config`` for the duration of
|
||||
the context. Used by the tuning benchmark to time specific configs."""
|
||||
global _ssm_config_override
|
||||
prev = _ssm_config_override
|
||||
_ssm_config_override = config
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_ssm_config_override = prev
|
||||
|
||||
|
||||
def try_get_optimal_ssm_config(
|
||||
headdim: int,
|
||||
dstate: int,
|
||||
batch: int,
|
||||
nheads: int,
|
||||
cache_dtype: str,
|
||||
is_blackwell: bool,
|
||||
) -> tuple[int, int]:
|
||||
"""Return (BLOCK_SIZE_M, num_warps) for the given kernel shape.
|
||||
|
||||
Tuning is keyed on ``effective_batch = batch * nheads`` (the kernel grid
|
||||
scales with the product), so configs transfer across (model, TP) combos
|
||||
sharing ``(headdim, dstate, cache_dtype)``.
|
||||
"""
|
||||
if _ssm_config_override is not None:
|
||||
return _ssm_config_override
|
||||
return _try_get_optimal_ssm_config_cached(
|
||||
headdim, dstate, batch, nheads, cache_dtype, is_blackwell
|
||||
)
|
||||
|
||||
|
||||
if TRITON3:
|
||||
|
||||
@triton.jit
|
||||
@@ -440,24 +614,11 @@ def selective_state_update(
|
||||
else (0, 0)
|
||||
)
|
||||
# We don't want autotune since it will overwrite the state.
|
||||
# We instead tune by hand based on dstate.
|
||||
|
||||
# Default
|
||||
BLOCK_SIZE_M, num_warps = 4, 8
|
||||
|
||||
if dstate <= 16:
|
||||
BLOCK_SIZE_M, num_warps = 32, 4
|
||||
elif dstate <= 32:
|
||||
BLOCK_SIZE_M, num_warps = 16, 4
|
||||
elif dstate <= 64:
|
||||
BLOCK_SIZE_M, num_warps = 8, 4
|
||||
else:
|
||||
# dstate > 64
|
||||
if is_blackwell:
|
||||
# Optimized for B200 with dstate>64
|
||||
BLOCK_SIZE_M, num_warps = 32, 8
|
||||
elif dstate <= 128:
|
||||
BLOCK_SIZE_M, num_warps = 4, 4
|
||||
# Load from JSON config if available, otherwise fall back to heuristic.
|
||||
cache_dtype = str(state.dtype).removeprefix("torch.")
|
||||
BLOCK_SIZE_M, num_warps = try_get_optimal_ssm_config(
|
||||
dim, dstate, N, nheads, cache_dtype, is_blackwell
|
||||
)
|
||||
|
||||
tie_hdim = (
|
||||
A.stride(-1) == 0
|
||||
|
||||
Reference in New Issue
Block a user