[Mamba] Flashinfer selective_state_update (#36162)
Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
@@ -123,14 +123,6 @@ class CacheConfig:
|
||||
- "align": only cache the mamba state of the last token of each scheduler step and
|
||||
when the token is at position i * block_size.
|
||||
"""
|
||||
enable_mamba_cache_stochastic_rounding: bool = False
|
||||
"""Enable stochastic rounding when writing SSM state to fp16 cache.
|
||||
Uses random bits to unbias the rounding error, which can improve
|
||||
numerical stability for long sequences."""
|
||||
mamba_cache_philox_rounds: int = 0
|
||||
"""Number of Philox PRNG rounds for stochastic rounding random number
|
||||
generation. 0 uses the Triton default. Higher values improve randomness
|
||||
quality at the cost of compute."""
|
||||
|
||||
# Will be set after profiling.
|
||||
num_gpu_blocks: int | None = field(default=None, init=False)
|
||||
@@ -258,29 +250,3 @@ class CacheConfig:
|
||||
str(cache_dtype),
|
||||
)
|
||||
return cache_dtype
|
||||
|
||||
def __post_init__(self):
|
||||
if self.enable_mamba_cache_stochastic_rounding:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
raise ValueError(
|
||||
"Stochastic rounding for Mamba cache is only supported "
|
||||
"on NVIDIA CUDA platforms. Please do not specify "
|
||||
"`--enable-mamba-cache-stochastic-rounding`."
|
||||
)
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
raise ValueError(
|
||||
"Stochastic rounding for Mamba cache requires compute "
|
||||
"capability 10.0 (data center Blackwell). The `cvt.rs` PTX "
|
||||
"instruction is not supported on your GPU. Please do not specify "
|
||||
"`--enable-mamba-cache-stochastic-rounding`."
|
||||
)
|
||||
if self.mamba_ssm_cache_dtype != "float16":
|
||||
raise ValueError(
|
||||
"Stochastic rounding for Mamba cache requires "
|
||||
"the SSM cache to be float16. Please set it explicitly, "
|
||||
"by specifying `--mamba-ssm-cache-dtype float16`, or disable "
|
||||
"stochastic rounding by not specifying "
|
||||
"`--enable-mamba-cache-stochastic-rounding`."
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user