[Mamba] Add stochastic rounding support (#35753)
Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
@@ -604,6 +604,10 @@ class EngineArgs:
|
||||
mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype
|
||||
mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
|
||||
mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode
|
||||
enable_mamba_cache_stochastic_rounding: bool = (
|
||||
CacheConfig.enable_mamba_cache_stochastic_rounding
|
||||
)
|
||||
mamba_cache_philox_rounds: int = CacheConfig.mamba_cache_philox_rounds
|
||||
|
||||
additional_config: dict[str, Any] = get_field(VllmConfig, "additional_config")
|
||||
|
||||
@@ -1024,6 +1028,13 @@ class EngineArgs:
|
||||
cache_group.add_argument(
|
||||
"--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--enable-mamba-cache-stochastic-rounding",
|
||||
**cache_kwargs["enable_mamba_cache_stochastic_rounding"],
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--mamba-cache-philox-rounds", **cache_kwargs["mamba_cache_philox_rounds"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--kv-offloading-size", **cache_kwargs["kv_offloading_size"]
|
||||
)
|
||||
@@ -1590,6 +1601,8 @@ class EngineArgs:
|
||||
mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
|
||||
mamba_block_size=self.mamba_block_size,
|
||||
mamba_cache_mode=self.mamba_cache_mode,
|
||||
enable_mamba_cache_stochastic_rounding=self.enable_mamba_cache_stochastic_rounding,
|
||||
mamba_cache_philox_rounds=self.mamba_cache_philox_rounds,
|
||||
kv_offloading_size=self.kv_offloading_size,
|
||||
kv_offloading_backend=self.kv_offloading_backend,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user