enable skipping of SW attention layers when using FP8 KV cache (#33695)
Signed-off-by: Jonas Kuebler <kuebj@amazon.com>
This commit is contained in:
@@ -597,6 +597,9 @@ class EngineArgs:
|
||||
attention_backend: AttentionBackendEnum | None = AttentionConfig.backend
|
||||
|
||||
calculate_kv_scales: bool = CacheConfig.calculate_kv_scales
|
||||
kv_cache_dtype_skip_layers: list[str] = get_field(
|
||||
CacheConfig, "kv_cache_dtype_skip_layers"
|
||||
)
|
||||
mamba_cache_dtype: MambaDType = CacheConfig.mamba_cache_dtype
|
||||
mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype
|
||||
mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
|
||||
@@ -1003,6 +1006,9 @@ class EngineArgs:
|
||||
cache_group.add_argument(
|
||||
"--calculate-kv-scales", **cache_kwargs["calculate_kv_scales"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--kv-cache-dtype-skip-layers", **cache_kwargs["kv_cache_dtype_skip_layers"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--kv-sharing-fast-prefill", **cache_kwargs["kv_sharing_fast_prefill"]
|
||||
)
|
||||
@@ -1578,6 +1584,7 @@ class EngineArgs:
|
||||
enable_prefix_caching=self.enable_prefix_caching,
|
||||
prefix_caching_hash_algo=self.prefix_caching_hash_algo,
|
||||
calculate_kv_scales=self.calculate_kv_scales,
|
||||
kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers,
|
||||
kv_sharing_fast_prefill=self.kv_sharing_fast_prefill,
|
||||
mamba_cache_dtype=self.mamba_cache_dtype,
|
||||
mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
|
||||
|
||||
Reference in New Issue
Block a user