[Attention] Add UltraQuant 4-bit KV cache backend (FlyDSL D=256) (#57057)
Signed-off-by: Aditi Rana <aditi.rana@amd.com>
This commit is contained in:
+24
-10
@@ -906,15 +906,29 @@ class Platform:
|
||||
cache_dtype_str=cache_config.cache_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
).page_size_bytes
|
||||
elif cache_config.cache_dtype.startswith("turboquant_"):
|
||||
# TQ has a packed K|V layout; the standard FullAttentionSpec
|
||||
# formula over-sizes it and trips unify_kv_cache_spec_page_size
|
||||
# when all attention layers are TQ. With mixed skip+TQ the skip
|
||||
# layers still use the standard layout — take max so mamba
|
||||
# padding covers the largest actual page.
|
||||
from vllm.v1.attention.backends.turboquant_attn import (
|
||||
TurboQuantAttentionBackend,
|
||||
)
|
||||
elif (
|
||||
cache_config.cache_dtype.startswith("turboquant_")
|
||||
or cache_config.cache_dtype == "ultraquant_4bit"
|
||||
):
|
||||
# TurboQuant and UltraQuant have packed K|V layouts; the standard
|
||||
# FullAttentionSpec formula over-sizes them and trips
|
||||
# unify_kv_cache_spec_page_size when all attention layers are
|
||||
# packed. With mixed skip+packed the skip layers still use the
|
||||
# standard layout — take max so mamba padding covers the largest
|
||||
# actual page. Each dtype packs via its own backend.
|
||||
_pack_backend: type[AttentionBackend]
|
||||
if cache_config.cache_dtype == "ultraquant_4bit":
|
||||
from vllm.v1.attention.backends.ultraquant_attn import (
|
||||
UltraQuantAttentionBackend,
|
||||
)
|
||||
|
||||
_pack_backend = UltraQuantAttentionBackend
|
||||
else:
|
||||
from vllm.v1.attention.backends.turboquant_attn import (
|
||||
TurboQuantAttentionBackend,
|
||||
)
|
||||
|
||||
_pack_backend = TurboQuantAttentionBackend
|
||||
|
||||
tq_spec = FullAttentionSpec(
|
||||
block_size=1,
|
||||
@@ -923,7 +937,7 @@ class Platform:
|
||||
dtype=kv_cache_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
)
|
||||
tq_page = TurboQuantAttentionBackend.customize_spec(tq_spec).page_size_bytes
|
||||
tq_page = _pack_backend.customize_spec(tq_spec).page_size_bytes
|
||||
if cache_config.kv_cache_dtype_skip_layers:
|
||||
skip_page = FullAttentionSpec(
|
||||
block_size=1,
|
||||
|
||||
Reference in New Issue
Block a user