[Attention] Add UltraQuant 4-bit KV cache backend (FlyDSL D=256) (#57057)

Signed-off-by: Aditi Rana <aditi.rana@amd.com>
This commit is contained in:
aditi-amd
2026-10-06 20:42:52 +00:00
committed by GitHub
parent 819852df6b
commit 385d86a6cf
22 changed files with 5377 additions and 24 deletions
+24 -10
View File
@@ -906,15 +906,29 @@ class Platform:
cache_dtype_str=cache_config.cache_dtype,
kv_quant_mode=kv_quant_mode,
).page_size_bytes
elif cache_config.cache_dtype.startswith("turboquant_"):
# TQ has a packed K|V layout; the standard FullAttentionSpec
# formula over-sizes it and trips unify_kv_cache_spec_page_size
# when all attention layers are TQ. With mixed skip+TQ the skip
# layers still use the standard layout — take max so mamba
# padding covers the largest actual page.
from vllm.v1.attention.backends.turboquant_attn import (
TurboQuantAttentionBackend,
)
elif (
cache_config.cache_dtype.startswith("turboquant_")
or cache_config.cache_dtype == "ultraquant_4bit"
):
# TurboQuant and UltraQuant have packed K|V layouts; the standard
# FullAttentionSpec formula over-sizes them and trips
# unify_kv_cache_spec_page_size when all attention layers are
# packed. With mixed skip+packed the skip layers still use the
# standard layout — take max so mamba padding covers the largest
# actual page. Each dtype packs via its own backend.
_pack_backend: type[AttentionBackend]
if cache_config.cache_dtype == "ultraquant_4bit":
from vllm.v1.attention.backends.ultraquant_attn import (
UltraQuantAttentionBackend,
)
_pack_backend = UltraQuantAttentionBackend
else:
from vllm.v1.attention.backends.turboquant_attn import (
TurboQuantAttentionBackend,
)
_pack_backend = TurboQuantAttentionBackend
tq_spec = FullAttentionSpec(
block_size=1,
@@ -923,7 +937,7 @@ class Platform:
dtype=kv_cache_dtype,
kv_quant_mode=kv_quant_mode,
)
tq_page = TurboQuantAttentionBackend.customize_spec(tq_spec).page_size_bytes
tq_page = _pack_backend.customize_spec(tq_spec).page_size_bytes
if cache_config.kv_cache_dtype_skip_layers:
skip_page = FullAttentionSpec(
block_size=1,