[Attention] Add UltraQuant 4-bit KV cache backend (FlyDSL D=256) (#57057)

Signed-off-by: Aditi Rana <aditi.rana@amd.com>
This commit is contained in:
aditi-amd
2026-10-06 20:42:52 +00:00
committed by GitHub
parent 819852df6b
commit 385d86a6cf
22 changed files with 5377 additions and 24 deletions
+14 -4
View File
@@ -2271,7 +2271,13 @@ class EngineArgs:
kv_offloading_backend=self.kv_offloading_backend,
)
if resolved_cache_dtype.startswith("turboquant_"):
# TurboQuant and UltraQuant both keep boundary attention layers at the
# native dtype, and compute those layers the same way.
uses_packed_kv_backend = (
resolved_cache_dtype.startswith("turboquant_")
or resolved_cache_dtype == "ultraquant_4bit"
)
if uses_packed_kv_backend:
from vllm.model_executor.layers.quantization.turboquant.config import (
TurboQuantConfig,
)
@@ -2681,14 +2687,18 @@ class EngineArgs:
# TurboQuant requires FlashAttention 2 — FA3 boundary layers assert
# FlashAttentionImpl which fails with TurboQuantAttentionImpl.
if resolved_cache_dtype.startswith("turboquant_") and (
# UltraQuant subclasses that impl, so it inherits the same limit.
if uses_packed_kv_backend and (
attention_config.flash_attn_version is None
or attention_config.flash_attn_version >= 3
):
logger.warning(
"TurboQuant is not yet compatible with FlashAttention >= 3. "
"%s is not yet compatible with FlashAttention >= 3. "
"Overriding flash_attn_version to 2. To silence this "
"warning, pass --attention-config.flash_attn_version=2"
"warning, pass --attention-config.flash_attn_version=2",
"TurboQuant"
if resolved_cache_dtype.startswith("turboquant_")
else "UltraQuant",
)
attention_config.flash_attn_version = 2