[Attention] Add UltraQuant 4-bit KV cache backend (FlyDSL D=256) (#57057)
Signed-off-by: Aditi Rana <aditi.rana@amd.com>
This commit is contained in:
@@ -2271,7 +2271,13 @@ class EngineArgs:
|
||||
kv_offloading_backend=self.kv_offloading_backend,
|
||||
)
|
||||
|
||||
if resolved_cache_dtype.startswith("turboquant_"):
|
||||
# TurboQuant and UltraQuant both keep boundary attention layers at the
|
||||
# native dtype, and compute those layers the same way.
|
||||
uses_packed_kv_backend = (
|
||||
resolved_cache_dtype.startswith("turboquant_")
|
||||
or resolved_cache_dtype == "ultraquant_4bit"
|
||||
)
|
||||
if uses_packed_kv_backend:
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
TurboQuantConfig,
|
||||
)
|
||||
@@ -2681,14 +2687,18 @@ class EngineArgs:
|
||||
|
||||
# TurboQuant requires FlashAttention 2 — FA3 boundary layers assert
|
||||
# FlashAttentionImpl which fails with TurboQuantAttentionImpl.
|
||||
if resolved_cache_dtype.startswith("turboquant_") and (
|
||||
# UltraQuant subclasses that impl, so it inherits the same limit.
|
||||
if uses_packed_kv_backend and (
|
||||
attention_config.flash_attn_version is None
|
||||
or attention_config.flash_attn_version >= 3
|
||||
):
|
||||
logger.warning(
|
||||
"TurboQuant is not yet compatible with FlashAttention >= 3. "
|
||||
"%s is not yet compatible with FlashAttention >= 3. "
|
||||
"Overriding flash_attn_version to 2. To silence this "
|
||||
"warning, pass --attention-config.flash_attn_version=2"
|
||||
"warning, pass --attention-config.flash_attn_version=2",
|
||||
"TurboQuant"
|
||||
if resolved_cache_dtype.startswith("turboquant_")
|
||||
else "UltraQuant",
|
||||
)
|
||||
attention_config.flash_attn_version = 2
|
||||
|
||||
|
||||
Reference in New Issue
Block a user