[Feature] TurboQuant: support hybrid models and uniform quantization (#39931)

Signed-off-by: JartX <sagformas@epdcenter.es>
Signed-off-by: Jim Smith <jhsmith0@me.com>
Co-authored-by: Jim Smith <jhsmith0@me.com>
Co-authored-by: Sandermage <sandermage@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
JartX
2026-05-04 20:14:01 -04:00
committed by GitHub
co-authored by Jim Smith Sandermage Claude
parent 577b9623e6
commit 4f2af1a7c0
4 changed files with 185 additions and 26 deletions
+36
View File
@@ -545,6 +545,42 @@ class Platform:
dtype=kv_cache_dtype,
kv_quant_mode=kv_quant_mode,
).page_size_bytes
elif cache_config.cache_dtype.startswith("turboquant_"):
# TQ has a packed K|V layout; the standard FullAttentionSpec
# formula over-sizes it and trips unify_kv_cache_spec_page_size
# when all attention layers are TQ. With mixed skip+TQ the skip
# layers still use the standard layout — take max so mamba
# padding covers the largest actual page.
from vllm.model_executor.layers.quantization.turboquant.config import (
TurboQuantConfig,
)
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
tq_cfg = TurboQuantConfig.from_cache_dtype(
cache_config.cache_dtype, model_config.get_head_size()
)
tq_page = TQFullAttentionSpec(
block_size=1,
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
head_size=model_config.get_head_size(),
head_size_v=model_config.get_head_size(),
dtype=kv_cache_dtype,
kv_quant_mode=kv_quant_mode,
tq_slot_size=tq_cfg.slot_size_aligned,
).page_size_bytes
if cache_config.kv_cache_dtype_skip_layers:
skip_page = FullAttentionSpec(
block_size=1,
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
head_size=model_config.get_head_size(),
dtype=model_config.dtype,
).page_size_bytes
# lcm, not max: skip_page is often not a multiple of
# tq_page, so max would leave per-layer page sizes
# un-unifiable downstream.
attn_page_size_1_token = lcm(tq_page, skip_page)
else:
attn_page_size_1_token = tq_page
else:
attn_page_size_1_token = FullAttentionSpec(
block_size=1,