[Feature] TurboQuant: support hybrid models and uniform quantization (#39931)
Signed-off-by: JartX <sagformas@epdcenter.es> Signed-off-by: Jim Smith <jhsmith0@me.com> Co-authored-by: Jim Smith <jhsmith0@me.com> Co-authored-by: Sandermage <sandermage@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jim Smith
Sandermage
Claude
parent
577b9623e6
commit
4f2af1a7c0
@@ -545,6 +545,42 @@ class Platform:
|
||||
dtype=kv_cache_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
).page_size_bytes
|
||||
elif cache_config.cache_dtype.startswith("turboquant_"):
|
||||
# TQ has a packed K|V layout; the standard FullAttentionSpec
|
||||
# formula over-sizes it and trips unify_kv_cache_spec_page_size
|
||||
# when all attention layers are TQ. With mixed skip+TQ the skip
|
||||
# layers still use the standard layout — take max so mamba
|
||||
# padding covers the largest actual page.
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
TurboQuantConfig,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
|
||||
|
||||
tq_cfg = TurboQuantConfig.from_cache_dtype(
|
||||
cache_config.cache_dtype, model_config.get_head_size()
|
||||
)
|
||||
tq_page = TQFullAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
head_size_v=model_config.get_head_size(),
|
||||
dtype=kv_cache_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
tq_slot_size=tq_cfg.slot_size_aligned,
|
||||
).page_size_bytes
|
||||
if cache_config.kv_cache_dtype_skip_layers:
|
||||
skip_page = FullAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=model_config.dtype,
|
||||
).page_size_bytes
|
||||
# lcm, not max: skip_page is often not a multiple of
|
||||
# tq_page, so max would leave per-layer page sizes
|
||||
# un-unifiable downstream.
|
||||
attn_page_size_1_token = lcm(tq_page, skip_page)
|
||||
else:
|
||||
attn_page_size_1_token = tq_page
|
||||
else:
|
||||
attn_page_size_1_token = FullAttentionSpec(
|
||||
block_size=1,
|
||||
|
||||
Reference in New Issue
Block a user