[Feature] Triton INT4 per-token-head KV cache quantization (#40835)
Signed-off-by: JartX <sagformas@epdcenter.es> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d20dbf921b
commit
70749fdcca
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for per-token-head KV cache quantization (INT8 and FP8).
|
||||
"""Tests for per-token-head KV cache quantization (INT4, INT8 and FP8).
|
||||
|
||||
Covers:
|
||||
- Per-token-head Triton reshape-and-cache kernel
|
||||
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops.int4_per_token_head import single_rht
|
||||
from vllm.v1.kv_cache_interface import KVQuantMode, is_quantized_kv_cache
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
@@ -81,11 +82,19 @@ FP8_CONFIG = QuantConfig(
|
||||
kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD,
|
||||
rounds_before_store=False,
|
||||
)
|
||||
|
||||
QUANT_CONFIGS = [INT8_CONFIG, FP8_CONFIG]
|
||||
INT4_CONFIG = QuantConfig(
|
||||
cache_dtype=torch.uint8,
|
||||
kv_cache_dtype_str="int4_per_token_head",
|
||||
quant_max=7.0,
|
||||
quant_min=-8.0,
|
||||
kv_quant_mode=KVQuantMode.INT4_PER_TOKEN_HEAD,
|
||||
# Unused for int4 (handled by its own rint path); kept for the dataclass.
|
||||
rounds_before_store=False,
|
||||
)
|
||||
QUANT_CONFIGS = [INT4_CONFIG, INT8_CONFIG, FP8_CONFIG]
|
||||
|
||||
|
||||
@pytest.fixture(params=QUANT_CONFIGS, ids=["int8", "fp8"])
|
||||
@pytest.fixture(params=QUANT_CONFIGS, ids=["int4", "int8", "fp8"])
|
||||
def qcfg(request) -> QuantConfig:
|
||||
return request.param
|
||||
|
||||
@@ -120,6 +129,9 @@ class TestIsQuantizedKvCache:
|
||||
assert is_quantized_kv_cache("fp8_e4m3")
|
||||
assert is_quantized_kv_cache("fp8_e5m2")
|
||||
|
||||
def test_int4_per_token_head(self):
|
||||
assert is_quantized_kv_cache("int4_per_token_head")
|
||||
|
||||
def test_int8_per_token_head(self):
|
||||
assert is_quantized_kv_cache("int8_per_token_head")
|
||||
|
||||
@@ -132,6 +144,13 @@ class TestIsQuantizedKvCache:
|
||||
def test_bfloat16(self):
|
||||
assert not is_quantized_kv_cache("bfloat16")
|
||||
|
||||
def test_kv_quant_mode_int4(self):
|
||||
from vllm.v1.kv_cache_interface import get_kv_quant_mode
|
||||
|
||||
assert (
|
||||
get_kv_quant_mode("int4_per_token_head") == KVQuantMode.INT4_PER_TOKEN_HEAD
|
||||
)
|
||||
|
||||
def test_kv_quant_mode_int8(self):
|
||||
from vllm.v1.kv_cache_interface import get_kv_quant_mode
|
||||
|
||||
@@ -171,15 +190,17 @@ def test_reshape_and_cache_per_token_head(
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 4
|
||||
is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
|
||||
cache_head_size = head_size // 2 if is_int4 else head_size
|
||||
|
||||
key = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
|
||||
value = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
|
||||
|
||||
key_cache = torch.zeros(
|
||||
num_blocks, block_size, num_heads, head_size, dtype=qcfg.cache_dtype
|
||||
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
|
||||
)
|
||||
value_cache = torch.zeros(
|
||||
num_blocks, block_size, num_heads, head_size, dtype=qcfg.cache_dtype
|
||||
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
|
||||
)
|
||||
k_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
|
||||
v_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
|
||||
@@ -197,45 +218,68 @@ def test_reshape_and_cache_per_token_head(
|
||||
k_scale_cache,
|
||||
v_scale_cache,
|
||||
slot_mapping,
|
||||
kv_quant_mode=qcfg.kv_quant_mode,
|
||||
)
|
||||
|
||||
# Reference
|
||||
ref_k_quant, ref_k_scales = _quantize_per_token_head_ref(key, qcfg)
|
||||
ref_v_quant, ref_v_scales = _quantize_per_token_head_ref(value, qcfg)
|
||||
# INT4 (RHT + asymmetric), INT8/FP8 have different dequant paths. Only
|
||||
# INT8/FP8 can be compared to a PyTorch reference.
|
||||
if not is_int4:
|
||||
ref_k_quant, ref_k_scales = _quantize_per_token_head_ref(key, qcfg)
|
||||
ref_v_quant, ref_v_scales = _quantize_per_token_head_ref(value, qcfg)
|
||||
|
||||
# Compare dequantized values rather than raw quantized values.
|
||||
# Triton and PyTorch reductions can differ at FP8 rounding boundaries
|
||||
# (up to 32 in quantized domain for fp8_e4m3), but the dequantized
|
||||
# error is bounded by the scale.
|
||||
for i, slot in enumerate(slot_mapping.tolist()):
|
||||
blk = slot // block_size
|
||||
off = slot % block_size
|
||||
|
||||
actual_k_scale = k_scale_cache[blk, off] # [num_heads]
|
||||
k_deq = key_cache[blk, off].float() * actual_k_scale[:, None]
|
||||
k_ref_deq = key[i].float()
|
||||
torch.testing.assert_close(
|
||||
k_deq,
|
||||
k_ref_deq,
|
||||
atol=0.1,
|
||||
rtol=0.1,
|
||||
)
|
||||
actual_v_scale = v_scale_cache[blk, off] # [num_heads]
|
||||
v_deq = value_cache[blk, off].float() * actual_v_scale[:, None]
|
||||
v_ref_deq = value[i].float()
|
||||
torch.testing.assert_close(
|
||||
v_deq,
|
||||
v_ref_deq,
|
||||
atol=0.1,
|
||||
rtol=0.1,
|
||||
)
|
||||
# Per-head scales: [num_heads]
|
||||
torch.testing.assert_close(
|
||||
k_scale_cache[blk, off], ref_k_scales[i], atol=1e-4, rtol=1e-3
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
v_scale_cache[blk, off], ref_v_scales[i], atol=1e-4, rtol=1e-3
|
||||
)
|
||||
if is_int4:
|
||||
# Coarser quantization → wider tolerance.
|
||||
deq_atol = deq_rtol = 0.5
|
||||
for label, data, cache, sc in [
|
||||
("key", key, key_cache, k_scale_cache),
|
||||
("val", value, value_cache, v_scale_cache),
|
||||
]:
|
||||
packed_scale = sc[blk, off] # [num_heads] float32
|
||||
scale_bits = packed_scale.view(torch.int32)
|
||||
zp = (scale_bits & 0xF).to(torch.float32)
|
||||
clean_scale = (scale_bits & -16).view(torch.float32)
|
||||
|
||||
packed = cache[blk, off]
|
||||
lo = (packed & 0xF).to(torch.float32)
|
||||
hi = ((packed >> 4) & 0xF).to(torch.float32)
|
||||
full = torch.zeros(num_heads, head_size, dtype=torch.float32)
|
||||
full[:, 0::2] = lo
|
||||
full[:, 1::2] = hi
|
||||
# Asymmetric dequant in RHT domain, then IRHT/d → original
|
||||
deq_rht = (full - zp[:, None]) * clean_scale[:, None]
|
||||
deq = single_rht(deq_rht, inverse=True) / head_size
|
||||
ref_deq = data[i].float()
|
||||
torch.testing.assert_close(deq, ref_deq, atol=deq_atol, rtol=deq_rtol)
|
||||
else:
|
||||
actual_k_scale = k_scale_cache[blk, off] # [num_heads]
|
||||
k_deq = key_cache[blk, off].float() * actual_k_scale[:, None]
|
||||
k_ref_deq = key[i].float()
|
||||
torch.testing.assert_close(
|
||||
k_deq,
|
||||
k_ref_deq,
|
||||
atol=0.1,
|
||||
rtol=0.1,
|
||||
)
|
||||
actual_v_scale = v_scale_cache[blk, off] # [num_heads]
|
||||
v_deq = value_cache[blk, off].float() * actual_v_scale[:, None]
|
||||
v_ref_deq = value[i].float()
|
||||
torch.testing.assert_close(
|
||||
v_deq,
|
||||
v_ref_deq,
|
||||
atol=0.1,
|
||||
rtol=0.1,
|
||||
)
|
||||
# Per-head scales: [num_heads]
|
||||
torch.testing.assert_close(
|
||||
k_scale_cache[blk, off], ref_k_scales[i], atol=1e-4, rtol=1e-3
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
v_scale_cache[blk, off], ref_v_scales[i], atol=1e-4, rtol=1e-3
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
@@ -265,16 +309,18 @@ def test_per_token_head_round_trip_accuracy(
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
set_random_seed(42)
|
||||
|
||||
is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 2
|
||||
cache_head_size = head_size // 2 if is_int4 else head_size
|
||||
|
||||
key = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16) * 0.5
|
||||
value = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16) * 0.5
|
||||
|
||||
key_cache = torch.zeros(
|
||||
num_blocks, block_size, num_heads, head_size, dtype=qcfg.cache_dtype
|
||||
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
|
||||
)
|
||||
value_cache = torch.zeros(
|
||||
num_blocks, block_size, num_heads, head_size, dtype=qcfg.cache_dtype
|
||||
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
|
||||
)
|
||||
k_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
|
||||
v_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
|
||||
@@ -289,8 +335,11 @@ def test_per_token_head_round_trip_accuracy(
|
||||
k_scale_cache,
|
||||
v_scale_cache,
|
||||
slot_mapping,
|
||||
kv_quant_mode=qcfg.kv_quant_mode,
|
||||
)
|
||||
|
||||
rt_atol = 0.5 if is_int4 else 0.1
|
||||
|
||||
for i in range(num_tokens):
|
||||
blk = i // block_size
|
||||
off = i % block_size
|
||||
@@ -300,18 +349,30 @@ def test_per_token_head_round_trip_accuracy(
|
||||
("val", value, value_cache, v_scale_cache),
|
||||
]:
|
||||
for h in range(num_heads):
|
||||
orig = data[i, h].float() # [head_size]
|
||||
|
||||
actual_q = cache[blk, off, h]
|
||||
orig = data[i, h].float()
|
||||
actual_sc = sc[blk, off, h]
|
||||
actual_deq = actual_q.float() * actual_sc
|
||||
|
||||
# Round-trip: dequantized should be close to original
|
||||
if is_int4:
|
||||
sc_bits = actual_sc.view(torch.int32)
|
||||
zp = (sc_bits & 0xF).to(torch.float32)
|
||||
clean_sc = (sc_bits & -16).view(torch.float32)
|
||||
packed = cache[blk, off, h]
|
||||
lo = (packed & 0xF).to(torch.float32)
|
||||
hi = ((packed >> 4) & 0xF).to(torch.float32)
|
||||
full = torch.zeros(head_size)
|
||||
full[0::2] = lo
|
||||
full[1::2] = hi
|
||||
deq_rht = (full - zp) * clean_sc
|
||||
actual_deq = (
|
||||
single_rht(deq_rht.unsqueeze(0), inverse=True).squeeze(0)
|
||||
/ head_size
|
||||
)
|
||||
else:
|
||||
actual_deq = cache[blk, off, h].float() * actual_sc
|
||||
torch.testing.assert_close(
|
||||
actual_deq,
|
||||
orig,
|
||||
atol=0.1,
|
||||
rtol=0.1,
|
||||
atol=rt_atol,
|
||||
rtol=rt_atol,
|
||||
)
|
||||
|
||||
|
||||
@@ -347,6 +408,7 @@ def test_int8_per_token_head_raw_cache_matches_round_reference():
|
||||
k_scale_cache,
|
||||
v_scale_cache,
|
||||
slot_mapping,
|
||||
kv_quant_mode=INT8_CONFIG.kv_quant_mode,
|
||||
)
|
||||
|
||||
ref_k_quant, ref_k_scales = _quantize_per_token_head_ref(key, INT8_CONFIG)
|
||||
@@ -377,15 +439,17 @@ def test_per_token_head_negative_slot_skipped(qcfg: QuantConfig):
|
||||
head_size = 64
|
||||
block_size = 16
|
||||
num_blocks = 2
|
||||
is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
|
||||
cache_head_size = head_size // 2 if is_int4 else head_size
|
||||
|
||||
key = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
|
||||
value = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
|
||||
|
||||
key_cache = torch.zeros(
|
||||
num_blocks, block_size, num_heads, head_size, dtype=qcfg.cache_dtype
|
||||
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
|
||||
)
|
||||
value_cache = torch.zeros(
|
||||
num_blocks, block_size, num_heads, head_size, dtype=qcfg.cache_dtype
|
||||
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
|
||||
)
|
||||
k_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
|
||||
v_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
|
||||
@@ -403,6 +467,7 @@ def test_per_token_head_negative_slot_skipped(qcfg: QuantConfig):
|
||||
k_scale_cache,
|
||||
v_scale_cache,
|
||||
slot_mapping,
|
||||
kv_quant_mode=qcfg.kv_quant_mode,
|
||||
)
|
||||
|
||||
# Slots 0 and 1 should have been written (tokens 0 and 2)
|
||||
@@ -420,7 +485,8 @@ def test_per_token_head_negative_slot_skipped(qcfg: QuantConfig):
|
||||
# 5. process_weights_after_loading -- per-token-head early return
|
||||
# ===========================================================================
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype", ["int8_per_token_head", "fp8_per_token_head"]
|
||||
"kv_cache_dtype",
|
||||
["int4_per_token_head", "int8_per_token_head", "fp8_per_token_head"],
|
||||
)
|
||||
def test_process_weights_sets_placeholder_scales(kv_cache_dtype: str):
|
||||
"""Per-token-head should set _k_scale=1.0, _v_scale=1.0
|
||||
@@ -454,7 +520,7 @@ def test_process_weights_sets_placeholder_scales(kv_cache_dtype: str):
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 6. Triton unified_attention -- per-token-head scale cache (INT8 and FP8)
|
||||
# 6. Triton unified_attention -- per-token-head scale cache (INT4/INT8/FP8)
|
||||
# ===========================================================================
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens",
|
||||
@@ -481,6 +547,8 @@ def test_triton_unified_attention_per_token_head_scale(
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
set_random_seed(0)
|
||||
|
||||
is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
|
||||
|
||||
num_seqs = len(seq_lens)
|
||||
query_lens = [s[0] for s in seq_lens]
|
||||
kv_lens = [s[1] for s in seq_lens]
|
||||
@@ -499,22 +567,68 @@ def test_triton_unified_attention_per_token_head_scale(
|
||||
)
|
||||
value_cache_bf16 = torch.randn_like(key_cache_bf16)
|
||||
|
||||
# Per-token-head quantization: one scale per (block, slot, head)
|
||||
k_absmax = key_cache_bf16.float().abs().amax(dim=-1) # [..., num_kv_heads]
|
||||
v_absmax = value_cache_bf16.float().abs().amax(dim=-1)
|
||||
k_scale_cache = (k_absmax / qcfg.quant_max).clamp(min=1e-6).to(torch.float32)
|
||||
v_scale_cache = (v_absmax / qcfg.quant_max).clamp(min=1e-6).to(torch.float32)
|
||||
if is_int4:
|
||||
# Asymmetric quantization reference (matches the Triton kernel).
|
||||
kf = key_cache_bf16.float()
|
||||
vf = value_cache_bf16.float()
|
||||
k_min = kf.amin(dim=-1)
|
||||
k_max = kf.amax(dim=-1)
|
||||
v_min = vf.amin(dim=-1)
|
||||
v_max = vf.amax(dim=-1)
|
||||
k_scale_cache = ((k_max - k_min) / 15.0).clamp(min=1e-6).to(torch.float32)
|
||||
v_scale_cache = ((v_max - v_min) / 15.0).clamp(min=1e-6).to(torch.float32)
|
||||
k_zp = (-k_min / k_scale_cache).round().clamp(0, 15)
|
||||
v_zp = (-v_min / v_scale_cache).round().clamp(0, 15)
|
||||
|
||||
scaled_k = key_cache_bf16.float() / k_scale_cache[:, :, :, None]
|
||||
scaled_v = value_cache_bf16.float() / v_scale_cache[:, :, :, None]
|
||||
if qcfg.rounds_before_store:
|
||||
key_cache_q = (
|
||||
scaled_k.round().clamp(qcfg.quant_min, qcfg.quant_max).to(qcfg.cache_dtype)
|
||||
key_cache_q_full = (
|
||||
(kf / k_scale_cache[..., None] + k_zp[..., None]).round().clamp(0, 15)
|
||||
)
|
||||
value_cache_q = (
|
||||
scaled_v.round().clamp(qcfg.quant_min, qcfg.quant_max).to(qcfg.cache_dtype)
|
||||
value_cache_q_full = (
|
||||
(vf / v_scale_cache[..., None] + v_zp[..., None]).round().clamp(0, 15)
|
||||
)
|
||||
|
||||
# Dequantized reference: x_hat = (q - zp) * scale
|
||||
key_cache_deq = (key_cache_q_full - k_zp[..., None]) * k_scale_cache[..., None]
|
||||
value_cache_deq = (value_cache_q_full - v_zp[..., None]) * v_scale_cache[
|
||||
..., None
|
||||
]
|
||||
|
||||
# Pack two uint4 values into one byte
|
||||
def _pack_int4(data_float):
|
||||
u = data_float.to(torch.uint8)
|
||||
lo = u[..., 0::2]
|
||||
hi = u[..., 1::2]
|
||||
return (lo & 0xF) | ((hi & 0xF) << 4)
|
||||
|
||||
key_cache_q = _pack_int4(key_cache_q_full)
|
||||
value_cache_q = _pack_int4(value_cache_q_full)
|
||||
|
||||
# Steganography: pack zp into low 4 bits of scale
|
||||
k_zp_int = k_zp.to(torch.int32)
|
||||
k_bits = k_scale_cache.view(torch.int32)
|
||||
k_scale_cache = ((k_bits & -16) | (k_zp_int & 0xF)).view(torch.float32)
|
||||
v_zp_int = v_zp.to(torch.int32)
|
||||
v_bits = v_scale_cache.view(torch.int32)
|
||||
v_scale_cache = ((v_bits & -16) | (v_zp_int & 0xF)).view(torch.float32)
|
||||
else:
|
||||
# Symmetric quantization for int8/fp8.
|
||||
k_absmax = key_cache_bf16.float().abs().amax(dim=-1)
|
||||
v_absmax = value_cache_bf16.float().abs().amax(dim=-1)
|
||||
k_scale_cache = (k_absmax / qcfg.quant_max).clamp(min=1e-6).to(torch.float32)
|
||||
v_scale_cache = (v_absmax / qcfg.quant_max).clamp(min=1e-6).to(torch.float32)
|
||||
scaled_k = key_cache_bf16.float() / k_scale_cache[:, :, :, None]
|
||||
scaled_v = value_cache_bf16.float() / v_scale_cache[:, :, :, None]
|
||||
|
||||
key_cache_q_full = scaled_k.round().clamp(qcfg.quant_min, qcfg.quant_max)
|
||||
value_cache_q_full = scaled_v.round().clamp(qcfg.quant_min, qcfg.quant_max)
|
||||
|
||||
key_cache_deq = key_cache_q_full * k_scale_cache[:, :, :, None]
|
||||
value_cache_deq = value_cache_q_full * v_scale_cache[:, :, :, None]
|
||||
|
||||
if not is_int4 and qcfg.rounds_before_store:
|
||||
key_cache_q = key_cache_q_full.to(qcfg.cache_dtype)
|
||||
value_cache_q = value_cache_q_full.to(qcfg.cache_dtype)
|
||||
elif not is_int4:
|
||||
key_cache_q = scaled_k.clamp(qcfg.quant_min, qcfg.quant_max).to(
|
||||
qcfg.cache_dtype
|
||||
)
|
||||
@@ -522,10 +636,6 @@ def test_triton_unified_attention_per_token_head_scale(
|
||||
qcfg.cache_dtype
|
||||
)
|
||||
|
||||
# Dequantized reference
|
||||
key_cache_deq = key_cache_q.float() * k_scale_cache[:, :, :, None]
|
||||
value_cache_deq = value_cache_q.float() * v_scale_cache[:, :, :, None]
|
||||
|
||||
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
|
||||
dim=0, dtype=torch.int32
|
||||
)
|
||||
@@ -605,4 +715,9 @@ def test_triton_unified_attention_per_token_head_scale(
|
||||
softmax_segm_expsum=softmax_segm_expsum,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output_q, output_ref, atol=5e-2, rtol=5e-2)
|
||||
# Coarser quantization → wider tolerance.
|
||||
if is_int4:
|
||||
atol, rtol = 0.5, 0.5
|
||||
else:
|
||||
atol, rtol = 5e-2, 5e-2
|
||||
torch.testing.assert_close(output_q, output_ref, atol=atol, rtol=rtol)
|
||||
|
||||
Reference in New Issue
Block a user