[Kernel][Bugfix] Fix INT8 per-token-head KV cache rounding in Triton reshape-and-cache (#45361)
Signed-off-by: ZedongLiu <113341356+Zedong-Liu@users.noreply.github.com>
This commit is contained in:
@@ -61,8 +61,8 @@ class QuantConfig:
|
||||
quant_max: float
|
||||
quant_min: float
|
||||
kv_quant_mode: KVQuantMode
|
||||
# INT8 Triton stores truncate; FP8 hardware casts round.
|
||||
uses_trunc: bool
|
||||
# INT8 rounds explicitly; FP8 relies on dtype cast rounding.
|
||||
rounds_before_store: bool
|
||||
|
||||
|
||||
INT8_CONFIG = QuantConfig(
|
||||
@@ -71,7 +71,7 @@ INT8_CONFIG = QuantConfig(
|
||||
quant_max=127.0,
|
||||
quant_min=-128.0,
|
||||
kv_quant_mode=KVQuantMode.INT8_PER_TOKEN_HEAD,
|
||||
uses_trunc=True,
|
||||
rounds_before_store=True,
|
||||
)
|
||||
FP8_CONFIG = QuantConfig(
|
||||
cache_dtype=FP8_DTYPE,
|
||||
@@ -79,7 +79,7 @@ FP8_CONFIG = QuantConfig(
|
||||
quant_max=FP8_MAX,
|
||||
quant_min=FP8_MIN,
|
||||
kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD,
|
||||
uses_trunc=False,
|
||||
rounds_before_store=False,
|
||||
)
|
||||
|
||||
QUANT_CONFIGS = [INT8_CONFIG, FP8_CONFIG]
|
||||
@@ -104,7 +104,7 @@ def _quantize_per_token_head_ref(
|
||||
absmax = data.float().abs().amax(dim=2) # [num_tokens, num_heads]
|
||||
scales = (absmax / cfg.quant_max).clamp(min=1e-6)
|
||||
scaled = data.float() * (1.0 / scales[:, :, None])
|
||||
if cfg.uses_trunc:
|
||||
if cfg.rounds_before_store:
|
||||
q = scaled.round().clamp(cfg.quant_min, cfg.quant_max).to(cfg.cache_dtype)
|
||||
else:
|
||||
q = scaled.clamp(cfg.quant_min, cfg.quant_max).to(cfg.cache_dtype)
|
||||
@@ -255,7 +255,7 @@ def test_per_token_head_round_trip_accuracy(
|
||||
):
|
||||
"""Verify per-token-head round-trip: kernel dequant matches reference.
|
||||
|
||||
INT8: Triton truncates on float->int8 store.
|
||||
INT8: round-to-nearest before int8 store.
|
||||
FP8: hardware cast (clamp then cast).
|
||||
"""
|
||||
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
||||
@@ -315,6 +315,52 @@ def test_per_token_head_round_trip_accuracy(
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_int8_per_token_head_raw_cache_matches_round_reference():
|
||||
"""INT8 cache writes should match round-to-nearest quantization exactly."""
|
||||
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
||||
triton_reshape_and_cache_flash_per_token_head_quant,
|
||||
)
|
||||
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
|
||||
head_size = 8
|
||||
block_size = 4
|
||||
|
||||
key = torch.tensor(
|
||||
[[[-127.0, -2.6, -2.4, -1.6, -1.4, -0.6, -0.4, 127.0]]],
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
value = -key
|
||||
|
||||
key_cache = torch.zeros(1, block_size, 1, head_size, dtype=torch.int8)
|
||||
value_cache = torch.zeros_like(key_cache)
|
||||
k_scale_cache = torch.ones(1, block_size, 1, dtype=torch.float32)
|
||||
v_scale_cache = torch.ones_like(k_scale_cache)
|
||||
slot_mapping = torch.tensor([2], dtype=torch.long)
|
||||
|
||||
triton_reshape_and_cache_flash_per_token_head_quant(
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
k_scale_cache,
|
||||
v_scale_cache,
|
||||
slot_mapping,
|
||||
)
|
||||
|
||||
ref_k_quant, ref_k_scales = _quantize_per_token_head_ref(key, INT8_CONFIG)
|
||||
ref_v_quant, ref_v_scales = _quantize_per_token_head_ref(value, INT8_CONFIG)
|
||||
|
||||
slot = slot_mapping.item()
|
||||
blk = slot // block_size
|
||||
off = slot % block_size
|
||||
assert torch.equal(key_cache[blk, off], ref_k_quant[0])
|
||||
assert torch.equal(value_cache[blk, off], ref_v_quant[0])
|
||||
torch.testing.assert_close(k_scale_cache[blk, off], ref_k_scales[0])
|
||||
torch.testing.assert_close(v_scale_cache[blk, off], ref_v_scales[0])
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 4. Negative slot mapping (padding tokens should be skipped)
|
||||
# ===========================================================================
|
||||
@@ -461,7 +507,7 @@ def test_triton_unified_attention_per_token_head_scale(
|
||||
|
||||
scaled_k = key_cache_bf16.float() / k_scale_cache[:, :, :, None]
|
||||
scaled_v = value_cache_bf16.float() / v_scale_cache[:, :, :, None]
|
||||
if qcfg.uses_trunc:
|
||||
if qcfg.rounds_before_store:
|
||||
key_cache_q = (
|
||||
scaled_k.round().clamp(qcfg.quant_min, qcfg.quant_max).to(qcfg.cache_dtype)
|
||||
)
|
||||
|
||||
@@ -181,6 +181,7 @@ def _reshape_cache_per_token_head(
|
||||
HEAD_SIZE_PADDED: tl.constexpr, # next_power_of_2(max(head_size, head_size_v))
|
||||
QUANT_MAX: tl.constexpr = 127.0,
|
||||
QUANT_MIN: tl.constexpr = -128.0,
|
||||
IS_INT_QUANT: tl.constexpr = False,
|
||||
):
|
||||
tok = tl.program_id(0)
|
||||
head = tl.program_id(1)
|
||||
@@ -211,7 +212,11 @@ def _reshape_cache_per_token_head(
|
||||
k_scale,
|
||||
)
|
||||
|
||||
k_q = tl.clamp(k_h * (1.0 / k_scale), QUANT_MIN, QUANT_MAX)
|
||||
k_q = k_h * (1.0 / k_scale)
|
||||
if IS_INT_QUANT:
|
||||
# Round half away from zero before the int8 store truncates.
|
||||
k_q = tl.where(k_q >= 0, k_q + 0.5, k_q - 0.5)
|
||||
k_q = tl.clamp(k_q, QUANT_MIN, QUANT_MAX)
|
||||
tl.store(
|
||||
key_cache_ptr
|
||||
+ blk * stride_kc_blk
|
||||
@@ -239,7 +244,11 @@ def _reshape_cache_per_token_head(
|
||||
v_scale,
|
||||
)
|
||||
|
||||
v_q = tl.clamp(v_h * (1.0 / v_scale), QUANT_MIN, QUANT_MAX)
|
||||
v_q = v_h * (1.0 / v_scale)
|
||||
if IS_INT_QUANT:
|
||||
# Round half away from zero before the int8 store truncates.
|
||||
v_q = tl.where(v_q >= 0, v_q + 0.5, v_q - 0.5)
|
||||
v_q = tl.clamp(v_q, QUANT_MIN, QUANT_MAX)
|
||||
tl.store(
|
||||
value_cache_ptr
|
||||
+ blk * stride_vc_blk
|
||||
@@ -327,6 +336,7 @@ def triton_reshape_and_cache_flash_per_token_head_quant(
|
||||
HEAD_SIZE_PADDED=head_size_padded,
|
||||
QUANT_MAX=quant_max,
|
||||
QUANT_MIN=quant_min,
|
||||
IS_INT_QUANT=cache_dtype == torch.int8,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user