diff --git a/tests/quantization/test_per_token_kv_cache.py b/tests/quantization/test_per_token_kv_cache.py index 254e284efb..b657c77a29 100644 --- a/tests/quantization/test_per_token_kv_cache.py +++ b/tests/quantization/test_per_token_kv_cache.py @@ -61,8 +61,8 @@ class QuantConfig: quant_max: float quant_min: float kv_quant_mode: KVQuantMode - # INT8 Triton stores truncate; FP8 hardware casts round. - uses_trunc: bool + # INT8 rounds explicitly; FP8 relies on dtype cast rounding. + rounds_before_store: bool INT8_CONFIG = QuantConfig( @@ -71,7 +71,7 @@ INT8_CONFIG = QuantConfig( quant_max=127.0, quant_min=-128.0, kv_quant_mode=KVQuantMode.INT8_PER_TOKEN_HEAD, - uses_trunc=True, + rounds_before_store=True, ) FP8_CONFIG = QuantConfig( cache_dtype=FP8_DTYPE, @@ -79,7 +79,7 @@ FP8_CONFIG = QuantConfig( quant_max=FP8_MAX, quant_min=FP8_MIN, kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD, - uses_trunc=False, + rounds_before_store=False, ) QUANT_CONFIGS = [INT8_CONFIG, FP8_CONFIG] @@ -104,7 +104,7 @@ def _quantize_per_token_head_ref( absmax = data.float().abs().amax(dim=2) # [num_tokens, num_heads] scales = (absmax / cfg.quant_max).clamp(min=1e-6) scaled = data.float() * (1.0 / scales[:, :, None]) - if cfg.uses_trunc: + if cfg.rounds_before_store: q = scaled.round().clamp(cfg.quant_min, cfg.quant_max).to(cfg.cache_dtype) else: q = scaled.clamp(cfg.quant_min, cfg.quant_max).to(cfg.cache_dtype) @@ -255,7 +255,7 @@ def test_per_token_head_round_trip_accuracy( ): """Verify per-token-head round-trip: kernel dequant matches reference. - INT8: Triton truncates on float->int8 store. + INT8: round-to-nearest before int8 store. FP8: hardware cast (clamp then cast). """ from vllm.v1.attention.ops.triton_reshape_and_cache_flash import ( @@ -315,6 +315,52 @@ def test_per_token_head_round_trip_accuracy( ) +@torch.inference_mode() +def test_int8_per_token_head_raw_cache_matches_round_reference(): + """INT8 cache writes should match round-to-nearest quantization exactly.""" + from vllm.v1.attention.ops.triton_reshape_and_cache_flash import ( + triton_reshape_and_cache_flash_per_token_head_quant, + ) + + torch.set_default_device(DEVICE_TYPE) + + head_size = 8 + block_size = 4 + + key = torch.tensor( + [[[-127.0, -2.6, -2.4, -1.6, -1.4, -0.6, -0.4, 127.0]]], + dtype=torch.bfloat16, + ) + value = -key + + key_cache = torch.zeros(1, block_size, 1, head_size, dtype=torch.int8) + value_cache = torch.zeros_like(key_cache) + k_scale_cache = torch.ones(1, block_size, 1, dtype=torch.float32) + v_scale_cache = torch.ones_like(k_scale_cache) + slot_mapping = torch.tensor([2], dtype=torch.long) + + triton_reshape_and_cache_flash_per_token_head_quant( + key, + value, + key_cache, + value_cache, + k_scale_cache, + v_scale_cache, + slot_mapping, + ) + + ref_k_quant, ref_k_scales = _quantize_per_token_head_ref(key, INT8_CONFIG) + ref_v_quant, ref_v_scales = _quantize_per_token_head_ref(value, INT8_CONFIG) + + slot = slot_mapping.item() + blk = slot // block_size + off = slot % block_size + assert torch.equal(key_cache[blk, off], ref_k_quant[0]) + assert torch.equal(value_cache[blk, off], ref_v_quant[0]) + torch.testing.assert_close(k_scale_cache[blk, off], ref_k_scales[0]) + torch.testing.assert_close(v_scale_cache[blk, off], ref_v_scales[0]) + + # =========================================================================== # 4. Negative slot mapping (padding tokens should be skipped) # =========================================================================== @@ -461,7 +507,7 @@ def test_triton_unified_attention_per_token_head_scale( scaled_k = key_cache_bf16.float() / k_scale_cache[:, :, :, None] scaled_v = value_cache_bf16.float() / v_scale_cache[:, :, :, None] - if qcfg.uses_trunc: + if qcfg.rounds_before_store: key_cache_q = ( scaled_k.round().clamp(qcfg.quant_min, qcfg.quant_max).to(qcfg.cache_dtype) ) diff --git a/vllm/v1/attention/ops/triton_reshape_and_cache_flash.py b/vllm/v1/attention/ops/triton_reshape_and_cache_flash.py index 320b7aa597..fb0c923055 100644 --- a/vllm/v1/attention/ops/triton_reshape_and_cache_flash.py +++ b/vllm/v1/attention/ops/triton_reshape_and_cache_flash.py @@ -181,6 +181,7 @@ def _reshape_cache_per_token_head( HEAD_SIZE_PADDED: tl.constexpr, # next_power_of_2(max(head_size, head_size_v)) QUANT_MAX: tl.constexpr = 127.0, QUANT_MIN: tl.constexpr = -128.0, + IS_INT_QUANT: tl.constexpr = False, ): tok = tl.program_id(0) head = tl.program_id(1) @@ -211,7 +212,11 @@ def _reshape_cache_per_token_head( k_scale, ) - k_q = tl.clamp(k_h * (1.0 / k_scale), QUANT_MIN, QUANT_MAX) + k_q = k_h * (1.0 / k_scale) + if IS_INT_QUANT: + # Round half away from zero before the int8 store truncates. + k_q = tl.where(k_q >= 0, k_q + 0.5, k_q - 0.5) + k_q = tl.clamp(k_q, QUANT_MIN, QUANT_MAX) tl.store( key_cache_ptr + blk * stride_kc_blk @@ -239,7 +244,11 @@ def _reshape_cache_per_token_head( v_scale, ) - v_q = tl.clamp(v_h * (1.0 / v_scale), QUANT_MIN, QUANT_MAX) + v_q = v_h * (1.0 / v_scale) + if IS_INT_QUANT: + # Round half away from zero before the int8 store truncates. + v_q = tl.where(v_q >= 0, v_q + 0.5, v_q - 0.5) + v_q = tl.clamp(v_q, QUANT_MIN, QUANT_MAX) tl.store( value_cache_ptr + blk * stride_vc_blk @@ -327,6 +336,7 @@ def triton_reshape_and_cache_flash_per_token_head_quant( HEAD_SIZE_PADDED=head_size_padded, QUANT_MAX=quant_max, QUANT_MIN=quant_min, + IS_INT_QUANT=cache_dtype == torch.int8, num_warps=num_warps, )