diff --git a/tests/kernels/attention/test_attention_selector.py b/tests/kernels/attention/test_attention_selector.py index 24f02836e9..c6f4094eb6 100644 --- a/tests/kernels/attention/test_attention_selector.py +++ b/tests/kernels/attention/test_attention_selector.py @@ -455,6 +455,31 @@ def test_per_head_quant_scales_backend_selection( assert backend_name in str(exc_info.value) +@pytest.mark.skipif( + CudaPlatform is None, reason="CUDA platform is required for this test" +) +@pytest.mark.parametrize("head_size", [64, 80, 96, 128, 192, 256]) +def test_int4_per_token_head_head_size_selection(head_size: int): + """INT4 per-token-head KV cache is selectable at any even head size. + + The Hadamard rotation the INT4 write path applies is now defined for + non-power-of-two rows as well (it runs block-diagonally, see + ``fast_hadamard_transform``), so a model such as Phi-3 (head_size=96) must + keep selecting TRITON_ATTN instead of being rejected. + """ + vllm_config = VllmConfig(cache_config=CacheConfig(block_size=64)) + with ( + set_current_vllm_config(vllm_config), + patch("vllm.platforms.current_platform", CudaPlatform()), + ): + backend = get_attn_backend( + head_size=head_size, + dtype=torch.float16, + kv_cache_dtype="int4_per_token_head", + ) + assert backend.get_name() == "TRITON_ATTN" + + @pytest.mark.parametrize( "backend_name,use_non_causal,should_succeed", [ diff --git a/vllm/v1/attention/ops/int4_per_token_head.py b/vllm/v1/attention/ops/int4_per_token_head.py index 322d32bb81..3567f49d95 100644 --- a/vllm/v1/attention/ops/int4_per_token_head.py +++ b/vllm/v1/attention/ops/int4_per_token_head.py @@ -18,6 +18,7 @@ import torch from vllm.platforms import current_platform from vllm.triton_utils import tl, triton +from vllm.utils.math_utils import largest_power_of_2_divisor from vllm.v1.attention.ops.triton_attention_helpers import ( apply_alibi_to_score, apply_softcap, @@ -908,14 +909,15 @@ def unified_attention_int4( ) -> None: """Paged attention over the INT4 packed cache, writing into *out*. - The forward RHT has norm ``sqrt(head_size)``, so ``softmax_scale`` is - divided by ``head_size`` and the inverse RHT divides the output by - ``head_size`` as well. + The forward RHT has norm ``sqrt(b)`` for ``b`` the largest power-of-two + divisor of ``head_size`` (``b == head_size`` for power-of-two sizes), so + ``softmax_scale`` is divided by ``b`` and the inverse RHT divides the output + by ``b`` as well. """ q_orig_dtype = q.dtype q = single_rht(q.float()).to(q_orig_dtype) - head_size = q.shape[2] - softmax_scale = softmax_scale / head_size + rht_norm = largest_power_of_2_divisor(q.shape[2]) + softmax_scale = softmax_scale / rht_norm _launch_packed_attn( q=q, @@ -945,7 +947,7 @@ def unified_attention_int4( packing_factor=_INT4_PACKING_FACTOR, ) - out_f = single_rht(out.float(), inverse=True) / head_size + out_f = single_rht(out.float(), inverse=True) / rht_norm out.copy_(out_f.to(q_orig_dtype)) @@ -1148,6 +1150,16 @@ def single_rht(x: torch.Tensor, inverse: bool = False) -> torch.Tensor: before asymmetric quantization. """ d = x.shape[-1] + block = largest_power_of_2_divisor(d) + if block < d: + # H is only defined for power-of-two rows, so apply it + # block-diagonally: H_b inside each of the d / b blocks, for b the + # largest power-of-two divisor of d. That keeps H @ H.T = b * I + # without padding the row, hence the norm is sqrt(b), not sqrt(d). + lead = x.shape[:-1] + return single_rht(x.reshape(*lead, d // block, block), inverse).reshape( + *lead, d + ) d1 = _get_rht_signs(d, 0, x.device) if inverse: return fast_hadamard_transform(x) * d1