[Bugfix][Attention] Support int4_per_token_head on non-power-of-two head sizes (#56198)
Signed-off-by: Gawr Gura 0721 <LiRunGuo@users.noreply.github.com> Co-authored-by: Gawr Gura 0721 <LiRunGuo@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Gawr Gura 0721
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Wentao Ye
parent
ac8c2130a8
commit
e8a53a55e6
@@ -455,6 +455,31 @@ def test_per_head_quant_scales_backend_selection(
|
||||
assert backend_name in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
CudaPlatform is None, reason="CUDA platform is required for this test"
|
||||
)
|
||||
@pytest.mark.parametrize("head_size", [64, 80, 96, 128, 192, 256])
|
||||
def test_int4_per_token_head_head_size_selection(head_size: int):
|
||||
"""INT4 per-token-head KV cache is selectable at any even head size.
|
||||
|
||||
The Hadamard rotation the INT4 write path applies is now defined for
|
||||
non-power-of-two rows as well (it runs block-diagonally, see
|
||||
``fast_hadamard_transform``), so a model such as Phi-3 (head_size=96) must
|
||||
keep selecting TRITON_ATTN instead of being rejected.
|
||||
"""
|
||||
vllm_config = VllmConfig(cache_config=CacheConfig(block_size=64))
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch("vllm.platforms.current_platform", CudaPlatform()),
|
||||
):
|
||||
backend = get_attn_backend(
|
||||
head_size=head_size,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="int4_per_token_head",
|
||||
)
|
||||
assert backend.get_name() == "TRITON_ATTN"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name,use_non_causal,should_succeed",
|
||||
[
|
||||
|
||||
@@ -18,6 +18,7 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import largest_power_of_2_divisor
|
||||
from vllm.v1.attention.ops.triton_attention_helpers import (
|
||||
apply_alibi_to_score,
|
||||
apply_softcap,
|
||||
@@ -908,14 +909,15 @@ def unified_attention_int4(
|
||||
) -> None:
|
||||
"""Paged attention over the INT4 packed cache, writing into *out*.
|
||||
|
||||
The forward RHT has norm ``sqrt(head_size)``, so ``softmax_scale`` is
|
||||
divided by ``head_size`` and the inverse RHT divides the output by
|
||||
``head_size`` as well.
|
||||
The forward RHT has norm ``sqrt(b)`` for ``b`` the largest power-of-two
|
||||
divisor of ``head_size`` (``b == head_size`` for power-of-two sizes), so
|
||||
``softmax_scale`` is divided by ``b`` and the inverse RHT divides the output
|
||||
by ``b`` as well.
|
||||
"""
|
||||
q_orig_dtype = q.dtype
|
||||
q = single_rht(q.float()).to(q_orig_dtype)
|
||||
head_size = q.shape[2]
|
||||
softmax_scale = softmax_scale / head_size
|
||||
rht_norm = largest_power_of_2_divisor(q.shape[2])
|
||||
softmax_scale = softmax_scale / rht_norm
|
||||
|
||||
_launch_packed_attn(
|
||||
q=q,
|
||||
@@ -945,7 +947,7 @@ def unified_attention_int4(
|
||||
packing_factor=_INT4_PACKING_FACTOR,
|
||||
)
|
||||
|
||||
out_f = single_rht(out.float(), inverse=True) / head_size
|
||||
out_f = single_rht(out.float(), inverse=True) / rht_norm
|
||||
out.copy_(out_f.to(q_orig_dtype))
|
||||
|
||||
|
||||
@@ -1148,6 +1150,16 @@ def single_rht(x: torch.Tensor, inverse: bool = False) -> torch.Tensor:
|
||||
before asymmetric quantization.
|
||||
"""
|
||||
d = x.shape[-1]
|
||||
block = largest_power_of_2_divisor(d)
|
||||
if block < d:
|
||||
# H is only defined for power-of-two rows, so apply it
|
||||
# block-diagonally: H_b inside each of the d / b blocks, for b the
|
||||
# largest power-of-two divisor of d. That keeps H @ H.T = b * I
|
||||
# without padding the row, hence the norm is sqrt(b), not sqrt(d).
|
||||
lead = x.shape[:-1]
|
||||
return single_rht(x.reshape(*lead, d // block, block), inverse).reshape(
|
||||
*lead, d
|
||||
)
|
||||
d1 = _get_rht_signs(d, 0, x.device)
|
||||
if inverse:
|
||||
return fast_hadamard_transform(x) * d1
|
||||
|
||||
Reference in New Issue
Block a user