[Attention] Add UltraQuant 4-bit KV cache backend (FlyDSL D=256) (#57057)

Signed-off-by: Aditi Rana <aditi.rana@amd.com>
This commit is contained in:
aditi-amd
2026-10-06 20:42:52 +00:00
committed by GitHub
parent 819852df6b
commit 385d86a6cf
22 changed files with 5377 additions and 24 deletions
+36
View File
@@ -0,0 +1,36 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU-only integration tests for the UltraQuant cache dtype."""
from vllm.v1.attention.backends.ultraquant_attn import (
UltraQuantAttentionBackend,
)
from vllm.v1.attention.ops.flydsl_ultraquant_decode import (
ultraquant_flydsl_decode_eligible,
)
def test_ultraquant_prefers_block_size_64():
# FlyDSL D=256 decode geometry requires a 64-token KV block.
assert UltraQuantAttentionBackend.get_preferred_block_size(16) == 64
assert UltraQuantAttentionBackend.get_preferred_block_size(128) == 64
def test_ultraquant_flydsl_decode_routing():
"""CPU check of FlyDSL vs unified-Triton decode eligibility."""
eligible = dict(
head_size=256,
num_kv_groups=8,
has_sinks=False,
sliding_window=None,
flydsl_loaded=True,
)
assert ultraquant_flydsl_decode_eligible(**eligible)
for gqa in (6, 8, 16):
assert ultraquant_flydsl_decode_eligible(**{**eligible, "num_kv_groups": gqa})
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "flydsl_loaded": False})
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "head_size": 128})
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "num_kv_groups": 4})
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "has_sinks": True})
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "sliding_window": 4096})
@@ -0,0 +1,111 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the UltraQuant Triton unified-attention fallback."""
from __future__ import annotations
import pytest
import torch
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.ops.ultraquant.format import slot_size
from vllm.v1.attention.ops.ultraquant.reference import reference_ultraquant_attention
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
from vllm.v1.attention.ops.ultraquant.triton_unified_attention import (
ultraquant_unified_attention,
)
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA/HIP device"
)
def _build_paged_cache(key, value, block_size):
batch_size, seq_len, num_kv_heads, head_size = key.shape
blocks_per_seq = (seq_len + block_size - 1) // block_size
kv_cache = torch.zeros(
batch_size * blocks_per_seq,
block_size,
num_kv_heads,
slot_size(head_size),
dtype=torch.uint8,
device=key.device,
)
block_table = torch.arange(
batch_size * blocks_per_seq,
dtype=torch.int32,
device=key.device,
).view(batch_size, blocks_per_seq)
slot_mapping = torch.arange(
batch_size * blocks_per_seq * block_size,
dtype=torch.int64,
device=key.device,
).view(batch_size, -1)[:, :seq_len]
ultraquant_store(
key.reshape(-1, num_kv_heads, head_size),
value.reshape(-1, num_kv_heads, head_size),
kv_cache,
slot_mapping.reshape(-1),
)
return kv_cache, block_table
def _attention_quality(actual: torch.Tensor, expected: torch.Tensor) -> dict:
a = actual.float().reshape(-1)
b = expected.float().reshape(-1)
diff = (a - b).abs()
return {
"cos_sim": torch.nn.functional.cosine_similarity(a, b, dim=0).item(),
"mean_abs": diff.mean().item(),
"max_abs": diff.max().item(),
"norm_ratio": (a.norm() / b.norm()).item(),
}
def _unified_decode(query, kv_cache, block_table, seq_lens, scale, sinks=None):
batch_size = query.shape[0]
return ultraquant_unified_attention(
query=query,
kv_cache=kv_cache,
block_table=block_table,
seq_lens=seq_lens,
query_start_loc=torch.arange(
batch_size + 1, dtype=torch.int32, device=query.device
),
scale=scale,
max_query_len=1,
max_seq_len=int(seq_lens.max().item()),
sinks=sinks,
)
@pytest.mark.parametrize("query_group_size", [6, 8])
def test_unified_decode_d256_matches_reference(query_group_size):
"""The D=256 fallback covers both target Qwen GQA layouts."""
device = torch.device("cuda")
set_random_seed(0xD256 + query_group_size)
B, Hk, N, D = 2, 1, 256, 256
Hq = query_group_size * Hk
block_size = 32
dtype = torch.bfloat16
scale = D**-0.5
query = torch.randn(B, Hq, D, dtype=dtype, device=device)
key = torch.randn(B, N, Hk, D, dtype=dtype, device=device)
value = torch.randn_like(key)
kv_cache, block_table = _build_paged_cache(key, value, block_size)
seq_lens = torch.full((B,), N, dtype=torch.int32, device=device)
actual = _unified_decode(query, kv_cache, block_table, seq_lens, scale)
expected = reference_ultraquant_attention(
query=query,
key=key,
value=value,
scale=scale,
)
quality = _attention_quality(actual, expected)
assert quality["cos_sim"] > 0.98, quality
assert quality["mean_abs"] < 0.08, quality
assert quality["max_abs"] < 0.8, quality
@@ -0,0 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the UltraQuant full-KV dequant kernel (continuation prefill).
The kernel reconstructs cached K/V from FP4 codes plus UE8M0 group scales.
Because dequant is an exact table lookup times a power of two, the result
must match the reference to within the output dtype's rounding.
"""
from __future__ import annotations
import pytest
import torch
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.ops.ultraquant.format import slot_size
from vllm.v1.attention.ops.ultraquant.reference import (
ultraquant_dequant,
ultraquant_encode,
)
from vllm.v1.attention.ops.ultraquant.triton_dequant import (
ultraquant_full_dequant_kv,
)
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA/HIP device"
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("head_dim", [128, 256])
@pytest.mark.parametrize("num_kv_heads", [1, 4])
@pytest.mark.parametrize("cached_len", [32, 96])
def test_full_dequant_matches_reference(dtype, head_dim, num_kv_heads, cached_len):
device = torch.device("cuda")
set_random_seed(0xD4)
block_size = 32
alloc_len = cached_len
num_blocks = alloc_len // block_size + 1
key = torch.randn(alloc_len, num_kv_heads, head_dim, dtype=dtype, device=device)
value = torch.randn(alloc_len, num_kv_heads, head_dim, dtype=dtype, device=device)
kv_cache = torch.zeros(
num_blocks,
block_size,
num_kv_heads,
slot_size(head_dim),
dtype=torch.uint8,
device=device,
)
# Leave block 0 unused so the block table exercises a non-identity mapping.
slot_mapping = torch.arange(
block_size, block_size + alloc_len, dtype=torch.int64, device=device
)
ultraquant_store(key, value, kv_cache, slot_mapping)
block_table = torch.arange(
1, num_blocks, dtype=torch.int32, device=device
).unsqueeze(0)
k_out = torch.zeros(
1, num_kv_heads, alloc_len, head_dim, dtype=dtype, device=device
)
v_out = torch.zeros_like(k_out)
ultraquant_full_dequant_kv(
kv_cache=kv_cache,
block_table=block_table,
k_out=k_out,
v_out=v_out,
alloc_len=alloc_len,
)
torch.accelerator.synchronize()
# K is stored Hadamard-rotated, V unrotated; the kernel returns them as
# stored, so the reference must not inverse-rotate either.
ref_k = ultraquant_dequant(ultraquant_encode(key, rotate=True), head_dim)
ref_v = ultraquant_dequant(ultraquant_encode(value, rotate=False), head_dim)
got_k = k_out[0].transpose(0, 1).to(torch.float32)
got_v = v_out[0].transpose(0, 1).to(torch.float32)
# Reference is fp32; the kernel rounds to the output dtype once.
tol = 1e-2 if dtype is torch.bfloat16 else 1e-3
torch.testing.assert_close(got_k, ref_k.to(torch.float32), rtol=tol, atol=tol)
torch.testing.assert_close(got_v, ref_v.to(torch.float32), rtol=tol, atol=tol)
@@ -0,0 +1,149 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the production UltraQuant D=256 FlyDSL decode."""
import pytest
import torch
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.ops.flydsl_ultraquant_decode import (
flydsl_ultraquant_decode_attention,
is_ultraquant_flydsl_available,
ultraquant_flydsl_decode_eligible,
)
from vllm.v1.attention.ops.ultraquant.format import slot_size
from vllm.v1.attention.ops.ultraquant.reference import (
hadamard_matrix,
reference_ultraquant_attention,
)
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA/HIP device"
)
def _build_paged_cache(key, value, block_size):
batch_size, seq_len, num_kv_heads, head_size = key.shape
blocks_per_seq = (seq_len + block_size - 1) // block_size
kv_cache = torch.zeros(
batch_size * blocks_per_seq,
block_size,
num_kv_heads,
slot_size(head_size),
dtype=torch.uint8,
device=key.device,
)
block_table = torch.arange(
batch_size * blocks_per_seq,
dtype=torch.int32,
device=key.device,
).view(batch_size, blocks_per_seq)
slot_mapping = torch.arange(
batch_size * blocks_per_seq * block_size,
dtype=torch.int64,
device=key.device,
).view(batch_size, -1)[:, :seq_len]
ultraquant_store(
key.reshape(-1, num_kv_heads, head_size),
value.reshape(-1, num_kv_heads, head_size),
kv_cache,
slot_mapping.reshape(-1),
)
return kv_cache, block_table
@pytest.mark.parametrize(
("batch_size", "query_group_size", "block_size", "contiguous_query"),
[
(1, 8, 32, True),
(8, 8, 64, True),
(9, 8, 256, True),
(1, 16, 32, True),
(1, 6, 16, True),
(1, 6, 32, True),
(2, 6, 64, True),
(1, 6, 128, True),
(1, 6, 256, True),
(1, 6, 32, False),
],
)
def test_flydsl_decode_matches_reference(
batch_size,
query_group_size,
block_size,
contiguous_query,
):
if not ultraquant_flydsl_decode_eligible(
head_size=256,
num_kv_groups=query_group_size,
has_sinks=False,
sliding_window=None,
flydsl_loaded=is_ultraquant_flydsl_available(),
):
pytest.skip("UltraQuant D=256 FlyDSL kernel is unavailable")
set_random_seed(0x57A1D + batch_size + block_size + query_group_size)
device = torch.device("cuda")
dtype = torch.bfloat16
num_kv_heads = 1
head_size = 256
seq_len = 2048
scale = head_size**-0.5
key = torch.randn(
batch_size,
seq_len,
num_kv_heads,
head_size,
dtype=dtype,
device=device,
)
value = torch.randn_like(key)
if contiguous_query:
qkv = torch.randn(
batch_size,
query_group_size + 2 * num_kv_heads,
head_size,
dtype=dtype,
device=device,
)
query = qkv[:, :query_group_size]
else:
query_storage = torch.randn(
batch_size,
query_group_size,
head_size * 2,
dtype=dtype,
device=device,
)
query = query_storage[..., ::2]
kv_cache, block_table = _build_paged_cache(key, value, block_size)
seq_lens = torch.full((batch_size,), seq_len, dtype=torch.int32, device=device)
hadamard = hadamard_matrix(head_size, device)
actual = flydsl_ultraquant_decode_attention(
query=query,
kv_cache=kv_cache,
block_table=block_table,
seq_lens=seq_lens,
scale=scale,
PiT=hadamard,
max_seq_len=seq_len,
)
expected = reference_ultraquant_attention(
query=query,
key=key,
value=value,
scale=scale,
)
actual_f32 = actual.float()
expected_f32 = expected.float()
cosine = torch.nn.functional.cosine_similarity(
actual_f32.flatten(), expected_f32.flatten(), dim=0
)
error = (actual_f32 - expected_f32).abs()
assert cosine > 0.995
assert error.mean() < 0.05
assert error.max() < 0.8
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""UltraQuant inherits TurboQuant's forward/do_kv_cache_update orchestration.
UltraQuant only overrides the store/decode/prefill seams, so a TurboQuant-side
change to a seam signature, to an inherited helper, or to the ``layer._tq_*``
attributes ``forward`` reads will break UltraQuant at serve time without
touching any UltraQuant kernel test. These checks pin that contract.
"""
import ast
import inspect
import textwrap
from vllm.v1.attention.backends import ultraquant_attn
from vllm.v1.attention.backends.turboquant_attn import TurboQuantAttentionImpl
from vllm.v1.attention.backends.ultraquant_attn import UltraQuantAttentionImpl
_SEAMS = ("_ensure_on_device", "_store_kv", "_decode_attention", "_prefill_attention")
def _method_ast(cls: type, name: str) -> ast.AST:
return ast.parse(textwrap.dedent(inspect.getsource(getattr(cls, name))))
def _tq_attrs_read(cls: type, name: str) -> set[str]:
"""``_tq_*`` attribute names read (not assigned) in ``cls.name``."""
tree = _method_ast(cls, name)
assigned = {
t.attr
for node in ast.walk(tree)
if isinstance(node, ast.Assign)
for t in node.targets
if isinstance(t, ast.Attribute)
}
read = {
node.attr
for node in ast.walk(tree)
if isinstance(node, ast.Attribute) and node.attr.startswith("_tq_")
}
return read - assigned
def _tq_attrs_written(cls: type, name: str) -> set[str]:
tree = _method_ast(cls, name)
return {
t.attr
for node in ast.walk(tree)
if isinstance(node, ast.Assign)
for t in node.targets
if isinstance(t, ast.Attribute) and t.attr.startswith("_tq_")
}
def test_ultraquant_matches_turboquant_seam_contract():
# 1. Each overridden seam accepts every parameter the base declares, so any
# call the inherited orchestration makes stays valid for UltraQuant.
for name in _SEAMS:
base = list(
inspect.signature(getattr(TurboQuantAttentionImpl, name)).parameters
)
override = list(
inspect.signature(getattr(UltraQuantAttentionImpl, name)).parameters
)
assert override[: len(base)] == base, (
f"UltraQuantAttentionImpl.{name} no longer accepts the arguments "
f"TurboQuantAttentionImpl passes: base={base} override={override}"
)
# 2. Keyword arguments UltraQuant passes to *inherited* TurboQuant helpers
# must exist on those helpers (e.g. _flash_attn_varlen).
own = vars(UltraQuantAttentionImpl)
module = ast.parse(inspect.getsource(ultraquant_attn))
for node in ast.walk(module):
if not isinstance(node, ast.Call):
continue
fn = node.func
if not (
isinstance(fn, ast.Attribute)
and isinstance(fn.value, ast.Name)
and fn.value.id == "self"
):
continue
target = getattr(TurboQuantAttentionImpl, fn.attr, None)
if target is None or fn.attr in own or not callable(target):
continue
params = inspect.signature(target).parameters
if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()):
continue
for kw in node.keywords:
if kw.arg is None:
continue
assert kw.arg in params, (
f"UltraQuant calls inherited {fn.attr}(..., {kw.arg}=...) but "
f"TurboQuantAttentionImpl.{fn.attr} accepts {list(params)}"
)
# 3. Every layer._tq_* attribute the inherited forward reads is set by
# UltraQuant's _ensure_on_device.
required = _tq_attrs_read(TurboQuantAttentionImpl, "forward")
provided = _tq_attrs_written(UltraQuantAttentionImpl, "_ensure_on_device")
assert required, "expected inherited forward to read layer._tq_* attributes"
assert required <= provided, (
f"UltraQuant._ensure_on_device does not set {sorted(required - provided)}, "
f"which the inherited TurboQuant forward reads"
)
@@ -0,0 +1,149 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the Triton UltraQuant store kernel.
The Triton kernel must produce bit-identical codes and scale bytes vs
the PyTorch reference (``reference.ultraquant_encode``).
"""
from __future__ import annotations
import pytest
import torch
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.ops.ultraquant.format import (
k_codes_offset,
k_scales_offset,
n_groups,
slot_size,
v_codes_offset,
v_scales_offset,
)
from vllm.v1.attention.ops.ultraquant.reference import ultraquant_encode
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA/HIP device"
)
def _make_cache(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_dim: int,
device: torch.device,
) -> torch.Tensor:
slot_b = slot_size(head_dim)
return torch.zeros(
num_blocks, block_size, num_kv_heads, slot_b, dtype=torch.uint8, device=device
)
def _read_codes_and_scales(
kv_cache: torch.Tensor,
slot_mapping: torch.Tensor,
num_kv_heads: int,
head_dim: int,
):
"""Pull (k_codes, k_scale_bytes, v_codes, v_scale_bytes) for each
(token, head) from the AoS slot layout. Scales come back as uint8
(one E8M0 byte per group)."""
block_size = kv_cache.shape[1]
N = slot_mapping.shape[0]
Gk = n_groups(head_dim)
k_codes_off = k_codes_offset(head_dim)
k_scales_off = k_scales_offset(head_dim)
v_codes_off = v_codes_offset(head_dim)
v_scales_off = v_scales_offset(head_dim)
codes_bytes = head_dim // 2
k_codes = torch.zeros(
N, num_kv_heads, codes_bytes, dtype=torch.uint8, device=kv_cache.device
)
k_scale_bytes = torch.zeros(
N, num_kv_heads, Gk, dtype=torch.uint8, device=kv_cache.device
)
v_codes = torch.zeros_like(k_codes)
v_scale_bytes = torch.zeros_like(k_scale_bytes)
for ti in range(N):
slot = int(slot_mapping[ti].item())
if slot < 0:
continue
blk = slot // block_size
off = slot % block_size
for h in range(num_kv_heads):
slot_bytes = kv_cache[blk, off, h] # [slot_size_aligned] uint8
k_codes[ti, h] = slot_bytes[k_codes_off : k_codes_off + codes_bytes]
k_scale_bytes[ti, h] = slot_bytes[k_scales_off : k_scales_off + Gk]
v_codes[ti, h] = slot_bytes[v_codes_off : v_codes_off + codes_bytes]
v_scale_bytes[ti, h] = slot_bytes[v_scales_off : v_scales_off + Gk]
return k_codes, k_scale_bytes, v_codes, v_scale_bytes
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("head_dim", [128, 256])
@pytest.mark.parametrize("num_kv_heads", [1, 4])
@pytest.mark.parametrize("num_tokens", [1, 16, 17])
def test_store_matches_reference(dtype, head_dim, num_kv_heads, num_tokens):
"""Triton store output (codes + E8M0 scale bytes) matches the reference
bit-exactly."""
device = torch.device("cuda")
set_random_seed(0xF8)
block_size = 32
num_blocks = max(2, (num_tokens + block_size - 1) // block_size + 1)
key = torch.randn(num_tokens, num_kv_heads, head_dim, dtype=dtype, device=device)
value = torch.randn(num_tokens, num_kv_heads, head_dim, dtype=dtype, device=device)
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
if num_tokens >= 4:
slot_mapping[1] = -1 # skip-path test
kv_cache = _make_cache(num_blocks, block_size, num_kv_heads, head_dim, device)
ultraquant_store(key, value, kv_cache, slot_mapping)
torch.accelerator.synchronize()
ref_k = ultraquant_encode(key, rotate=True)
ref_v = ultraquant_encode(value, rotate=False)
k_codes, k_scale_bytes, v_codes, v_scale_bytes = _read_codes_and_scales(
kv_cache, slot_mapping, num_kv_heads, head_dim
)
if num_tokens >= 4:
assert (kv_cache[0, 1, :, :] == 0).all(), (
"skipped slot's physical cache region was unexpectedly written"
)
for ti in range(num_tokens):
slot = int(slot_mapping[ti].item())
if slot < 0:
continue
assert torch.equal(k_codes[ti], ref_k.codes_packed[ti]), (
f"K codes mismatch at token {ti}\n"
f"got: {k_codes[ti].cpu().numpy().tolist()[:16]}\n"
f"expected: {ref_k.codes_packed[ti].cpu().numpy().tolist()[:16]}"
)
assert torch.equal(k_scale_bytes[ti], ref_k.scale_bytes[ti]), (
f"K scale bytes mismatch at token {ti}\n"
f"got: {k_scale_bytes[ti].cpu().numpy().tolist()}\n"
f"expected: {ref_k.scale_bytes[ti].cpu().numpy().tolist()}"
)
assert torch.equal(v_codes[ti], ref_v.codes_packed[ti]), (
f"V codes mismatch at token {ti}"
)
assert torch.equal(v_scale_bytes[ti], ref_v.scale_bytes[ti]), (
f"V scale bytes mismatch at token {ti}\n"
f"got: {v_scale_bytes[ti].cpu().numpy().tolist()}\n"
f"expected: {ref_v.scale_bytes[ti].cpu().numpy().tolist()}"
)