[Attention] Add UltraQuant 4-bit KV cache backend (FlyDSL D=256) (#57057)
Signed-off-by: Aditi Rana <aditi.rana@amd.com>
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU-only integration tests for the UltraQuant cache dtype."""
|
||||
|
||||
from vllm.v1.attention.backends.ultraquant_attn import (
|
||||
UltraQuantAttentionBackend,
|
||||
)
|
||||
from vllm.v1.attention.ops.flydsl_ultraquant_decode import (
|
||||
ultraquant_flydsl_decode_eligible,
|
||||
)
|
||||
|
||||
|
||||
def test_ultraquant_prefers_block_size_64():
|
||||
# FlyDSL D=256 decode geometry requires a 64-token KV block.
|
||||
assert UltraQuantAttentionBackend.get_preferred_block_size(16) == 64
|
||||
assert UltraQuantAttentionBackend.get_preferred_block_size(128) == 64
|
||||
|
||||
|
||||
def test_ultraquant_flydsl_decode_routing():
|
||||
"""CPU check of FlyDSL vs unified-Triton decode eligibility."""
|
||||
eligible = dict(
|
||||
head_size=256,
|
||||
num_kv_groups=8,
|
||||
has_sinks=False,
|
||||
sliding_window=None,
|
||||
flydsl_loaded=True,
|
||||
)
|
||||
assert ultraquant_flydsl_decode_eligible(**eligible)
|
||||
for gqa in (6, 8, 16):
|
||||
assert ultraquant_flydsl_decode_eligible(**{**eligible, "num_kv_groups": gqa})
|
||||
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "flydsl_loaded": False})
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "head_size": 128})
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "num_kv_groups": 4})
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "has_sinks": True})
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "sliding_window": 4096})
|
||||
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the UltraQuant Triton unified-attention fallback."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops.ultraquant.format import slot_size
|
||||
from vllm.v1.attention.ops.ultraquant.reference import reference_ultraquant_attention
|
||||
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
|
||||
from vllm.v1.attention.ops.ultraquant.triton_unified_attention import (
|
||||
ultraquant_unified_attention,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA/HIP device"
|
||||
)
|
||||
|
||||
|
||||
def _build_paged_cache(key, value, block_size):
|
||||
batch_size, seq_len, num_kv_heads, head_size = key.shape
|
||||
blocks_per_seq = (seq_len + block_size - 1) // block_size
|
||||
kv_cache = torch.zeros(
|
||||
batch_size * blocks_per_seq,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
slot_size(head_size),
|
||||
dtype=torch.uint8,
|
||||
device=key.device,
|
||||
)
|
||||
block_table = torch.arange(
|
||||
batch_size * blocks_per_seq,
|
||||
dtype=torch.int32,
|
||||
device=key.device,
|
||||
).view(batch_size, blocks_per_seq)
|
||||
slot_mapping = torch.arange(
|
||||
batch_size * blocks_per_seq * block_size,
|
||||
dtype=torch.int64,
|
||||
device=key.device,
|
||||
).view(batch_size, -1)[:, :seq_len]
|
||||
ultraquant_store(
|
||||
key.reshape(-1, num_kv_heads, head_size),
|
||||
value.reshape(-1, num_kv_heads, head_size),
|
||||
kv_cache,
|
||||
slot_mapping.reshape(-1),
|
||||
)
|
||||
return kv_cache, block_table
|
||||
|
||||
|
||||
def _attention_quality(actual: torch.Tensor, expected: torch.Tensor) -> dict:
|
||||
a = actual.float().reshape(-1)
|
||||
b = expected.float().reshape(-1)
|
||||
diff = (a - b).abs()
|
||||
return {
|
||||
"cos_sim": torch.nn.functional.cosine_similarity(a, b, dim=0).item(),
|
||||
"mean_abs": diff.mean().item(),
|
||||
"max_abs": diff.max().item(),
|
||||
"norm_ratio": (a.norm() / b.norm()).item(),
|
||||
}
|
||||
|
||||
|
||||
def _unified_decode(query, kv_cache, block_table, seq_lens, scale, sinks=None):
|
||||
batch_size = query.shape[0]
|
||||
return ultraquant_unified_attention(
|
||||
query=query,
|
||||
kv_cache=kv_cache,
|
||||
block_table=block_table,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=torch.arange(
|
||||
batch_size + 1, dtype=torch.int32, device=query.device
|
||||
),
|
||||
scale=scale,
|
||||
max_query_len=1,
|
||||
max_seq_len=int(seq_lens.max().item()),
|
||||
sinks=sinks,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query_group_size", [6, 8])
|
||||
def test_unified_decode_d256_matches_reference(query_group_size):
|
||||
"""The D=256 fallback covers both target Qwen GQA layouts."""
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(0xD256 + query_group_size)
|
||||
|
||||
B, Hk, N, D = 2, 1, 256, 256
|
||||
Hq = query_group_size * Hk
|
||||
block_size = 32
|
||||
dtype = torch.bfloat16
|
||||
scale = D**-0.5
|
||||
|
||||
query = torch.randn(B, Hq, D, dtype=dtype, device=device)
|
||||
key = torch.randn(B, N, Hk, D, dtype=dtype, device=device)
|
||||
value = torch.randn_like(key)
|
||||
kv_cache, block_table = _build_paged_cache(key, value, block_size)
|
||||
seq_lens = torch.full((B,), N, dtype=torch.int32, device=device)
|
||||
|
||||
actual = _unified_decode(query, kv_cache, block_table, seq_lens, scale)
|
||||
expected = reference_ultraquant_attention(
|
||||
query=query,
|
||||
key=key,
|
||||
value=value,
|
||||
scale=scale,
|
||||
)
|
||||
|
||||
quality = _attention_quality(actual, expected)
|
||||
assert quality["cos_sim"] > 0.98, quality
|
||||
assert quality["mean_abs"] < 0.08, quality
|
||||
assert quality["max_abs"] < 0.8, quality
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the UltraQuant full-KV dequant kernel (continuation prefill).
|
||||
|
||||
The kernel reconstructs cached K/V from FP4 codes plus UE8M0 group scales.
|
||||
Because dequant is an exact table lookup times a power of two, the result
|
||||
must match the reference to within the output dtype's rounding.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops.ultraquant.format import slot_size
|
||||
from vllm.v1.attention.ops.ultraquant.reference import (
|
||||
ultraquant_dequant,
|
||||
ultraquant_encode,
|
||||
)
|
||||
from vllm.v1.attention.ops.ultraquant.triton_dequant import (
|
||||
ultraquant_full_dequant_kv,
|
||||
)
|
||||
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA/HIP device"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("head_dim", [128, 256])
|
||||
@pytest.mark.parametrize("num_kv_heads", [1, 4])
|
||||
@pytest.mark.parametrize("cached_len", [32, 96])
|
||||
def test_full_dequant_matches_reference(dtype, head_dim, num_kv_heads, cached_len):
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(0xD4)
|
||||
|
||||
block_size = 32
|
||||
alloc_len = cached_len
|
||||
num_blocks = alloc_len // block_size + 1
|
||||
|
||||
key = torch.randn(alloc_len, num_kv_heads, head_dim, dtype=dtype, device=device)
|
||||
value = torch.randn(alloc_len, num_kv_heads, head_dim, dtype=dtype, device=device)
|
||||
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
slot_size(head_dim),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
# Leave block 0 unused so the block table exercises a non-identity mapping.
|
||||
slot_mapping = torch.arange(
|
||||
block_size, block_size + alloc_len, dtype=torch.int64, device=device
|
||||
)
|
||||
ultraquant_store(key, value, kv_cache, slot_mapping)
|
||||
|
||||
block_table = torch.arange(
|
||||
1, num_blocks, dtype=torch.int32, device=device
|
||||
).unsqueeze(0)
|
||||
|
||||
k_out = torch.zeros(
|
||||
1, num_kv_heads, alloc_len, head_dim, dtype=dtype, device=device
|
||||
)
|
||||
v_out = torch.zeros_like(k_out)
|
||||
ultraquant_full_dequant_kv(
|
||||
kv_cache=kv_cache,
|
||||
block_table=block_table,
|
||||
k_out=k_out,
|
||||
v_out=v_out,
|
||||
alloc_len=alloc_len,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# K is stored Hadamard-rotated, V unrotated; the kernel returns them as
|
||||
# stored, so the reference must not inverse-rotate either.
|
||||
ref_k = ultraquant_dequant(ultraquant_encode(key, rotate=True), head_dim)
|
||||
ref_v = ultraquant_dequant(ultraquant_encode(value, rotate=False), head_dim)
|
||||
|
||||
got_k = k_out[0].transpose(0, 1).to(torch.float32)
|
||||
got_v = v_out[0].transpose(0, 1).to(torch.float32)
|
||||
|
||||
# Reference is fp32; the kernel rounds to the output dtype once.
|
||||
tol = 1e-2 if dtype is torch.bfloat16 else 1e-3
|
||||
torch.testing.assert_close(got_k, ref_k.to(torch.float32), rtol=tol, atol=tol)
|
||||
torch.testing.assert_close(got_v, ref_v.to(torch.float32), rtol=tol, atol=tol)
|
||||
@@ -0,0 +1,149 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the production UltraQuant D=256 FlyDSL decode."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops.flydsl_ultraquant_decode import (
|
||||
flydsl_ultraquant_decode_attention,
|
||||
is_ultraquant_flydsl_available,
|
||||
ultraquant_flydsl_decode_eligible,
|
||||
)
|
||||
from vllm.v1.attention.ops.ultraquant.format import slot_size
|
||||
from vllm.v1.attention.ops.ultraquant.reference import (
|
||||
hadamard_matrix,
|
||||
reference_ultraquant_attention,
|
||||
)
|
||||
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA/HIP device"
|
||||
)
|
||||
|
||||
|
||||
def _build_paged_cache(key, value, block_size):
|
||||
batch_size, seq_len, num_kv_heads, head_size = key.shape
|
||||
blocks_per_seq = (seq_len + block_size - 1) // block_size
|
||||
kv_cache = torch.zeros(
|
||||
batch_size * blocks_per_seq,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
slot_size(head_size),
|
||||
dtype=torch.uint8,
|
||||
device=key.device,
|
||||
)
|
||||
block_table = torch.arange(
|
||||
batch_size * blocks_per_seq,
|
||||
dtype=torch.int32,
|
||||
device=key.device,
|
||||
).view(batch_size, blocks_per_seq)
|
||||
slot_mapping = torch.arange(
|
||||
batch_size * blocks_per_seq * block_size,
|
||||
dtype=torch.int64,
|
||||
device=key.device,
|
||||
).view(batch_size, -1)[:, :seq_len]
|
||||
ultraquant_store(
|
||||
key.reshape(-1, num_kv_heads, head_size),
|
||||
value.reshape(-1, num_kv_heads, head_size),
|
||||
kv_cache,
|
||||
slot_mapping.reshape(-1),
|
||||
)
|
||||
return kv_cache, block_table
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch_size", "query_group_size", "block_size", "contiguous_query"),
|
||||
[
|
||||
(1, 8, 32, True),
|
||||
(8, 8, 64, True),
|
||||
(9, 8, 256, True),
|
||||
(1, 16, 32, True),
|
||||
(1, 6, 16, True),
|
||||
(1, 6, 32, True),
|
||||
(2, 6, 64, True),
|
||||
(1, 6, 128, True),
|
||||
(1, 6, 256, True),
|
||||
(1, 6, 32, False),
|
||||
],
|
||||
)
|
||||
def test_flydsl_decode_matches_reference(
|
||||
batch_size,
|
||||
query_group_size,
|
||||
block_size,
|
||||
contiguous_query,
|
||||
):
|
||||
if not ultraquant_flydsl_decode_eligible(
|
||||
head_size=256,
|
||||
num_kv_groups=query_group_size,
|
||||
has_sinks=False,
|
||||
sliding_window=None,
|
||||
flydsl_loaded=is_ultraquant_flydsl_available(),
|
||||
):
|
||||
pytest.skip("UltraQuant D=256 FlyDSL kernel is unavailable")
|
||||
|
||||
set_random_seed(0x57A1D + batch_size + block_size + query_group_size)
|
||||
device = torch.device("cuda")
|
||||
dtype = torch.bfloat16
|
||||
num_kv_heads = 1
|
||||
head_size = 256
|
||||
seq_len = 2048
|
||||
scale = head_size**-0.5
|
||||
|
||||
key = torch.randn(
|
||||
batch_size,
|
||||
seq_len,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
value = torch.randn_like(key)
|
||||
if contiguous_query:
|
||||
qkv = torch.randn(
|
||||
batch_size,
|
||||
query_group_size + 2 * num_kv_heads,
|
||||
head_size,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
query = qkv[:, :query_group_size]
|
||||
else:
|
||||
query_storage = torch.randn(
|
||||
batch_size,
|
||||
query_group_size,
|
||||
head_size * 2,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
query = query_storage[..., ::2]
|
||||
kv_cache, block_table = _build_paged_cache(key, value, block_size)
|
||||
seq_lens = torch.full((batch_size,), seq_len, dtype=torch.int32, device=device)
|
||||
hadamard = hadamard_matrix(head_size, device)
|
||||
|
||||
actual = flydsl_ultraquant_decode_attention(
|
||||
query=query,
|
||||
kv_cache=kv_cache,
|
||||
block_table=block_table,
|
||||
seq_lens=seq_lens,
|
||||
scale=scale,
|
||||
PiT=hadamard,
|
||||
max_seq_len=seq_len,
|
||||
)
|
||||
expected = reference_ultraquant_attention(
|
||||
query=query,
|
||||
key=key,
|
||||
value=value,
|
||||
scale=scale,
|
||||
)
|
||||
|
||||
actual_f32 = actual.float()
|
||||
expected_f32 = expected.float()
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
actual_f32.flatten(), expected_f32.flatten(), dim=0
|
||||
)
|
||||
error = (actual_f32 - expected_f32).abs()
|
||||
assert cosine > 0.995
|
||||
assert error.mean() < 0.05
|
||||
assert error.max() < 0.8
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""UltraQuant inherits TurboQuant's forward/do_kv_cache_update orchestration.
|
||||
|
||||
UltraQuant only overrides the store/decode/prefill seams, so a TurboQuant-side
|
||||
change to a seam signature, to an inherited helper, or to the ``layer._tq_*``
|
||||
attributes ``forward`` reads will break UltraQuant at serve time without
|
||||
touching any UltraQuant kernel test. These checks pin that contract.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
|
||||
from vllm.v1.attention.backends import ultraquant_attn
|
||||
from vllm.v1.attention.backends.turboquant_attn import TurboQuantAttentionImpl
|
||||
from vllm.v1.attention.backends.ultraquant_attn import UltraQuantAttentionImpl
|
||||
|
||||
_SEAMS = ("_ensure_on_device", "_store_kv", "_decode_attention", "_prefill_attention")
|
||||
|
||||
|
||||
def _method_ast(cls: type, name: str) -> ast.AST:
|
||||
return ast.parse(textwrap.dedent(inspect.getsource(getattr(cls, name))))
|
||||
|
||||
|
||||
def _tq_attrs_read(cls: type, name: str) -> set[str]:
|
||||
"""``_tq_*`` attribute names read (not assigned) in ``cls.name``."""
|
||||
tree = _method_ast(cls, name)
|
||||
assigned = {
|
||||
t.attr
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Assign)
|
||||
for t in node.targets
|
||||
if isinstance(t, ast.Attribute)
|
||||
}
|
||||
read = {
|
||||
node.attr
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Attribute) and node.attr.startswith("_tq_")
|
||||
}
|
||||
return read - assigned
|
||||
|
||||
|
||||
def _tq_attrs_written(cls: type, name: str) -> set[str]:
|
||||
tree = _method_ast(cls, name)
|
||||
return {
|
||||
t.attr
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Assign)
|
||||
for t in node.targets
|
||||
if isinstance(t, ast.Attribute) and t.attr.startswith("_tq_")
|
||||
}
|
||||
|
||||
|
||||
def test_ultraquant_matches_turboquant_seam_contract():
|
||||
# 1. Each overridden seam accepts every parameter the base declares, so any
|
||||
# call the inherited orchestration makes stays valid for UltraQuant.
|
||||
for name in _SEAMS:
|
||||
base = list(
|
||||
inspect.signature(getattr(TurboQuantAttentionImpl, name)).parameters
|
||||
)
|
||||
override = list(
|
||||
inspect.signature(getattr(UltraQuantAttentionImpl, name)).parameters
|
||||
)
|
||||
assert override[: len(base)] == base, (
|
||||
f"UltraQuantAttentionImpl.{name} no longer accepts the arguments "
|
||||
f"TurboQuantAttentionImpl passes: base={base} override={override}"
|
||||
)
|
||||
|
||||
# 2. Keyword arguments UltraQuant passes to *inherited* TurboQuant helpers
|
||||
# must exist on those helpers (e.g. _flash_attn_varlen).
|
||||
own = vars(UltraQuantAttentionImpl)
|
||||
module = ast.parse(inspect.getsource(ultraquant_attn))
|
||||
for node in ast.walk(module):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
fn = node.func
|
||||
if not (
|
||||
isinstance(fn, ast.Attribute)
|
||||
and isinstance(fn.value, ast.Name)
|
||||
and fn.value.id == "self"
|
||||
):
|
||||
continue
|
||||
target = getattr(TurboQuantAttentionImpl, fn.attr, None)
|
||||
if target is None or fn.attr in own or not callable(target):
|
||||
continue
|
||||
params = inspect.signature(target).parameters
|
||||
if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()):
|
||||
continue
|
||||
for kw in node.keywords:
|
||||
if kw.arg is None:
|
||||
continue
|
||||
assert kw.arg in params, (
|
||||
f"UltraQuant calls inherited {fn.attr}(..., {kw.arg}=...) but "
|
||||
f"TurboQuantAttentionImpl.{fn.attr} accepts {list(params)}"
|
||||
)
|
||||
|
||||
# 3. Every layer._tq_* attribute the inherited forward reads is set by
|
||||
# UltraQuant's _ensure_on_device.
|
||||
required = _tq_attrs_read(TurboQuantAttentionImpl, "forward")
|
||||
provided = _tq_attrs_written(UltraQuantAttentionImpl, "_ensure_on_device")
|
||||
assert required, "expected inherited forward to read layer._tq_* attributes"
|
||||
assert required <= provided, (
|
||||
f"UltraQuant._ensure_on_device does not set {sorted(required - provided)}, "
|
||||
f"which the inherited TurboQuant forward reads"
|
||||
)
|
||||
@@ -0,0 +1,149 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the Triton UltraQuant store kernel.
|
||||
|
||||
The Triton kernel must produce bit-identical codes and scale bytes vs
|
||||
the PyTorch reference (``reference.ultraquant_encode``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops.ultraquant.format import (
|
||||
k_codes_offset,
|
||||
k_scales_offset,
|
||||
n_groups,
|
||||
slot_size,
|
||||
v_codes_offset,
|
||||
v_scales_offset,
|
||||
)
|
||||
from vllm.v1.attention.ops.ultraquant.reference import ultraquant_encode
|
||||
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA/HIP device"
|
||||
)
|
||||
|
||||
|
||||
def _make_cache(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
slot_b = slot_size(head_dim)
|
||||
return torch.zeros(
|
||||
num_blocks, block_size, num_kv_heads, slot_b, dtype=torch.uint8, device=device
|
||||
)
|
||||
|
||||
|
||||
def _read_codes_and_scales(
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
):
|
||||
"""Pull (k_codes, k_scale_bytes, v_codes, v_scale_bytes) for each
|
||||
(token, head) from the AoS slot layout. Scales come back as uint8
|
||||
(one E8M0 byte per group)."""
|
||||
block_size = kv_cache.shape[1]
|
||||
N = slot_mapping.shape[0]
|
||||
Gk = n_groups(head_dim)
|
||||
|
||||
k_codes_off = k_codes_offset(head_dim)
|
||||
k_scales_off = k_scales_offset(head_dim)
|
||||
v_codes_off = v_codes_offset(head_dim)
|
||||
v_scales_off = v_scales_offset(head_dim)
|
||||
codes_bytes = head_dim // 2
|
||||
|
||||
k_codes = torch.zeros(
|
||||
N, num_kv_heads, codes_bytes, dtype=torch.uint8, device=kv_cache.device
|
||||
)
|
||||
k_scale_bytes = torch.zeros(
|
||||
N, num_kv_heads, Gk, dtype=torch.uint8, device=kv_cache.device
|
||||
)
|
||||
v_codes = torch.zeros_like(k_codes)
|
||||
v_scale_bytes = torch.zeros_like(k_scale_bytes)
|
||||
|
||||
for ti in range(N):
|
||||
slot = int(slot_mapping[ti].item())
|
||||
if slot < 0:
|
||||
continue
|
||||
blk = slot // block_size
|
||||
off = slot % block_size
|
||||
for h in range(num_kv_heads):
|
||||
slot_bytes = kv_cache[blk, off, h] # [slot_size_aligned] uint8
|
||||
|
||||
k_codes[ti, h] = slot_bytes[k_codes_off : k_codes_off + codes_bytes]
|
||||
k_scale_bytes[ti, h] = slot_bytes[k_scales_off : k_scales_off + Gk]
|
||||
v_codes[ti, h] = slot_bytes[v_codes_off : v_codes_off + codes_bytes]
|
||||
v_scale_bytes[ti, h] = slot_bytes[v_scales_off : v_scales_off + Gk]
|
||||
|
||||
return k_codes, k_scale_bytes, v_codes, v_scale_bytes
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("head_dim", [128, 256])
|
||||
@pytest.mark.parametrize("num_kv_heads", [1, 4])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 16, 17])
|
||||
def test_store_matches_reference(dtype, head_dim, num_kv_heads, num_tokens):
|
||||
"""Triton store output (codes + E8M0 scale bytes) matches the reference
|
||||
bit-exactly."""
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(0xF8)
|
||||
|
||||
block_size = 32
|
||||
num_blocks = max(2, (num_tokens + block_size - 1) // block_size + 1)
|
||||
|
||||
key = torch.randn(num_tokens, num_kv_heads, head_dim, dtype=dtype, device=device)
|
||||
value = torch.randn(num_tokens, num_kv_heads, head_dim, dtype=dtype, device=device)
|
||||
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
if num_tokens >= 4:
|
||||
slot_mapping[1] = -1 # skip-path test
|
||||
|
||||
kv_cache = _make_cache(num_blocks, block_size, num_kv_heads, head_dim, device)
|
||||
|
||||
ultraquant_store(key, value, kv_cache, slot_mapping)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
ref_k = ultraquant_encode(key, rotate=True)
|
||||
ref_v = ultraquant_encode(value, rotate=False)
|
||||
|
||||
k_codes, k_scale_bytes, v_codes, v_scale_bytes = _read_codes_and_scales(
|
||||
kv_cache, slot_mapping, num_kv_heads, head_dim
|
||||
)
|
||||
|
||||
if num_tokens >= 4:
|
||||
assert (kv_cache[0, 1, :, :] == 0).all(), (
|
||||
"skipped slot's physical cache region was unexpectedly written"
|
||||
)
|
||||
|
||||
for ti in range(num_tokens):
|
||||
slot = int(slot_mapping[ti].item())
|
||||
if slot < 0:
|
||||
continue
|
||||
|
||||
assert torch.equal(k_codes[ti], ref_k.codes_packed[ti]), (
|
||||
f"K codes mismatch at token {ti}\n"
|
||||
f"got: {k_codes[ti].cpu().numpy().tolist()[:16]}\n"
|
||||
f"expected: {ref_k.codes_packed[ti].cpu().numpy().tolist()[:16]}"
|
||||
)
|
||||
assert torch.equal(k_scale_bytes[ti], ref_k.scale_bytes[ti]), (
|
||||
f"K scale bytes mismatch at token {ti}\n"
|
||||
f"got: {k_scale_bytes[ti].cpu().numpy().tolist()}\n"
|
||||
f"expected: {ref_k.scale_bytes[ti].cpu().numpy().tolist()}"
|
||||
)
|
||||
|
||||
assert torch.equal(v_codes[ti], ref_v.codes_packed[ti]), (
|
||||
f"V codes mismatch at token {ti}"
|
||||
)
|
||||
assert torch.equal(v_scale_bytes[ti], ref_v.scale_bytes[ti]), (
|
||||
f"V scale bytes mismatch at token {ti}\n"
|
||||
f"got: {v_scale_bytes[ti].cpu().numpy().tolist()}\n"
|
||||
f"expected: {ref_v.scale_bytes[ti].cpu().numpy().tolist()}"
|
||||
)
|
||||
Reference in New Issue
Block a user