Add nvfp4 kv cache support (#40177)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
@@ -5,12 +5,17 @@ import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.quantization.nvfp4_utils import (
|
||||
dequant_nvfp4_kv_cache,
|
||||
dequantize_nvfp4_to_dtype,
|
||||
get_nvfp4_global_scale,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.utils.torch_utils import (
|
||||
nvfp4_kv_cache_full_dim,
|
||||
nvfp4_kv_cache_split_views,
|
||||
set_random_seed,
|
||||
)
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip(
|
||||
@@ -33,6 +38,117 @@ def to_float8(x, dtype=torch.float8_e4m3fn):
|
||||
return x_scl_sat.to(dtype), scale.float().reciprocal()
|
||||
|
||||
|
||||
def build_paged_kv_metadata(
|
||||
seq_lens: torch.Tensor,
|
||||
block_tables: torch.Tensor,
|
||||
block_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Build paged-KV indptr/indices/last_page_lens from seq_lens + block_tables."""
|
||||
kv_indptr = [0]
|
||||
kv_indices = []
|
||||
kv_last_page_lens = []
|
||||
for i in range(len(seq_lens)):
|
||||
sl = int(seq_lens[i])
|
||||
assert sl > 0
|
||||
nb = (sl + block_size - 1) // block_size
|
||||
kv_indices.extend(block_tables[i, :nb].tolist())
|
||||
kv_indptr.append(kv_indptr[-1] + nb)
|
||||
kv_last_page_lens.append(sl % block_size or block_size)
|
||||
return (
|
||||
torch.tensor(kv_indptr, dtype=torch.int32),
|
||||
torch.tensor(kv_indices, dtype=torch.int32),
|
||||
torch.tensor(kv_last_page_lens, dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def make_nvfp4_kv_cache(
|
||||
kv_bf16_hnd: torch.Tensor, block_size: int, head_size: int
|
||||
) -> tuple:
|
||||
"""Quantize bf16 KV cache to nvfp4 via reshape_and_cache_flash.
|
||||
|
||||
Returns (k_data, v_data), (k_scales, v_scales), kv_scale, ref_kv_bf16.
|
||||
"""
|
||||
num_blocks, _, num_kv_heads, _, _ = kv_bf16_hnd.shape
|
||||
kv_scale_val = (kv_bf16_hnd.abs().amax() / 448.0).item()
|
||||
kv_scale_tensor = torch.tensor(
|
||||
kv_scale_val, dtype=torch.float32, device=kv_bf16_hnd.device
|
||||
)
|
||||
|
||||
# Allocate in HND physical order, permute to NHD logical order.
|
||||
# hnd_order swaps dims 2↔3; it is its own inverse.
|
||||
full_dim = nvfp4_kv_cache_full_dim(head_size)
|
||||
hnd_order = (0, 1, 3, 2, 4)
|
||||
kv_cache = torch.zeros(
|
||||
(num_blocks, 2, num_kv_heads, block_size, full_dim),
|
||||
dtype=torch.uint8,
|
||||
device=kv_bf16_hnd.device,
|
||||
).permute(*hnd_order)
|
||||
|
||||
# Flatten NHD [N, T, H, D] → token tensors [N*T, H, D] for the kernel.
|
||||
num_tokens = num_blocks * block_size
|
||||
k_tokens = (
|
||||
kv_bf16_hnd[:, 0]
|
||||
.permute(0, 2, 1, 3)
|
||||
.reshape(num_tokens, num_kv_heads, head_size)
|
||||
)
|
||||
v_tokens = (
|
||||
kv_bf16_hnd[:, 1]
|
||||
.permute(0, 2, 1, 3)
|
||||
.reshape(num_tokens, num_kv_heads, head_size)
|
||||
)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_bf16_hnd.device)
|
||||
|
||||
# reshape_and_cache_flash: kernel receives kv_cache[:, 0] and [:, 1]
|
||||
# (full K/V buffers containing both data and scale).
|
||||
torch.ops._C_cache_ops.reshape_and_cache_flash(
|
||||
k_tokens,
|
||||
v_tokens,
|
||||
kv_cache[:, 0],
|
||||
kv_cache[:, 1],
|
||||
slot_mapping,
|
||||
"nvfp4",
|
||||
kv_scale_tensor,
|
||||
kv_scale_tensor,
|
||||
)
|
||||
|
||||
# Split in HND order for trtllm kernel (expects HND numTokensPerPage).
|
||||
kv_cache_hnd = kv_cache.permute(*hnd_order)
|
||||
(k_data, v_data), (k_scales, v_scales) = nvfp4_kv_cache_split_views(kv_cache_hnd)
|
||||
|
||||
# Dequantize for the FA2 reference baseline.
|
||||
ref_k = dequant_nvfp4_kv_cache(
|
||||
k_data, k_scales, kv_scale_val, head_size, block_size
|
||||
).to(torch.bfloat16)
|
||||
ref_v = dequant_nvfp4_kv_cache(
|
||||
v_data, v_scales, kv_scale_val, head_size, block_size
|
||||
).to(torch.bfloat16)
|
||||
ref_kv_bf16 = torch.stack([ref_k, ref_v], dim=1) # [N, 2, H, T, D]
|
||||
|
||||
return (k_data, v_data), (k_scales, v_scales), kv_scale_val, ref_kv_bf16
|
||||
|
||||
|
||||
def make_quantized_kv_cache(
|
||||
kv_cache: torch.Tensor,
|
||||
kv_quant_dtype: torch.dtype,
|
||||
block_size: int,
|
||||
head_size: int,
|
||||
) -> tuple:
|
||||
"""Quantize kv_cache based on dtype. Returns (kv_cache, kv_cache_sf,
|
||||
kv_scale, ref_kv_cache, is_nvfp4_kv)."""
|
||||
is_nvfp4_kv = kv_quant_dtype == FP4_DTYPE
|
||||
if is_nvfp4_kv:
|
||||
data, scales, kv_scale, ref = make_nvfp4_kv_cache(
|
||||
kv_cache, block_size, head_size
|
||||
)
|
||||
return data, scales, kv_scale, ref, True
|
||||
elif kv_quant_dtype == FP8_DTYPE:
|
||||
kv_fp8, kv_scale = to_float8(kv_cache)
|
||||
ref = kv_fp8.to(kv_cache.dtype) * kv_scale
|
||||
return kv_fp8, None, kv_scale, ref, False
|
||||
else:
|
||||
return kv_cache, None, 1.0, kv_cache, False
|
||||
|
||||
|
||||
DTYPE = [torch.bfloat16]
|
||||
QUANT_DTYPES = [
|
||||
# (q_quant_dtype, kv_quant_dtype, o_quant_dtype)
|
||||
@@ -41,6 +157,7 @@ QUANT_DTYPES = [
|
||||
(FP8_DTYPE, FP8_DTYPE, None),
|
||||
(FP8_DTYPE, FP8_DTYPE, FP8_DTYPE),
|
||||
(FP8_DTYPE, FP8_DTYPE, FP4_DTYPE),
|
||||
(FP8_DTYPE, FP4_DTYPE, FP8_DTYPE), # nvfp4 KV cache
|
||||
]
|
||||
BATCH_SIZE = [4, 12]
|
||||
MAX_SEQ_LENS = [(1024, 4096)]
|
||||
@@ -127,35 +244,19 @@ def test_flashinfer_trtllm_decode_with_baseline(
|
||||
max_seq_len = torch.max(seq_lens).item()
|
||||
|
||||
kv_cache = torch.randn(kv_cache_shape, dtype=dtype)
|
||||
if kv_quant_dtype == FP8_DTYPE:
|
||||
kv_cache, kv_scale = to_float8(kv_cache)
|
||||
ref_kv_cache = kv_cache.to(dtype) * kv_scale
|
||||
else:
|
||||
kv_scale = 1.0
|
||||
ref_kv_cache = kv_cache
|
||||
kv_cache, kv_cache_sf, kv_scale, ref_kv_cache, is_nvfp4_kv = (
|
||||
make_quantized_kv_cache(kv_cache, kv_quant_dtype, block_size, head_size)
|
||||
)
|
||||
|
||||
k_scale = v_scale = kv_scale
|
||||
|
||||
max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, NUM_BLOCKS, (batch_size, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
kv_indptr = [0]
|
||||
kv_indices = []
|
||||
kv_last_page_lens = []
|
||||
for i in range(batch_size):
|
||||
seq_len = seq_lens[i]
|
||||
assert seq_len > 0
|
||||
num_blocks = (seq_len + block_size - 1) // block_size
|
||||
kv_indices.extend(block_tables[i, :num_blocks])
|
||||
kv_indptr.append(kv_indptr[-1] + num_blocks)
|
||||
kv_last_page_len = seq_len % block_size
|
||||
if kv_last_page_len == 0:
|
||||
kv_last_page_len = block_size
|
||||
kv_last_page_lens.append(kv_last_page_len)
|
||||
|
||||
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
|
||||
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
|
||||
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
|
||||
kv_indptr, kv_indices, kv_last_page_lens = build_paged_kv_metadata(
|
||||
seq_lens, block_tables, block_size
|
||||
)
|
||||
workspace_buffer = torch.zeros(128 * 1024 * 1024, dtype=torch.int8)
|
||||
|
||||
# Baseline Decode
|
||||
@@ -225,6 +326,7 @@ def test_flashinfer_trtllm_decode_with_baseline(
|
||||
sinks=sinks,
|
||||
o_sf_scale=o_sf_scale_float,
|
||||
out=output_trtllm,
|
||||
kv_cache_sf=kv_cache_sf,
|
||||
)
|
||||
if o_quant_dtype == FP8_DTYPE:
|
||||
output_trtllm = output_trtllm.to(dtype) * o_scale
|
||||
@@ -237,7 +339,9 @@ def test_flashinfer_trtllm_decode_with_baseline(
|
||||
)
|
||||
output_trtllm = output_trtllm.reshape(-1, query.shape[1], query.shape[2])
|
||||
|
||||
if q_quant_dtype == FP8_DTYPE and o_quant_dtype == FP4_DTYPE:
|
||||
if is_nvfp4_kv:
|
||||
rtol, atol = 1.0, 1.0 # nvfp4 has higher quantization error
|
||||
elif q_quant_dtype == FP8_DTYPE and o_quant_dtype == FP4_DTYPE:
|
||||
rtol, atol = 7e-2, 9e-2
|
||||
elif q_quant_dtype == FP8_DTYPE and o_quant_dtype == FP8_DTYPE:
|
||||
rtol, atol = 3e-2, 4e-2
|
||||
@@ -287,7 +391,12 @@ def test_flashinfer_trtllm_prefill_with_baseline(
|
||||
kv_quant_dtype = kv_quant_dtype or dtype
|
||||
o_quant_dtype = o_quant_dtype or dtype
|
||||
|
||||
if q_quant_dtype != kv_quant_dtype:
|
||||
# FP8 Q + nvfp4 KV is the required combination for the nvfp4 KV path.
|
||||
# All other mixed Q/KV dtype combinations are unsupported.
|
||||
is_nvfp4_kv = kv_quant_dtype == FP4_DTYPE
|
||||
if q_quant_dtype != kv_quant_dtype and not (
|
||||
q_quant_dtype == FP8_DTYPE and is_nvfp4_kv
|
||||
):
|
||||
pytest.skip("Skipped mixed QKV dtypes for prefill")
|
||||
|
||||
max_q_len, max_kv_len = max_seq_lens
|
||||
@@ -329,35 +438,19 @@ def test_flashinfer_trtllm_prefill_with_baseline(
|
||||
max_seq_len = torch.max(seq_lens).item()
|
||||
|
||||
kv_cache = torch.randn(kv_cache_shape, dtype=dtype)
|
||||
if kv_quant_dtype == FP8_DTYPE:
|
||||
kv_cache, kv_scale = to_float8(kv_cache)
|
||||
ref_kv_cache = kv_cache.to(dtype) * kv_scale
|
||||
else:
|
||||
kv_scale = 1.0
|
||||
ref_kv_cache = kv_cache
|
||||
kv_cache, kv_cache_sf, kv_scale, ref_kv_cache, is_nvfp4_kv = (
|
||||
make_quantized_kv_cache(kv_cache, kv_quant_dtype, block_size, head_size)
|
||||
)
|
||||
|
||||
k_scale = v_scale = kv_scale
|
||||
|
||||
max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, NUM_BLOCKS, (batch_size, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
kv_indptr = [0]
|
||||
kv_indices = []
|
||||
kv_last_page_lens = []
|
||||
for i in range(batch_size):
|
||||
seq_len = seq_lens[i]
|
||||
assert seq_len > 0
|
||||
num_blocks = (seq_len + block_size - 1) // block_size
|
||||
kv_indices.extend(block_tables[i, :num_blocks])
|
||||
kv_indptr.append(kv_indptr[-1] + num_blocks)
|
||||
kv_last_page_len = seq_len % block_size
|
||||
if kv_last_page_len == 0:
|
||||
kv_last_page_len = block_size
|
||||
kv_last_page_lens.append(kv_last_page_len)
|
||||
|
||||
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
|
||||
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
|
||||
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
|
||||
kv_indptr, kv_indices, kv_last_page_lens = build_paged_kv_metadata(
|
||||
seq_lens, block_tables, block_size
|
||||
)
|
||||
workspace_buffer = torch.zeros(128 * 1024 * 1024, dtype=torch.int8)
|
||||
|
||||
# Baseline Prefill
|
||||
@@ -431,6 +524,7 @@ def test_flashinfer_trtllm_prefill_with_baseline(
|
||||
sinks=sinks,
|
||||
o_sf_scale=o_sf_scale_float,
|
||||
out=output_trtllm,
|
||||
kv_cache_sf=kv_cache_sf,
|
||||
)
|
||||
if o_quant_dtype == FP8_DTYPE:
|
||||
output_trtllm = output_trtllm.to(dtype) * o_scale
|
||||
@@ -443,7 +537,9 @@ def test_flashinfer_trtllm_prefill_with_baseline(
|
||||
)
|
||||
output_trtllm = output_trtllm.reshape(-1, query.shape[1], query.shape[2])
|
||||
|
||||
if q_quant_dtype == FP8_DTYPE and o_quant_dtype == FP4_DTYPE:
|
||||
if is_nvfp4_kv:
|
||||
rtol, atol = 1.0, 1.5 # nvfp4 has higher quantization error
|
||||
elif q_quant_dtype == FP8_DTYPE and o_quant_dtype == FP4_DTYPE:
|
||||
rtol, atol = 3e-1, 4e-1
|
||||
elif q_quant_dtype == FP8_DTYPE and o_quant_dtype == FP8_DTYPE:
|
||||
rtol, atol = 4e-2, 6e-2
|
||||
|
||||
@@ -17,13 +17,13 @@ from tests.v1.attention.utils import (
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.utils.torch_utils import nvfp4_kv_cache_full_dim, set_random_seed
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
PerLayerParameters,
|
||||
get_kv_cache_layout,
|
||||
set_kv_cache_layout,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVQuantMode
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip(
|
||||
@@ -53,6 +53,7 @@ class MockAttentionLayer:
|
||||
|
||||
|
||||
MODEL = "Qwen/Qwen2.5-0.5B"
|
||||
MODEL_NVFP4 = "Qwen/Qwen3-4B" # nvfp4 needs head_dim >= 128 (or 80)
|
||||
BLOCK_SIZE = 16
|
||||
NUM_GPU_BLOCKS = 8192
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
@@ -169,19 +170,129 @@ def _create_hnd_kv_cache(
|
||||
return kv_cache
|
||||
|
||||
|
||||
def _run_trtllm_integration(batch_spec):
|
||||
def _create_nvfp4_hnd_kv_cache(
|
||||
k_contexts,
|
||||
v_contexts,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
dtype,
|
||||
device,
|
||||
num_blocks,
|
||||
common_attn_metadata,
|
||||
kv_scale_val,
|
||||
):
|
||||
"""Create an nvfp4 KV cache by quantizing bf16 context via
|
||||
reshape_and_cache_flash, using the same block-table layout as
|
||||
_create_hnd_kv_cache.
|
||||
|
||||
The returned tensor is dtype ``uint8`` with shape
|
||||
``(num_blocks, 2, block_size, num_kv_heads, full_dim)`` in logical
|
||||
(NHD) order, but physically permuted to HND layout via stride order
|
||||
``(0, 1, 3, 2, 4)`` (i.e. ``num_kv_heads`` before ``block_size``).
|
||||
|
||||
The last dimension ``full_dim = head_size // 2 + head_size // 16``
|
||||
packs two regions contiguously:
|
||||
- **FP4 data** (``head_size // 2`` bytes): pairs of E2M1 values,
|
||||
two per byte.
|
||||
- **FP8 block scales** (``head_size // 16`` bytes): one E4M3
|
||||
scale per 16-element block.
|
||||
|
||||
Dimension 1 indexes K (``[:, 0]``) and V (``[:, 1]``).
|
||||
|
||||
Args:
|
||||
k_contexts: List of key context tensors, one per sequence.
|
||||
v_contexts: List of value context tensors, one per sequence.
|
||||
block_size: Number of tokens per cache block.
|
||||
num_kv_heads: Number of key/value heads.
|
||||
head_size: Head dimension (must be divisible by 16).
|
||||
dtype: Source data type for the bf16 intermediate cache.
|
||||
device: Target device.
|
||||
num_blocks: Total number of blocks to allocate.
|
||||
common_attn_metadata: Metadata containing block tables and
|
||||
sequence lengths.
|
||||
kv_scale_val: Scalar float used as both k_scale and v_scale
|
||||
during quantization.
|
||||
|
||||
Returns:
|
||||
``torch.Tensor``: The nvfp4 kv_cache tensor (uint8, HND-strided).
|
||||
"""
|
||||
# First create a bf16 HND cache so block tables are populated.
|
||||
bf16_cache = _create_hnd_kv_cache(
|
||||
k_contexts,
|
||||
v_contexts,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
dtype,
|
||||
device,
|
||||
num_blocks,
|
||||
common_attn_metadata,
|
||||
)
|
||||
|
||||
# Allocate nvfp4 cache: same shape but with full_dim (data + scale).
|
||||
full_dim = nvfp4_kv_cache_full_dim(head_size)
|
||||
hnd_order = (0, 1, 3, 2, 4)
|
||||
nvfp4_cache = torch.zeros(
|
||||
(num_blocks, 2, num_kv_heads, block_size, full_dim),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
).permute(*hnd_order)
|
||||
|
||||
# Flatten bf16 context into tokens and quantize via reshape_and_cache_flash.
|
||||
# bf16_cache is (num_blocks, 2, block_size, num_kv_heads, head_size) logical
|
||||
# with HND physical strides.
|
||||
block_table = common_attn_metadata.block_table_tensor
|
||||
seq_lens = common_attn_metadata.seq_lens.cpu()
|
||||
query_lens = (
|
||||
common_attn_metadata.query_start_loc_cpu[1:]
|
||||
- common_attn_metadata.query_start_loc_cpu[:-1]
|
||||
)
|
||||
kv_scale_t = torch.tensor(kv_scale_val, dtype=torch.float32, device=device)
|
||||
|
||||
for i in range(len(k_contexts)):
|
||||
ctx_len = int(seq_lens[i]) - int(query_lens[i])
|
||||
if ctx_len == 0:
|
||||
continue
|
||||
# Gather context tokens from the bf16 cache using block table.
|
||||
n_ctx_blocks = (ctx_len + block_size - 1) // block_size
|
||||
blocks = block_table[i, :n_ctx_blocks]
|
||||
# bf16_cache[:, kv_idx] is (num_blocks, block_size, num_kv_heads, head_size)
|
||||
k_ctx = bf16_cache[blocks, 0].reshape(-1, num_kv_heads, head_size)[:ctx_len]
|
||||
v_ctx = bf16_cache[blocks, 1].reshape(-1, num_kv_heads, head_size)[:ctx_len]
|
||||
# Build slot mapping for these context tokens.
|
||||
token_offsets = torch.arange(ctx_len, device=device)
|
||||
block_indices = token_offsets // block_size
|
||||
intra_offsets = token_offsets % block_size
|
||||
slots = block_table[i, block_indices] * block_size + intra_offsets
|
||||
torch.ops._C_cache_ops.reshape_and_cache_flash(
|
||||
k_ctx,
|
||||
v_ctx,
|
||||
nvfp4_cache[:, 0],
|
||||
nvfp4_cache[:, 1],
|
||||
slots,
|
||||
"nvfp4",
|
||||
kv_scale_t,
|
||||
kv_scale_t,
|
||||
)
|
||||
|
||||
return nvfp4_cache
|
||||
|
||||
|
||||
def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL):
|
||||
"""Run TRTLLM attention through the full FlashInfer pipeline
|
||||
and compare against an SDPA reference."""
|
||||
set_random_seed(42)
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name=MODEL,
|
||||
model_name=model_name,
|
||||
max_model_len=max(batch_spec.seq_lens),
|
||||
block_size=BLOCK_SIZE,
|
||||
num_gpu_blocks=NUM_GPU_BLOCKS,
|
||||
)
|
||||
vllm_config.attention_config.use_trtllm_attention = True
|
||||
vllm_config.cache_config.cache_dtype = kv_cache_dtype
|
||||
|
||||
num_q_heads = vllm_config.model_config.get_num_attention_heads(
|
||||
vllm_config.parallel_config
|
||||
@@ -248,28 +359,51 @@ def _run_trtllm_integration(batch_spec):
|
||||
common_attn_metadata = create_common_attn_metadata(batch_spec, BLOCK_SIZE, device)
|
||||
|
||||
# 2. Create HND KV cache
|
||||
kv_cache = _create_hnd_kv_cache(
|
||||
k_contexts,
|
||||
v_contexts,
|
||||
BLOCK_SIZE,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
dtype,
|
||||
device,
|
||||
NUM_GPU_BLOCKS,
|
||||
common_attn_metadata,
|
||||
)
|
||||
is_nvfp4 = kv_cache_dtype == "nvfp4"
|
||||
if is_nvfp4:
|
||||
# Compute a global scale from the context data.
|
||||
all_ctx = torch.cat(k_contexts + v_contexts, dim=0)
|
||||
kv_scale_val = (all_ctx.abs().amax() / 448.0).item()
|
||||
kv_cache = _create_nvfp4_hnd_kv_cache(
|
||||
k_contexts,
|
||||
v_contexts,
|
||||
BLOCK_SIZE,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
dtype,
|
||||
device,
|
||||
NUM_GPU_BLOCKS,
|
||||
common_attn_metadata,
|
||||
kv_scale_val,
|
||||
)
|
||||
else:
|
||||
kv_scale_val = 1.0
|
||||
kv_cache = _create_hnd_kv_cache(
|
||||
k_contexts,
|
||||
v_contexts,
|
||||
BLOCK_SIZE,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
dtype,
|
||||
device,
|
||||
NUM_GPU_BLOCKS,
|
||||
common_attn_metadata,
|
||||
)
|
||||
|
||||
# 3. Run through FlashInfer with TRTLLM enabled
|
||||
set_kv_cache_layout("HND")
|
||||
get_kv_cache_layout.cache_clear()
|
||||
|
||||
try:
|
||||
is_nvfp4 = kv_cache_dtype == "nvfp4"
|
||||
kv_quant_mode = KVQuantMode.NVFP4 if is_nvfp4 else KVQuantMode.NONE
|
||||
spec_dtype = torch.uint8 if is_nvfp4 else dtype
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
dtype=dtype,
|
||||
dtype=spec_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
)
|
||||
layer_names = ["test_layer_0"]
|
||||
|
||||
@@ -312,10 +446,20 @@ def _run_trtllm_integration(batch_spec):
|
||||
num_kv_heads=num_kv_heads,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype="auto",
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
)
|
||||
|
||||
mock_layer = MockAttentionLayer(device)
|
||||
if is_nvfp4:
|
||||
# For nvfp4, k_scale/v_scale are the global quantization
|
||||
# scales (amax/448) used by reshape_and_cache_flash.
|
||||
kv_scale_t = torch.tensor(
|
||||
kv_scale_val, dtype=torch.float32, device=device
|
||||
)
|
||||
mock_layer._k_scale = kv_scale_t
|
||||
mock_layer._v_scale = kv_scale_t
|
||||
mock_layer._k_scale_float = kv_scale_val
|
||||
mock_layer._v_scale_float = kv_scale_val
|
||||
output = torch.empty_like(query_vllm)
|
||||
|
||||
impl.do_kv_cache_update(
|
||||
@@ -326,6 +470,23 @@ def _run_trtllm_integration(batch_spec):
|
||||
attn_metadata.slot_mapping,
|
||||
)
|
||||
|
||||
# nvfp4 trtllm kernel requires FP8 queries. In the real
|
||||
# pipeline the attention layer handles this; here we
|
||||
# quantize manually.
|
||||
if is_nvfp4:
|
||||
finfo = torch.finfo(torch.float8_e4m3fn)
|
||||
q_amax = query_vllm.abs().amax().clamp(min=1e-12)
|
||||
q_s = (finfo.max / q_amax * 0.1).item()
|
||||
query_vllm = (
|
||||
(query_vllm * q_s)
|
||||
.clamp(finfo.min, finfo.max)
|
||||
.to(torch.float8_e4m3fn)
|
||||
)
|
||||
mock_layer._q_scale = torch.tensor(
|
||||
1.0 / q_s, dtype=torch.float32, device=device
|
||||
)
|
||||
mock_layer._q_scale_float = 1.0 / q_s
|
||||
|
||||
output = impl.forward(
|
||||
mock_layer,
|
||||
query_vllm,
|
||||
@@ -337,12 +498,11 @@ def _run_trtllm_integration(batch_spec):
|
||||
)
|
||||
|
||||
# 4. Compare against SDPA reference
|
||||
torch.testing.assert_close(
|
||||
output,
|
||||
sdpa_output,
|
||||
atol=1e-2,
|
||||
rtol=1e-2,
|
||||
)
|
||||
if is_nvfp4:
|
||||
atol, rtol = 1.0, 1.0 # nvfp4 has higher quantization error
|
||||
else:
|
||||
atol, rtol = 1e-2, 1e-2
|
||||
torch.testing.assert_close(output, sdpa_output, atol=atol, rtol=rtol)
|
||||
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
@@ -359,3 +519,18 @@ def test_trtllm_gen_full_attention_integration(batch_spec_name: str):
|
||||
MetadataBuilder.build() -> FlashInferImpl.forward() pipeline,
|
||||
with real TRTLLM kernels on Blackwell."""
|
||||
_run_trtllm_integration(BATCH_SPECS[batch_spec_name])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"batch_spec_name",
|
||||
list(BATCH_SPECS.keys()),
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_gen_nvfp4_kv_integration(batch_spec_name: str):
|
||||
"""Test TRTLLM attention with nvfp4 KV cache through the full
|
||||
FlashInfer MetadataBuilder.build() -> FlashInferImpl.forward() pipeline."""
|
||||
_run_trtllm_integration(
|
||||
BATCH_SPECS[batch_spec_name],
|
||||
kv_cache_dtype="nvfp4",
|
||||
model_name=MODEL_NVFP4,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user