[Model][Hardware][AMD][Kernel] Part 2/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_ATTN (#42755)
Signed-off-by: Jack Hu <Jack.Hu@amd.com> Signed-off-by: Jack Hu <jingxujaster@hotmail.com> Signed-off-by: Jack Hu <jack.hu@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
@@ -157,10 +157,17 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
|
||||
)
|
||||
|
||||
if self.kv_cache_dtype != self.dtype:
|
||||
self.attn._k_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
self.attn._v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
self.attn._k_scale_float = 1.0
|
||||
self.attn._v_scale_float = 1.0
|
||||
# Non-unit scales so the fused and unfused writers must agree on them.
|
||||
self.attn._k_scale = torch.tensor(0.5, dtype=torch.float32, device=device)
|
||||
self.attn._v_scale = torch.tensor(0.25, dtype=torch.float32, device=device)
|
||||
self.attn._k_scale_cpu = torch.tensor(
|
||||
0.5, dtype=torch.float32, device="cpu"
|
||||
)
|
||||
self.attn._v_scale_cpu = torch.tensor(
|
||||
0.25, dtype=torch.float32, device="cpu"
|
||||
)
|
||||
self.attn._k_scale_float = 0.5
|
||||
self.attn._v_scale_float = 0.25
|
||||
else:
|
||||
self.attn._k_scale = self.attn._k_scale.to(device)
|
||||
self.attn._v_scale = self.attn._v_scale.to(device)
|
||||
@@ -527,16 +534,6 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
else:
|
||||
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
|
||||
|
||||
if not expect_fusion or (not is_fp8_cache and mrope_section is None):
|
||||
# The AITER PTS kernel populates k_out only for non-FP8 caches.
|
||||
# With FP8, the kernel writes quantized K directly to the cache
|
||||
# and may leave k_out uninitialised. In production this is fine
|
||||
# because downstream attention reads K from the cache.
|
||||
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
|
||||
|
||||
# Should be bit exact since no processing had been done on v for both paths
|
||||
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
|
||||
|
||||
# Fused and unfused arithmetic can straddle an FP8 rounding boundary.
|
||||
# Allow one E4M3 quantization step while still comparing every block.
|
||||
if is_fp8_cache:
|
||||
@@ -544,6 +541,18 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
else:
|
||||
cache_atol, cache_rtol = ATOL, RTOL
|
||||
|
||||
# The fused k_out is the dequantized cache write, so it carries the same
|
||||
# rounding; FA and ROCM_ATTN prefill consume it directly. Unified reads K
|
||||
# from the cache and asks the kernel not to write k_out at all.
|
||||
if attn_backend != AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN:
|
||||
torch.testing.assert_close(
|
||||
k_unfused, k_fused, atol=cache_atol, rtol=cache_rtol
|
||||
)
|
||||
|
||||
# Should be bit exact since no processing had been done on v for both paths
|
||||
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
|
||||
|
||||
# Whole cache, so the interleaved V write (ROCM_ATTN kv_cache[1]) is checked.
|
||||
torch.testing.assert_close(
|
||||
kv_cache_unfused.float(),
|
||||
kv_cache_fused.float(),
|
||||
@@ -570,20 +579,52 @@ _FUSION_CONFIGS = [
|
||||
_FUSION_CONFIGS,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
"attn_backend, use_shuffle_kv_layout, kv_layout",
|
||||
[
|
||||
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
# ROCM_ATTN always writes the shuffled, interleaved-V layout and needs each
|
||||
# K/V side contiguous per block, so head-major only.
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_ATTN,
|
||||
"0",
|
||||
KVCacheLayout.LBHNC,
|
||||
id="rocm_attn",
|
||||
),
|
||||
# Unified never reads the shuffle env; its strided views take any layout.
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
"0",
|
||||
KVCacheLayout.LBHNC,
|
||||
id="unified-head_major",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
"0",
|
||||
KVCacheLayout.LBNHC,
|
||||
id="unified-token_major",
|
||||
),
|
||||
# FA takes any layout unshuffled; its shuffle writer needs each K/V side
|
||||
# contiguous per block, so head-major only.
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
"0",
|
||||
KVCacheLayout.LBHNC,
|
||||
id="fa-head_major",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
"0",
|
||||
KVCacheLayout.LBNHC,
|
||||
id="fa-token_major",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
"1",
|
||||
KVCacheLayout.LBHNC,
|
||||
id="fa-shuffle",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [5, 2048])
|
||||
@pytest.mark.parametrize(
|
||||
"kv_layout",
|
||||
[
|
||||
pytest.param(KVCacheLayout.LBHNC, id="head_major"),
|
||||
pytest.param(KVCacheLayout.LBNHC, id="token_major"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
|
||||
@pytest.mark.parametrize("block_size", [16, 32, 64])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@@ -615,6 +656,7 @@ def test_qk_norm_rope_kvcache_fusion(
|
||||
kv_cache_dtype: CacheDType,
|
||||
rms_norm_eps: float,
|
||||
custom_op: str,
|
||||
use_shuffle_kv_layout: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
@@ -627,7 +669,7 @@ def test_qk_norm_rope_kvcache_fusion(
|
||||
rotary_dim=rotary_dim,
|
||||
block_size=block_size,
|
||||
is_neox=is_neox,
|
||||
use_shuffle_kv_layout="0",
|
||||
use_shuffle_kv_layout=use_shuffle_kv_layout,
|
||||
kv_layout=kv_layout,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
|
||||
@@ -122,6 +122,7 @@ def ref_single_query_cached_kv_attention(
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("use_interleaved_v_cache", [False, True])
|
||||
def test_paged_attention(
|
||||
kv_cache_factory,
|
||||
num_seqs: int,
|
||||
@@ -133,9 +134,14 @@ def test_paged_attention(
|
||||
kv_cache_dtype: str,
|
||||
seed: int,
|
||||
device: str,
|
||||
use_interleaved_v_cache: bool,
|
||||
) -> None:
|
||||
if current_platform.is_navi() and (
|
||||
kv_cache_dtype == "fp8" or head_size != 128 or block_size != 16 or use_alibi
|
||||
kv_cache_dtype == "fp8"
|
||||
or head_size != 128
|
||||
or block_size != 16
|
||||
or use_alibi
|
||||
or use_interleaved_v_cache
|
||||
):
|
||||
pytest.skip()
|
||||
|
||||
@@ -201,6 +207,20 @@ def test_paged_attention(
|
||||
last_block_idx = block_tables_lst[seq_idx][seq_len // block_size]
|
||||
value_cache[last_block_idx, :, :, padding_start:] = padding_nan
|
||||
|
||||
standard_value_cache = value_cache
|
||||
if use_interleaved_v_cache:
|
||||
# Hand the kernel V in the interleaved layout (same 4D shape); the
|
||||
# reference below gets the standard layout back.
|
||||
x_v = 16 // value_cache.element_size()
|
||||
value_cache = (
|
||||
value_cache.view(
|
||||
NUM_BLOCKS, num_kv_heads, head_size, block_size // x_v, x_v
|
||||
)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(NUM_BLOCKS, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
|
||||
# Call the paged attention kernel.
|
||||
output = torch.empty_like(query)
|
||||
num_partitions = (max_seq_len + PARTITION_SIZE_ROCM - 1) // PARTITION_SIZE_ROCM
|
||||
@@ -234,6 +254,7 @@ def test_paged_attention(
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
use_interleaved_v_cache=use_interleaved_v_cache,
|
||||
)
|
||||
|
||||
opcheck(
|
||||
@@ -259,10 +280,13 @@ def test_paged_attention(
|
||||
v_scale,
|
||||
None,
|
||||
"f16",
|
||||
use_interleaved_v_cache,
|
||||
),
|
||||
cond=(head_size == 64 and block_size == BLOCK_SIZES[0]),
|
||||
)
|
||||
|
||||
value_cache = standard_value_cache
|
||||
|
||||
# Run the reference implementation.
|
||||
if kv_cache_dtype == "fp8":
|
||||
# Convert cache data back to dtype.
|
||||
|
||||
@@ -109,6 +109,7 @@ def create_alibi_causal_mask(
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention(
|
||||
num_heads: int,
|
||||
@@ -119,6 +120,7 @@ def test_contexted_kv_attention(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
block_size: int = 32,
|
||||
) -> None:
|
||||
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
|
||||
@@ -225,6 +227,23 @@ def test_contexted_kv_attention(
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
extra_op_kwargs = {}
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
|
||||
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
|
||||
# read it back interleaved.
|
||||
x_v = 16 // v_cache.element_size()
|
||||
v_cache = (
|
||||
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(cache_size, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
if op is context_attention_fwd:
|
||||
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
|
||||
else:
|
||||
extra_op_kwargs["use_interleaved_v_cache"] = True
|
||||
|
||||
# Warm up the Triton kernel by calling it once before actually measuring
|
||||
# generation time
|
||||
op(
|
||||
@@ -243,6 +262,7 @@ def test_contexted_kv_attention(
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=sliding_window,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
start_time = time.time()
|
||||
@@ -262,6 +282,7 @@ def test_contexted_kv_attention(
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=sliding_window,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
end_time = time.time()
|
||||
@@ -386,6 +407,7 @@ def test_rocm_backend_sliding_window_includes_boundary_token(
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_cached_kv(
|
||||
num_heads: int,
|
||||
@@ -394,6 +416,7 @@ def test_contexted_kv_attention_cached_kv(
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
interleave_v: bool,
|
||||
block_size: int = 32,
|
||||
) -> None:
|
||||
# Exercises the KV_FROM_CACHE path of context_attention_fwd: the current
|
||||
@@ -487,6 +510,18 @@ def test_contexted_kv_attention_cached_kv(
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
interleaved_v_pack_factor = 0
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape).
|
||||
interleaved_v_pack_factor = 16 // v_cache.element_size()
|
||||
x_v = interleaved_v_pack_factor
|
||||
v_cache = (
|
||||
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(cache_size, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
|
||||
# Cached-K/V path: current-chunk k and v are None.
|
||||
context_attention_fwd(
|
||||
query,
|
||||
@@ -504,6 +539,7 @@ def test_contexted_kv_attention_cached_kv(
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=0,
|
||||
interleaved_v_pack_factor=interleaved_v_pack_factor,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -544,8 +580,11 @@ def test_contexted_kv_attention_cached_kv(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> None:
|
||||
def test_contexted_kv_attention_cached_kv_block_table_boundary(
|
||||
device: str, interleave_v: bool
|
||||
) -> None:
|
||||
# Boundary guard for the KV_FROM_CACHE block-table load. With an
|
||||
# exact-sized block table (row length == number of blocks for the
|
||||
# sequence) and a query that ends the sequence, the last K/V tile has
|
||||
@@ -611,6 +650,18 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
interleaved_v_pack_factor = 0
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape).
|
||||
interleaved_v_pack_factor = 16 // v_cache.element_size()
|
||||
x_v = interleaved_v_pack_factor
|
||||
v_cache = (
|
||||
v_cache.view(num_blocks, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(num_blocks, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
|
||||
# Cached-K/V path: current-chunk k and v are None.
|
||||
context_attention_fwd(
|
||||
query,
|
||||
@@ -628,6 +679,7 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=0,
|
||||
interleaved_v_pack_factor=interleaved_v_pack_factor,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -723,6 +775,7 @@ def test_contexted_kv_attention_cached_kv_alibi_unsupported(device: str) -> None
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_alibi(
|
||||
num_heads: int,
|
||||
@@ -732,6 +785,7 @@ def test_contexted_kv_attention_alibi(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
block_size: int = 32,
|
||||
) -> None:
|
||||
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
|
||||
@@ -860,6 +914,23 @@ def test_contexted_kv_attention_alibi(
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
extra_op_kwargs = {}
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
|
||||
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
|
||||
# read it back interleaved.
|
||||
x_v = 16 // v_cache.element_size()
|
||||
v_cache = (
|
||||
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(cache_size, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
if op is context_attention_fwd:
|
||||
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
|
||||
else:
|
||||
extra_op_kwargs["use_interleaved_v_cache"] = True
|
||||
|
||||
# Warm up the Triton kernel by calling it once before actually measuring
|
||||
# generation time
|
||||
op(
|
||||
@@ -878,6 +949,7 @@ def test_contexted_kv_attention_alibi(
|
||||
k_scale,
|
||||
v_scale,
|
||||
alibi_slopes=alibi_slopes,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
start_time = time.time()
|
||||
@@ -897,6 +969,7 @@ def test_contexted_kv_attention_alibi(
|
||||
k_scale,
|
||||
v_scale,
|
||||
alibi_slopes=alibi_slopes,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
end_time = time.time()
|
||||
@@ -992,6 +1065,7 @@ def test_contexted_kv_attention_alibi(
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_f32(
|
||||
num_heads: int,
|
||||
@@ -1002,6 +1076,7 @@ def test_contexted_kv_attention_f32(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
) -> None:
|
||||
test_contexted_kv_attention(
|
||||
num_heads,
|
||||
@@ -1012,6 +1087,7 @@ def test_contexted_kv_attention_f32(
|
||||
kv_cache_dtype,
|
||||
device,
|
||||
op,
|
||||
interleave_v=interleave_v,
|
||||
)
|
||||
|
||||
|
||||
@@ -1023,6 +1099,7 @@ def test_contexted_kv_attention_f32(
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_alibi_f32(
|
||||
num_heads: int,
|
||||
@@ -1032,9 +1109,17 @@ def test_contexted_kv_attention_alibi_f32(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
) -> None:
|
||||
test_contexted_kv_attention_alibi(
|
||||
num_heads, num_queries_per_kv, head_size, dtype, kv_cache_dtype, device, op
|
||||
num_heads,
|
||||
num_queries_per_kv,
|
||||
head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
device,
|
||||
op,
|
||||
interleave_v=interleave_v,
|
||||
)
|
||||
|
||||
|
||||
@@ -1052,6 +1137,7 @@ NONSTANDARD_BLOCK_SIZE_SHAPES = [
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_qwen3_nonstandard_block_size(
|
||||
num_heads: int,
|
||||
@@ -1061,6 +1147,7 @@ def test_qwen3_nonstandard_block_size(
|
||||
dtype: torch.dtype,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
) -> None:
|
||||
"""Non-power-of-2 pages must match, even when a tile straddles a page."""
|
||||
if not current_platform.is_rocm():
|
||||
@@ -1076,4 +1163,5 @@ def test_qwen3_nonstandard_block_size(
|
||||
kv_cache_dtype="auto",
|
||||
device=device,
|
||||
op=op,
|
||||
interleave_v=interleave_v,
|
||||
)
|
||||
|
||||
@@ -1061,3 +1061,25 @@ def test_aiter_mha_varlen_fp8_kv(dtype):
|
||||
rtol=rtol,
|
||||
)
|
||||
torch.testing.assert_close(output, ref, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
def test_asm_kv_scale_tables_shared_across_builders():
|
||||
"""Builders share one AsmKvScaleTables; equal (shape, scale) requests must
|
||||
return the same dense table, a different scale a different one."""
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import (
|
||||
AiterFlashAttentionMetadataBuilder,
|
||||
AsmKvScaleTables,
|
||||
)
|
||||
|
||||
tables = AsmKvScaleTables()
|
||||
device = torch.device("cuda")
|
||||
table = tables.get(device, 8, 2, 16, 0.5)
|
||||
assert tables.get(device, 8, 2, 16, 0.5) is table
|
||||
assert tables.get(device, 8, 2, 16, 1.0) is not table
|
||||
assert table.shape == (8, 2, 16)
|
||||
assert table.dtype == torch.float32
|
||||
assert torch.all(table == 0.5)
|
||||
# The builder class holds the single process-wide instance.
|
||||
assert isinstance(
|
||||
AiterFlashAttentionMetadataBuilder.asm_kv_scale_tables, AsmKvScaleTables
|
||||
)
|
||||
|
||||
@@ -368,9 +368,11 @@ async def test_dp_pause_late_request_does_not_block_drain():
|
||||
# down first. The front-end sets its own copy optimistically when it
|
||||
# forwards the wake, so sample only after several coordinator
|
||||
# publishes (every 100ms while stats change) have overwritten it.
|
||||
# Rank 0 reports the wave only after its first step, which can outlast
|
||||
# the 2s on a cold device, so wait for the report rather than sample it.
|
||||
await asyncio.sleep(2)
|
||||
assert not long_request.done(), "the warmup request was too short to sample"
|
||||
assert engine.engine_core.dp_engines_running(), (
|
||||
assert await _poll_flag(engine, True, timeout=30), (
|
||||
"the coordinator does not report the engines as running while they are"
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user