[ROCm][Perf] Kimi-K3 enable prefill checkpoints on ROCm (#58344)

Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
This commit is contained in:
kliuae
2026-10-02 06:58:04 +00:00
committed by GitHub
parent 4056c8ac1f
commit 0154979656
7 changed files with 748 additions and 14 deletions
+56 -6
View File
@@ -13,17 +13,17 @@ import torch
from vllm.platforms import current_platform
def _on_gfx950() -> bool:
if not current_platform.is_rocm():
def _fused_chunk_available() -> bool:
if not (current_platform.is_rocm() and torch.cuda.is_available()):
return False
from vllm.platforms.rocm import on_gfx950
from vllm.models.kimi_k3.amd.ops.kda_chunk import is_fused_kda_chunk_supported
return on_gfx950()
return is_fused_kda_chunk_supported()
pytestmark = pytest.mark.skipif(
not _on_gfx950(),
reason="The fused KDA chunk kernel is only built for gfx950",
not _fused_chunk_available(),
reason="The fused KDA chunk kernel needs a gfx950 build that includes it",
)
HEAD_DIM = 128
@@ -701,6 +701,56 @@ def test_cold_rows_start_from_zero_not_from_cache_junk(use_fused: bool) -> None:
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
@pytest.mark.parametrize("use_fused", [False, True])
def test_state_indices_accept_a_block_table_column(use_fused: bool) -> None:
# The layer passes block_table[:, 0], which is contiguous only while the
# table has one column. Speculative decoding reserves a state page per
# draft token, so the table widens and that column becomes strided.
_requires_kernel()
seqlens = [512, 320, 64]
inp = _inputs(seqlens, seed=43)
n = len(seqlens)
rows = [3 * i + 5 for i in range(n)]
cache, idx, warm = _paged_cache(inp, [True] * n, rows, slots=3 * n + 12)
ref_cache = cache.clone()
table = torch.zeros(n, 9, dtype=torch.int32, device="cuda")
table[:, 0] = idx
column = table[:, 0]
assert not column.is_contiguous()
o_ref, _ = _run_paged(inp, ref_cache, idx, warm, use_fused=use_fused)
o_got, _ = _run_paged(inp, cache, column, warm, use_fused=use_fused)
torch.testing.assert_close(o_got, o_ref)
torch.testing.assert_close(cache, ref_cache)
@pytest.mark.parametrize("use_fused", [False, True])
def test_padded_zero_length_sequence_is_inert(use_fused: bool) -> None:
# A batch carrying speculative decodes pads its non-spec group with
# zero-length sequences whose cache row still belongs to a request, so
# consuming no tokens has to mean writing nothing and perturbing nobody.
_requires_kernel()
live = [200, 1, 128]
# _inputs draws h0 last, so the padded batch's live rows are the same draw
# as the unpadded one's and the two runs compare elementwise.
padded, unpadded = _inputs([*live, 0], seed=41), _inputs(live, seed=41)
rows = [3 * i + 5 for i in range(4)]
cache, idx, warm = _paged_cache(padded, [True] * 4, rows, slots=24)
before = cache.clone()
_run_paged(padded, cache, idx, warm, use_fused=use_fused)
ref_cache, ref_idx, ref_warm = _paged_cache(unpadded, [True] * 3, rows[:3], 24)
_run_paged(unpadded, ref_cache, ref_idx, ref_warm, use_fused=use_fused)
assert torch.equal(cache[rows[3]], before[rows[3]]), (
"the padded sequence's cache row was written"
)
torch.testing.assert_close(cache[ref_idx.long()], ref_cache[ref_idx.long()])
@pytest.mark.parametrize("groups", [2, 4])
def test_paged_state_survives_the_group_split(groups: int, monkeypatch) -> None:
"""At G>1 the read and the write are different blocks of one launch."""