[ROCm][Perf][Kimi-K3] Use AITER Gluon chunk_kda for KDA prefill (#59552)

Signed-off-by: Omar Muhammad <omar.muhammad@amd.com>
Signed-off-by: Nicholas Susanto <nicholas.susanto@amd.com>
Co-authored-by: Nicholas Susanto <nicholas.susanto@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Omar Muhammad
2026-10-16 22:58:56 +00:00
committed by GitHub
co-authored by Nicholas Susanto Douglas Lehr Kevin H. Luu
parent b088eba944
commit 28e92eeff2
6 changed files with 197 additions and 20 deletions
@@ -482,7 +482,7 @@ def _spec_for(monkeypatch, **attrs) -> MambaSpec:
layer = KimiK3DeltaAttention.__new__(KimiK3DeltaAttention)
for name, value in dict(
use_prefill_checkpoint=True,
use_fused_chunk=True,
kda_prefill_backend="fused",
num_spec=0,
use_safe_gate=True,
).items():
@@ -506,7 +506,8 @@ def test_opt_in_publishes_chunk_walk_alignment(monkeypatch) -> None:
"attr,value",
[
("use_prefill_checkpoint", False),
("use_fused_chunk", False),
("kda_prefill_backend", "triton"),
("kda_prefill_backend", "gluon"),
("use_safe_gate", False),
],
)
+23 -2
View File
@@ -77,7 +77,7 @@ def _run(inp: dict, use_fused: bool, out: torch.Tensor | None = None, **kw):
output_final_state=True,
use_qk_l2norm_in_kernel=True,
cu_seqlens=inp["cu"],
use_fused_chunk=use_fused,
backend=kw.pop("backend", "fused" if use_fused else "triton"),
out=out,
**kw,
)
@@ -112,6 +112,27 @@ def test_fused_chunk_matches_triton(seqlens: list[int]) -> None:
torch.testing.assert_close(state_fused, state_ref, rtol=1e-3, atol=1e-3)
@pytest.mark.parametrize(
"seqlens",
[[1], [64], [1024], [4096], [513, 64, 1, 1200], [512] * 6],
)
def test_gluon_chunk_matches_triton(seqlens: list[int]) -> None:
"""AITER's Gluon chunk_kda must match the Triton chunk path too."""
from vllm._aiter_ops import rocm_aiter_ops
if not rocm_aiter_ops.is_kda_gluon_prefill_available():
pytest.skip("AITER Gluon chunk_kda is not available here")
inp = _inputs(seqlens, seed=len(seqlens))
o_ref, state_ref = _run(inp, use_fused=False)
o_gluon, state_gluon = _run(inp, use_fused=False, backend="gluon")
torch.testing.assert_close(o_gluon.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(state_gluon, state_ref, rtol=5e-3, atol=5e-3)
state_rel = (state_gluon - state_ref).norm() / state_ref.norm()
assert state_rel < 5e-3, f"final state relative L2 error {state_rel:.2e}"
@pytest.mark.parametrize("use_fused", [False, True])
def test_out_argument_receives_the_result(use_fused: bool) -> None:
"""``out`` is a promise about where the result lands.
@@ -648,7 +669,7 @@ def _run_paged(inp: dict, cache, idx, warm, use_fused: bool, **kw):
has_initial_state=warm,
use_qk_l2norm_in_kernel=True,
cu_seqlens=inp["cu"],
use_fused_chunk=use_fused,
backend="fused" if use_fused else "triton",
**kw,
)