[ROCm][Kimi-K3] Optimize low-concurrency speculative KDA (#58045)

Signed-off-by: jiacao-amd <jiahui.cao@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
jiacao-amd
2026-09-25 12:33:30 -05:00
committed by GitHub
co-authored by OpenAI Codex Andreas Karatzas
parent c89b5b4176
commit 1417022c3d
6 changed files with 201 additions and 27 deletions
+56 -9
View File
@@ -22,6 +22,9 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_upd
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
gather_initial_states,
)
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
fused_recurrent_kda as fused_recurrent_kda_amd,
)
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
fused_recurrent_kda_packed_decode as fused_recurrent_kda_packed_decode_amd,
)
@@ -44,10 +47,12 @@ from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
chunk_kda,
chunk_kda_with_fused_gate,
fused_kda_gate,
fused_recurrent_kda,
fused_recurrent_kda_fwd,
fused_recurrent_kda_packed_decode,
)
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
fused_recurrent_kda as fused_recurrent_kda_nvidia,
)
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
from vllm.utils.flashinfer import flashinfer_fused_kda_decode
@@ -66,6 +71,10 @@ PACKED_DECODE_IMPLS = {
"nvidia": fused_recurrent_kda_packed_decode,
"amd": fused_recurrent_kda_packed_decode_amd,
}
SPEC_DECODE_IMPLS = {
"nvidia": fused_recurrent_kda_nvidia,
"amd": fused_recurrent_kda_amd,
}
def test_kda_warmup_skips_missing_metadata(monkeypatch):
@@ -468,17 +477,51 @@ def test_packed_kda_decode_correctness(
@pytest.mark.parametrize(
("H", "fuse_gate"),
[(12, True), (12, False), (12, None), (96, None)],
("impl", "H", "fuse_gate", "num_seqs", "query_len"),
[
*[
pytest.param(
impl,
H,
fuse_gate,
3,
3,
id=f"{impl}-H{H}-fuse-{fuse_gate}",
)
for impl in SPEC_DECODE_IMPLS
for H, fuse_gate in [
(12, True),
(12, False),
(12, None),
(96, None),
]
],
pytest.param("amd", 12, True, 1, 1, id="amd-single-token"),
pytest.param("amd", 12, True, 1, 8, id="amd-single-sequence-short"),
pytest.param(
"amd",
96,
None,
1,
8,
id="amd-single-sequence-many-heads",
),
pytest.param("amd", 12, True, 2, 4, id="amd-uniform-two-sequences"),
pytest.param("amd", 12, True, 4, 4, id="amd-uniform-four-sequences"),
pytest.param("amd", 12, True, 8, 7, id="amd-uniform-eight-sequences"),
],
)
@pytest.mark.parametrize("lower_bound", [-5.0, None])
@torch.inference_mode()
def test_kda_spec_decode_correctness(
H: int,
fuse_gate: bool | None,
num_seqs: int,
query_len: int,
lower_bound: float | None,
impl: str,
):
num_seqs, query_len, D = 3, 3, 128
D = 128
T = num_seqs * query_len
torch.manual_seed(1234)
@@ -522,10 +565,8 @@ def test_kda_spec_decode_correctness(
dtype=torch.int32,
device=DEVICE,
).view(num_seqs, query_len)
num_accepted_tokens = torch.tensor(
[1, 2, 3],
dtype=torch.int32,
device=DEVICE,
num_accepted_tokens = (
torch.arange(num_seqs, dtype=torch.int32, device=DEVICE) % query_len + 1
)
state_storage = 0.01 * torch.randn(
T + 1,
@@ -578,7 +619,12 @@ def test_kda_spec_decode_correctness(
expected = torch.cat(expected_outputs, dim=1)
actual_state = state.clone()
actual, _ = fused_recurrent_kda(
extra_args = (
{"uniform_sequence_length": query_len}
if impl == "amd" and num_seqs in (2, 4, 8)
else {}
)
actual, _ = SPEC_DECODE_IMPLS[impl](
q=q,
k=k,
v=v,
@@ -593,6 +639,7 @@ def test_kda_spec_decode_correctness(
num_accepted_tokens=num_accepted_tokens,
out=output,
fuse_gate=fuse_gate,
**extra_args,
)
assert actual.data_ptr() == output.data_ptr()
+19
View File
@@ -49,6 +49,7 @@ PRUNED_METADATA_FIELDS = {
"prefill_state_indices",
"prefill_has_initial_state",
"spec_sequence_masks",
"uniform_spec_sequence_length",
"flashinfer_prefill_query_start_loc",
"flashinfer_prefill_seq_order",
}
@@ -409,6 +410,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset():
"num_speculative_tokens",
"full_cuda_graph",
"is_prefilling",
"expected_uniform_spec_sequence_length",
),
[
pytest.param(
@@ -417,6 +419,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset():
2,
False,
[False, False],
3,
id="pure-spec-decode",
),
pytest.param(
@@ -425,6 +428,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset():
2,
False,
[True, False, False],
3,
id="mixed-prefill-and-spec-decode",
),
pytest.param(
@@ -433,6 +437,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset():
0,
False,
[False, False],
None,
id="regular-decode",
),
pytest.param(
@@ -441,8 +446,18 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset():
2,
False,
[False, False],
None,
id="no-scheduled-draft-tokens",
),
pytest.param(
BatchSpec(seq_lens=[50, 30], query_lens=[3, 2]),
[2, 1],
2,
False,
[False, False],
None,
id="ragged-spec-decode",
),
],
)
def test_kimi_k3_kda_metadata_matches_shared_gdn(
@@ -451,6 +466,7 @@ def test_kimi_k3_kda_metadata_matches_shared_gdn(
num_speculative_tokens: int,
full_cuda_graph: bool,
is_prefilling: list[bool],
expected_uniform_spec_sequence_length: int | None,
):
kwargs: dict[str, torch.Tensor] = {}
if num_decode_draft_tokens is not None:
@@ -483,6 +499,9 @@ def test_kimi_k3_kda_metadata_matches_shared_gdn(
assert isinstance(actual, KimiK3KDAMetadata)
_assert_matches_shared_gdn(reference, actual)
assert (
reference.uniform_spec_sequence_length == expected_uniform_spec_sequence_length
)
def test_mixed_regular_and_spec_decode_uses_packed_decode_metadata():