[ROCm][Kimi-K3] Optimize low-concurrency speculative KDA (#58045)
Signed-off-by: jiacao-amd <jiahui.cao@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
OpenAI Codex
Andreas Karatzas
parent
c89b5b4176
commit
1417022c3d
@@ -22,6 +22,9 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_upd
|
||||
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
gather_initial_states,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
fused_recurrent_kda as fused_recurrent_kda_amd,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
fused_recurrent_kda_packed_decode as fused_recurrent_kda_packed_decode_amd,
|
||||
)
|
||||
@@ -44,10 +47,12 @@ from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
|
||||
chunk_kda,
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_kda_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_fwd,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
|
||||
fused_recurrent_kda as fused_recurrent_kda_nvidia,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
|
||||
from vllm.utils.flashinfer import flashinfer_fused_kda_decode
|
||||
@@ -66,6 +71,10 @@ PACKED_DECODE_IMPLS = {
|
||||
"nvidia": fused_recurrent_kda_packed_decode,
|
||||
"amd": fused_recurrent_kda_packed_decode_amd,
|
||||
}
|
||||
SPEC_DECODE_IMPLS = {
|
||||
"nvidia": fused_recurrent_kda_nvidia,
|
||||
"amd": fused_recurrent_kda_amd,
|
||||
}
|
||||
|
||||
|
||||
def test_kda_warmup_skips_missing_metadata(monkeypatch):
|
||||
@@ -468,17 +477,51 @@ def test_packed_kda_decode_correctness(
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("H", "fuse_gate"),
|
||||
[(12, True), (12, False), (12, None), (96, None)],
|
||||
("impl", "H", "fuse_gate", "num_seqs", "query_len"),
|
||||
[
|
||||
*[
|
||||
pytest.param(
|
||||
impl,
|
||||
H,
|
||||
fuse_gate,
|
||||
3,
|
||||
3,
|
||||
id=f"{impl}-H{H}-fuse-{fuse_gate}",
|
||||
)
|
||||
for impl in SPEC_DECODE_IMPLS
|
||||
for H, fuse_gate in [
|
||||
(12, True),
|
||||
(12, False),
|
||||
(12, None),
|
||||
(96, None),
|
||||
]
|
||||
],
|
||||
pytest.param("amd", 12, True, 1, 1, id="amd-single-token"),
|
||||
pytest.param("amd", 12, True, 1, 8, id="amd-single-sequence-short"),
|
||||
pytest.param(
|
||||
"amd",
|
||||
96,
|
||||
None,
|
||||
1,
|
||||
8,
|
||||
id="amd-single-sequence-many-heads",
|
||||
),
|
||||
pytest.param("amd", 12, True, 2, 4, id="amd-uniform-two-sequences"),
|
||||
pytest.param("amd", 12, True, 4, 4, id="amd-uniform-four-sequences"),
|
||||
pytest.param("amd", 12, True, 8, 7, id="amd-uniform-eight-sequences"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("lower_bound", [-5.0, None])
|
||||
@torch.inference_mode()
|
||||
def test_kda_spec_decode_correctness(
|
||||
H: int,
|
||||
fuse_gate: bool | None,
|
||||
num_seqs: int,
|
||||
query_len: int,
|
||||
lower_bound: float | None,
|
||||
impl: str,
|
||||
):
|
||||
num_seqs, query_len, D = 3, 3, 128
|
||||
D = 128
|
||||
T = num_seqs * query_len
|
||||
torch.manual_seed(1234)
|
||||
|
||||
@@ -522,10 +565,8 @@ def test_kda_spec_decode_correctness(
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
).view(num_seqs, query_len)
|
||||
num_accepted_tokens = torch.tensor(
|
||||
[1, 2, 3],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
num_accepted_tokens = (
|
||||
torch.arange(num_seqs, dtype=torch.int32, device=DEVICE) % query_len + 1
|
||||
)
|
||||
state_storage = 0.01 * torch.randn(
|
||||
T + 1,
|
||||
@@ -578,7 +619,12 @@ def test_kda_spec_decode_correctness(
|
||||
expected = torch.cat(expected_outputs, dim=1)
|
||||
|
||||
actual_state = state.clone()
|
||||
actual, _ = fused_recurrent_kda(
|
||||
extra_args = (
|
||||
{"uniform_sequence_length": query_len}
|
||||
if impl == "amd" and num_seqs in (2, 4, 8)
|
||||
else {}
|
||||
)
|
||||
actual, _ = SPEC_DECODE_IMPLS[impl](
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
@@ -593,6 +639,7 @@ def test_kda_spec_decode_correctness(
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
out=output,
|
||||
fuse_gate=fuse_gate,
|
||||
**extra_args,
|
||||
)
|
||||
|
||||
assert actual.data_ptr() == output.data_ptr()
|
||||
|
||||
@@ -49,6 +49,7 @@ PRUNED_METADATA_FIELDS = {
|
||||
"prefill_state_indices",
|
||||
"prefill_has_initial_state",
|
||||
"spec_sequence_masks",
|
||||
"uniform_spec_sequence_length",
|
||||
"flashinfer_prefill_query_start_loc",
|
||||
"flashinfer_prefill_seq_order",
|
||||
}
|
||||
@@ -409,6 +410,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset():
|
||||
"num_speculative_tokens",
|
||||
"full_cuda_graph",
|
||||
"is_prefilling",
|
||||
"expected_uniform_spec_sequence_length",
|
||||
),
|
||||
[
|
||||
pytest.param(
|
||||
@@ -417,6 +419,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset():
|
||||
2,
|
||||
False,
|
||||
[False, False],
|
||||
3,
|
||||
id="pure-spec-decode",
|
||||
),
|
||||
pytest.param(
|
||||
@@ -425,6 +428,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset():
|
||||
2,
|
||||
False,
|
||||
[True, False, False],
|
||||
3,
|
||||
id="mixed-prefill-and-spec-decode",
|
||||
),
|
||||
pytest.param(
|
||||
@@ -433,6 +437,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset():
|
||||
0,
|
||||
False,
|
||||
[False, False],
|
||||
None,
|
||||
id="regular-decode",
|
||||
),
|
||||
pytest.param(
|
||||
@@ -441,8 +446,18 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset():
|
||||
2,
|
||||
False,
|
||||
[False, False],
|
||||
None,
|
||||
id="no-scheduled-draft-tokens",
|
||||
),
|
||||
pytest.param(
|
||||
BatchSpec(seq_lens=[50, 30], query_lens=[3, 2]),
|
||||
[2, 1],
|
||||
2,
|
||||
False,
|
||||
[False, False],
|
||||
None,
|
||||
id="ragged-spec-decode",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_kimi_k3_kda_metadata_matches_shared_gdn(
|
||||
@@ -451,6 +466,7 @@ def test_kimi_k3_kda_metadata_matches_shared_gdn(
|
||||
num_speculative_tokens: int,
|
||||
full_cuda_graph: bool,
|
||||
is_prefilling: list[bool],
|
||||
expected_uniform_spec_sequence_length: int | None,
|
||||
):
|
||||
kwargs: dict[str, torch.Tensor] = {}
|
||||
if num_decode_draft_tokens is not None:
|
||||
@@ -483,6 +499,9 @@ def test_kimi_k3_kda_metadata_matches_shared_gdn(
|
||||
|
||||
assert isinstance(actual, KimiK3KDAMetadata)
|
||||
_assert_matches_shared_gdn(reference, actual)
|
||||
assert (
|
||||
reference.uniform_spec_sequence_length == expected_uniform_spec_sequence_length
|
||||
)
|
||||
|
||||
|
||||
def test_mixed_regular_and_spec_decode_uses_packed_decode_metadata():
|
||||
|
||||
@@ -438,6 +438,14 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1)
|
||||
)
|
||||
spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1]
|
||||
uniform_sequence_length = m.uniform_spec_sequence_length
|
||||
# Token padding can make the packed tensor larger than the
|
||||
# request-local metadata represented by this length.
|
||||
if (
|
||||
uniform_sequence_length is not None
|
||||
and q_spec.shape[1] != m.num_spec_decodes * uniform_sequence_length
|
||||
):
|
||||
uniform_sequence_length = None
|
||||
# Spec-only batches write directly into core_attn_out.
|
||||
spec_out = (
|
||||
core_attn_out[:, : q_spec.shape[1]]
|
||||
@@ -457,6 +465,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
cu_seqlens=spec_cu_seqlens,
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
uniform_sequence_length=uniform_sequence_length,
|
||||
out=spec_out,
|
||||
)
|
||||
|
||||
|
||||
+4
-3
@@ -16,9 +16,10 @@
|
||||
# - OOB-mask correctness fix: present (all tl.load use mask=..., other=0).
|
||||
# Validated on gfx950: no core-dump, gsm8k 94.1%.
|
||||
#
|
||||
# AMD-specific deltas vs the NVIDIA copy: NONE yet (byte-identical). Keep in sync
|
||||
# with the NVIDIA copy on FLA updates; any divergence should be an intentional,
|
||||
# documented gfx950-specific change (a #869-style AMD-only fix).
|
||||
# AMD-specific deltas vs the NVIDIA copy:
|
||||
# - fused recurrent gated speculative-decode launch geometry is tuned for
|
||||
# gfx950.
|
||||
# Keep the copies in sync on FLA updates outside documented platform tuning.
|
||||
|
||||
from .chunk import (
|
||||
chunk_kda,
|
||||
|
||||
+103
-15
@@ -164,15 +164,24 @@ def fused_recurrent_kda_fwd_kernel(
|
||||
APPLY_BETA_SIGMOID: tl.constexpr,
|
||||
HAS_DT_BIAS: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
IS_SINGLE_SEQUENCE: tl.constexpr,
|
||||
SEQUENCE_LENGTH: tl.constexpr,
|
||||
num_stages: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
i_v = pid % tl.cdiv(V, BV)
|
||||
i_nh = pid // tl.cdiv(V, BV)
|
||||
i_n, i_h = i_nh // H, i_nh % H
|
||||
bos = tl.load(cu_seqlens + i_n).to(tl.int64)
|
||||
eos = tl.load(cu_seqlens + i_n + 1).to(tl.int64)
|
||||
sequence_length = eos - bos
|
||||
if IS_SINGLE_SEQUENCE:
|
||||
i_n, i_h = 0, i_nh
|
||||
else:
|
||||
i_n, i_h = i_nh // H, i_nh % H
|
||||
if SEQUENCE_LENGTH:
|
||||
bos = i_n * SEQUENCE_LENGTH
|
||||
sequence_length = SEQUENCE_LENGTH
|
||||
else:
|
||||
bos = tl.load(cu_seqlens + i_n).to(tl.int64)
|
||||
eos = tl.load(cu_seqlens + i_n + 1).to(tl.int64)
|
||||
sequence_length = eos - bos
|
||||
if sequence_length == 0:
|
||||
return
|
||||
|
||||
@@ -208,6 +217,14 @@ def fused_recurrent_kda_fwd_kernel(
|
||||
p_v = v + bos * stride_qkv_token + i_h * V + o_v
|
||||
p_g = g + bos * stride_g_token + i_h * K + o_k
|
||||
p_beta = beta + bos * stride_beta_token + i_h
|
||||
if USE_GATE_IN_KERNEL:
|
||||
b_a = exp(tl.load(A_log + i_h).to(tl.float32))
|
||||
if HAS_DT_BIAS:
|
||||
b_bias = tl.load(
|
||||
dt_bias + i_h * K + o_k,
|
||||
mask=m_k,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
for i_t in tl.range(0, sequence_length, num_stages=num_stages):
|
||||
b_q = tl.load(p_q, mask=m_k, other=0.0, eviction_policy="evict_last").to(
|
||||
tl.float32
|
||||
@@ -219,8 +236,8 @@ def fused_recurrent_kda_fwd_kernel(
|
||||
tl.float32
|
||||
)
|
||||
if USE_QK_L2NORM_IN_KERNEL:
|
||||
b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
|
||||
b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
|
||||
b_q *= tl.rsqrt(tl.sum(b_q * b_q) + 1e-6)
|
||||
b_k *= tl.rsqrt(tl.sum(b_k * b_k) + 1e-6)
|
||||
b_q *= scale
|
||||
|
||||
b_gate = tl.load(
|
||||
@@ -231,13 +248,7 @@ def fused_recurrent_kda_fwd_kernel(
|
||||
).to(tl.float32)
|
||||
if USE_GATE_IN_KERNEL:
|
||||
if HAS_DT_BIAS:
|
||||
b_bias = tl.load(
|
||||
dt_bias + i_h * K + o_k,
|
||||
mask=m_k,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
b_gate += b_bias
|
||||
b_a = exp(tl.load(A_log + i_h).to(tl.float32))
|
||||
if USE_LOWER_BOUND:
|
||||
b_gate = lower_bound * tl.sigmoid(b_a * b_gate)
|
||||
else:
|
||||
@@ -288,6 +299,60 @@ def fused_recurrent_kda_fwd_kernel(
|
||||
p_out += stride_out_token
|
||||
|
||||
|
||||
def _get_static_sequence_length(
|
||||
total_tokens: int,
|
||||
head_sequences: int,
|
||||
is_single_sequence: bool,
|
||||
uniform_sequence_length: int | None,
|
||||
) -> int:
|
||||
if is_single_sequence:
|
||||
return total_tokens
|
||||
# MI355X performance gate: 192 head-sequences is 16 sequences at H=12.
|
||||
# Static loops win through 16; dynamic loops win at 32 for short queries.
|
||||
if (
|
||||
uniform_sequence_length is not None
|
||||
and uniform_sequence_length > 1
|
||||
and head_sequences <= 192
|
||||
):
|
||||
return uniform_sequence_length
|
||||
return 0
|
||||
|
||||
|
||||
def _select_kda_launch_config(
|
||||
use_gate_in_kernel: bool,
|
||||
is_spec_decoding: bool,
|
||||
head_sequences: int,
|
||||
static_sequence_length: int,
|
||||
has_multi_token_sequence: bool,
|
||||
) -> tuple[int, int, int]:
|
||||
if static_sequence_length:
|
||||
BV = 4 if head_sequences <= 24 else 8
|
||||
if static_sequence_length == 1:
|
||||
num_stages = 2
|
||||
elif static_sequence_length <= 3:
|
||||
num_stages = 8 if head_sequences <= 24 else 4
|
||||
elif static_sequence_length <= 6:
|
||||
num_stages = 6
|
||||
elif static_sequence_length <= 10:
|
||||
num_stages = 10
|
||||
else:
|
||||
num_stages = 12
|
||||
return BV, 1, num_stages
|
||||
|
||||
if use_gate_in_kernel and is_spec_decoding:
|
||||
if head_sequences <= 24:
|
||||
return 4, 1, 4 if has_multi_token_sequence else 2
|
||||
if has_multi_token_sequence and head_sequences <= 48:
|
||||
return 8, 1, 4
|
||||
if has_multi_token_sequence:
|
||||
return 8, 1, 2
|
||||
return 32, 4, 2
|
||||
|
||||
if use_gate_in_kernel:
|
||||
return 32, 4, 2
|
||||
return 8, 1, 2
|
||||
|
||||
|
||||
def fused_recurrent_kda_fwd(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
@@ -300,6 +365,7 @@ def fused_recurrent_kda_fwd(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
ssm_state_indices: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
uniform_sequence_length: int | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
A_log: torch.Tensor | None = None,
|
||||
dt_bias: torch.Tensor | None = None,
|
||||
@@ -336,6 +402,10 @@ def fused_recurrent_kda_fwd(
|
||||
assert num_accepted_tokens is None
|
||||
else:
|
||||
assert ssm_state_indices.stride(1) == 1
|
||||
if uniform_sequence_length is not None:
|
||||
assert num_accepted_tokens is not None
|
||||
assert uniform_sequence_length > 0
|
||||
assert N * uniform_sequence_length == T
|
||||
assert cu_seqlens.is_contiguous()
|
||||
if use_gate_in_kernel:
|
||||
assert A_log is not None and A_log.is_contiguous()
|
||||
@@ -344,8 +414,22 @@ def fused_recurrent_kda_fwd(
|
||||
if scale is None:
|
||||
scale = K**-0.5
|
||||
|
||||
BV = 32 if use_gate_in_kernel else 8
|
||||
num_warps = 4 if use_gate_in_kernel else 1
|
||||
is_spec_decoding = num_accepted_tokens is not None
|
||||
is_single_sequence = is_spec_decoding and N == 1
|
||||
head_sequences = H * N
|
||||
static_sequence_length = _get_static_sequence_length(
|
||||
total_tokens=T,
|
||||
head_sequences=head_sequences,
|
||||
is_single_sequence=is_single_sequence,
|
||||
uniform_sequence_length=uniform_sequence_length,
|
||||
)
|
||||
BV, num_warps, num_stages = _select_kda_launch_config(
|
||||
use_gate_in_kernel=use_gate_in_kernel,
|
||||
is_spec_decoding=is_spec_decoding,
|
||||
head_sequences=head_sequences,
|
||||
static_sequence_length=static_sequence_length,
|
||||
has_multi_token_sequence=T > N,
|
||||
)
|
||||
grid = (cdiv(V, BV) * N * H,)
|
||||
fused_recurrent_kda_fwd_kernel[grid](
|
||||
q=q,
|
||||
@@ -379,8 +463,10 @@ def fused_recurrent_kda_fwd(
|
||||
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
|
||||
USE_GATE_IN_KERNEL=use_gate_in_kernel,
|
||||
APPLY_BETA_SIGMOID=use_beta_sigmoid_in_kernel,
|
||||
IS_SINGLE_SEQUENCE=is_single_sequence,
|
||||
SEQUENCE_LENGTH=static_sequence_length,
|
||||
num_warps=num_warps,
|
||||
num_stages=2,
|
||||
num_stages=num_stages,
|
||||
)
|
||||
return out, initial_state
|
||||
|
||||
@@ -398,6 +484,7 @@ def fused_recurrent_kda(
|
||||
cu_seqlens: torch.Tensor,
|
||||
ssm_state_indices: torch.Tensor,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
uniform_sequence_length: int | None = None,
|
||||
out: torch.Tensor | None = None,
|
||||
fuse_gate: bool | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -433,6 +520,7 @@ def fused_recurrent_kda(
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
uniform_sequence_length=uniform_sequence_length,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
A_log=A_log if fuse_gate else None,
|
||||
dt_bias=dt_bias if fuse_gate else None,
|
||||
|
||||
@@ -64,6 +64,7 @@ class GDNAttentionMetadata:
|
||||
non_spec_token_indx: torch.Tensor | None = None
|
||||
|
||||
num_accepted_tokens: torch.Tensor | None = None # shape: [batch,]
|
||||
uniform_spec_sequence_length: int | None = None # None for ragged batches
|
||||
|
||||
# Pre-computed FLA chunk metadata (avoids GPU->CPU sync in prepare_chunk_indices)
|
||||
chunk_indices: torch.Tensor | None = None
|
||||
@@ -226,6 +227,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
self.vllm_config.cache_config.mamba_cache_mode,
|
||||
)
|
||||
|
||||
uniform_spec_sequence_length = None
|
||||
spec_sequence_masks_cpu: torch.Tensor | None = None
|
||||
if not self.use_spec_decode or num_decode_draft_tokens_cpu is None:
|
||||
spec_sequence_masks = None
|
||||
@@ -289,6 +291,13 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
assert spec_sequence_masks_cpu is not None
|
||||
non_spec_sequence_masks_cpu = ~spec_sequence_masks_cpu
|
||||
query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
||||
spec_query_lens_cpu = query_lens_cpu[spec_sequence_masks_cpu]
|
||||
if spec_query_lens_cpu.numel() > 0:
|
||||
first_spec_sequence_length = int(spec_query_lens_cpu[0])
|
||||
if first_spec_sequence_length > 0 and bool(
|
||||
torch.all(spec_query_lens_cpu == first_spec_sequence_length)
|
||||
):
|
||||
uniform_spec_sequence_length = first_spec_sequence_length
|
||||
|
||||
# Use CPU tensors to avoid CPU-GPU sync
|
||||
non_spec_query_lens_cpu = query_lens_cpu[non_spec_sequence_masks_cpu]
|
||||
@@ -540,6 +549,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
spec_token_indx=spec_token_indx,
|
||||
non_spec_token_indx=non_spec_token_indx,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
uniform_spec_sequence_length=uniform_spec_sequence_length,
|
||||
nums_dict=nums_dict,
|
||||
batch_ptr=batch_ptr,
|
||||
token_chunk_offset_ptr=token_chunk_offset_ptr,
|
||||
|
||||
Reference in New Issue
Block a user