[Bugfix][Kimi-K3] Do not classify a stateless first chunk as a decode (#51483)

Signed-off-by: Oleksandr Zakharchuk <oleksandr.zakharchuk@gmail.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
sashko-zakharchuk
2026-09-16 21:24:48 +08:00
committed by GitHub
co-authored by zjy0516
parent 975dca5bb5
commit f8b5c11468
2 changed files with 114 additions and 3 deletions
+26 -3
View File
@@ -427,10 +427,33 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
spec_token_start = None
non_spec_token_start = None
if num_spec_decodes == 0:
# The runner orders ordinary decodes before prefills.
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
split_decodes_and_prefills(m, decode_threshold=1)
# V2 already excludes prefills from full decode graphs via has_prefill.
# Classify first chunks as prefills to mask recycled state;
# resumed one-token chunks can still use the decode kernels.
assert m.seq_lens_cpu_upper_bound is not None
query_lens_cpu = query_start_loc_cpu.diff()
no_prior_state = (query_lens_cpu > 0) & (
m.seq_lens_cpu_upper_bound <= query_lens_cpu
)
# Capture batches also have seq_len == query_len, but are not prefills.
if m.is_prefilling is not None:
no_prior_state &= m.is_prefilling
else:
no_prior_state = torch.zeros_like(no_prior_state)
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
split_decodes_and_prefills(
m.replace(is_prefilling=no_prior_state),
decode_threshold=1,
treat_short_extends_as_decodes=False,
)
)
# Exclude trailing padding from both prefill counts.
if num_prefills:
num_prefills -= int((query_lens_cpu[num_decodes:] == 0).sum())
num_prefill_tokens = (
int(query_start_loc_cpu[num_decodes + num_prefills])
- num_decode_tokens
)
num_spec_decode_tokens = 0
spec_token_indx = None
non_spec_token_indx = None