[Bugfix][Kimi-K3] Do not classify a stateless first chunk as a decode (#51483)
Signed-off-by: Oleksandr Zakharchuk <oleksandr.zakharchuk@gmail.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
zjy0516
parent
975dca5bb5
commit
f8b5c11468
@@ -427,10 +427,33 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
spec_token_start = None
|
||||
non_spec_token_start = None
|
||||
if num_spec_decodes == 0:
|
||||
# The runner orders ordinary decodes before prefills.
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(m, decode_threshold=1)
|
||||
# V2 already excludes prefills from full decode graphs via has_prefill.
|
||||
# Classify first chunks as prefills to mask recycled state;
|
||||
# resumed one-token chunks can still use the decode kernels.
|
||||
assert m.seq_lens_cpu_upper_bound is not None
|
||||
query_lens_cpu = query_start_loc_cpu.diff()
|
||||
no_prior_state = (query_lens_cpu > 0) & (
|
||||
m.seq_lens_cpu_upper_bound <= query_lens_cpu
|
||||
)
|
||||
# Capture batches also have seq_len == query_len, but are not prefills.
|
||||
if m.is_prefilling is not None:
|
||||
no_prior_state &= m.is_prefilling
|
||||
else:
|
||||
no_prior_state = torch.zeros_like(no_prior_state)
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(
|
||||
m.replace(is_prefilling=no_prior_state),
|
||||
decode_threshold=1,
|
||||
treat_short_extends_as_decodes=False,
|
||||
)
|
||||
)
|
||||
# Exclude trailing padding from both prefill counts.
|
||||
if num_prefills:
|
||||
num_prefills -= int((query_lens_cpu[num_decodes:] == 0).sum())
|
||||
num_prefill_tokens = (
|
||||
int(query_start_loc_cpu[num_decodes + num_prefills])
|
||||
- num_decode_tokens
|
||||
)
|
||||
num_spec_decode_tokens = 0
|
||||
spec_token_indx = None
|
||||
non_spec_token_indx = None
|
||||
|
||||
Reference in New Issue
Block a user