[Bugfix][GDN] Reset speculative decode count for an empty draft schedule (#53077)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
This commit is contained in:
@@ -239,6 +239,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
|||||||
or num_decode_draft_tokens_cpu[spec_sequence_masks_cpu].sum().item()
|
or num_decode_draft_tokens_cpu[spec_sequence_masks_cpu].sum().item()
|
||||||
== 0
|
== 0
|
||||||
):
|
):
|
||||||
|
num_spec_decodes = 0
|
||||||
spec_sequence_masks = None
|
spec_sequence_masks = None
|
||||||
spec_sequence_masks_cpu = None
|
spec_sequence_masks_cpu = None
|
||||||
else:
|
else:
|
||||||
|
|||||||
Reference in New Issue
Block a user