[BUGFIX] Raise an error for no draft token case when draft_tp>1 (#6369)

This commit is contained in:
Woo-Yeon Lee
2024-07-19 06:01:09 -07:00
committed by GitHub
parent 6366efc67b
commit a921e86392
4 changed files with 85 additions and 5 deletions
+3
View File
@@ -22,6 +22,9 @@ class SpeculativeProposals:
# The valid length of each proposal; can be zero.
proposal_lens: torch.Tensor
# A flag to mark that there's no available proposals
no_proposals: bool = False
def __repr__(self):
return (f"SpeculativeProposals("
f"proposal_token_ids={self.proposal_token_ids}, "