[Bugfix][MRV2][Spec Decode] Accept num_speculative_tokens in NgramGPUSpeculator.propose (#60300)
Signed-off-by: cheesecake <farzanaman99@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ae19af883a
commit
b6d8e8afd9
@@ -231,6 +231,7 @@ class NgramGPUSpeculator(BaseSpeculator):
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
is_profile: bool = False,
|
||||
num_speculative_tokens: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
num_reqs = input_batch.num_reqs
|
||||
if dummy_run:
|
||||
|
||||
Reference in New Issue
Block a user