[Spec Decode] Efficient padded speculation (#24539)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
Benjamin Chislett
2025-09-18 01:07:24 -04:00
committed by GitHub
parent 5c65a72bb1
commit b7433ca1a4
5 changed files with 507 additions and 104 deletions
+4 -1
View File
@@ -64,7 +64,10 @@ class CachedRequestState:
def get_token_id(self, idx: int) -> int:
if idx < self.num_prompt_tokens:
return self.prompt_token_ids[idx]
return self.output_token_ids[idx - self.num_prompt_tokens]
elif idx - self.num_prompt_tokens < len(self.output_token_ids):
return self.output_token_ids[idx - self.num_prompt_tokens]
else:
return -1
class InputBatch: