[Model] MLPSpeculator speculative decoding support (#4947)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>

Co-authored-by: Thomas Parnell <tpa@zurich.ibm.com>
Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Co-authored-by: Davis Wertheimer <Davis.Wertheimer@ibm.com>
This commit is contained in:
Joshua Rosenkranz
2024-06-20 20:23:12 -04:00
committed by GitHub
co-authored by Thomas Parnell Nick Hill Davis Wertheimer
parent 6c5b7af152
commit b12518d3cf
18 changed files with 523 additions and 40 deletions
+4
View File
@@ -1,5 +1,6 @@
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Optional
import torch
@@ -46,6 +47,9 @@ class SpeculativeScores:
# tokens and also non-speculative normal decoding.
token_ids: torch.Tensor
# Optional last hidden states from the scoring model.
hidden_states: Optional[torch.Tensor] = None
def __repr__(self):
return (f"SpeculativeScores("
f"probs={self.probs.shape}, "