Add: SupportsEagle3 interface for explicit EAGLE3 support (#22642)
Signed-off-by: Rahul Tuli <rtuli@redhat.com>
This commit is contained in:
@@ -823,3 +823,56 @@ def supports_v0_only(
|
||||
model: Union[type[object], object],
|
||||
) -> Union[TypeIs[type[SupportsV0Only]], TypeIs[SupportsV0Only]]:
|
||||
return getattr(model, "supports_v0_only", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsEagle3(Protocol):
|
||||
"""The interface required for models that support
|
||||
EAGLE3 speculative decoding."""
|
||||
|
||||
supports_eagle3: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
A flag that indicates this model supports EAGLE3
|
||||
speculative decoding.
|
||||
|
||||
Note:
|
||||
There is no need to redefine this flag if this class is in the
|
||||
MRO of your model class.
|
||||
"""
|
||||
|
||||
def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
|
||||
"""
|
||||
Set which layers should output auxiliary
|
||||
hidden states for EAGLE3.
|
||||
|
||||
Args:
|
||||
layers: Tuple of layer indices that should output auxiliary
|
||||
hidden states.
|
||||
"""
|
||||
...
|
||||
|
||||
def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]:
|
||||
"""
|
||||
Get the layer indices that should output auxiliary hidden states
|
||||
for EAGLE3.
|
||||
|
||||
Returns:
|
||||
Tuple of layer indices for auxiliary hidden state outputs.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_eagle3(model: type[object]) -> TypeIs[type[SupportsEagle3]]:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_eagle3(model: object) -> TypeIs[SupportsEagle3]:
|
||||
...
|
||||
|
||||
|
||||
def supports_eagle3(
|
||||
model: Union[type[object], object],
|
||||
) -> Union[TypeIs[type[SupportsEagle3]], TypeIs[SupportsEagle3]]:
|
||||
return isinstance(model, SupportsEagle3)
|
||||
|
||||
Reference in New Issue
Block a user