[Bugfix][MRV2][Spec Decode] Honor EAGLE3 single-state draft configuration (#61563)
Signed-off-by: Sandeep Agrawal <sandeepa@nvidia.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Kevin H. Luu
parent
06265beb25
commit
21e207efa1
@@ -12,6 +12,16 @@ from vllm.model_executor.models.interfaces import SupportsEagle3, supports_eagle
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def requires_aux_hidden_states(spec_config: SpeculativeConfig) -> bool:
|
||||
if spec_config.method == "eagle3":
|
||||
eagle_config = (
|
||||
getattr(spec_config.draft_model_config.hf_config, "eagle_config", None)
|
||||
or {}
|
||||
)
|
||||
return eagle_config.get("use_aux_hidden_state", True)
|
||||
return spec_config.method in ("dflash", "dspark", "extract_hidden_states")
|
||||
|
||||
|
||||
def set_eagle3_aux_hidden_state_layers(
|
||||
model: nn.Module,
|
||||
spec_config: SpeculativeConfig,
|
||||
|
||||
Reference in New Issue
Block a user