[Speculative Decoding] Add DFlash speculators config parsing (#38300)
Signed-off-by: Zhanqiu Hu <zhu@redhat.com>
This commit is contained in:
@@ -41,3 +41,34 @@ def update_eagle3(config_dict: dict, pre_trained_config: dict) -> None:
|
||||
pre_trained_config["eagle_aux_hidden_state_layer_ids"] = config_dict[
|
||||
"eagle_aux_hidden_state_layer_ids"
|
||||
]
|
||||
|
||||
|
||||
@register_speculator("dflash")
|
||||
def update_dflash(config_dict: dict, pre_trained_config: dict) -> None:
|
||||
"""
|
||||
Apply DFlash specific configuration transformations to the `dict` used to
|
||||
construct the Transformers PreTrainedConfig.
|
||||
|
||||
DFlash specific fields:
|
||||
- draft_vocab_size: Size of the draft model's vocabulary
|
||||
- target_hidden_size: Hidden size of the target model
|
||||
- mask_token_id (required): Token ID used for parallel drafting mask
|
||||
placeholders
|
||||
- aux_hidden_state_layer_ids (required): Layer indices from the target
|
||||
model whose intermediate hidden states are used as context for the
|
||||
DFlash drafter. Mapped to both eagle_aux_hidden_state_layer_ids
|
||||
(for gpu_model_runner) and dflash_config.target_layer_ids (for the
|
||||
DFlash model).
|
||||
"""
|
||||
pre_trained_config["architectures"] = ["DFlashDraftModel"]
|
||||
pre_trained_config["draft_vocab_size"] = config_dict.get("draft_vocab_size")
|
||||
if config_dict.get("target_hidden_size") is not None:
|
||||
pre_trained_config["target_hidden_size"] = config_dict["target_hidden_size"]
|
||||
|
||||
aux_layer_ids = config_dict["aux_hidden_state_layer_ids"]
|
||||
pre_trained_config["eagle_aux_hidden_state_layer_ids"] = aux_layer_ids
|
||||
|
||||
pre_trained_config["dflash_config"] = {
|
||||
"mask_token_id": config_dict["mask_token_id"],
|
||||
"target_layer_ids": aux_layer_ids,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user