Support loading sample_from_anchor flag from speculators config (#48639)
Signed-off-by: Fynn Schmitt-Ulms <fschmitt@redhat.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Michael Goin
parent
5feb3950e5
commit
642076d26c
@@ -106,6 +106,9 @@ def update_dflash(config_dict: dict, pre_trained_config: dict) -> None:
|
||||
DFlash drafter. Mapped to both eagle_aux_hidden_state_layer_ids
|
||||
(for gpu_model_runner) and dflash_config.target_layer_ids (for the
|
||||
DFlash model).
|
||||
- sample_from_anchor: Whether to sample from the anchor position. Default
|
||||
False (anchor is a bonus token, only mask tokens predict, yielding
|
||||
block_size - 1 speculative tokens).
|
||||
"""
|
||||
pre_trained_config["architectures"] = ["DFlashDraftModel"]
|
||||
pre_trained_config["draft_vocab_size"] = config_dict.get("draft_vocab_size")
|
||||
@@ -119,6 +122,7 @@ def update_dflash(config_dict: dict, pre_trained_config: dict) -> None:
|
||||
pre_trained_config["dflash_config"] = {
|
||||
"mask_token_id": config_dict["mask_token_id"],
|
||||
"target_layer_ids": [i - 1 for i in aux_layer_ids],
|
||||
"sample_from_anchor": config_dict.get("sample_from_anchor", False),
|
||||
}
|
||||
# Enable causal masking in SWA for vllm-project/speculators models
|
||||
pre_trained_config["dflash_config"]["causal"] = not config_dict.get(
|
||||
@@ -146,10 +150,14 @@ def update_dspark(config_dict: dict, pre_trained_config: dict) -> None:
|
||||
- aux_hidden_state_layer_ids (required): target layer indices feeding the
|
||||
drafter. Mapped to both eagle_aux_hidden_state_layer_ids and
|
||||
target_layer_ids (DSpark's i-1 layer semantics).
|
||||
- sample_from_anchor: Whether to sample from the anchor position. Default
|
||||
False (anchor is a bonus token, only mask tokens predict, yielding
|
||||
block_size - 1 speculative tokens).
|
||||
"""
|
||||
pre_trained_config["architectures"] = ["Qwen3DSparkModel"]
|
||||
# Speculators DSpark uses the 1+N fill-in block (anchor is a bonus token).
|
||||
pre_trained_config["dspark_bonus_anchor"] = True
|
||||
pre_trained_config["sample_from_anchor"] = config_dict.get(
|
||||
"sample_from_anchor", False
|
||||
)
|
||||
|
||||
aux_layer_ids = config_dict["aux_hidden_state_layer_ids"]
|
||||
pre_trained_config["eagle_aux_hidden_state_layer_ids"] = aux_layer_ids
|
||||
|
||||
Reference in New Issue
Block a user