Enable causal masking for SWA in vllm-project/speculators models (#47745)
Signed-off-by: Eldar Kurtic <8884008+eldarkurtic@users.noreply.github.com>
This commit is contained in:
@@ -120,6 +120,10 @@ def update_dflash(config_dict: dict, pre_trained_config: dict) -> None:
|
|||||||
"mask_token_id": config_dict["mask_token_id"],
|
"mask_token_id": config_dict["mask_token_id"],
|
||||||
"target_layer_ids": [i - 1 for i in aux_layer_ids],
|
"target_layer_ids": [i - 1 for i in aux_layer_ids],
|
||||||
}
|
}
|
||||||
|
# Enable causal masking in SWA for vllm-project/speculators models
|
||||||
|
pre_trained_config["dflash_config"]["causal"] = not config_dict.get(
|
||||||
|
"sliding_window_non_causal", True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@register_speculator("dspark")
|
@register_speculator("dspark")
|
||||||
|
|||||||
Reference in New Issue
Block a user