[SpecDec] Remove Batch Expansion (2/3) (#9298)

This commit is contained in:
Lily Liu
2024-10-12 05:13:37 +00:00
committed by GitHub
parent ec10cb8511
commit 89feb4c84d
8 changed files with 122 additions and 70 deletions
-6
View File
@@ -190,12 +190,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
"[Speculative Decoding] Disabling MQA scorer as the "
"MQA is only available with flash attn backend.")
if ngram_prompt_lookup_max > 0:
disable_mqa_scorer = True
logger.info(
"[Speculative Decoding] Disabling MQA scorer as the "
"NGramWorker does not support MQA scorer.")
if "model_config" in draft_worker_kwargs and \
draft_worker_kwargs["model_config"].max_model_len < \
scorer_worker.model_config.max_model_len: