[Bug fix][Core] fixup ngram not setup correctly (#4551)

Co-authored-by: Lei Wen <wenlei03@qiyi.com>
Co-authored-by: Cade Daniel <edacih@gmail.com>
Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
This commit is contained in:
leiwen83
2024-05-07 11:40:18 -07:00
committed by GitHub
co-authored by Lei Wen Cade Daniel Cody Yu
parent 469f85c782
commit 8344f7742b
3 changed files with 29 additions and 13 deletions
+7 -7
View File
@@ -57,13 +57,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
draft_worker_kwargs,
) -> "SpecDecodeWorker":
if "ngram_prompt_lookup_max" in draft_worker_kwargs:
ngram_prompt_lookup_max = (
draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
ngram_prompt_lookup_min = (
draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
else:
ngram_prompt_lookup_max = 0
ngram_prompt_lookup_max = (
draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
ngram_prompt_lookup_min = (
draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
if ngram_prompt_lookup_max > 0:
proposer_worker = NGramWorker(**draft_worker_kwargs)
@@ -72,6 +69,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
else:
proposer_worker = MultiStepWorker(**draft_worker_kwargs)
logger.info("Configuring SpecDecodeWorker with proposer=%s",
type(proposer_worker))
return SpecDecodeWorker(
proposer_worker,
scorer_worker,