[Bug fix][Core] fixup ngram not setup correctly (#4551)
Co-authored-by: Lei Wen <wenlei03@qiyi.com> Co-authored-by: Cade Daniel <edacih@gmail.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
This commit is contained in:
co-authored by
Lei Wen
Cade Daniel
Cody Yu
parent
469f85c782
commit
8344f7742b
@@ -82,6 +82,10 @@ class GPUExecutor(ExecutorBase):
|
||||
draft_worker_kwargs.update(
|
||||
model_config=self.speculative_config.draft_model_config,
|
||||
parallel_config=self.speculative_config.draft_parallel_config,
|
||||
ngram_prompt_lookup_max=self.speculative_config.
|
||||
ngram_prompt_lookup_max,
|
||||
ngram_prompt_lookup_min=self.speculative_config.
|
||||
ngram_prompt_lookup_min,
|
||||
# TODO allow draft-model specific load config.
|
||||
#load_config=self.load_config,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user