[Bugfix][LoRA][Spec Decode] Support LoRA with speculative decoding (#21068)
Signed-off-by: Sean Chen <xiaohong_chen1991@hotmail.com> Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Danielle Robinson <dcmaddix@gmail.com> Co-authored-by: Haipeng Li <li2haipeng@gmail.com> Co-authored-by: li2haipeng <44383182+li2haipeng@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
Danielle Robinson
Haipeng Li
li2haipeng
parent
b158df2813
commit
d0c7792004
@@ -1574,6 +1574,20 @@ class EngineArgs:
|
||||
else None
|
||||
)
|
||||
|
||||
if (
|
||||
lora_config is not None
|
||||
and speculative_config is not None
|
||||
and scheduler_config.max_num_batched_tokens
|
||||
< (
|
||||
scheduler_config.max_num_seqs
|
||||
* (speculative_config.num_speculative_tokens + 1)
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"Consider increasing max_num_batched_tokens or "
|
||||
"decreasing num_speculative_tokens"
|
||||
)
|
||||
|
||||
# bitsandbytes pre-quantized model need a specific model loader
|
||||
if model_config.quantization == "bitsandbytes":
|
||||
self.quantization = self.load_format = "bitsandbytes"
|
||||
|
||||
Reference in New Issue
Block a user