[Bugfix][Spec Decode] Honour the draft's moe_backend on Model Runner V2 (#54788)
Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Nick Hill
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c0d8d5413e
commit
9c2d21046b
@@ -75,6 +75,16 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
if speculative_config.moe_backend is not None:
|
||||
# Otherwise the draft inherits the target's --moe-backend, which
|
||||
# fails when the draft is unquantized and that backend is not.
|
||||
vllm_config = replace(
|
||||
vllm_config,
|
||||
kernel_config=replace(
|
||||
vllm_config.kernel_config,
|
||||
moe_backend=speculative_config.moe_backend,
|
||||
),
|
||||
)
|
||||
if speculative_config.kv_cache_dtype is not None:
|
||||
vllm_config = replace(
|
||||
vllm_config,
|
||||
|
||||
Reference in New Issue
Block a user