[torch.compile] Don't do the fast moe cold start optimization if there is speculative decoding (#33624)
Signed-off-by: Richard Zou <zou3519@gmail.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
1b60b45d0d
commit
5eac9a1b34
+14
-1
@@ -286,9 +286,22 @@ def create_forward_context(
|
||||
additional_kwargs: dict[str, Any] | None = None,
|
||||
skip_compiled: bool = False,
|
||||
):
|
||||
if vllm_config.compilation_config.fast_moe_cold_start:
|
||||
if vllm_config.speculative_config is None:
|
||||
all_moe_layers = vllm_config.compilation_config.static_all_moe_layers
|
||||
else:
|
||||
logger.warning_once(
|
||||
"vllm_config.compilation_config.fast_moe_cold_start is not "
|
||||
"compatible with speculative decoding so we are ignoring "
|
||||
"fast_moe_cold_start."
|
||||
)
|
||||
all_moe_layers = None
|
||||
else:
|
||||
all_moe_layers = None
|
||||
|
||||
return ForwardContext(
|
||||
no_compile_layers=vllm_config.compilation_config.static_forward_context,
|
||||
all_moe_layers=vllm_config.compilation_config.static_all_moe_layers,
|
||||
all_moe_layers=all_moe_layers,
|
||||
virtual_engine=virtual_engine,
|
||||
attn_metadata=attn_metadata,
|
||||
slot_mapping=slot_mapping or {},
|
||||
|
||||
Reference in New Issue
Block a user