[torch.compile] Improve Cold Start for MoEs (#32805)

Signed-off-by: Richard Zou <zou3519@gmail.com>
This commit is contained in:
Richard Zou
2026-01-22 10:44:40 -05:00
committed by GitHub
parent 15e302dfce
commit 654a71fc3c
3 changed files with 71 additions and 9 deletions
+34 -1
View File
@@ -210,6 +210,30 @@ class ForwardContext:
# If True, bypass the compiled model call, e.g. by using .forward() directly
skip_compiled: bool = False
# For torch.compile cold start times, we need to avoid hard-coding
# any strings into the graph. Right now, the vllm.moe_forward
# and vllm.moe_forward_shared custom operators hard-code strings into
# the graph.
#
# The workaround is to store a list of the strings that each of those
# custom ops needs, in reverse order, in the ForwardContext.
# The ForwardContext object is alive for the duration of the forward pass.
# When the custom op needs the string, pop the string from this list.
#
# This assumes that the custom operators will always be executed in
# order and that torch.compile will not try to reorder these
# operations with respect to each other.
#
# TODO(https://github.com/vllm-project/vllm/issues/31985):
# There are longer-term solutions, like unwrapping the moe custom operator,
# that aren't ready yet.
# We could also treat the string as a "symbolic input" to the graph but
# the PyTorch-side bits for that aren't ready yet either.
#
# If this value is None (like in some tests), then we end up baking the string
# into the graph. Otherwise, the moe custom ops will pop a string from this list.
remaining_moe_layers: list[str] | None = None
additional_kwargs: dict[str, Any] = field(default_factory=dict)
def __post_init__(self):
@@ -245,8 +269,17 @@ def create_forward_context(
additional_kwargs: dict[str, Any] | None = None,
skip_compiled: bool = False,
):
no_compile_layers = vllm_config.compilation_config.static_forward_context
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
remaining_moe_layers = [
name for name, layer in no_compile_layers.items() if isinstance(layer, FusedMoE)
]
remaining_moe_layers.reverse()
return ForwardContext(
no_compile_layers=vllm_config.compilation_config.static_forward_context,
no_compile_layers=no_compile_layers,
remaining_moe_layers=remaining_moe_layers,
virtual_engine=virtual_engine,
attn_metadata=attn_metadata,
dp_metadata=dp_metadata,