[torch.compile] Speed up MOE handling in forward_context (#33184)

Signed-off-by: Richard Zou <zou3519@gmail.com>
This commit is contained in:
Richard Zou
2026-01-27 15:17:54 -08:00
committed by GitHub
parent 3a6d5cbefd
commit d9aa39a3bb
4 changed files with 22 additions and 18 deletions
+8 -13
View File
@@ -217,9 +217,11 @@ class ForwardContext:
# the graph.
#
# The workaround is to store a list of the strings that each of those
# custom ops needs, in reverse order, in the ForwardContext.
# custom ops needs in the ForwardContext (all_moe_layers)
# as well as a counter (moe_layer_index).
# The ForwardContext object is alive for the duration of the forward pass.
# When the custom op needs the string, pop the string from this list.
# When the custom op needs a layer string, get the next string
# from all_moe_layers and increment the counter.
#
# This assumes that the custom operators will always be executed in
# order and that torch.compile will not try to reorder these
@@ -233,7 +235,8 @@ class ForwardContext:
#
# If this value is None (like in some tests), then we end up baking the string
# into the graph. Otherwise, the moe custom ops will pop a string from this list.
remaining_moe_layers: list[str] | None = None
all_moe_layers: list[str] | None = None
moe_layer_index: int = 0
additional_kwargs: dict[str, Any] = field(default_factory=dict)
@@ -271,17 +274,9 @@ def create_forward_context(
additional_kwargs: dict[str, Any] | None = None,
skip_compiled: bool = False,
):
no_compile_layers = vllm_config.compilation_config.static_forward_context
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
remaining_moe_layers = [
name for name, layer in no_compile_layers.items() if isinstance(layer, FusedMoE)
]
remaining_moe_layers.reverse()
return ForwardContext(
no_compile_layers=no_compile_layers,
remaining_moe_layers=remaining_moe_layers,
no_compile_layers=vllm_config.compilation_config.static_forward_context,
all_moe_layers=vllm_config.compilation_config.static_all_moe_layers,
virtual_engine=virtual_engine,
attn_metadata=attn_metadata,
slot_mapping=slot_mapping or {},