[torch.compile] Speed up MOE handling in forward_context (#33184)
Signed-off-by: Richard Zou <zou3519@gmail.com>
This commit is contained in:
+8
-13
@@ -217,9 +217,11 @@ class ForwardContext:
|
||||
# the graph.
|
||||
#
|
||||
# The workaround is to store a list of the strings that each of those
|
||||
# custom ops needs, in reverse order, in the ForwardContext.
|
||||
# custom ops needs in the ForwardContext (all_moe_layers)
|
||||
# as well as a counter (moe_layer_index).
|
||||
# The ForwardContext object is alive for the duration of the forward pass.
|
||||
# When the custom op needs the string, pop the string from this list.
|
||||
# When the custom op needs a layer string, get the next string
|
||||
# from all_moe_layers and increment the counter.
|
||||
#
|
||||
# This assumes that the custom operators will always be executed in
|
||||
# order and that torch.compile will not try to reorder these
|
||||
@@ -233,7 +235,8 @@ class ForwardContext:
|
||||
#
|
||||
# If this value is None (like in some tests), then we end up baking the string
|
||||
# into the graph. Otherwise, the moe custom ops will pop a string from this list.
|
||||
remaining_moe_layers: list[str] | None = None
|
||||
all_moe_layers: list[str] | None = None
|
||||
moe_layer_index: int = 0
|
||||
|
||||
additional_kwargs: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@@ -271,17 +274,9 @@ def create_forward_context(
|
||||
additional_kwargs: dict[str, Any] | None = None,
|
||||
skip_compiled: bool = False,
|
||||
):
|
||||
no_compile_layers = vllm_config.compilation_config.static_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
|
||||
remaining_moe_layers = [
|
||||
name for name, layer in no_compile_layers.items() if isinstance(layer, FusedMoE)
|
||||
]
|
||||
remaining_moe_layers.reverse()
|
||||
|
||||
return ForwardContext(
|
||||
no_compile_layers=no_compile_layers,
|
||||
remaining_moe_layers=remaining_moe_layers,
|
||||
no_compile_layers=vllm_config.compilation_config.static_forward_context,
|
||||
all_moe_layers=vllm_config.compilation_config.static_all_moe_layers,
|
||||
virtual_engine=virtual_engine,
|
||||
attn_metadata=attn_metadata,
|
||||
slot_mapping=slot_mapping or {},
|
||||
|
||||
Reference in New Issue
Block a user