[Transformers] Generalize merged-column linear fusion (#55301)
Signed-off-by: BANANASJIM <bananasjim1@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Harry Mellor
parent
60ad959b6f
commit
c0d8d5413e
@@ -9,6 +9,9 @@ from vllm.model_executor.models.transformers.fusers.base import (
|
||||
StackedFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
|
||||
from vllm.model_executor.models.transformers.fusers.merged_column import (
|
||||
MergedColumnParallelFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fusers.mla import MLAFuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.transformers.fusers.packed_qkv import PackedQKVFuser
|
||||
@@ -21,6 +24,7 @@ __all__ = [
|
||||
"RewriteFuser",
|
||||
"StackedFuser",
|
||||
"GLUFuser",
|
||||
"MergedColumnParallelFuser",
|
||||
"MLAFuser",
|
||||
"MoEBlockFuser",
|
||||
"PackedQKVFuser",
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Base classes for the Transformers backend fusers."""
|
||||
|
||||
import ast
|
||||
import types
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable
|
||||
@@ -10,6 +11,7 @@ from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.model_executor.models.transformers.fx_utils import replace_expr
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -124,14 +126,14 @@ class StackedFuser(RewriteFuser):
|
||||
|
||||
merged_name: ClassVar[str]
|
||||
"""Attribute name of the merged module created by `update_attrs`."""
|
||||
merged_cls: ClassVar[str]
|
||||
merged_cls_name: ClassVar[str]
|
||||
"""Name of the vLLM class the merged projection becomes (for logging)."""
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
sources = " + ".join(shard for shard, _ in self.shards)
|
||||
return (
|
||||
f"Fused: {sources} ({name}: {self.source_cls}) -> "
|
||||
f"{self.merged_name} ({self.merged_cls})"
|
||||
f"{self.merged_name} ({self.merged_cls_name})"
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -157,3 +159,36 @@ class StackedFuser(RewriteFuser):
|
||||
"""`{merged_name: [projection names]}` so quantization can unpack the
|
||||
fused layer into its per-shard configs."""
|
||||
return {self.merged_name: [name for name, _ in self.shards]}
|
||||
|
||||
def _splice_merged_split(
|
||||
self,
|
||||
funcdef: ast.FunctionDef,
|
||||
calls: list[ast.Call],
|
||||
block: list[ast.stmt],
|
||||
index: int,
|
||||
) -> None:
|
||||
"""Insert `temps = self.<merged_name>(arg).split(sizes, -1)` at
|
||||
`block[index]` and replace each of `calls` with its temp name.
|
||||
|
||||
`calls` must share one input argument; `block[index]` must be where
|
||||
they are (or would be) evaluated. Raises if a generated temporary
|
||||
would shadow an existing name in `funcdef`.
|
||||
"""
|
||||
temps = [f"_vllm_merged_{i}" for i in range(len(calls))]
|
||||
names = {node.id for node in ast.walk(funcdef) if isinstance(node, ast.Name)}
|
||||
if names & set(temps):
|
||||
raise ValueError("fused temporaries would shadow existing names")
|
||||
targets = ", ".join(temps)
|
||||
sections = local_output_sizes(self.merged_name)
|
||||
source = f"{targets} = self.{self.merged_name}(__arg__).split({sections}, -1)"
|
||||
assign = ast.parse(source).body[0]
|
||||
arg = next(
|
||||
node
|
||||
for node in ast.walk(assign)
|
||||
if isinstance(node, ast.Name) and node.id == "__arg__"
|
||||
)
|
||||
replace_expr(assign, arg, calls[0].args[0])
|
||||
ast.copy_location(assign, block[index])
|
||||
block.insert(index, assign)
|
||||
for call, temp in zip(calls, temps):
|
||||
replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load()))
|
||||
|
||||
@@ -12,8 +12,9 @@ from transformers.activations import ACT2CLS
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import _ACTIVATION_AND_MUL_REGISTRY
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
|
||||
from vllm.model_executor.models.transformers.fusers.merged_column import (
|
||||
MergedColumnParallelFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
compile_forward,
|
||||
find_node,
|
||||
@@ -28,7 +29,7 @@ from vllm.model_executor.models.transformers.utils import (
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -46,19 +47,20 @@ ACT_AND_MUL_NAMES = frozenset(_ACTIVATION_AND_MUL_REGISTRY.keys())
|
||||
|
||||
|
||||
@dataclass
|
||||
class GLUFuser(StackedFuser):
|
||||
class GLUFuser(MergedColumnParallelFuser):
|
||||
"""Fuser for the GLU pattern `act(gate(x)) * up(x)`."""
|
||||
|
||||
act_name: str
|
||||
gate_name: str
|
||||
up_name: str
|
||||
down_name: str | None
|
||||
merged_name: ClassVar[str] = "gate_up_proj"
|
||||
merged_cls: ClassVar[str] = "MergedColumnParallelLinear"
|
||||
|
||||
@property
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
return [(self.gate_name, 0), (self.up_name, 1)]
|
||||
def gate_name(self) -> str:
|
||||
return self.linear_names[0]
|
||||
|
||||
@property
|
||||
def up_name(self) -> str:
|
||||
return self.linear_names[1]
|
||||
|
||||
@classmethod
|
||||
def _is_act_of_gate(cls, node: fx.Node, module: nn.Module) -> bool:
|
||||
@@ -122,22 +124,14 @@ class GLUFuser(StackedFuser):
|
||||
return None
|
||||
act_node, gate_node, up_node, mul_node = glu_nodes
|
||||
|
||||
gate = module.get_submodule(gate_node.target)
|
||||
up = module.get_submodule(up_node.target)
|
||||
# Shapes must be compatible for a single merged GEMM.
|
||||
if gate.in_features == up.in_features and (gate.bias is None) == (
|
||||
up.bias is None
|
||||
):
|
||||
predicate = lambda n: is_linear(n, module) and peel(n.args[0]) is mul_node
|
||||
down_node = find_node(graph, predicate)
|
||||
return cls(
|
||||
source_cls=type(module).__name__,
|
||||
act_name=act_node.target,
|
||||
gate_name=gate_node.target,
|
||||
up_name=up_node.target,
|
||||
down_name=down_node.target if down_node is not None else None,
|
||||
)
|
||||
return None
|
||||
predicate = lambda n: is_linear(n, module) and peel(n.args[0]) is mul_node
|
||||
down_node = find_node(graph, predicate)
|
||||
return cls(
|
||||
source_cls=type(module).__name__,
|
||||
act_name=act_node.target,
|
||||
linear_names=(gate_node.target, up_node.target),
|
||||
down_name=down_node.target if down_node is not None else None,
|
||||
)
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Replace `act(gate(x)) * up(x)` with `act(gate_up(x))` in source."""
|
||||
@@ -166,47 +160,32 @@ class GLUFuser(StackedFuser):
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
if not super().validate(module, vllm_config):
|
||||
return False
|
||||
# An AndMul kernel splits its input in half, so gate and up must match.
|
||||
gate = module.get_submodule(self.gate_name)
|
||||
up = module.get_submodule(self.up_name)
|
||||
if gate.out_features != up.out_features:
|
||||
logger.debug("%s and %s differ in size; skipping fusion", gate, up)
|
||||
return False
|
||||
act = module.get_submodule(self.act_name)
|
||||
if self._get_act_and_mul_name(act) is None:
|
||||
logger.debug("No AndMul equivalent for %s; skipping fusion", type(act))
|
||||
logger.debug("%s has no AndMul equivalent; skipping fusion", act)
|
||||
return False
|
||||
return True
|
||||
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
super().update_attrs(module, prefix, vllm_config)
|
||||
act_fn = self._get_act_and_mul(module.get_submodule(self.act_name))
|
||||
gate = module.get_submodule(self.gate_name)
|
||||
up = module.get_submodule(self.up_name)
|
||||
merged = MergedColumnParallelLinear(
|
||||
input_size=gate.in_features,
|
||||
output_sizes=[gate.out_features, up.out_features],
|
||||
bias=gate.bias is not None,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, self.merged_name),
|
||||
return_bias=False,
|
||||
)
|
||||
logger.debug(
|
||||
"%s: %s, %s: %s -> %s: %s",
|
||||
self.gate_name,
|
||||
gate,
|
||||
self.up_name,
|
||||
up,
|
||||
self.merged_name,
|
||||
merged,
|
||||
)
|
||||
setattr(module, self.merged_name, merged)
|
||||
setattr(module, self.act_name, act_fn)
|
||||
# Drop the consumed submodules so their (meta) params are not expected.
|
||||
delattr(module, self.gate_name)
|
||||
delattr(module, self.up_name)
|
||||
# If there is a down projection, we know it must be rowwise.
|
||||
if self.down_name is not None:
|
||||
down_prefix = maybe_prefix(prefix, self.down_name)
|
||||
down = module.get_submodule(self.down_name)
|
||||
new_down = replace_linear_class(
|
||||
down, "rowwise", quant_config, prefix=down_prefix
|
||||
down, "rowwise", vllm_config.quant_config, prefix=down_prefix
|
||||
)
|
||||
setattr(module, self.down_name, new_down)
|
||||
log_replacement(down_prefix, down, new_down)
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fuser for parallel linear projections."""
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
compile_forward,
|
||||
innermost_block,
|
||||
is_linear,
|
||||
recover_forward,
|
||||
single_self_call,
|
||||
)
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MergedColumnParallelFuser(StackedFuser):
|
||||
"""Fuser for merging column-parallel linear projections."""
|
||||
|
||||
linear_names: tuple[str, ...]
|
||||
merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear"
|
||||
|
||||
@property
|
||||
def merged_name(self) -> str:
|
||||
"""Programmatic name for the merged projection, based on the original names."""
|
||||
# len is used to disambiguate names like `a_b` + `c` vs `a` + `b_c`
|
||||
parts = "_".join(f"{len(name)}_{name}" for name in self.linear_names)
|
||||
return f"merged_proj_{parts}"
|
||||
|
||||
@property
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
return [(name, index) for index, name in enumerate(self.linear_names)]
|
||||
|
||||
@staticmethod
|
||||
def _names(group: list[fx.Node]) -> tuple[str, ...] | None:
|
||||
names = tuple(str(node.target) for node in group)
|
||||
if len(set(names)) != len(names) or any("." in name for name in names):
|
||||
return None
|
||||
return names
|
||||
|
||||
@classmethod
|
||||
def match(
|
||||
cls, graph: fx.Graph, module: nn.Module
|
||||
) -> "MergedColumnParallelFuser | None":
|
||||
"""Fuse the module's sibling linears when there is only one such group."""
|
||||
by_input: dict[fx.Node, list[fx.Node]] = {}
|
||||
for node in graph.nodes:
|
||||
if (
|
||||
is_linear(node, module)
|
||||
and len(node.args) == 1
|
||||
and not node.kwargs
|
||||
and isinstance(node.args[0], fx.Node)
|
||||
# Like QKVFuser/PackedQKVFuser: this fuser has no head or
|
||||
# TP-replication awareness, so it must not absorb a QKV-shaped
|
||||
# pattern (e.g. behind a norm) a more specific fuser declined.
|
||||
and node.args[0].op == "placeholder"
|
||||
):
|
||||
by_input.setdefault(node.args[0], []).append(node)
|
||||
# A group whose members are not distinct direct children is dropped:
|
||||
# the source rewrite addresses each projection as `self.<name>` once.
|
||||
groups = [
|
||||
(group, names)
|
||||
for group in by_input.values()
|
||||
if len(group) >= 2 and (names := cls._names(group)) is not None
|
||||
]
|
||||
if len(groups) > 1:
|
||||
logger.debug(
|
||||
"%s has %d fusable sibling-linear groups; skipping fusion "
|
||||
"since which one to merge is ambiguous",
|
||||
type(module).__name__,
|
||||
len(groups),
|
||||
)
|
||||
if len(groups) != 1:
|
||||
return None
|
||||
_, names = groups[0]
|
||||
fuser = cls(source_cls=type(module).__name__, linear_names=names)
|
||||
return None if hasattr(module, fuser.merged_name) else fuser
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Replace the parallel calls with one merged call and split."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
calls = [single_self_call(funcdef, name) for name in self.linear_names]
|
||||
if len(set(ast.dump(call.args[0]) for call in calls)) != 1:
|
||||
raise ValueError("parallel linears read different inputs")
|
||||
blocks = [innermost_block(funcdef.body, call) for call in calls]
|
||||
if any(found is None for found in blocks):
|
||||
raise ValueError("parallel linear calls not found in the function body")
|
||||
if len(set(id(block) for block, _ in blocks)) != 1:
|
||||
raise ValueError("parallel linear calls are in different blocks")
|
||||
|
||||
block = blocks[0][0]
|
||||
index = min(index for _, index in blocks)
|
||||
calls_have_name_arg = all(isinstance(call.args[0], ast.Name) for call in calls)
|
||||
# Moving projections must not cross operations that can change their input.
|
||||
for statement in block[index : max(index for _, index in blocks) + 1]:
|
||||
if not isinstance(statement, (ast.Assign, ast.Return)):
|
||||
raise ValueError("parallel projections cross other operations")
|
||||
if isinstance(statement, ast.Assign) and any(
|
||||
not isinstance(target, ast.Name) for target in statement.targets
|
||||
):
|
||||
raise ValueError("parallel projection assignment has side effects")
|
||||
value = statement.value
|
||||
values = value.elts if isinstance(value, ast.Tuple) else [value]
|
||||
if any(value not in calls for value in values) or not calls_have_name_arg:
|
||||
raise ValueError("parallel projections cross other operations")
|
||||
|
||||
# l1(x), l2(x), ... -> merged(x).split(merged.output_sizes / merged.tp_size, -1)
|
||||
self._splice_merged_split(funcdef, calls, block, index)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Check that the parallel linears are compatible for merging."""
|
||||
linear_layers = [module.get_submodule(name) for name in self.linear_names]
|
||||
tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
return (
|
||||
len(linear_layers) >= 2
|
||||
and len(set(linear.in_features for linear in linear_layers)) == 1
|
||||
and len(set(linear.bias is None for linear in linear_layers)) == 1
|
||||
and all(linear.out_features % tp_size == 0 for linear in linear_layers)
|
||||
)
|
||||
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
"""Replace the module's parallel linears with one merged projection."""
|
||||
linear_modules = [module.get_submodule(name) for name in self.linear_names]
|
||||
merged = MergedColumnParallelLinear(
|
||||
input_size=linear_modules[0].in_features,
|
||||
output_sizes=[linear.out_features for linear in linear_modules],
|
||||
bias=linear_modules[0].bias is not None,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, self.merged_name),
|
||||
return_bias=False,
|
||||
)
|
||||
setattr(module, self.merged_name, merged)
|
||||
for name in self.linear_names:
|
||||
delattr(module, name)
|
||||
logger.debug(
|
||||
"%s -> %s: %s",
|
||||
", ".join(f"{n}: {m}" for n, m in zip(self.linear_names, linear_modules)),
|
||||
self.merged_name,
|
||||
merged,
|
||||
)
|
||||
@@ -35,10 +35,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Temporaries the fused down-projection binds in the rewritten forward.
|
||||
_Q_A_TEMP = "__q_a_fused"
|
||||
_KV_A_TEMP = "__kv_a_fused"
|
||||
|
||||
|
||||
def _consumes_placeholder(node: fx.Node) -> bool:
|
||||
"""Whether `node` is a linear applied directly to a `forward` input."""
|
||||
@@ -131,7 +127,7 @@ class MLAFuser(StackedFuser):
|
||||
kv_b_proj_name: str
|
||||
o_proj_name: str | None
|
||||
merged_name: ClassVar[str] = "fused_qkv_a_proj"
|
||||
merged_cls: ClassVar[str] = "MergedColumnParallelLinear"
|
||||
merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear"
|
||||
|
||||
@property
|
||||
def has_q_lora(self) -> bool:
|
||||
@@ -242,31 +238,12 @@ class MLAFuser(StackedFuser):
|
||||
if self.has_q_lora:
|
||||
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
|
||||
# The fused call is inserted at the top-level statement preceding both.
|
||||
q_call = single_self_call(funcdef, self.q_a_proj_name)
|
||||
kv_call = single_self_call(funcdef, self.kv_a_proj_name)
|
||||
if ast.dump(q_call.args[0]) != ast.dump(kv_call.args[0]):
|
||||
names = [self.q_a_proj_name, self.kv_a_proj_name]
|
||||
calls = [single_self_call(funcdef, name) for name in names]
|
||||
if ast.dump(calls[0].args[0]) != ast.dump(calls[1].args[0]):
|
||||
raise ValueError("down-projections read different inputs")
|
||||
names = {n.id for n in ast.walk(funcdef) if isinstance(n, ast.Name)}
|
||||
if names & {_Q_A_TEMP, _KV_A_TEMP}:
|
||||
raise ValueError("fused temporaries would shadow existing names")
|
||||
|
||||
merged = f"self.{self.merged_name}"
|
||||
targets = f"{_Q_A_TEMP}, {_KV_A_TEMP}"
|
||||
sections = f"[s // {merged}.tp_size for s in {merged}.output_sizes]"
|
||||
source = f"{targets} = {merged}(__arg__).split({sections}, -1)"
|
||||
assign = ast.parse(source=source).body[0]
|
||||
placeholder = next(
|
||||
node
|
||||
for node in ast.walk(assign)
|
||||
if isinstance(node, ast.Name) and node.id == "__arg__"
|
||||
)
|
||||
replace_expr(assign, placeholder, q_call.args[0])
|
||||
|
||||
index = min(_top_level_index(funcdef, call) for call in (q_call, kv_call))
|
||||
ast.copy_location(assign, funcdef.body[index])
|
||||
funcdef.body.insert(index, assign)
|
||||
replace_expr(funcdef, q_call, ast.Name(id=_Q_A_TEMP, ctx=ast.Load()))
|
||||
replace_expr(funcdef, kv_call, ast.Name(id=_KV_A_TEMP, ctx=ast.Load()))
|
||||
index = min(_top_level_index(funcdef, call) for call in calls)
|
||||
self._splice_merged_split(funcdef, calls, funcdef.body, index)
|
||||
|
||||
# Transformers expands the latent into full key/value in a dedicated method.
|
||||
# `MLAAttention` consumes the latent directly (absorbing `kv_b_proj`),
|
||||
|
||||
@@ -13,14 +13,12 @@ from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
StackedFuser,
|
||||
fused_head_size,
|
||||
local_output_sizes,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
compile_forward,
|
||||
innermost_block,
|
||||
is_linear,
|
||||
recover_forward,
|
||||
replace_expr,
|
||||
returned_linear,
|
||||
single_self_call,
|
||||
)
|
||||
@@ -45,7 +43,7 @@ class QKVFuser(StackedFuser):
|
||||
v_name: str
|
||||
o_name: str | None
|
||||
merged_name: ClassVar[str] = "qkv_proj"
|
||||
merged_cls: ClassVar[str] = "QKVParallelLinear"
|
||||
merged_cls_name: ClassVar[str] = "QKVParallelLinear"
|
||||
|
||||
@property
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
@@ -133,25 +131,9 @@ class QKVFuser(StackedFuser):
|
||||
raise ValueError("projection calls are in different blocks")
|
||||
|
||||
# q(x), k(x), v(x) -> q, k, v = qkv(x).split(qkv.output_sizes / qkv.tp_size, -1)
|
||||
names = {node.id for node in ast.walk(funcdef) if isinstance(node, ast.Name)}
|
||||
temps = [f"{name}_fused" for name in (self.q_name, self.k_name, self.v_name)]
|
||||
if names & set(temps):
|
||||
raise ValueError("fused temporaries would shadow existing names")
|
||||
merged = f"self.{self.merged_name}"
|
||||
sections = local_output_sizes(self.merged_name)
|
||||
template = f"{', '.join(temps)} = {merged}(__arg__).split({sections}, -1)"
|
||||
assign = ast.parse(template).body[0]
|
||||
arg = next(
|
||||
node
|
||||
for node in ast.walk(assign)
|
||||
if isinstance(node, ast.Name) and node.id == "__arg__"
|
||||
)
|
||||
replace_expr(assign, arg, calls[0].args[0])
|
||||
block, index = blocks[0]
|
||||
ast.copy_location(assign, block[index])
|
||||
block.insert(min(index for _, index in blocks), assign)
|
||||
for call, temp in zip(calls, temps):
|
||||
replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load()))
|
||||
block = blocks[0][0]
|
||||
index = min(index for _, index in blocks)
|
||||
self._splice_merged_split(funcdef, calls, block, index)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
|
||||
Reference in New Issue
Block a user