Improve Transformers modelling backend fx tracer (#49957)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -13,8 +13,7 @@ from torch import fx, nn
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -36,16 +35,12 @@ class BaseFuser(ABC):
|
||||
"""Match the pattern in `graph`, returning a fuser if found."""
|
||||
|
||||
@abstractmethod
|
||||
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Whether this fuser can be applied to this `module` instance."""
|
||||
|
||||
@abstractmethod
|
||||
def fuse(
|
||||
self,
|
||||
module: nn.Module,
|
||||
prefix: str,
|
||||
model_config: "ModelConfig",
|
||||
quant_config: "QuantizationConfig",
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Apply the fusion to an already-validated `module`, returning the
|
||||
module to install in its place (mutated in place, or freshly built)."""
|
||||
@@ -123,24 +118,16 @@ class StackedFuser(BaseFuser):
|
||||
|
||||
@abstractmethod
|
||||
def update_attrs(
|
||||
self,
|
||||
module: nn.Module,
|
||||
prefix: str,
|
||||
model_config: "ModelConfig",
|
||||
quant_config: "QuantizationConfig",
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
"""Replace `module`'s submodules with the merged module."""
|
||||
|
||||
def fuse(
|
||||
self,
|
||||
module: nn.Module,
|
||||
prefix: str,
|
||||
model_config: "ModelConfig",
|
||||
quant_config: "QuantizationConfig",
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
|
||||
|
||||
Builds the merged submodule and binds the compiled forward."""
|
||||
self.update_attrs(module, prefix, model_config, quant_config)
|
||||
self.update_attrs(module, prefix, vllm_config)
|
||||
module.forward = types.MethodType(self.fused_forward, module)
|
||||
return module
|
||||
|
||||
@@ -33,8 +33,7 @@ from vllm.model_executor.models.transformers.utils import (
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -168,7 +167,7 @@ class GLUFuser(StackedFuser):
|
||||
replace_expr(funcdef, muls[0], act_call)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
act = module.get_submodule(self.act_name)
|
||||
if self._get_act_and_mul_name(act) is None:
|
||||
logger.debug("No AndMul equivalent for %s; skipping fusion", type(act))
|
||||
@@ -176,12 +175,9 @@ class GLUFuser(StackedFuser):
|
||||
return True
|
||||
|
||||
def update_attrs(
|
||||
self,
|
||||
module: nn.Module,
|
||||
prefix: str,
|
||||
model_config: "ModelConfig",
|
||||
quant_config: "QuantizationConfig",
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
act_fn = self._get_act_and_mul(module.get_submodule(self.act_name))
|
||||
gate = module.get_submodule(self.gate_name)
|
||||
up = module.get_submodule(self.up_name)
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
is_linear,
|
||||
recover_forward,
|
||||
replace_expr,
|
||||
returned_linear,
|
||||
single_self_call,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
@@ -26,8 +27,7 @@ from vllm.model_executor.models.transformers.utils import (
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -84,15 +84,16 @@ class QKVFuser(StackedFuser):
|
||||
return None
|
||||
q, k, v = qkv_nodes
|
||||
names = dict(q_name=q.target, k_name=k.target, v_name=v.target)
|
||||
attn_width = module.get_submodule(q.target).out_features
|
||||
candidates = [
|
||||
name
|
||||
for name, child in module.named_children()
|
||||
if isinstance(child, nn.Linear)
|
||||
and name not in names.values()
|
||||
and child.in_features == attn_width
|
||||
]
|
||||
names["o_name"] = candidates[0] if len(candidates) == 1 else None
|
||||
# o_proj produces the module's output.
|
||||
o_name = returned_linear(graph, module)
|
||||
# o_proj must be compatible with the q/k/v projections.
|
||||
if o_name in names.values() or (
|
||||
o_name is not None
|
||||
and module.get_submodule(o_name).in_features
|
||||
!= module.get_submodule(q.target).out_features
|
||||
):
|
||||
o_name = None
|
||||
names["o_name"] = o_name
|
||||
return cls(source_cls=type(module).__name__, **names)
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
@@ -149,12 +150,12 @@ class QKVFuser(StackedFuser):
|
||||
replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load()))
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Shapes must be compatible for a single merged, head-sharded GEMM."""
|
||||
q = module.get_submodule(self.q_name)
|
||||
k = module.get_submodule(self.k_name)
|
||||
v = module.get_submodule(self.v_name)
|
||||
head_size = model_config.get_head_size()
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
compatible = (
|
||||
q.in_features == k.in_features == v.in_features
|
||||
and len({proj.bias is None for proj in (q, k, v)}) == 1
|
||||
@@ -167,13 +168,10 @@ class QKVFuser(StackedFuser):
|
||||
return compatible
|
||||
|
||||
def update_attrs(
|
||||
self,
|
||||
module: nn.Module,
|
||||
prefix: str,
|
||||
model_config: "ModelConfig",
|
||||
quant_config: "QuantizationConfig",
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
head_size = model_config.get_head_size()
|
||||
quant_config = vllm_config.quant_config
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
q = module.get_submodule(self.q_name)
|
||||
k = module.get_submodule(self.k_name)
|
||||
merged = QKVParallelLinear(
|
||||
|
||||
@@ -21,13 +21,13 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
forward_input_count,
|
||||
is_op,
|
||||
output_value,
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
def _is_squared(node: object, x: fx.Node) -> bool:
|
||||
@@ -71,10 +71,8 @@ def _is_one_plus(node: object) -> bool:
|
||||
|
||||
def _has_trailing_compute(graph: fx.Graph, node: fx.Node) -> bool:
|
||||
"""Does the forward compute anything after `node` before returning?"""
|
||||
output = find_node(graph, lambda n: n.op == "output")
|
||||
if output is None or not output.args:
|
||||
return False
|
||||
return peel(output.args[0]) is not node
|
||||
value = output_value(graph)
|
||||
return value is not None and peel(value) is not node
|
||||
|
||||
|
||||
class TPAwareNormMixin(nn.Module):
|
||||
@@ -185,17 +183,14 @@ class RMSNormFuser(BaseFuser):
|
||||
return eps
|
||||
return None
|
||||
|
||||
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
return True
|
||||
|
||||
def fuse(
|
||||
self,
|
||||
module: nn.Module,
|
||||
prefix: str,
|
||||
model_config: "ModelConfig",
|
||||
quant_config: "QuantizationConfig",
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
|
||||
model_config = vllm_config.model_config
|
||||
weight = getattr(module, "weight", None)
|
||||
hidden_size = (
|
||||
weight.size(0) if weight is not None else model_config.get_hidden_size()
|
||||
|
||||
Reference in New Issue
Block a user