From b8baa31a28650b6f3189bc7fcb051e1360c9270c Mon Sep 17 00:00:00 2001 From: Thomas Ortner Date: Thu, 13 Aug 2026 12:21:50 +0200 Subject: [PATCH] Hardware-agnostic model definition via HF transformer backend (1/N) (#49458) Signed-off-by: Thomas Ortner --- .../transformers/test_layer_registry.py | 177 ++++++++++ vllm/envs.py | 5 + vllm/model_executor/hw_agnostic/__init__.py | 2 + vllm/model_executor/hw_agnostic/custom_op.py | 318 ++++++++++++++++++ .../hw_agnostic/layers/__init__.py | 2 + .../hw_agnostic/layers/activation.py | 27 ++ .../hw_agnostic/layers/layernorm.py | 90 +++++ .../models/transformers/fusers/glu.py | 6 +- .../models/transformers/fusers/rms_norm.py | 5 +- .../models/transformers/layers.py | 64 ++++ 10 files changed, 691 insertions(+), 5 deletions(-) create mode 100644 tests/models/transformers/test_layer_registry.py create mode 100644 vllm/model_executor/hw_agnostic/__init__.py create mode 100644 vllm/model_executor/hw_agnostic/custom_op.py create mode 100644 vllm/model_executor/hw_agnostic/layers/__init__.py create mode 100644 vllm/model_executor/hw_agnostic/layers/activation.py create mode 100644 vllm/model_executor/hw_agnostic/layers/layernorm.py create mode 100644 vllm/model_executor/models/transformers/layers.py diff --git a/tests/models/transformers/test_layer_registry.py b/tests/models/transformers/test_layer_registry.py new file mode 100644 index 0000000000..9e20e2ba54 --- /dev/null +++ b/tests/models/transformers/test_layer_registry.py @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the Transformers backend's hw-agnostic layer resolution. + +`layers._resolve` imports a layer symbol from +`vllm.model_executor.hw_agnostic.layers.` when `VLLM_USE_HW_AGNOSTIC` +is set and the symbol exists, and otherwise falls back to +`vllm.model_executor.layers.`. These tests pin that contract and the +logging that reports which source was used. +""" + +import logging +import sys +import types + +import pytest +import torch + +from vllm.model_executor.models.transformers import layers + +HW_MODULE = "vllm.model_executor.hw_agnostic.layers.layernorm" + + +@pytest.fixture +def fake_hw_layernorm(monkeypatch): + """Inject a hw-agnostic `layernorm` module exposing a sentinel `RMSNorm`. + + A `SimpleNamespace` stands in for the module: `importlib.import_module` + returns it from `sys.modules` and `getattr` resolves `RMSNorm`, while its + attributes are set at construction (no `ModuleType` attribute-set that mypy + rejects, no constant `setattr` that ruff rejects).""" + module = types.SimpleNamespace(RMSNorm=type("HwRMSNorm", (), {})) + monkeypatch.setitem(sys.modules, HW_MODULE, module) + return module + + +def test_falls_back_to_vllm_when_disabled(monkeypatch, fake_hw_layernorm): + """Disabled: the vLLM class is used even if a hw-agnostic one exists.""" + monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0") + from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm + + assert layers._resolve("layernorm", "RMSNorm") is VllmRMSNorm + + +def test_uses_hw_agnostic_when_enabled(monkeypatch, fake_hw_layernorm, caplog): + """Enabled and available: the hw-agnostic class is used and logged.""" + monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1") + with caplog.at_level(logging.INFO): + resolved = layers._resolve("layernorm", "RMSNorm") + assert resolved is fake_hw_layernorm.RMSNorm + assert "Using hw-agnostic layer: RMSNorm" in caplog.text + + +def test_falls_back_when_symbol_missing(monkeypatch, caplog): + """Enabled but the symbol is not ported: fall back to vLLM and warn.""" + monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1") + # A hw-agnostic module without the requested attribute triggers fallback. + empty = types.ModuleType(HW_MODULE) + monkeypatch.setitem(sys.modules, HW_MODULE, empty) + from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm + + with caplog.at_level(logging.WARNING): + resolved = layers._resolve("layernorm", "RMSNorm") + assert resolved is VllmRMSNorm + assert "falling back to default" in caplog.text + + +def test_act_and_mul_falls_back_for_unknown_activation( + monkeypatch, default_vllm_config +): + """An activation with no hw-agnostic equivalent falls back to vLLM's. + + `default_vllm_config` supplies the config context the CustomOp needs. + """ + monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1") + from vllm.model_executor.layers.activation import GeluAndMul + + assert isinstance(layers.get_act_and_mul_fn("gelu"), GeluAndMul) + + +@pytest.fixture(scope="module") +def tiny_llama_path(tmp_path_factory): + """A randomly-initialized microscopic Llama saved to disk (with an ungated + tokenizer) so vLLM can load it like any local checkpoint.""" + from transformers import AutoTokenizer, LlamaConfig, LlamaForCausalLM + + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") + config = LlamaConfig( + vocab_size=tokenizer.vocab_size, + hidden_size=128, + intermediate_size=256, + num_hidden_layers=2, + num_attention_heads=4, + rms_norm_eps=1e-6, + hidden_act="silu", + ) + torch.manual_seed(0) + model = LlamaForCausalLM(config) + + path = tmp_path_factory.mktemp("tiny_llama") + model.save_pretrained(path) + tokenizer.save_pretrained(path) + return str(path) + + +# Registered names of the layers the backend can +# currently route to hw-agnostic implementations. +_COVERED_LAYERS = ("rms_norm", "silu_and_mul") + + +def _layer_providers(model) -> dict[str, str]: + """Map each covered layer type present in the model to the provider its + implementation came from (``hw_agnostic`` or ``vllm``). + """ + + def provider_of(module) -> str | None: + for cls in type(module).__mro__: + if "hw_agnostic.layers" in cls.__module__: + return "hw_agnostic" + if ".model_executor.layers." in cls.__module__: + return "vllm" + return None + + providers: dict[str, str] = {} + for module in model.modules(): + name = getattr(module, "name", None) + if ( + name in _COVERED_LAYERS + and name not in providers + and (prov := provider_of(module)) is not None + ): + providers[name] = prov + return providers + + +def _serve(vllm_runner, model_path, prompts): + """Serve the model through the backend; return (layer_providers, logprobs).""" + with vllm_runner( + model_path, + model_impl="transformers", + max_model_len=64, + enforce_eager=True, + gpu_memory_utilization=0.3, + ) as runner: + assert runner.llm.llm_engine.model_config.using_transformers_backend() + providers = runner.apply_model(_layer_providers)[0] + outputs = runner.generate_greedy_logprobs( + prompts, max_tokens=32, num_logprobs=5 + ) + return providers, outputs + + +def test_hw_agnostic_matches_vllm_end_to_end(monkeypatch, vllm_runner, tiny_llama_path): + """Serving the tiny model with hw-agnostic layers matches the vLLM baseline.""" + # spawn: worker re-imports layers with the env set (see docstring). + monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + # apply_model pickles the introspection function. + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + from ..utils import check_logprobs_close + + prompts = ["The capital of France is", "vLLM is"] + + monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0") + vllm_providers, vllm_outputs = _serve(vllm_runner, tiny_llama_path, prompts) + # Both replaceable layers present in a Llama block must be vLLM's here. + assert vllm_providers == {"rms_norm": "vllm", "silu_and_mul": "vllm"} + + monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1") + hw_providers, hw_outputs = _serve(vllm_runner, tiny_llama_path, prompts) + assert hw_providers == {"rms_norm": "hw_agnostic", "silu_and_mul": "hw_agnostic"} + + check_logprobs_close( + outputs_0_lst=vllm_outputs, + outputs_1_lst=hw_outputs, + name_0="vllm", + name_1="hw_agnostic", + ) diff --git a/vllm/envs.py b/vllm/envs.py index f8401f53c3..3382fa1d67 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -125,6 +125,7 @@ if TYPE_CHECKING: VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False VLLM_SKIP_P2P_CHECK: bool = False VLLM_DISABLED_KERNELS: list[str] = [] + VLLM_USE_HW_AGNOSTIC: bool = False VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True VLLM_DISABLE_PYNCCL: bool = False VLLM_USE_OINK_OPS: bool = False @@ -1200,6 +1201,10 @@ environment_variables: dict[str, Callable[[], Any]] = { if "VLLM_DISABLED_KERNELS" not in os.environ else os.environ["VLLM_DISABLED_KERNELS"].split(",") ), + # Selects hw-agnostic layers for HF transformer backend + "VLLM_USE_HW_AGNOSTIC": lambda: ( + os.getenv("VLLM_USE_HW_AGNOSTIC", "False").lower() in ("true", "1") + ), "VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool( int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1")) ), diff --git a/vllm/model_executor/hw_agnostic/__init__.py b/vllm/model_executor/hw_agnostic/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/model_executor/hw_agnostic/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/model_executor/hw_agnostic/custom_op.py b/vllm/model_executor/hw_agnostic/custom_op.py new file mode 100644 index 0000000000..72d3f8522f --- /dev/null +++ b/vllm/model_executor/hw_agnostic/custom_op.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import functools +import inspect + +import torch +import torch.nn as nn + +from vllm.config import get_cached_compilation_config +from vllm.logger import init_logger +from vllm.model_executor.utils import maybe_disable_graph_partition +from vllm.platforms import current_platform + +logger = init_logger(__name__) + +# Dictionary of all custom ops (classes, indexed by registered name). +# To check if an op with a name is enabled, call .enabled() on the class. +# Examples: +# - MyOp.enabled() +# - op_registry["my_op"].enabled() +op_registry: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {} +op_registry_oot: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {} + + +def maybe_get_oot_by_class(class_type: type) -> type: + class_name = class_type.__name__ + if class_name in op_registry_oot: + return op_registry_oot[class_name] + return class_type + + +class PluggableLayer(nn.Module): + """ + Base class for pluggable layers. + + A PluggableLayer is a *module-composing* abstraction: it may instantiate other + ``torch.nn.Module`` objects as sub-layers, and its functionality depends on + these sub-layers following a generalized invocation sequence. Also, it is stateful + and may hold parameters or buffers. + + Unlike :class:`CustomOp`, PluggableLayer does NOT provide per-platform + ``forward_*`` dispatch. Instead, it supports out-of-tree (OOT) replacement + of the entire layer class at instantiation time, allowing customized + initialization and submodule composition. + """ + + def __new__(cls, *args, **kwargs): + try: + layer_class_name = cls.__name__ + except AttributeError: + raise TypeError( + f"Cannot instantiate '{cls.__name__}': its 'name' attribute " + f"was not set, possibly because it was not decorated with " + f"@PluggableLayer.register, or it's the PluggableLayer itself." + ) from None + + if layer_class_name not in op_registry_oot: + layer_cls_to_instantiate = cls + else: + layer_cls_to_instantiate = op_registry_oot[layer_class_name] + logger.debug( + "Instantiating pluggable layer: %s using %s", + layer_class_name, + str(layer_cls_to_instantiate), + ) + return super().__new__(layer_cls_to_instantiate) + + # Decorator to register pluggable layers. + @classmethod + def register(cls, name: str): + def decorator(op_cls): + assert name not in op_registry, f"Duplicate op name: {name}" + op_cls.name = name + op_registry[name] = op_cls + return op_cls + + return decorator + + # Decorator to register out-of-tree(oot) pluggable layers. + # For OOT pluggable layers: + # if in-tree layer class is registered with an oot_custom_layer, + # the oot_custom_layer will be used instead. + @classmethod + def register_oot(cls, _decorated_layer_cls=None, name: str | None = None): + def decorator(layer_cls): + reg_name = name if name is not None else cls.__name__ + assert reg_name not in op_registry_oot, f"Duplicate layer name: {reg_name}" + layer_cls.name = reg_name + op_registry_oot[reg_name] = layer_cls + return layer_cls + + if _decorated_layer_cls is None: + # Called with parentheses: @PluggableLayer.register_oot() + # or @PluggableLayer.register_oot(name="...") + return decorator + elif isinstance(_decorated_layer_cls, type): # Check if it's a class + # Called without parentheses: @PluggableLayer.register_oot + return decorator(_decorated_layer_cls) + else: + raise TypeError("Decorator can only be applied to classes.") + + +class CustomOp(nn.Module): + """ + Base class for custom ops. + Dispatches the forward method to the appropriate backend. + """ + + def __new__(cls, *args, **kwargs): + try: + op_name = cls.__name__ + except AttributeError: + raise TypeError( + f"Cannot instantiate '{cls.__name__}': its 'name' attribute " + f"was not set, possibly because it was not decorated with " + f"@CustomOp.register, or it's the CustomOp base class itself." + ) from None + + if op_name not in op_registry_oot: + op_cls_to_instantiate = cls + else: + op_cls_to_instantiate = op_registry_oot[op_name] + logger.debug( + "Instantiating custom op: %s using %s", + op_name, + str(op_cls_to_instantiate), + ) + return super().__new__(op_cls_to_instantiate) + + def __init__(self, *, enforce_enable: bool = False, compile_native: bool = False): + super().__init__() + self._enforce_enable = enforce_enable + self._forward_method = self.dispatch_forward(compile_native=compile_native) + + def forward(self, *args, **kwargs): + return self._forward_method(*args, **kwargs) + + def forward_native(self, *args, **kwargs): + """PyTorch-native implementation; OOT plugins override via + :meth:`forward_oot`.""" + raise NotImplementedError + + def forward_oot(self, *args, **kwargs): + # By default, OOT ops fall back to the PyTorch-native implementation. + return self.forward_native(*args, **kwargs) + + def dispatch_forward(self, compile_native: bool): + # The hw-agnostic CustomOp dispatches only between forward_native + # (in-tree, used on every platform) and forward_oot (overridden by + # an OOT plugin via register_oot). + compilation_config = get_cached_compilation_config() + + # NOTE(shen-shanshan): CustomOp object can be enforce enabled, e.g., + # enable device-specific kernels in ViT models when enabling graph + # mode. By default, it will follow the compilation_config to determine + # whether enable itself. + # This enforce_enable mechanism will be removed after we adding a + # separate compilation_config for multi-modal part. + enabled = self._enforce_enable or self.enabled() + if enabled: + compilation_config.enabled_custom_ops.update([self.__class__.name]) + else: + compilation_config.disabled_custom_ops.update([self.__class__.name]) + + if not enabled: + # Compile forward_native to avoid eager torch ops if inside + # opaque torch custom op (e.g. fused_moe, unified_attention, etc.) + return self.maybe_compile(self.forward_native, enable=compile_native) + + if current_platform.is_out_of_tree(): + return self.forward_oot + return self.forward_native + + def maybe_compile(self, fn, *, enable: bool = True): + """ + Compile fn if compilation enabled. + Useful for CustomOp instances called from within a torch custom op, + meaning the forward call is hidden from the model-level torch.compile. + + NOTE: this does not enable fusion across ops, so opaque custom ops + should still be unwrapped wherever possible. + """ + from vllm.config.compilation import CompilationMode + + # Do not compile if compilation disabled + if not enable: + return fn + + # Do not compile if global compilation disabled + compilation_config = get_cached_compilation_config() + if compilation_config.mode == CompilationMode.NONE: + return fn + + # If eager backend is used, do not compile either + if compilation_config.backend == "eager": + return fn + + compile_options = maybe_disable_graph_partition( + current_platform.simple_compile_backend + ) + backend = current_platform.simple_compile_backend + + dynamic_arg_dims = getattr(self.__class__, "_dynamic_arg_dims", None) + if dynamic_arg_dims is not None: + compiled_fn = torch.compile( + fn, + dynamic=False, + backend=backend, + options=compile_options, + ) + sig = inspect.signature(fn) + + @functools.wraps(fn) + def wrapper(*args, **kwargs): + bound = sig.bind(*args, **kwargs) + bound.apply_defaults() + for name, dims in dynamic_arg_dims.items(): + arg = bound.arguments.get(name) + if arg is not None and isinstance(arg, torch.Tensor): + dims_list = [dims] if isinstance(dims, int) else dims + for d in dims_list: + real_d = arg.ndim + d if d < 0 else d + torch._dynamo.mark_dynamic(arg, real_d) + return compiled_fn(*args, **kwargs) + + return wrapper + + # dynamic=True to avoid recompilations + return torch.compile( + fn, + dynamic=True, + backend=backend, + options=compile_options, + ) + + @classmethod + def enabled(cls) -> bool: + # if no name, then it was not registered + compilation_config = get_cached_compilation_config() + custom_ops = compilation_config.custom_ops + if not hasattr(cls, "name"): + logger.warning_once( + "Custom op %s was not registered, which means it won't appear " + "in the op registry. It will be enabled/disabled based on the " + "global settings.", + cls.__name__, + ) + return CustomOp.default_on() + + enabled = f"+{cls.name}" in custom_ops + disabled = f"-{cls.name}" in custom_ops + assert not (enabled and disabled), f"Cannot enable and disable {cls.name}" + + return (CustomOp.default_on() or enabled) and not disabled + + @staticmethod + def default_on() -> bool: + """ + Behavior controlled by `CompilationConfig.custom_ops`: On by default if + 'all', off by default if 'none'. + When PyTorch Inductor is used, 'none' is the default value, + otherwise 'all'. + """ + compilation_config = get_cached_compilation_config() + count_none = compilation_config.custom_ops.count("none") + count_all = compilation_config.custom_ops.count("all") + assert count_none + count_all == 1 + + return not count_none > 0 or count_all > 0 + + # Decorator to register custom ops. + @classmethod + def register( + cls, + name: str, + dynamic_arg_dims: dict[str, int | list[int]] | None = None, + ): + def decorator(op_cls): + assert name not in op_registry, f"Duplicate op name: {name}" + op_cls.name = name + op_cls._dynamic_arg_dims = dynamic_arg_dims + op_registry[name] = op_cls + return op_cls + + return decorator + + # Decorator to register out-of-tree(oot) custom ops. + # For OOT custom ops: + # if in-tree layer class is registered with an oot_custom_op layer, + # the oot_custom_op layer will be used instead. + # Example: + # - @UnquantizedFusedMoEMethod.register_oot + # class HPUUnquantizedFusedMoEMethod(UnquantizedFusedMoEMethod) + # or + # - @CustomOP.register_oot(name="UnquantizedFusedMoEMethod") + @classmethod + def register_oot(cls, _decorated_op_cls=None, name: str | None = None): + def decorator(op_cls): + reg_name = name if name is not None else cls.__name__ + assert reg_name not in op_registry_oot, f"Duplicate op name: {reg_name}" + op_cls.name = reg_name + op_registry_oot[reg_name] = op_cls + return op_cls + + if _decorated_op_cls is None: + # Called with parentheses: @CustomOP.register_oot() + # or @CustomOP.register_oot(name="...") + # So, _decorated_op_cls is None. + # We return the actual decorator function. + return decorator + elif isinstance(_decorated_op_cls, type): # Check if it's a class + # Called without parentheses: @CustomOP.register_oot + # The first argument is the class itself. + # We call the 'decorator' function immediately with the class. + return decorator(_decorated_op_cls) + else: + # Handle other unexpected cases if necessary + raise TypeError("Decorator can only be applied to classes.") diff --git a/vllm/model_executor/hw_agnostic/layers/__init__.py b/vllm/model_executor/hw_agnostic/layers/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/model_executor/hw_agnostic/layers/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/model_executor/hw_agnostic/layers/activation.py b/vllm/model_executor/hw_agnostic/layers/activation.py new file mode 100644 index 0000000000..c9282b4027 --- /dev/null +++ b/vllm/model_executor/hw_agnostic/layers/activation.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import torch +import torch.nn.functional as F + +from vllm.model_executor.hw_agnostic.custom_op import CustomOp + + +@CustomOp.register("silu_and_mul") +class SiluAndMul(CustomOp): + """SwiGLU: ``x -> silu(x[:d]) * x[d:]`` where ``d = x.shape[-1] // 2``.""" + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + return F.silu(x[..., :d]) * x[..., d:] + + +# Activation-and-mul ops keyed by HF activation name. +_ACTIVATION_AND_MUL_REGISTRY = { + "silu": SiluAndMul, + "swish": SiluAndMul, +} + + +def get_act_and_mul_fn(act_fn_name: str) -> CustomOp: + """Build the hw-agnostic activation-and-mul op named `act_fn_name`.""" + return _ACTIVATION_AND_MUL_REGISTRY[act_fn_name.lower()]() diff --git a/vllm/model_executor/hw_agnostic/layers/layernorm.py b/vllm/model_executor/hw_agnostic/layers/layernorm.py new file mode 100644 index 0000000000..d2c854afd3 --- /dev/null +++ b/vllm/model_executor/hw_agnostic/layers/layernorm.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import torch +import torch.nn as nn +from torch import Tensor + +from vllm.model_executor.hw_agnostic.custom_op import CustomOp + + +@CustomOp.register("rms_norm") +class RMSNorm(CustomOp): + """``x -> w * x / sqrt(E[x^2] + eps)``. With ``residual``, fuses + ``residual += x`` then RMSNorm and returns ``(normalized, residual)``.""" + + def __init__( + self, + hidden_size: int, + eps: float = 1e-6, + var_hidden_size: int | None = None, + has_weight: bool = True, + dtype: torch.dtype | None = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.variance_epsilon = eps + self.variance_size_override = ( + None if var_hidden_size == hidden_size else var_hidden_size + ) + weight_dtype = dtype or torch.get_default_dtype() + self.has_weight = has_weight + weight = torch.ones(hidden_size, dtype=weight_dtype) + if has_weight: + self.weight = nn.Parameter(weight) + else: + self.register_buffer("weight", weight, persistent=False) + + def _rms_norm( + self, + x: Tensor, + weight: Tensor | None, + epsilon: float, + variance_size: int | None = None, + ) -> Tensor: + """Weighted root-mean-square layer normalization""" + orig_dtype = x.dtype + x = x.to(torch.float32) + x_var = x if variance_size is None else x[..., :variance_size] + variance = x_var.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + epsilon) + if weight is not None: + x = x.to(weight.dtype) * weight + return x.to(orig_dtype) + + def _fused_add_rms_norm( + self, + x: Tensor, + x_residual: Tensor, + weight: Tensor | None, + epsilon: float, + variance_size: int | None = None, + ) -> tuple[Tensor, Tensor]: + """Fused add and weighted root-mean-square layer normalization""" + orig_dtype = x.dtype + x = x.to(torch.float32) + x = x + x_residual.to(torch.float32) + x_residual = x.to(orig_dtype) + + x_var = x if variance_size is None else x[..., :variance_size] + variance = x_var.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + epsilon) + if weight is not None: + x = x.to(weight.dtype) * weight + return x.to(orig_dtype), x_residual + + def forward_native( + self, + x: torch.Tensor, + residual: torch.Tensor | None = None, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + weight = self.weight if self.has_weight else None + epsilon = self.variance_epsilon + variance_size = self.variance_size_override + + if residual is None: + return self._rms_norm(x, weight, epsilon, variance_size) + else: + return self._fused_add_rms_norm(x, residual, weight, epsilon, variance_size) + + def extra_repr(self) -> str: + return f"hidden_size={self.hidden_size}, eps={self.variance_epsilon}" diff --git a/vllm/model_executor/models/transformers/fusers/glu.py b/vllm/model_executor/models/transformers/fusers/glu.py index 2da84addea..e19f5b71df 100644 --- a/vllm/model_executor/models/transformers/fusers/glu.py +++ b/vllm/model_executor/models/transformers/fusers/glu.py @@ -11,10 +11,7 @@ from torch import fx, nn from transformers.activations import ACT2CLS from vllm.logger import init_logger -from vllm.model_executor.layers.activation import ( - _ACTIVATION_AND_MUL_REGISTRY, - get_act_and_mul_fn, -) +from vllm.model_executor.layers.activation import _ACTIVATION_AND_MUL_REGISTRY from vllm.model_executor.layers.linear import MergedColumnParallelLinear from vllm.model_executor.models.transformers.fusers.base import StackedFuser from vllm.model_executor.models.transformers.fx_utils import ( @@ -26,6 +23,7 @@ from vllm.model_executor.models.transformers.fx_utils import ( replace_expr, single_self_call, ) +from vllm.model_executor.models.transformers.layers import get_act_and_mul_fn from vllm.model_executor.models.transformers.utils import ( log_replacement, replace_linear_class, diff --git a/vllm/model_executor/models/transformers/fusers/rms_norm.py b/vllm/model_executor/models/transformers/fusers/rms_norm.py index 396ae4c5b2..7241f8539c 100644 --- a/vllm/model_executor/models/transformers/fusers/rms_norm.py +++ b/vllm/model_executor/models/transformers/fusers/rms_norm.py @@ -15,7 +15,6 @@ from vllm.distributed import ( ) from vllm.distributed.parallel_state import model_parallel_is_initialized from vllm.distributed.utils import split_tensor_along_last_dim -from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm from vllm.model_executor.models.transformers.fusers.base import BaseFuser from vllm.model_executor.models.transformers.fx_utils import ( find_node, @@ -25,6 +24,10 @@ from vllm.model_executor.models.transformers.fx_utils import ( peel, trace, ) +from vllm.model_executor.models.transformers.layers import ( + GemmaRMSNorm, + RMSNorm, +) if TYPE_CHECKING: from vllm.config import VllmConfig diff --git a/vllm/model_executor/models/transformers/layers.py b/vllm/model_executor/models/transformers/layers.py new file mode 100644 index 0000000000..4ff3651dfc --- /dev/null +++ b/vllm/model_executor/models/transformers/layers.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Layer provider resolution for the Transformers modeling backend. + +When ``VLLM_USE_HW_AGNOSTIC`` is set, layer symbols are imported from +``vllm.model_executor.hw_agnostic.layers.``, falling back to +``vllm.model_executor.layers.`` for anything not yet ported. The +resolved source of every symbol is logged so it is clear which layers run +hw-agnostic and which fell back to vLLM. +""" + +import importlib + +import vllm.envs as envs +from vllm.logger import init_logger + +logger = init_logger(__name__) + +_HW_PKG = "vllm.model_executor.hw_agnostic.layers" +_VLLM_PKG = "vllm.model_executor.layers" + + +def _resolve(module: str, name: str): + """Return `name` from the hw-agnostic `module` when enabled and available, + else from vLLM. Logs which source was used.""" + if envs.VLLM_USE_HW_AGNOSTIC: + try: + obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name) + logger.info("Using hw-agnostic layer: %s", name) + return obj + except (ImportError, AttributeError): + logger.warning( + "hw-agnostic layer %s is not available; falling back to default", name + ) + return getattr(importlib.import_module(f"{_VLLM_PKG}.{module}"), name) + + +RMSNorm = _resolve("layernorm", "RMSNorm") +GemmaRMSNorm = _resolve("layernorm", "GemmaRMSNorm") + + +def get_act_and_mul_fn(act_fn_name: str): + """Fused activation-and-mul op for `act_fn_name`, preferring hw-agnostic. + + Resolved per call because the op is name-parameterized: an activation with + no hw-agnostic equivalent falls back to vLLM individually. + """ + if envs.VLLM_USE_HW_AGNOSTIC: + try: + from vllm.model_executor.hw_agnostic.layers.activation import ( + get_act_and_mul_fn as hw_fn, + ) + + fn = hw_fn(act_fn_name) + logger.info_once("Using hw-agnostic activation: %s", act_fn_name) + return fn + except (ImportError, KeyError): + logger.warning_once( + "hw-agnostic activation %s is not available; falling back to vLLM", + act_fn_name, + ) + from vllm.model_executor.layers.activation import get_act_and_mul_fn as vllm_fn + + return vllm_fn(act_fn_name)