Hardware-agnostic model definition via HF transformer backend (1/N) (#49458)

Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
This commit is contained in:
Thomas Ortner
2026-08-13 11:21:50 +01:00
committed by GitHub
parent 903da602f3
commit b8baa31a28
10 changed files with 691 additions and 5 deletions
@@ -0,0 +1,177 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Transformers backend's hw-agnostic layer resolution.
`layers._resolve` imports a layer symbol from
`vllm.model_executor.hw_agnostic.layers.<module>` when `VLLM_USE_HW_AGNOSTIC`
is set and the symbol exists, and otherwise falls back to
`vllm.model_executor.layers.<module>`. These tests pin that contract and the
logging that reports which source was used.
"""
import logging
import sys
import types
import pytest
import torch
from vllm.model_executor.models.transformers import layers
HW_MODULE = "vllm.model_executor.hw_agnostic.layers.layernorm"
@pytest.fixture
def fake_hw_layernorm(monkeypatch):
"""Inject a hw-agnostic `layernorm` module exposing a sentinel `RMSNorm`.
A `SimpleNamespace` stands in for the module: `importlib.import_module`
returns it from `sys.modules` and `getattr` resolves `RMSNorm`, while its
attributes are set at construction (no `ModuleType` attribute-set that mypy
rejects, no constant `setattr` that ruff rejects)."""
module = types.SimpleNamespace(RMSNorm=type("HwRMSNorm", (), {}))
monkeypatch.setitem(sys.modules, HW_MODULE, module)
return module
def test_falls_back_to_vllm_when_disabled(monkeypatch, fake_hw_layernorm):
"""Disabled: the vLLM class is used even if a hw-agnostic one exists."""
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0")
from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm
assert layers._resolve("layernorm", "RMSNorm") is VllmRMSNorm
def test_uses_hw_agnostic_when_enabled(monkeypatch, fake_hw_layernorm, caplog):
"""Enabled and available: the hw-agnostic class is used and logged."""
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
with caplog.at_level(logging.INFO):
resolved = layers._resolve("layernorm", "RMSNorm")
assert resolved is fake_hw_layernorm.RMSNorm
assert "Using hw-agnostic layer: RMSNorm" in caplog.text
def test_falls_back_when_symbol_missing(monkeypatch, caplog):
"""Enabled but the symbol is not ported: fall back to vLLM and warn."""
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
# A hw-agnostic module without the requested attribute triggers fallback.
empty = types.ModuleType(HW_MODULE)
monkeypatch.setitem(sys.modules, HW_MODULE, empty)
from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm
with caplog.at_level(logging.WARNING):
resolved = layers._resolve("layernorm", "RMSNorm")
assert resolved is VllmRMSNorm
assert "falling back to default" in caplog.text
def test_act_and_mul_falls_back_for_unknown_activation(
monkeypatch, default_vllm_config
):
"""An activation with no hw-agnostic equivalent falls back to vLLM's.
`default_vllm_config` supplies the config context the CustomOp needs.
"""
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
from vllm.model_executor.layers.activation import GeluAndMul
assert isinstance(layers.get_act_and_mul_fn("gelu"), GeluAndMul)
@pytest.fixture(scope="module")
def tiny_llama_path(tmp_path_factory):
"""A randomly-initialized microscopic Llama saved to disk (with an ungated
tokenizer) so vLLM can load it like any local checkpoint."""
from transformers import AutoTokenizer, LlamaConfig, LlamaForCausalLM
tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
config = LlamaConfig(
vocab_size=tokenizer.vocab_size,
hidden_size=128,
intermediate_size=256,
num_hidden_layers=2,
num_attention_heads=4,
rms_norm_eps=1e-6,
hidden_act="silu",
)
torch.manual_seed(0)
model = LlamaForCausalLM(config)
path = tmp_path_factory.mktemp("tiny_llama")
model.save_pretrained(path)
tokenizer.save_pretrained(path)
return str(path)
# Registered names of the layers the backend can
# currently route to hw-agnostic implementations.
_COVERED_LAYERS = ("rms_norm", "silu_and_mul")
def _layer_providers(model) -> dict[str, str]:
"""Map each covered layer type present in the model to the provider its
implementation came from (``hw_agnostic`` or ``vllm``).
"""
def provider_of(module) -> str | None:
for cls in type(module).__mro__:
if "hw_agnostic.layers" in cls.__module__:
return "hw_agnostic"
if ".model_executor.layers." in cls.__module__:
return "vllm"
return None
providers: dict[str, str] = {}
for module in model.modules():
name = getattr(module, "name", None)
if (
name in _COVERED_LAYERS
and name not in providers
and (prov := provider_of(module)) is not None
):
providers[name] = prov
return providers
def _serve(vllm_runner, model_path, prompts):
"""Serve the model through the backend; return (layer_providers, logprobs)."""
with vllm_runner(
model_path,
model_impl="transformers",
max_model_len=64,
enforce_eager=True,
gpu_memory_utilization=0.3,
) as runner:
assert runner.llm.llm_engine.model_config.using_transformers_backend()
providers = runner.apply_model(_layer_providers)[0]
outputs = runner.generate_greedy_logprobs(
prompts, max_tokens=32, num_logprobs=5
)
return providers, outputs
def test_hw_agnostic_matches_vllm_end_to_end(monkeypatch, vllm_runner, tiny_llama_path):
"""Serving the tiny model with hw-agnostic layers matches the vLLM baseline."""
# spawn: worker re-imports layers with the env set (see docstring).
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
# apply_model pickles the introspection function.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
from ..utils import check_logprobs_close
prompts = ["The capital of France is", "vLLM is"]
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0")
vllm_providers, vllm_outputs = _serve(vllm_runner, tiny_llama_path, prompts)
# Both replaceable layers present in a Llama block must be vLLM's here.
assert vllm_providers == {"rms_norm": "vllm", "silu_and_mul": "vllm"}
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
hw_providers, hw_outputs = _serve(vllm_runner, tiny_llama_path, prompts)
assert hw_providers == {"rms_norm": "hw_agnostic", "silu_and_mul": "hw_agnostic"}
check_logprobs_close(
outputs_0_lst=vllm_outputs,
outputs_1_lst=hw_outputs,
name_0="vllm",
name_1="hw_agnostic",
)
+5
View File
@@ -125,6 +125,7 @@ if TYPE_CHECKING:
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
VLLM_SKIP_P2P_CHECK: bool = False
VLLM_DISABLED_KERNELS: list[str] = []
VLLM_USE_HW_AGNOSTIC: bool = False
VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True
VLLM_DISABLE_PYNCCL: bool = False
VLLM_USE_OINK_OPS: bool = False
@@ -1200,6 +1201,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
if "VLLM_DISABLED_KERNELS" not in os.environ
else os.environ["VLLM_DISABLED_KERNELS"].split(",")
),
# Selects hw-agnostic layers for HF transformer backend
"VLLM_USE_HW_AGNOSTIC": lambda: (
os.getenv("VLLM_USE_HW_AGNOSTIC", "False").lower() in ("true", "1")
),
"VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool(
int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1"))
),
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,318 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import inspect
import torch
import torch.nn as nn
from vllm.config import get_cached_compilation_config
from vllm.logger import init_logger
from vllm.model_executor.utils import maybe_disable_graph_partition
from vllm.platforms import current_platform
logger = init_logger(__name__)
# Dictionary of all custom ops (classes, indexed by registered name).
# To check if an op with a name is enabled, call .enabled() on the class.
# Examples:
# - MyOp.enabled()
# - op_registry["my_op"].enabled()
op_registry: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {}
op_registry_oot: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {}
def maybe_get_oot_by_class(class_type: type) -> type:
class_name = class_type.__name__
if class_name in op_registry_oot:
return op_registry_oot[class_name]
return class_type
class PluggableLayer(nn.Module):
"""
Base class for pluggable layers.
A PluggableLayer is a *module-composing* abstraction: it may instantiate other
``torch.nn.Module`` objects as sub-layers, and its functionality depends on
these sub-layers following a generalized invocation sequence. Also, it is stateful
and may hold parameters or buffers.
Unlike :class:`CustomOp`, PluggableLayer does NOT provide per-platform
``forward_*`` dispatch. Instead, it supports out-of-tree (OOT) replacement
of the entire layer class at instantiation time, allowing customized
initialization and submodule composition.
"""
def __new__(cls, *args, **kwargs):
try:
layer_class_name = cls.__name__
except AttributeError:
raise TypeError(
f"Cannot instantiate '{cls.__name__}': its 'name' attribute "
f"was not set, possibly because it was not decorated with "
f"@PluggableLayer.register, or it's the PluggableLayer itself."
) from None
if layer_class_name not in op_registry_oot:
layer_cls_to_instantiate = cls
else:
layer_cls_to_instantiate = op_registry_oot[layer_class_name]
logger.debug(
"Instantiating pluggable layer: %s using %s",
layer_class_name,
str(layer_cls_to_instantiate),
)
return super().__new__(layer_cls_to_instantiate)
# Decorator to register pluggable layers.
@classmethod
def register(cls, name: str):
def decorator(op_cls):
assert name not in op_registry, f"Duplicate op name: {name}"
op_cls.name = name
op_registry[name] = op_cls
return op_cls
return decorator
# Decorator to register out-of-tree(oot) pluggable layers.
# For OOT pluggable layers:
# if in-tree layer class is registered with an oot_custom_layer,
# the oot_custom_layer will be used instead.
@classmethod
def register_oot(cls, _decorated_layer_cls=None, name: str | None = None):
def decorator(layer_cls):
reg_name = name if name is not None else cls.__name__
assert reg_name not in op_registry_oot, f"Duplicate layer name: {reg_name}"
layer_cls.name = reg_name
op_registry_oot[reg_name] = layer_cls
return layer_cls
if _decorated_layer_cls is None:
# Called with parentheses: @PluggableLayer.register_oot()
# or @PluggableLayer.register_oot(name="...")
return decorator
elif isinstance(_decorated_layer_cls, type): # Check if it's a class
# Called without parentheses: @PluggableLayer.register_oot
return decorator(_decorated_layer_cls)
else:
raise TypeError("Decorator can only be applied to classes.")
class CustomOp(nn.Module):
"""
Base class for custom ops.
Dispatches the forward method to the appropriate backend.
"""
def __new__(cls, *args, **kwargs):
try:
op_name = cls.__name__
except AttributeError:
raise TypeError(
f"Cannot instantiate '{cls.__name__}': its 'name' attribute "
f"was not set, possibly because it was not decorated with "
f"@CustomOp.register, or it's the CustomOp base class itself."
) from None
if op_name not in op_registry_oot:
op_cls_to_instantiate = cls
else:
op_cls_to_instantiate = op_registry_oot[op_name]
logger.debug(
"Instantiating custom op: %s using %s",
op_name,
str(op_cls_to_instantiate),
)
return super().__new__(op_cls_to_instantiate)
def __init__(self, *, enforce_enable: bool = False, compile_native: bool = False):
super().__init__()
self._enforce_enable = enforce_enable
self._forward_method = self.dispatch_forward(compile_native=compile_native)
def forward(self, *args, **kwargs):
return self._forward_method(*args, **kwargs)
def forward_native(self, *args, **kwargs):
"""PyTorch-native implementation; OOT plugins override via
:meth:`forward_oot`."""
raise NotImplementedError
def forward_oot(self, *args, **kwargs):
# By default, OOT ops fall back to the PyTorch-native implementation.
return self.forward_native(*args, **kwargs)
def dispatch_forward(self, compile_native: bool):
# The hw-agnostic CustomOp dispatches only between forward_native
# (in-tree, used on every platform) and forward_oot (overridden by
# an OOT plugin via register_oot).
compilation_config = get_cached_compilation_config()
# NOTE(shen-shanshan): CustomOp object can be enforce enabled, e.g.,
# enable device-specific kernels in ViT models when enabling graph
# mode. By default, it will follow the compilation_config to determine
# whether enable itself.
# This enforce_enable mechanism will be removed after we adding a
# separate compilation_config for multi-modal part.
enabled = self._enforce_enable or self.enabled()
if enabled:
compilation_config.enabled_custom_ops.update([self.__class__.name])
else:
compilation_config.disabled_custom_ops.update([self.__class__.name])
if not enabled:
# Compile forward_native to avoid eager torch ops if inside
# opaque torch custom op (e.g. fused_moe, unified_attention, etc.)
return self.maybe_compile(self.forward_native, enable=compile_native)
if current_platform.is_out_of_tree():
return self.forward_oot
return self.forward_native
def maybe_compile(self, fn, *, enable: bool = True):
"""
Compile fn if compilation enabled.
Useful for CustomOp instances called from within a torch custom op,
meaning the forward call is hidden from the model-level torch.compile.
NOTE: this does not enable fusion across ops, so opaque custom ops
should still be unwrapped wherever possible.
"""
from vllm.config.compilation import CompilationMode
# Do not compile if compilation disabled
if not enable:
return fn
# Do not compile if global compilation disabled
compilation_config = get_cached_compilation_config()
if compilation_config.mode == CompilationMode.NONE:
return fn
# If eager backend is used, do not compile either
if compilation_config.backend == "eager":
return fn
compile_options = maybe_disable_graph_partition(
current_platform.simple_compile_backend
)
backend = current_platform.simple_compile_backend
dynamic_arg_dims = getattr(self.__class__, "_dynamic_arg_dims", None)
if dynamic_arg_dims is not None:
compiled_fn = torch.compile(
fn,
dynamic=False,
backend=backend,
options=compile_options,
)
sig = inspect.signature(fn)
@functools.wraps(fn)
def wrapper(*args, **kwargs):
bound = sig.bind(*args, **kwargs)
bound.apply_defaults()
for name, dims in dynamic_arg_dims.items():
arg = bound.arguments.get(name)
if arg is not None and isinstance(arg, torch.Tensor):
dims_list = [dims] if isinstance(dims, int) else dims
for d in dims_list:
real_d = arg.ndim + d if d < 0 else d
torch._dynamo.mark_dynamic(arg, real_d)
return compiled_fn(*args, **kwargs)
return wrapper
# dynamic=True to avoid recompilations
return torch.compile(
fn,
dynamic=True,
backend=backend,
options=compile_options,
)
@classmethod
def enabled(cls) -> bool:
# if no name, then it was not registered
compilation_config = get_cached_compilation_config()
custom_ops = compilation_config.custom_ops
if not hasattr(cls, "name"):
logger.warning_once(
"Custom op %s was not registered, which means it won't appear "
"in the op registry. It will be enabled/disabled based on the "
"global settings.",
cls.__name__,
)
return CustomOp.default_on()
enabled = f"+{cls.name}" in custom_ops
disabled = f"-{cls.name}" in custom_ops
assert not (enabled and disabled), f"Cannot enable and disable {cls.name}"
return (CustomOp.default_on() or enabled) and not disabled
@staticmethod
def default_on() -> bool:
"""
Behavior controlled by `CompilationConfig.custom_ops`: On by default if
'all', off by default if 'none'.
When PyTorch Inductor is used, 'none' is the default value,
otherwise 'all'.
"""
compilation_config = get_cached_compilation_config()
count_none = compilation_config.custom_ops.count("none")
count_all = compilation_config.custom_ops.count("all")
assert count_none + count_all == 1
return not count_none > 0 or count_all > 0
# Decorator to register custom ops.
@classmethod
def register(
cls,
name: str,
dynamic_arg_dims: dict[str, int | list[int]] | None = None,
):
def decorator(op_cls):
assert name not in op_registry, f"Duplicate op name: {name}"
op_cls.name = name
op_cls._dynamic_arg_dims = dynamic_arg_dims
op_registry[name] = op_cls
return op_cls
return decorator
# Decorator to register out-of-tree(oot) custom ops.
# For OOT custom ops:
# if in-tree layer class is registered with an oot_custom_op layer,
# the oot_custom_op layer will be used instead.
# Example:
# - @UnquantizedFusedMoEMethod.register_oot
# class HPUUnquantizedFusedMoEMethod(UnquantizedFusedMoEMethod)
# or
# - @CustomOP.register_oot(name="UnquantizedFusedMoEMethod")
@classmethod
def register_oot(cls, _decorated_op_cls=None, name: str | None = None):
def decorator(op_cls):
reg_name = name if name is not None else cls.__name__
assert reg_name not in op_registry_oot, f"Duplicate op name: {reg_name}"
op_cls.name = reg_name
op_registry_oot[reg_name] = op_cls
return op_cls
if _decorated_op_cls is None:
# Called with parentheses: @CustomOP.register_oot()
# or @CustomOP.register_oot(name="...")
# So, _decorated_op_cls is None.
# We return the actual decorator function.
return decorator
elif isinstance(_decorated_op_cls, type): # Check if it's a class
# Called without parentheses: @CustomOP.register_oot
# The first argument is the class itself.
# We call the 'decorator' function immediately with the class.
return decorator(_decorated_op_cls)
else:
# Handle other unexpected cases if necessary
raise TypeError("Decorator can only be applied to classes.")
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn.functional as F
from vllm.model_executor.hw_agnostic.custom_op import CustomOp
@CustomOp.register("silu_and_mul")
class SiluAndMul(CustomOp):
"""SwiGLU: ``x -> silu(x[:d]) * x[d:]`` where ``d = x.shape[-1] // 2``."""
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
d = x.shape[-1] // 2
return F.silu(x[..., :d]) * x[..., d:]
# Activation-and-mul ops keyed by HF activation name.
_ACTIVATION_AND_MUL_REGISTRY = {
"silu": SiluAndMul,
"swish": SiluAndMul,
}
def get_act_and_mul_fn(act_fn_name: str) -> CustomOp:
"""Build the hw-agnostic activation-and-mul op named `act_fn_name`."""
return _ACTIVATION_AND_MUL_REGISTRY[act_fn_name.lower()]()
@@ -0,0 +1,90 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn as nn
from torch import Tensor
from vllm.model_executor.hw_agnostic.custom_op import CustomOp
@CustomOp.register("rms_norm")
class RMSNorm(CustomOp):
"""``x -> w * x / sqrt(E[x^2] + eps)``. With ``residual``, fuses
``residual += x`` then RMSNorm and returns ``(normalized, residual)``."""
def __init__(
self,
hidden_size: int,
eps: float = 1e-6,
var_hidden_size: int | None = None,
has_weight: bool = True,
dtype: torch.dtype | None = None,
) -> None:
super().__init__()
self.hidden_size = hidden_size
self.variance_epsilon = eps
self.variance_size_override = (
None if var_hidden_size == hidden_size else var_hidden_size
)
weight_dtype = dtype or torch.get_default_dtype()
self.has_weight = has_weight
weight = torch.ones(hidden_size, dtype=weight_dtype)
if has_weight:
self.weight = nn.Parameter(weight)
else:
self.register_buffer("weight", weight, persistent=False)
def _rms_norm(
self,
x: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> Tensor:
"""Weighted root-mean-square layer normalization"""
orig_dtype = x.dtype
x = x.to(torch.float32)
x_var = x if variance_size is None else x[..., :variance_size]
variance = x_var.pow(2).mean(dim=-1, keepdim=True)
x = x * torch.rsqrt(variance + epsilon)
if weight is not None:
x = x.to(weight.dtype) * weight
return x.to(orig_dtype)
def _fused_add_rms_norm(
self,
x: Tensor,
x_residual: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> tuple[Tensor, Tensor]:
"""Fused add and weighted root-mean-square layer normalization"""
orig_dtype = x.dtype
x = x.to(torch.float32)
x = x + x_residual.to(torch.float32)
x_residual = x.to(orig_dtype)
x_var = x if variance_size is None else x[..., :variance_size]
variance = x_var.pow(2).mean(dim=-1, keepdim=True)
x = x * torch.rsqrt(variance + epsilon)
if weight is not None:
x = x.to(weight.dtype) * weight
return x.to(orig_dtype), x_residual
def forward_native(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
weight = self.weight if self.has_weight else None
epsilon = self.variance_epsilon
variance_size = self.variance_size_override
if residual is None:
return self._rms_norm(x, weight, epsilon, variance_size)
else:
return self._fused_add_rms_norm(x, residual, weight, epsilon, variance_size)
def extra_repr(self) -> str:
return f"hidden_size={self.hidden_size}, eps={self.variance_epsilon}"
@@ -11,10 +11,7 @@ from torch import fx, nn
from transformers.activations import ACT2CLS
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import (
_ACTIVATION_AND_MUL_REGISTRY,
get_act_and_mul_fn,
)
from vllm.model_executor.layers.activation import _ACTIVATION_AND_MUL_REGISTRY
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
from vllm.model_executor.models.transformers.fx_utils import (
@@ -26,6 +23,7 @@ from vllm.model_executor.models.transformers.fx_utils import (
replace_expr,
single_self_call,
)
from vllm.model_executor.models.transformers.layers import get_act_and_mul_fn
from vllm.model_executor.models.transformers.utils import (
log_replacement,
replace_linear_class,
@@ -15,7 +15,6 @@ from vllm.distributed import (
)
from vllm.distributed.parallel_state import model_parallel_is_initialized
from vllm.distributed.utils import split_tensor_along_last_dim
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
@@ -25,6 +24,10 @@ from vllm.model_executor.models.transformers.fx_utils import (
peel,
trace,
)
from vllm.model_executor.models.transformers.layers import (
GemmaRMSNorm,
RMSNorm,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -0,0 +1,64 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Layer provider resolution for the Transformers modeling backend.
When ``VLLM_USE_HW_AGNOSTIC`` is set, layer symbols are imported from
``vllm.model_executor.hw_agnostic.layers.<module>``, falling back to
``vllm.model_executor.layers.<module>`` for anything not yet ported. The
resolved source of every symbol is logged so it is clear which layers run
hw-agnostic and which fell back to vLLM.
"""
import importlib
import vllm.envs as envs
from vllm.logger import init_logger
logger = init_logger(__name__)
_HW_PKG = "vllm.model_executor.hw_agnostic.layers"
_VLLM_PKG = "vllm.model_executor.layers"
def _resolve(module: str, name: str):
"""Return `name` from the hw-agnostic `module` when enabled and available,
else from vLLM. Logs which source was used."""
if envs.VLLM_USE_HW_AGNOSTIC:
try:
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
logger.info("Using hw-agnostic layer: %s", name)
return obj
except (ImportError, AttributeError):
logger.warning(
"hw-agnostic layer %s is not available; falling back to default", name
)
return getattr(importlib.import_module(f"{_VLLM_PKG}.{module}"), name)
RMSNorm = _resolve("layernorm", "RMSNorm")
GemmaRMSNorm = _resolve("layernorm", "GemmaRMSNorm")
def get_act_and_mul_fn(act_fn_name: str):
"""Fused activation-and-mul op for `act_fn_name`, preferring hw-agnostic.
Resolved per call because the op is name-parameterized: an activation with
no hw-agnostic equivalent falls back to vLLM individually.
"""
if envs.VLLM_USE_HW_AGNOSTIC:
try:
from vllm.model_executor.hw_agnostic.layers.activation import (
get_act_and_mul_fn as hw_fn,
)
fn = hw_fn(act_fn_name)
logger.info_once("Using hw-agnostic activation: %s", act_fn_name)
return fn
except (ImportError, KeyError):
logger.warning_once(
"hw-agnostic activation %s is not available; falling back to vLLM",
act_fn_name,
)
from vllm.model_executor.layers.activation import get_act_and_mul_fn as vllm_fn
return vllm_fn(act_fn_name)