Hardware-agnostic model definition via HF transformer backend (1/N) (#49458)
Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
This commit is contained in:
@@ -0,0 +1,177 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the Transformers backend's hw-agnostic layer resolution.
|
||||
|
||||
`layers._resolve` imports a layer symbol from
|
||||
`vllm.model_executor.hw_agnostic.layers.<module>` when `VLLM_USE_HW_AGNOSTIC`
|
||||
is set and the symbol exists, and otherwise falls back to
|
||||
`vllm.model_executor.layers.<module>`. These tests pin that contract and the
|
||||
logging that reports which source was used.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.transformers import layers
|
||||
|
||||
HW_MODULE = "vllm.model_executor.hw_agnostic.layers.layernorm"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_hw_layernorm(monkeypatch):
|
||||
"""Inject a hw-agnostic `layernorm` module exposing a sentinel `RMSNorm`.
|
||||
|
||||
A `SimpleNamespace` stands in for the module: `importlib.import_module`
|
||||
returns it from `sys.modules` and `getattr` resolves `RMSNorm`, while its
|
||||
attributes are set at construction (no `ModuleType` attribute-set that mypy
|
||||
rejects, no constant `setattr` that ruff rejects)."""
|
||||
module = types.SimpleNamespace(RMSNorm=type("HwRMSNorm", (), {}))
|
||||
monkeypatch.setitem(sys.modules, HW_MODULE, module)
|
||||
return module
|
||||
|
||||
|
||||
def test_falls_back_to_vllm_when_disabled(monkeypatch, fake_hw_layernorm):
|
||||
"""Disabled: the vLLM class is used even if a hw-agnostic one exists."""
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0")
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm
|
||||
|
||||
assert layers._resolve("layernorm", "RMSNorm") is VllmRMSNorm
|
||||
|
||||
|
||||
def test_uses_hw_agnostic_when_enabled(monkeypatch, fake_hw_layernorm, caplog):
|
||||
"""Enabled and available: the hw-agnostic class is used and logged."""
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
|
||||
with caplog.at_level(logging.INFO):
|
||||
resolved = layers._resolve("layernorm", "RMSNorm")
|
||||
assert resolved is fake_hw_layernorm.RMSNorm
|
||||
assert "Using hw-agnostic layer: RMSNorm" in caplog.text
|
||||
|
||||
|
||||
def test_falls_back_when_symbol_missing(monkeypatch, caplog):
|
||||
"""Enabled but the symbol is not ported: fall back to vLLM and warn."""
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
|
||||
# A hw-agnostic module without the requested attribute triggers fallback.
|
||||
empty = types.ModuleType(HW_MODULE)
|
||||
monkeypatch.setitem(sys.modules, HW_MODULE, empty)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
resolved = layers._resolve("layernorm", "RMSNorm")
|
||||
assert resolved is VllmRMSNorm
|
||||
assert "falling back to default" in caplog.text
|
||||
|
||||
|
||||
def test_act_and_mul_falls_back_for_unknown_activation(
|
||||
monkeypatch, default_vllm_config
|
||||
):
|
||||
"""An activation with no hw-agnostic equivalent falls back to vLLM's.
|
||||
|
||||
`default_vllm_config` supplies the config context the CustomOp needs.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
|
||||
from vllm.model_executor.layers.activation import GeluAndMul
|
||||
|
||||
assert isinstance(layers.get_act_and_mul_fn("gelu"), GeluAndMul)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tiny_llama_path(tmp_path_factory):
|
||||
"""A randomly-initialized microscopic Llama saved to disk (with an ungated
|
||||
tokenizer) so vLLM can load it like any local checkpoint."""
|
||||
from transformers import AutoTokenizer, LlamaConfig, LlamaForCausalLM
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
|
||||
config = LlamaConfig(
|
||||
vocab_size=tokenizer.vocab_size,
|
||||
hidden_size=128,
|
||||
intermediate_size=256,
|
||||
num_hidden_layers=2,
|
||||
num_attention_heads=4,
|
||||
rms_norm_eps=1e-6,
|
||||
hidden_act="silu",
|
||||
)
|
||||
torch.manual_seed(0)
|
||||
model = LlamaForCausalLM(config)
|
||||
|
||||
path = tmp_path_factory.mktemp("tiny_llama")
|
||||
model.save_pretrained(path)
|
||||
tokenizer.save_pretrained(path)
|
||||
return str(path)
|
||||
|
||||
|
||||
# Registered names of the layers the backend can
|
||||
# currently route to hw-agnostic implementations.
|
||||
_COVERED_LAYERS = ("rms_norm", "silu_and_mul")
|
||||
|
||||
|
||||
def _layer_providers(model) -> dict[str, str]:
|
||||
"""Map each covered layer type present in the model to the provider its
|
||||
implementation came from (``hw_agnostic`` or ``vllm``).
|
||||
"""
|
||||
|
||||
def provider_of(module) -> str | None:
|
||||
for cls in type(module).__mro__:
|
||||
if "hw_agnostic.layers" in cls.__module__:
|
||||
return "hw_agnostic"
|
||||
if ".model_executor.layers." in cls.__module__:
|
||||
return "vllm"
|
||||
return None
|
||||
|
||||
providers: dict[str, str] = {}
|
||||
for module in model.modules():
|
||||
name = getattr(module, "name", None)
|
||||
if (
|
||||
name in _COVERED_LAYERS
|
||||
and name not in providers
|
||||
and (prov := provider_of(module)) is not None
|
||||
):
|
||||
providers[name] = prov
|
||||
return providers
|
||||
|
||||
|
||||
def _serve(vllm_runner, model_path, prompts):
|
||||
"""Serve the model through the backend; return (layer_providers, logprobs)."""
|
||||
with vllm_runner(
|
||||
model_path,
|
||||
model_impl="transformers",
|
||||
max_model_len=64,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.3,
|
||||
) as runner:
|
||||
assert runner.llm.llm_engine.model_config.using_transformers_backend()
|
||||
providers = runner.apply_model(_layer_providers)[0]
|
||||
outputs = runner.generate_greedy_logprobs(
|
||||
prompts, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
return providers, outputs
|
||||
|
||||
|
||||
def test_hw_agnostic_matches_vllm_end_to_end(monkeypatch, vllm_runner, tiny_llama_path):
|
||||
"""Serving the tiny model with hw-agnostic layers matches the vLLM baseline."""
|
||||
# spawn: worker re-imports layers with the env set (see docstring).
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
# apply_model pickles the introspection function.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
prompts = ["The capital of France is", "vLLM is"]
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0")
|
||||
vllm_providers, vllm_outputs = _serve(vllm_runner, tiny_llama_path, prompts)
|
||||
# Both replaceable layers present in a Llama block must be vLLM's here.
|
||||
assert vllm_providers == {"rms_norm": "vllm", "silu_and_mul": "vllm"}
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
|
||||
hw_providers, hw_outputs = _serve(vllm_runner, tiny_llama_path, prompts)
|
||||
assert hw_providers == {"rms_norm": "hw_agnostic", "silu_and_mul": "hw_agnostic"}
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=vllm_outputs,
|
||||
outputs_1_lst=hw_outputs,
|
||||
name_0="vllm",
|
||||
name_1="hw_agnostic",
|
||||
)
|
||||
@@ -125,6 +125,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
|
||||
VLLM_SKIP_P2P_CHECK: bool = False
|
||||
VLLM_DISABLED_KERNELS: list[str] = []
|
||||
VLLM_USE_HW_AGNOSTIC: bool = False
|
||||
VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True
|
||||
VLLM_DISABLE_PYNCCL: bool = False
|
||||
VLLM_USE_OINK_OPS: bool = False
|
||||
@@ -1200,6 +1201,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
if "VLLM_DISABLED_KERNELS" not in os.environ
|
||||
else os.environ["VLLM_DISABLED_KERNELS"].split(",")
|
||||
),
|
||||
# Selects hw-agnostic layers for HF transformer backend
|
||||
"VLLM_USE_HW_AGNOSTIC": lambda: (
|
||||
os.getenv("VLLM_USE_HW_AGNOSTIC", "False").lower() in ("true", "1")
|
||||
),
|
||||
"VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1"))
|
||||
),
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,318 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import functools
|
||||
import inspect
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import get_cached_compilation_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.utils import maybe_disable_graph_partition
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Dictionary of all custom ops (classes, indexed by registered name).
|
||||
# To check if an op with a name is enabled, call .enabled() on the class.
|
||||
# Examples:
|
||||
# - MyOp.enabled()
|
||||
# - op_registry["my_op"].enabled()
|
||||
op_registry: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {}
|
||||
op_registry_oot: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {}
|
||||
|
||||
|
||||
def maybe_get_oot_by_class(class_type: type) -> type:
|
||||
class_name = class_type.__name__
|
||||
if class_name in op_registry_oot:
|
||||
return op_registry_oot[class_name]
|
||||
return class_type
|
||||
|
||||
|
||||
class PluggableLayer(nn.Module):
|
||||
"""
|
||||
Base class for pluggable layers.
|
||||
|
||||
A PluggableLayer is a *module-composing* abstraction: it may instantiate other
|
||||
``torch.nn.Module`` objects as sub-layers, and its functionality depends on
|
||||
these sub-layers following a generalized invocation sequence. Also, it is stateful
|
||||
and may hold parameters or buffers.
|
||||
|
||||
Unlike :class:`CustomOp`, PluggableLayer does NOT provide per-platform
|
||||
``forward_*`` dispatch. Instead, it supports out-of-tree (OOT) replacement
|
||||
of the entire layer class at instantiation time, allowing customized
|
||||
initialization and submodule composition.
|
||||
"""
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
try:
|
||||
layer_class_name = cls.__name__
|
||||
except AttributeError:
|
||||
raise TypeError(
|
||||
f"Cannot instantiate '{cls.__name__}': its 'name' attribute "
|
||||
f"was not set, possibly because it was not decorated with "
|
||||
f"@PluggableLayer.register, or it's the PluggableLayer itself."
|
||||
) from None
|
||||
|
||||
if layer_class_name not in op_registry_oot:
|
||||
layer_cls_to_instantiate = cls
|
||||
else:
|
||||
layer_cls_to_instantiate = op_registry_oot[layer_class_name]
|
||||
logger.debug(
|
||||
"Instantiating pluggable layer: %s using %s",
|
||||
layer_class_name,
|
||||
str(layer_cls_to_instantiate),
|
||||
)
|
||||
return super().__new__(layer_cls_to_instantiate)
|
||||
|
||||
# Decorator to register pluggable layers.
|
||||
@classmethod
|
||||
def register(cls, name: str):
|
||||
def decorator(op_cls):
|
||||
assert name not in op_registry, f"Duplicate op name: {name}"
|
||||
op_cls.name = name
|
||||
op_registry[name] = op_cls
|
||||
return op_cls
|
||||
|
||||
return decorator
|
||||
|
||||
# Decorator to register out-of-tree(oot) pluggable layers.
|
||||
# For OOT pluggable layers:
|
||||
# if in-tree layer class is registered with an oot_custom_layer,
|
||||
# the oot_custom_layer will be used instead.
|
||||
@classmethod
|
||||
def register_oot(cls, _decorated_layer_cls=None, name: str | None = None):
|
||||
def decorator(layer_cls):
|
||||
reg_name = name if name is not None else cls.__name__
|
||||
assert reg_name not in op_registry_oot, f"Duplicate layer name: {reg_name}"
|
||||
layer_cls.name = reg_name
|
||||
op_registry_oot[reg_name] = layer_cls
|
||||
return layer_cls
|
||||
|
||||
if _decorated_layer_cls is None:
|
||||
# Called with parentheses: @PluggableLayer.register_oot()
|
||||
# or @PluggableLayer.register_oot(name="...")
|
||||
return decorator
|
||||
elif isinstance(_decorated_layer_cls, type): # Check if it's a class
|
||||
# Called without parentheses: @PluggableLayer.register_oot
|
||||
return decorator(_decorated_layer_cls)
|
||||
else:
|
||||
raise TypeError("Decorator can only be applied to classes.")
|
||||
|
||||
|
||||
class CustomOp(nn.Module):
|
||||
"""
|
||||
Base class for custom ops.
|
||||
Dispatches the forward method to the appropriate backend.
|
||||
"""
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
try:
|
||||
op_name = cls.__name__
|
||||
except AttributeError:
|
||||
raise TypeError(
|
||||
f"Cannot instantiate '{cls.__name__}': its 'name' attribute "
|
||||
f"was not set, possibly because it was not decorated with "
|
||||
f"@CustomOp.register, or it's the CustomOp base class itself."
|
||||
) from None
|
||||
|
||||
if op_name not in op_registry_oot:
|
||||
op_cls_to_instantiate = cls
|
||||
else:
|
||||
op_cls_to_instantiate = op_registry_oot[op_name]
|
||||
logger.debug(
|
||||
"Instantiating custom op: %s using %s",
|
||||
op_name,
|
||||
str(op_cls_to_instantiate),
|
||||
)
|
||||
return super().__new__(op_cls_to_instantiate)
|
||||
|
||||
def __init__(self, *, enforce_enable: bool = False, compile_native: bool = False):
|
||||
super().__init__()
|
||||
self._enforce_enable = enforce_enable
|
||||
self._forward_method = self.dispatch_forward(compile_native=compile_native)
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
return self._forward_method(*args, **kwargs)
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
"""PyTorch-native implementation; OOT plugins override via
|
||||
:meth:`forward_oot`."""
|
||||
raise NotImplementedError
|
||||
|
||||
def forward_oot(self, *args, **kwargs):
|
||||
# By default, OOT ops fall back to the PyTorch-native implementation.
|
||||
return self.forward_native(*args, **kwargs)
|
||||
|
||||
def dispatch_forward(self, compile_native: bool):
|
||||
# The hw-agnostic CustomOp dispatches only between forward_native
|
||||
# (in-tree, used on every platform) and forward_oot (overridden by
|
||||
# an OOT plugin via register_oot).
|
||||
compilation_config = get_cached_compilation_config()
|
||||
|
||||
# NOTE(shen-shanshan): CustomOp object can be enforce enabled, e.g.,
|
||||
# enable device-specific kernels in ViT models when enabling graph
|
||||
# mode. By default, it will follow the compilation_config to determine
|
||||
# whether enable itself.
|
||||
# This enforce_enable mechanism will be removed after we adding a
|
||||
# separate compilation_config for multi-modal part.
|
||||
enabled = self._enforce_enable or self.enabled()
|
||||
if enabled:
|
||||
compilation_config.enabled_custom_ops.update([self.__class__.name])
|
||||
else:
|
||||
compilation_config.disabled_custom_ops.update([self.__class__.name])
|
||||
|
||||
if not enabled:
|
||||
# Compile forward_native to avoid eager torch ops if inside
|
||||
# opaque torch custom op (e.g. fused_moe, unified_attention, etc.)
|
||||
return self.maybe_compile(self.forward_native, enable=compile_native)
|
||||
|
||||
if current_platform.is_out_of_tree():
|
||||
return self.forward_oot
|
||||
return self.forward_native
|
||||
|
||||
def maybe_compile(self, fn, *, enable: bool = True):
|
||||
"""
|
||||
Compile fn if compilation enabled.
|
||||
Useful for CustomOp instances called from within a torch custom op,
|
||||
meaning the forward call is hidden from the model-level torch.compile.
|
||||
|
||||
NOTE: this does not enable fusion across ops, so opaque custom ops
|
||||
should still be unwrapped wherever possible.
|
||||
"""
|
||||
from vllm.config.compilation import CompilationMode
|
||||
|
||||
# Do not compile if compilation disabled
|
||||
if not enable:
|
||||
return fn
|
||||
|
||||
# Do not compile if global compilation disabled
|
||||
compilation_config = get_cached_compilation_config()
|
||||
if compilation_config.mode == CompilationMode.NONE:
|
||||
return fn
|
||||
|
||||
# If eager backend is used, do not compile either
|
||||
if compilation_config.backend == "eager":
|
||||
return fn
|
||||
|
||||
compile_options = maybe_disable_graph_partition(
|
||||
current_platform.simple_compile_backend
|
||||
)
|
||||
backend = current_platform.simple_compile_backend
|
||||
|
||||
dynamic_arg_dims = getattr(self.__class__, "_dynamic_arg_dims", None)
|
||||
if dynamic_arg_dims is not None:
|
||||
compiled_fn = torch.compile(
|
||||
fn,
|
||||
dynamic=False,
|
||||
backend=backend,
|
||||
options=compile_options,
|
||||
)
|
||||
sig = inspect.signature(fn)
|
||||
|
||||
@functools.wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
bound = sig.bind(*args, **kwargs)
|
||||
bound.apply_defaults()
|
||||
for name, dims in dynamic_arg_dims.items():
|
||||
arg = bound.arguments.get(name)
|
||||
if arg is not None and isinstance(arg, torch.Tensor):
|
||||
dims_list = [dims] if isinstance(dims, int) else dims
|
||||
for d in dims_list:
|
||||
real_d = arg.ndim + d if d < 0 else d
|
||||
torch._dynamo.mark_dynamic(arg, real_d)
|
||||
return compiled_fn(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
# dynamic=True to avoid recompilations
|
||||
return torch.compile(
|
||||
fn,
|
||||
dynamic=True,
|
||||
backend=backend,
|
||||
options=compile_options,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def enabled(cls) -> bool:
|
||||
# if no name, then it was not registered
|
||||
compilation_config = get_cached_compilation_config()
|
||||
custom_ops = compilation_config.custom_ops
|
||||
if not hasattr(cls, "name"):
|
||||
logger.warning_once(
|
||||
"Custom op %s was not registered, which means it won't appear "
|
||||
"in the op registry. It will be enabled/disabled based on the "
|
||||
"global settings.",
|
||||
cls.__name__,
|
||||
)
|
||||
return CustomOp.default_on()
|
||||
|
||||
enabled = f"+{cls.name}" in custom_ops
|
||||
disabled = f"-{cls.name}" in custom_ops
|
||||
assert not (enabled and disabled), f"Cannot enable and disable {cls.name}"
|
||||
|
||||
return (CustomOp.default_on() or enabled) and not disabled
|
||||
|
||||
@staticmethod
|
||||
def default_on() -> bool:
|
||||
"""
|
||||
Behavior controlled by `CompilationConfig.custom_ops`: On by default if
|
||||
'all', off by default if 'none'.
|
||||
When PyTorch Inductor is used, 'none' is the default value,
|
||||
otherwise 'all'.
|
||||
"""
|
||||
compilation_config = get_cached_compilation_config()
|
||||
count_none = compilation_config.custom_ops.count("none")
|
||||
count_all = compilation_config.custom_ops.count("all")
|
||||
assert count_none + count_all == 1
|
||||
|
||||
return not count_none > 0 or count_all > 0
|
||||
|
||||
# Decorator to register custom ops.
|
||||
@classmethod
|
||||
def register(
|
||||
cls,
|
||||
name: str,
|
||||
dynamic_arg_dims: dict[str, int | list[int]] | None = None,
|
||||
):
|
||||
def decorator(op_cls):
|
||||
assert name not in op_registry, f"Duplicate op name: {name}"
|
||||
op_cls.name = name
|
||||
op_cls._dynamic_arg_dims = dynamic_arg_dims
|
||||
op_registry[name] = op_cls
|
||||
return op_cls
|
||||
|
||||
return decorator
|
||||
|
||||
# Decorator to register out-of-tree(oot) custom ops.
|
||||
# For OOT custom ops:
|
||||
# if in-tree layer class is registered with an oot_custom_op layer,
|
||||
# the oot_custom_op layer will be used instead.
|
||||
# Example:
|
||||
# - @UnquantizedFusedMoEMethod.register_oot
|
||||
# class HPUUnquantizedFusedMoEMethod(UnquantizedFusedMoEMethod)
|
||||
# or
|
||||
# - @CustomOP.register_oot(name="UnquantizedFusedMoEMethod")
|
||||
@classmethod
|
||||
def register_oot(cls, _decorated_op_cls=None, name: str | None = None):
|
||||
def decorator(op_cls):
|
||||
reg_name = name if name is not None else cls.__name__
|
||||
assert reg_name not in op_registry_oot, f"Duplicate op name: {reg_name}"
|
||||
op_cls.name = reg_name
|
||||
op_registry_oot[reg_name] = op_cls
|
||||
return op_cls
|
||||
|
||||
if _decorated_op_cls is None:
|
||||
# Called with parentheses: @CustomOP.register_oot()
|
||||
# or @CustomOP.register_oot(name="...")
|
||||
# So, _decorated_op_cls is None.
|
||||
# We return the actual decorator function.
|
||||
return decorator
|
||||
elif isinstance(_decorated_op_cls, type): # Check if it's a class
|
||||
# Called without parentheses: @CustomOP.register_oot
|
||||
# The first argument is the class itself.
|
||||
# We call the 'decorator' function immediately with the class.
|
||||
return decorator(_decorated_op_cls)
|
||||
else:
|
||||
# Handle other unexpected cases if necessary
|
||||
raise TypeError("Decorator can only be applied to classes.")
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,27 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.hw_agnostic.custom_op import CustomOp
|
||||
|
||||
|
||||
@CustomOp.register("silu_and_mul")
|
||||
class SiluAndMul(CustomOp):
|
||||
"""SwiGLU: ``x -> silu(x[:d]) * x[d:]`` where ``d = x.shape[-1] // 2``."""
|
||||
|
||||
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
return F.silu(x[..., :d]) * x[..., d:]
|
||||
|
||||
|
||||
# Activation-and-mul ops keyed by HF activation name.
|
||||
_ACTIVATION_AND_MUL_REGISTRY = {
|
||||
"silu": SiluAndMul,
|
||||
"swish": SiluAndMul,
|
||||
}
|
||||
|
||||
|
||||
def get_act_and_mul_fn(act_fn_name: str) -> CustomOp:
|
||||
"""Build the hw-agnostic activation-and-mul op named `act_fn_name`."""
|
||||
return _ACTIVATION_AND_MUL_REGISTRY[act_fn_name.lower()]()
|
||||
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch import Tensor
|
||||
|
||||
from vllm.model_executor.hw_agnostic.custom_op import CustomOp
|
||||
|
||||
|
||||
@CustomOp.register("rms_norm")
|
||||
class RMSNorm(CustomOp):
|
||||
"""``x -> w * x / sqrt(E[x^2] + eps)``. With ``residual``, fuses
|
||||
``residual += x`` then RMSNorm and returns ``(normalized, residual)``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
eps: float = 1e-6,
|
||||
var_hidden_size: int | None = None,
|
||||
has_weight: bool = True,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.variance_epsilon = eps
|
||||
self.variance_size_override = (
|
||||
None if var_hidden_size == hidden_size else var_hidden_size
|
||||
)
|
||||
weight_dtype = dtype or torch.get_default_dtype()
|
||||
self.has_weight = has_weight
|
||||
weight = torch.ones(hidden_size, dtype=weight_dtype)
|
||||
if has_weight:
|
||||
self.weight = nn.Parameter(weight)
|
||||
else:
|
||||
self.register_buffer("weight", weight, persistent=False)
|
||||
|
||||
def _rms_norm(
|
||||
self,
|
||||
x: Tensor,
|
||||
weight: Tensor | None,
|
||||
epsilon: float,
|
||||
variance_size: int | None = None,
|
||||
) -> Tensor:
|
||||
"""Weighted root-mean-square layer normalization"""
|
||||
orig_dtype = x.dtype
|
||||
x = x.to(torch.float32)
|
||||
x_var = x if variance_size is None else x[..., :variance_size]
|
||||
variance = x_var.pow(2).mean(dim=-1, keepdim=True)
|
||||
x = x * torch.rsqrt(variance + epsilon)
|
||||
if weight is not None:
|
||||
x = x.to(weight.dtype) * weight
|
||||
return x.to(orig_dtype)
|
||||
|
||||
def _fused_add_rms_norm(
|
||||
self,
|
||||
x: Tensor,
|
||||
x_residual: Tensor,
|
||||
weight: Tensor | None,
|
||||
epsilon: float,
|
||||
variance_size: int | None = None,
|
||||
) -> tuple[Tensor, Tensor]:
|
||||
"""Fused add and weighted root-mean-square layer normalization"""
|
||||
orig_dtype = x.dtype
|
||||
x = x.to(torch.float32)
|
||||
x = x + x_residual.to(torch.float32)
|
||||
x_residual = x.to(orig_dtype)
|
||||
|
||||
x_var = x if variance_size is None else x[..., :variance_size]
|
||||
variance = x_var.pow(2).mean(dim=-1, keepdim=True)
|
||||
x = x * torch.rsqrt(variance + epsilon)
|
||||
if weight is not None:
|
||||
x = x.to(weight.dtype) * weight
|
||||
return x.to(orig_dtype), x_residual
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
weight = self.weight if self.has_weight else None
|
||||
epsilon = self.variance_epsilon
|
||||
variance_size = self.variance_size_override
|
||||
|
||||
if residual is None:
|
||||
return self._rms_norm(x, weight, epsilon, variance_size)
|
||||
else:
|
||||
return self._fused_add_rms_norm(x, residual, weight, epsilon, variance_size)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"hidden_size={self.hidden_size}, eps={self.variance_epsilon}"
|
||||
@@ -11,10 +11,7 @@ from torch import fx, nn
|
||||
from transformers.activations import ACT2CLS
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import (
|
||||
_ACTIVATION_AND_MUL_REGISTRY,
|
||||
get_act_and_mul_fn,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import _ACTIVATION_AND_MUL_REGISTRY
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
@@ -26,6 +23,7 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
replace_expr,
|
||||
single_self_call,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.layers import get_act_and_mul_fn
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
|
||||
@@ -15,7 +15,6 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.distributed.parallel_state import model_parallel_is_initialized
|
||||
from vllm.distributed.utils import split_tensor_along_last_dim
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
@@ -25,6 +24,10 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.layers import (
|
||||
GemmaRMSNorm,
|
||||
RMSNorm,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Layer provider resolution for the Transformers modeling backend.
|
||||
|
||||
When ``VLLM_USE_HW_AGNOSTIC`` is set, layer symbols are imported from
|
||||
``vllm.model_executor.hw_agnostic.layers.<module>``, falling back to
|
||||
``vllm.model_executor.layers.<module>`` for anything not yet ported. The
|
||||
resolved source of every symbol is logged so it is clear which layers run
|
||||
hw-agnostic and which fell back to vLLM.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_HW_PKG = "vllm.model_executor.hw_agnostic.layers"
|
||||
_VLLM_PKG = "vllm.model_executor.layers"
|
||||
|
||||
|
||||
def _resolve(module: str, name: str):
|
||||
"""Return `name` from the hw-agnostic `module` when enabled and available,
|
||||
else from vLLM. Logs which source was used."""
|
||||
if envs.VLLM_USE_HW_AGNOSTIC:
|
||||
try:
|
||||
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
|
||||
logger.info("Using hw-agnostic layer: %s", name)
|
||||
return obj
|
||||
except (ImportError, AttributeError):
|
||||
logger.warning(
|
||||
"hw-agnostic layer %s is not available; falling back to default", name
|
||||
)
|
||||
return getattr(importlib.import_module(f"{_VLLM_PKG}.{module}"), name)
|
||||
|
||||
|
||||
RMSNorm = _resolve("layernorm", "RMSNorm")
|
||||
GemmaRMSNorm = _resolve("layernorm", "GemmaRMSNorm")
|
||||
|
||||
|
||||
def get_act_and_mul_fn(act_fn_name: str):
|
||||
"""Fused activation-and-mul op for `act_fn_name`, preferring hw-agnostic.
|
||||
|
||||
Resolved per call because the op is name-parameterized: an activation with
|
||||
no hw-agnostic equivalent falls back to vLLM individually.
|
||||
"""
|
||||
if envs.VLLM_USE_HW_AGNOSTIC:
|
||||
try:
|
||||
from vllm.model_executor.hw_agnostic.layers.activation import (
|
||||
get_act_and_mul_fn as hw_fn,
|
||||
)
|
||||
|
||||
fn = hw_fn(act_fn_name)
|
||||
logger.info_once("Using hw-agnostic activation: %s", act_fn_name)
|
||||
return fn
|
||||
except (ImportError, KeyError):
|
||||
logger.warning_once(
|
||||
"hw-agnostic activation %s is not available; falling back to vLLM",
|
||||
act_fn_name,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import get_act_and_mul_fn as vllm_fn
|
||||
|
||||
return vllm_fn(act_fn_name)
|
||||
Reference in New Issue
Block a user