Hardware-agnostic model definition via HF transformer backend (1/N) (#49458)

Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
This commit is contained in:
Thomas Ortner
2026-08-13 11:21:50 +01:00
committed by GitHub
parent 903da602f3
commit b8baa31a28
10 changed files with 691 additions and 5 deletions
@@ -11,10 +11,7 @@ from torch import fx, nn
from transformers.activations import ACT2CLS
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import (
_ACTIVATION_AND_MUL_REGISTRY,
get_act_and_mul_fn,
)
from vllm.model_executor.layers.activation import _ACTIVATION_AND_MUL_REGISTRY
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
from vllm.model_executor.models.transformers.fx_utils import (
@@ -26,6 +23,7 @@ from vllm.model_executor.models.transformers.fx_utils import (
replace_expr,
single_self_call,
)
from vllm.model_executor.models.transformers.layers import get_act_and_mul_fn
from vllm.model_executor.models.transformers.utils import (
log_replacement,
replace_linear_class,
@@ -15,7 +15,6 @@ from vllm.distributed import (
)
from vllm.distributed.parallel_state import model_parallel_is_initialized
from vllm.distributed.utils import split_tensor_along_last_dim
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
@@ -25,6 +24,10 @@ from vllm.model_executor.models.transformers.fx_utils import (
peel,
trace,
)
from vllm.model_executor.models.transformers.layers import (
GemmaRMSNorm,
RMSNorm,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -0,0 +1,64 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Layer provider resolution for the Transformers modeling backend.
When ``VLLM_USE_HW_AGNOSTIC`` is set, layer symbols are imported from
``vllm.model_executor.hw_agnostic.layers.<module>``, falling back to
``vllm.model_executor.layers.<module>`` for anything not yet ported. The
resolved source of every symbol is logged so it is clear which layers run
hw-agnostic and which fell back to vLLM.
"""
import importlib
import vllm.envs as envs
from vllm.logger import init_logger
logger = init_logger(__name__)
_HW_PKG = "vllm.model_executor.hw_agnostic.layers"
_VLLM_PKG = "vllm.model_executor.layers"
def _resolve(module: str, name: str):
"""Return `name` from the hw-agnostic `module` when enabled and available,
else from vLLM. Logs which source was used."""
if envs.VLLM_USE_HW_AGNOSTIC:
try:
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
logger.info("Using hw-agnostic layer: %s", name)
return obj
except (ImportError, AttributeError):
logger.warning(
"hw-agnostic layer %s is not available; falling back to default", name
)
return getattr(importlib.import_module(f"{_VLLM_PKG}.{module}"), name)
RMSNorm = _resolve("layernorm", "RMSNorm")
GemmaRMSNorm = _resolve("layernorm", "GemmaRMSNorm")
def get_act_and_mul_fn(act_fn_name: str):
"""Fused activation-and-mul op for `act_fn_name`, preferring hw-agnostic.
Resolved per call because the op is name-parameterized: an activation with
no hw-agnostic equivalent falls back to vLLM individually.
"""
if envs.VLLM_USE_HW_AGNOSTIC:
try:
from vllm.model_executor.hw_agnostic.layers.activation import (
get_act_and_mul_fn as hw_fn,
)
fn = hw_fn(act_fn_name)
logger.info_once("Using hw-agnostic activation: %s", act_fn_name)
return fn
except (ImportError, KeyError):
logger.warning_once(
"hw-agnostic activation %s is not available; falling back to vLLM",
act_fn_name,
)
from vllm.model_executor.layers.activation import get_act_and_mul_fn as vllm_fn
return vllm_fn(act_fn_name)