Hardware-agnostic model definition via HF transformer backend (1/N) (#49458)
Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
This commit is contained in:
@@ -11,10 +11,7 @@ from torch import fx, nn
|
||||
from transformers.activations import ACT2CLS
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import (
|
||||
_ACTIVATION_AND_MUL_REGISTRY,
|
||||
get_act_and_mul_fn,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import _ACTIVATION_AND_MUL_REGISTRY
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
@@ -26,6 +23,7 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
replace_expr,
|
||||
single_self_call,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.layers import get_act_and_mul_fn
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
|
||||
@@ -15,7 +15,6 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.distributed.parallel_state import model_parallel_is_initialized
|
||||
from vllm.distributed.utils import split_tensor_along_last_dim
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
@@ -25,6 +24,10 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.layers import (
|
||||
GemmaRMSNorm,
|
||||
RMSNorm,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Layer provider resolution for the Transformers modeling backend.
|
||||
|
||||
When ``VLLM_USE_HW_AGNOSTIC`` is set, layer symbols are imported from
|
||||
``vllm.model_executor.hw_agnostic.layers.<module>``, falling back to
|
||||
``vllm.model_executor.layers.<module>`` for anything not yet ported. The
|
||||
resolved source of every symbol is logged so it is clear which layers run
|
||||
hw-agnostic and which fell back to vLLM.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_HW_PKG = "vllm.model_executor.hw_agnostic.layers"
|
||||
_VLLM_PKG = "vllm.model_executor.layers"
|
||||
|
||||
|
||||
def _resolve(module: str, name: str):
|
||||
"""Return `name` from the hw-agnostic `module` when enabled and available,
|
||||
else from vLLM. Logs which source was used."""
|
||||
if envs.VLLM_USE_HW_AGNOSTIC:
|
||||
try:
|
||||
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
|
||||
logger.info("Using hw-agnostic layer: %s", name)
|
||||
return obj
|
||||
except (ImportError, AttributeError):
|
||||
logger.warning(
|
||||
"hw-agnostic layer %s is not available; falling back to default", name
|
||||
)
|
||||
return getattr(importlib.import_module(f"{_VLLM_PKG}.{module}"), name)
|
||||
|
||||
|
||||
RMSNorm = _resolve("layernorm", "RMSNorm")
|
||||
GemmaRMSNorm = _resolve("layernorm", "GemmaRMSNorm")
|
||||
|
||||
|
||||
def get_act_and_mul_fn(act_fn_name: str):
|
||||
"""Fused activation-and-mul op for `act_fn_name`, preferring hw-agnostic.
|
||||
|
||||
Resolved per call because the op is name-parameterized: an activation with
|
||||
no hw-agnostic equivalent falls back to vLLM individually.
|
||||
"""
|
||||
if envs.VLLM_USE_HW_AGNOSTIC:
|
||||
try:
|
||||
from vllm.model_executor.hw_agnostic.layers.activation import (
|
||||
get_act_and_mul_fn as hw_fn,
|
||||
)
|
||||
|
||||
fn = hw_fn(act_fn_name)
|
||||
logger.info_once("Using hw-agnostic activation: %s", act_fn_name)
|
||||
return fn
|
||||
except (ImportError, KeyError):
|
||||
logger.warning_once(
|
||||
"hw-agnostic activation %s is not available; falling back to vLLM",
|
||||
act_fn_name,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import get_act_and_mul_fn as vllm_fn
|
||||
|
||||
return vllm_fn(act_fn_name)
|
||||
Reference in New Issue
Block a user