Deprecations for v0.23 and v0.24 (#44992)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -212,6 +212,9 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
"flashinfer_cudnn": {
|
||||
FlashInferCudnnNvFp4LinearKernel,
|
||||
},
|
||||
"flashinfer_b12x": {
|
||||
FlashInferB12xNvFp4LinearKernel,
|
||||
},
|
||||
"marlin": {
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
MarlinLinearKernel,
|
||||
@@ -392,7 +395,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
|
||||
PlatformEnum.CUDA: [
|
||||
# FlashInferB12xNvFp4LinearKernel excluded from auto-selection until
|
||||
# upstream CUTLASS SM121 MMA op guard is resolved; use
|
||||
# VLLM_NVFP4_GEMM_BACKEND=flashinfer-b12x to opt in explicitly.
|
||||
# --linear-backend flashinfer_b12x to opt in explicitly.
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
CutlassNvFp4LinearKernel,
|
||||
MarlinNvFp4LinearKernel,
|
||||
@@ -752,20 +755,6 @@ def init_mxfp4_linear_kernel() -> MxFp4LinearKernel:
|
||||
current platform."""
|
||||
linear_backend = _get_linear_backend()
|
||||
|
||||
force_kernel: type[MxFp4LinearKernel] | None = None
|
||||
if linear_backend == "auto" and envs.VLLM_MXFP4_USE_MARLIN:
|
||||
force_kernel = MarlinMxFp4LinearKernel
|
||||
|
||||
if force_kernel is not None:
|
||||
is_supported, reason = force_kernel.is_supported()
|
||||
if not is_supported:
|
||||
raise ValueError(
|
||||
f"Forced MXFP4 kernel {force_kernel.__name__} is not "
|
||||
f"supported: {reason}"
|
||||
)
|
||||
logger.info_once("Using %s for MXFP4 GEMM", force_kernel.__name__)
|
||||
return force_kernel(MxFp4LinearLayerConfig())
|
||||
|
||||
platform = current_platform._enum
|
||||
possible = list(_POSSIBLE_MXFP4_KERNELS.get(platform, []))
|
||||
|
||||
@@ -836,18 +825,6 @@ def init_wfp8_a16_linear_kernel(
|
||||
)
|
||||
|
||||
|
||||
# Maps VLLM_NVFP4_GEMM_BACKEND env var values to kernel classes.
|
||||
_NVFP4_BACKEND_TO_KERNEL: dict[str, type[NvFp4LinearKernel]] = {
|
||||
"flashinfer-b12x": FlashInferB12xNvFp4LinearKernel,
|
||||
"flashinfer-cutlass": FlashInferCutlassNvFp4LinearKernel,
|
||||
"cutlass": CutlassNvFp4LinearKernel,
|
||||
"marlin": MarlinNvFp4LinearKernel,
|
||||
"flashinfer-trtllm": FlashInferTrtllmNvFp4LinearKernel,
|
||||
"flashinfer-cudnn": FlashInferCudnnNvFp4LinearKernel,
|
||||
"emulation": EmulationNvFp4LinearKernel,
|
||||
}
|
||||
|
||||
|
||||
def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
||||
"""Select and instantiate the best NVFP4 linear kernel for the
|
||||
current platform."""
|
||||
@@ -855,8 +832,7 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
||||
|
||||
# VLLM_BATCH_INVARIANT forces deterministic execution. Prefer the
|
||||
# batch-invariant CUTLASS implementation when available, otherwise fall
|
||||
# back to emulation. It overrides both --linear-backend and the deprecated
|
||||
# env vars below.
|
||||
# back to emulation. It overrides --linear-backend.
|
||||
force_kernel: type[NvFp4LinearKernel] | None = None
|
||||
linear_backend = _get_linear_backend()
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
@@ -888,24 +864,9 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
||||
reason,
|
||||
)
|
||||
force_kernel = EmulationNvFp4LinearKernel
|
||||
elif linear_backend == "auto":
|
||||
# Deprecated env-var overrides — only honoured when --linear-backend
|
||||
# is "auto". Deprecation warnings are emitted from vllm/envs.py.
|
||||
if use_a16: # force a16 if running weight-only quantization
|
||||
force_kernel = MarlinNvFp4LinearKernel
|
||||
elif envs.VLLM_USE_FBGEMM:
|
||||
force_kernel = FbgemmNvFp4LinearKernel
|
||||
elif envs.VLLM_USE_NVFP4_CT_EMULATIONS:
|
||||
force_kernel = EmulationNvFp4LinearKernel
|
||||
elif envs.VLLM_NVFP4_GEMM_BACKEND is not None:
|
||||
backend_name = envs.VLLM_NVFP4_GEMM_BACKEND
|
||||
force_kernel = _NVFP4_BACKEND_TO_KERNEL.get(backend_name)
|
||||
if force_kernel is None:
|
||||
raise ValueError(
|
||||
f"Unknown VLLM_NVFP4_GEMM_BACKEND={backend_name!r}. "
|
||||
f"Valid choices: "
|
||||
f"{list(_NVFP4_BACKEND_TO_KERNEL.keys())}"
|
||||
)
|
||||
elif linear_backend == "auto" and use_a16:
|
||||
# Force a16 (Marlin) when running weight-only quantization.
|
||||
force_kernel = MarlinNvFp4LinearKernel
|
||||
|
||||
if force_kernel is not None:
|
||||
is_supported, reason = force_kernel.is_supported()
|
||||
|
||||
@@ -20,8 +20,6 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
FlashinferMoeBackend,
|
||||
get_flashinfer_moe_backend,
|
||||
prepare_fp8_moe_layer_for_fi,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
@@ -321,54 +319,6 @@ def select_fp8_moe_backend(
|
||||
requested_backend, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
|
||||
# Handle explicit FlashInfer FP8 configuration.
|
||||
if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP8"):
|
||||
if not envs.VLLM_USE_FLASHINFER_MOE_FP8:
|
||||
# If the user rejects FlashInfer remove those backends.
|
||||
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.FLASHINFER_TRTLLM)
|
||||
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.FLASHINFER_CUTLASS)
|
||||
|
||||
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
|
||||
# If user is explicit about backend, validate it.
|
||||
fi_backend = get_flashinfer_moe_backend()
|
||||
if fi_backend == FlashinferMoeBackend.CUTLASS:
|
||||
backend = Fp8MoeBackend.FLASHINFER_CUTLASS
|
||||
elif fi_backend == FlashinferMoeBackend.TENSORRT_LLM:
|
||||
backend = Fp8MoeBackend.FLASHINFER_TRTLLM
|
||||
else:
|
||||
raise ValueError(
|
||||
f"FlashInfer MOE backend {fi_backend} does not support FP8 MoE."
|
||||
)
|
||||
k_cls = backend_to_kernel_cls(backend)[0]
|
||||
return _return_or_raise(
|
||||
backend, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
else:
|
||||
# If the user is not explicit about the backend, try both.
|
||||
for backend in [
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
Fp8MoeBackend.FLASHINFER_CUTLASS,
|
||||
]:
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls,
|
||||
config,
|
||||
weight_key,
|
||||
activation_key,
|
||||
activation_format,
|
||||
)
|
||||
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
return backend, k_cls
|
||||
else:
|
||||
logger.debug_once(_make_log_unsupported(backend, reason))
|
||||
|
||||
raise NotImplementedError(
|
||||
"Found VLLM_USE_FLASHINFER_MOE_FP8=1, but no "
|
||||
"FlashInfer FP8 MoE backend supports the configuration."
|
||||
)
|
||||
|
||||
# Handle explicit DeepGEMM FP8 configuration.
|
||||
if envs.is_set("VLLM_USE_DEEP_GEMM") or envs.is_set("VLLM_MOE_USE_DEEP_GEMM"):
|
||||
if not envs.VLLM_USE_DEEP_GEMM or not envs.VLLM_MOE_USE_DEEP_GEMM:
|
||||
|
||||
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Literal, Union
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
@@ -465,74 +464,6 @@ def select_mxfp4_moe_backend(
|
||||
_get_priority_backends_for_gpt_oss(), requested_activation_key
|
||||
)
|
||||
|
||||
# Handle explicit FlashInfer MXFP4 BF16 configuration.
|
||||
if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"):
|
||||
if not envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16:
|
||||
for _b in (
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
):
|
||||
if _b in AVAILABLE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(_b)
|
||||
else:
|
||||
if current_platform.is_device_capability(90):
|
||||
return _return_or_raise(
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
None,
|
||||
activation_format,
|
||||
)
|
||||
if current_platform.is_device_capability_family(100):
|
||||
return _return_or_raise(
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
None,
|
||||
activation_format,
|
||||
)
|
||||
raise ValueError(
|
||||
"VLLM_USE_FLASHINFER_MOE_MXFP4_BF16=1 is set but the "
|
||||
"current device capability is not supported. "
|
||||
"Only SM90 (CUTLASS) and SM100+ (TRTLLM) are supported."
|
||||
)
|
||||
|
||||
# Handle explicit FlashInfer MXFP4 MXFP8 TRTLLM configuration.
|
||||
if (
|
||||
envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8")
|
||||
and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8
|
||||
):
|
||||
return _return_or_raise(
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
kMxfp8Dynamic,
|
||||
activation_format,
|
||||
)
|
||||
|
||||
# Handle explicit FlashInfer MXFP4 MXFP8 CUTLASS configuration.
|
||||
if (
|
||||
envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS")
|
||||
and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS
|
||||
):
|
||||
return _return_or_raise(
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
kMxfp8Dynamic,
|
||||
activation_format,
|
||||
)
|
||||
|
||||
# Handle explicit Marlin MXFP4 configuration.
|
||||
if envs.is_set("VLLM_MXFP4_USE_MARLIN") and envs.VLLM_MXFP4_USE_MARLIN:
|
||||
return _return_or_raise(
|
||||
Mxfp4MoeBackend.MARLIN,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
None,
|
||||
activation_format,
|
||||
)
|
||||
|
||||
for backend in AVAILABLE_BACKENDS:
|
||||
# Use requested_activation_key if provided, otherwise use backend default
|
||||
act_key = (
|
||||
|
||||
@@ -22,10 +22,6 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
|
||||
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
FlashinferMoeBackend,
|
||||
get_flashinfer_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
prepare_nvfp4_moe_layer_for_marlin,
|
||||
)
|
||||
@@ -58,12 +54,6 @@ FLASHINFER_NVFP4_MOE_BACKENDS = [
|
||||
NvFp4MoeBackend.FLASHINFER_B12X,
|
||||
]
|
||||
|
||||
fi_2_vllm_backend_map: dict[FlashinferMoeBackend, NvFp4MoeBackend] = {
|
||||
FlashinferMoeBackend.CUTLASS: NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
||||
FlashinferMoeBackend.TENSORRT_LLM: NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
FlashinferMoeBackend.CUTEDSL: NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
}
|
||||
|
||||
|
||||
def is_global_sf_supported_for_nvfp4_backend(backend: NvFp4MoeBackend) -> bool:
|
||||
# Checks whether `backend` supports quantizing with scaling factors
|
||||
@@ -258,55 +248,6 @@ def select_nvfp4_moe_backend(
|
||||
requested_backend, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
|
||||
if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP4"):
|
||||
if not envs.VLLM_USE_FLASHINFER_MOE_FP4:
|
||||
# If the user rejects FlashInfer remove those backends.
|
||||
for b in FLASHINFER_NVFP4_MOE_BACKENDS:
|
||||
if b in AVAILABLE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(b)
|
||||
|
||||
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
|
||||
# If user is explicit about backend, validate it.
|
||||
backend = fi_2_vllm_backend_map[get_flashinfer_moe_backend()]
|
||||
if (
|
||||
config.swiglu_limit is not None
|
||||
and backend not in NVFP4_BACKENDS_WITH_CLAMP
|
||||
):
|
||||
raise ValueError(
|
||||
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
|
||||
f"FlashInfer backend selected via VLLM_FLASHINFER_MOE_BACKEND "
|
||||
f"({backend.value}) does not apply the SwiGLU clamp."
|
||||
)
|
||||
return _return_or_raise(
|
||||
backend, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
else:
|
||||
# If the user is not explicit about the backend, try each.
|
||||
fi_backends = [
|
||||
b
|
||||
for b in FLASHINFER_NVFP4_MOE_BACKENDS
|
||||
if config.swiglu_limit is None or b in NVFP4_BACKENDS_WITH_CLAMP
|
||||
]
|
||||
for backend in fi_backends:
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls,
|
||||
config,
|
||||
weight_key,
|
||||
activation_key,
|
||||
activation_format,
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
return backend, k_cls
|
||||
else:
|
||||
logger.debug_once(_make_log_unsupported(backend, reason))
|
||||
|
||||
raise NotImplementedError(
|
||||
"Found VLLM_USE_FLASHINFER_MOE_FP4=1, but no "
|
||||
"FlashInfer NVFP4 MoE backend supports the configuration."
|
||||
)
|
||||
|
||||
if envs.VLLM_TEST_FORCE_FP8_MARLIN:
|
||||
backend = NvFp4MoeBackend.MARLIN
|
||||
return _return_or_raise(
|
||||
|
||||
@@ -19,9 +19,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
FlashinferMoeBackend,
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
||||
get_flashinfer_moe_backend,
|
||||
swap_w13_to_w31,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -230,49 +228,6 @@ def select_unquantized_moe_backend(
|
||||
|
||||
return _return_or_raise(requested_backend, moe_config, activation_format)
|
||||
|
||||
# Handle explicit FlashInfer FP16 configuration.
|
||||
if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP16"):
|
||||
if not envs.VLLM_USE_FLASHINFER_MOE_FP16:
|
||||
if UnquantizedMoeBackend.FLASHINFER_TRTLLM in AVAILABLE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.FLASHINFER_TRTLLM)
|
||||
if UnquantizedMoeBackend.FLASHINFER_CUTLASS in AVAILABLE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.FLASHINFER_CUTLASS)
|
||||
|
||||
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
|
||||
# If user is explicit about backend, validate it.
|
||||
fi_backend = get_flashinfer_moe_backend()
|
||||
if fi_backend == FlashinferMoeBackend.CUTLASS:
|
||||
backend = UnquantizedMoeBackend.FLASHINFER_CUTLASS
|
||||
elif fi_backend == FlashinferMoeBackend.TENSORRT_LLM:
|
||||
backend = UnquantizedMoeBackend.FLASHINFER_TRTLLM
|
||||
else:
|
||||
raise ValueError(
|
||||
f"FlashInfer MOE backend {fi_backend} "
|
||||
"does not support unquantized MoE."
|
||||
)
|
||||
k_cls = backend_to_kernel_cls(backend)
|
||||
return _return_or_raise(backend, moe_config, activation_format)
|
||||
else:
|
||||
# If the user is not explicit about the backend, try both.
|
||||
for backend in [
|
||||
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
|
||||
UnquantizedMoeBackend.FLASHINFER_CUTLASS,
|
||||
]:
|
||||
k_cls = backend_to_kernel_cls(backend)
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, moe_config, None, None, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
return backend, k_cls
|
||||
else:
|
||||
logger.debug_once(_make_log_unsupported(backend, reason))
|
||||
|
||||
raise NotImplementedError(
|
||||
"Found VLLM_USE_FLASHINFER_MOE_FP16=1, but no "
|
||||
"FlashInfer unquantized MoE backend supports the configuration."
|
||||
)
|
||||
|
||||
# Handle explicit AITER FP8 configuration.
|
||||
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
|
||||
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
|
||||
|
||||
@@ -6,7 +6,7 @@ import nixl_ep
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.distributed.device_communicators.all2all import NixlEPAll2AllManager
|
||||
from vllm.logger import init_logger
|
||||
@@ -192,10 +192,11 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
x = x.view((-1, hidden_dim))
|
||||
q_dtype = quant_config.quant_dtype
|
||||
|
||||
if envs.VLLM_FLASHINFER_MOE_BACKEND == "masked_gemm":
|
||||
moe_backend = get_current_vllm_config().kernel_config.moe_backend
|
||||
if moe_backend == "flashinfer_cutedsl":
|
||||
logger.info_once(
|
||||
"Skip quantization when using FlashInfer CUTEDSL(masked_gemm) "
|
||||
"for ModelOptNvFp4FusedMoE."
|
||||
"Skip quantization when using FlashInfer CUTEDSL "
|
||||
"(--moe-backend flashinfer_cutedsl) for ModelOptNvFp4FusedMoE."
|
||||
)
|
||||
q_dtype = None
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
align_fp4_moe_weights_for_fi,
|
||||
@@ -15,10 +14,6 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
swizzle_blockscale,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
has_flashinfer_cutlass_fused_moe,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
@@ -34,16 +29,6 @@ __all__ = [
|
||||
]
|
||||
|
||||
|
||||
def is_flashinfer_fp4_cutlass_moe_available() -> bool:
|
||||
"""Return `True` when FlashInfer CUTLASS NV-FP4 kernels can be used."""
|
||||
return (
|
||||
envs.VLLM_USE_FLASHINFER_MOE_FP4
|
||||
and has_flashinfer_cutlass_fused_moe()
|
||||
and current_platform.is_cuda()
|
||||
and current_platform.has_device_capability(100)
|
||||
)
|
||||
|
||||
|
||||
def reorder_w1w3_to_w3w1(
|
||||
weight: torch.Tensor, scale: torch.Tensor, dim: int = -2
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
@@ -5,10 +5,8 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -95,34 +93,6 @@ def rotate_weights_for_fi_trtllm_fp8_per_tensor_moe(
|
||||
)
|
||||
|
||||
|
||||
def get_flashinfer_moe_backend() -> FlashinferMoeBackend:
|
||||
backend_map = {
|
||||
"throughput": FlashinferMoeBackend.CUTLASS,
|
||||
"latency": FlashinferMoeBackend.TENSORRT_LLM,
|
||||
"masked_gemm": FlashinferMoeBackend.CUTEDSL,
|
||||
}
|
||||
|
||||
flashinfer_moe_backend = envs.VLLM_FLASHINFER_MOE_BACKEND
|
||||
if flashinfer_moe_backend in backend_map:
|
||||
if (
|
||||
flashinfer_moe_backend == "latency"
|
||||
and not current_platform.is_device_capability_family(100)
|
||||
):
|
||||
logger.info_once(
|
||||
"Flashinfer TRTLLM MOE backend is only supported on "
|
||||
"SM100 and later, using CUTLASS backend instead",
|
||||
)
|
||||
return FlashinferMoeBackend.CUTLASS
|
||||
return backend_map[flashinfer_moe_backend]
|
||||
elif current_platform.is_device_capability(90):
|
||||
return FlashinferMoeBackend.CUTLASS
|
||||
|
||||
raise ValueError(
|
||||
f"Unknown flashinfer moe backend: {flashinfer_moe_backend!r}. "
|
||||
f"Expected one of {list(backend_map.keys())}."
|
||||
)
|
||||
|
||||
|
||||
def is_flashinfer_supporting_global_sf(backend: FlashinferMoeBackend | None) -> bool:
|
||||
# TODO(shuw@nvidia): Update when new backends are added.
|
||||
backends_supporting_global_sf = (
|
||||
|
||||
Reference in New Issue
Block a user