[Quantization] Rework quantization_config to use QuantKey and allow for activation override (#41566)
This commit is contained in:
@@ -1,13 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING, Union
|
||||
from typing import TYPE_CHECKING, Literal, Union
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
@@ -207,25 +209,38 @@ def backend_to_kernel_cls(
|
||||
raise ValueError(f"Unknown MXFP4 MoE backend: {backend.value}")
|
||||
|
||||
|
||||
def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend:
|
||||
"""Map user's moe_backend string to Mxfp4MoeBackend."""
|
||||
mapping: dict[str, Mxfp4MoeBackend] = {
|
||||
"deep_gemm": Mxfp4MoeBackend.DEEPGEMM_MXFP4,
|
||||
"flashinfer_trtllm": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
"flashinfer_trtllm_afp8": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
"flashinfer_cutlass": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
"flashinfer_cutlass_afp8": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
"triton": Mxfp4MoeBackend.TRITON,
|
||||
"triton_unfused": Mxfp4MoeBackend.TRITON_UNFUSED,
|
||||
"humming": Mxfp4MoeBackend.HUMMING,
|
||||
"marlin": Mxfp4MoeBackend.MARLIN,
|
||||
"aiter": Mxfp4MoeBackend.AITER_MXFP4_BF16,
|
||||
"aiter_mxfp4_fp8": Mxfp4MoeBackend.AITER_MXFP4_FP8,
|
||||
"xpu": Mxfp4MoeBackend.XPU,
|
||||
"emulation": Mxfp4MoeBackend.EMULATION,
|
||||
def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
|
||||
"""Map a moe_backend string to its candidate Mxfp4MoeBackends.
|
||||
|
||||
Vendor families return all activation variants; the caller picks one
|
||||
via ``activation_key`` and ``is_supported_config``.
|
||||
"""
|
||||
mapping: dict[str, list[Mxfp4MoeBackend]] = {
|
||||
"deep_gemm": [Mxfp4MoeBackend.DEEPGEMM_MXFP4],
|
||||
"flashinfer_trtllm": [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
],
|
||||
"flashinfer_trtllm_afp8": [Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8],
|
||||
"flashinfer_cutlass": [
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
],
|
||||
"flashinfer_cutlass_afp8": [Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8],
|
||||
"triton": [Mxfp4MoeBackend.TRITON],
|
||||
"triton_unfused": [Mxfp4MoeBackend.TRITON_UNFUSED],
|
||||
"humming": [Mxfp4MoeBackend.HUMMING],
|
||||
"marlin": [Mxfp4MoeBackend.MARLIN],
|
||||
"aiter": [
|
||||
Mxfp4MoeBackend.AITER_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.AITER_MXFP4_FP8,
|
||||
],
|
||||
"aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8],
|
||||
"xpu": [Mxfp4MoeBackend.XPU],
|
||||
"emulation": [Mxfp4MoeBackend.EMULATION],
|
||||
}
|
||||
if backend := mapping.get(runner_backend):
|
||||
return backend
|
||||
if backends := mapping.get(runner_backend):
|
||||
return backends
|
||||
raise ValueError(
|
||||
f"moe_backend='{runner_backend}' is not supported for MXFP4 MoE. "
|
||||
f"Expected one of {list(mapping.keys())}."
|
||||
@@ -233,16 +248,16 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend:
|
||||
|
||||
|
||||
def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
"""
|
||||
Get available backends in priority order based on platform and config.
|
||||
Only includes BF16 backends. MXFP8 backends are selected via env vars.
|
||||
"""
|
||||
"""Available backends in priority order, BF16-act variant before
|
||||
activation-quantized variant within each vendor family."""
|
||||
_AVAILABLE_BACKENDS = [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.AITER_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.AITER_MXFP4_FP8,
|
||||
Mxfp4MoeBackend.TRITON,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
# TRITON_UNFUSED has bug with MTP support
|
||||
# TODO re-enable after kernel is fixed
|
||||
# TRITON_UNFUSED
|
||||
@@ -288,6 +303,79 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
|
||||
return None # BF16 activation
|
||||
|
||||
|
||||
def _user_moe_activation_override() -> QuantKey | None:
|
||||
"""User's MoE activation override from quantization_config, or None."""
|
||||
args = get_current_vllm_config().model_config.quantization_config
|
||||
if not isinstance(args, QuantizationConfigArgs) or args.moe is None:
|
||||
return None
|
||||
return args.moe.activation
|
||||
|
||||
|
||||
def _resolve_activation_key(
|
||||
model_activation_key: QuantKey | None,
|
||||
) -> QuantKey | None:
|
||||
"""Combine the model-supplied activation key with the user override.
|
||||
Raises on conflict (both set and disagreeing)."""
|
||||
user_override = _user_moe_activation_override()
|
||||
if user_override is None:
|
||||
return model_activation_key
|
||||
if model_activation_key is None or model_activation_key == user_override:
|
||||
return user_override
|
||||
raise ValueError(
|
||||
f"checkpoint declares MoE activation={model_activation_key} but "
|
||||
f"quantization_config.moe.activation={user_override}; remove the "
|
||||
f"override or align it with the checkpoint."
|
||||
)
|
||||
|
||||
|
||||
def _make_log_backend(backend: Mxfp4MoeBackend) -> str:
|
||||
return f"Using '{backend.value}' Mxfp4 MoE backend."
|
||||
|
||||
|
||||
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
|
||||
base = (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
f"deployment configuration"
|
||||
)
|
||||
return f"{base} since {reason}." if reason else f"{base}."
|
||||
|
||||
|
||||
def _return_or_raise(
|
||||
backend: Mxfp4MoeBackend,
|
||||
config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
scope: Literal["process", "global", "local"] = "local",
|
||||
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
reason: str | None = None
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend), scope=scope)
|
||||
return backend, k_cls
|
||||
raise ValueError(_make_log_unsupported(backend, reason))
|
||||
|
||||
|
||||
def _filter_by_activation(
|
||||
backends: list[Mxfp4MoeBackend],
|
||||
requested_activation_key: QuantKey | None,
|
||||
) -> list[Mxfp4MoeBackend]:
|
||||
"""Pick variants matching ``requested_activation_key``; without one,
|
||||
prefer BF16 if the list has any, else keep the list as-is so explicit
|
||||
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
|
||||
if requested_activation_key is not None:
|
||||
return [
|
||||
b
|
||||
for b in backends
|
||||
if _backend_activation_key(b) == requested_activation_key
|
||||
]
|
||||
bf16 = [b for b in backends if _backend_activation_key(b) is None]
|
||||
return bf16 if bf16 else backends
|
||||
|
||||
|
||||
def select_mxfp4_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
activation_key: QuantKey | None = None,
|
||||
@@ -303,8 +391,7 @@ def select_mxfp4_moe_backend(
|
||||
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
"""
|
||||
# If activation_key is explicitly provided (e.g., W4A8), use it
|
||||
requested_activation_key = activation_key
|
||||
requested_activation_key = _resolve_activation_key(activation_key)
|
||||
|
||||
activation_format = (
|
||||
mk.FusedMoEActivationFormat.BatchedExperts
|
||||
@@ -312,67 +399,55 @@ def select_mxfp4_moe_backend(
|
||||
else mk.FusedMoEActivationFormat.Standard
|
||||
)
|
||||
|
||||
def _make_log_backend(backend: Mxfp4MoeBackend):
|
||||
return f"Using '{backend.value}' Mxfp4 MoE backend."
|
||||
|
||||
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
|
||||
if reason:
|
||||
return (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
f"deployment configuration since {reason}."
|
||||
)
|
||||
return (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
"deployment configuration."
|
||||
)
|
||||
|
||||
def _return_or_raise(
|
||||
backend: Mxfp4MoeBackend,
|
||||
config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
reason: str | None = None
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
return backend, k_cls
|
||||
raise ValueError(_make_log_unsupported(backend, reason))
|
||||
|
||||
runner_backend = config.moe_backend
|
||||
if runner_backend != "auto":
|
||||
requested_backend = map_mxfp4_backend(runner_backend)
|
||||
if (
|
||||
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
|
||||
and requested_backend == Mxfp4MoeBackend.MARLIN
|
||||
):
|
||||
requested_backend = Mxfp4MoeBackend.BATCHED_MARLIN
|
||||
# Use requested_activation_key if provided, otherwise use backend default
|
||||
act_key = (
|
||||
requested_activation_key
|
||||
if requested_activation_key is not None
|
||||
else _backend_activation_key(requested_backend)
|
||||
)
|
||||
return _return_or_raise(
|
||||
requested_backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
act_key,
|
||||
activation_format,
|
||||
)
|
||||
requested_backends = map_mxfp4_backend(runner_backend)
|
||||
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
requested_backends = [
|
||||
Mxfp4MoeBackend.BATCHED_MARLIN if b == Mxfp4MoeBackend.MARLIN else b
|
||||
for b in requested_backends
|
||||
]
|
||||
candidates = _filter_by_activation(requested_backends, requested_activation_key)
|
||||
if not candidates:
|
||||
raise ValueError(
|
||||
f"moe_backend={runner_backend!r} does not support "
|
||||
f"activation={requested_activation_key}; supported variants: "
|
||||
f"{[b.name for b in requested_backends]}"
|
||||
)
|
||||
last_error: Exception | None = None
|
||||
for requested_backend in candidates:
|
||||
act_key = (
|
||||
requested_activation_key
|
||||
if requested_activation_key is not None
|
||||
else _backend_activation_key(requested_backend)
|
||||
)
|
||||
try:
|
||||
return _return_or_raise(
|
||||
requested_backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
act_key,
|
||||
activation_format,
|
||||
)
|
||||
except ValueError as e:
|
||||
last_error = e
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
# Select kernels in order of backend.
|
||||
AVAILABLE_BACKENDS = _get_priority_backends_for_gpt_oss()
|
||||
AVAILABLE_BACKENDS = _filter_by_activation(
|
||||
_get_priority_backends_for_gpt_oss(), requested_activation_key
|
||||
)
|
||||
|
||||
# Handle explicit FlashInfer MXFP4 BF16 configuration.
|
||||
if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"):
|
||||
if not envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16:
|
||||
AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16)
|
||||
AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16)
|
||||
for _b in (
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
):
|
||||
if _b in AVAILABLE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(_b)
|
||||
else:
|
||||
if current_platform.is_device_capability(90):
|
||||
return _return_or_raise(
|
||||
@@ -481,54 +556,30 @@ def select_deepseek_v4_mxfp4_moe_backend(
|
||||
else mk.FusedMoEActivationFormat.Standard
|
||||
)
|
||||
|
||||
def _make_log_backend(backend: Mxfp4MoeBackend):
|
||||
return f"Using '{backend.value}' Mxfp4 MoE backend."
|
||||
|
||||
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
|
||||
if reason:
|
||||
return (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
f"deployment configuration since {reason}."
|
||||
)
|
||||
return (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
"deployment configuration."
|
||||
)
|
||||
|
||||
def _return_or_raise(
|
||||
backend: Mxfp4MoeBackend,
|
||||
config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
reason: str | None = None
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend), scope="local")
|
||||
return backend, k_cls
|
||||
raise ValueError(_make_log_unsupported(backend, reason))
|
||||
|
||||
# Honor explicit moe_backend (e.g. "marlin", "triton_unfused") before
|
||||
# falling back to the auto priority list.
|
||||
runner_backend = config.moe_backend
|
||||
if runner_backend != "auto":
|
||||
requested_backend = map_mxfp4_backend(runner_backend)
|
||||
if (
|
||||
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
|
||||
and requested_backend == Mxfp4MoeBackend.MARLIN
|
||||
):
|
||||
requested_backend = Mxfp4MoeBackend.BATCHED_MARLIN
|
||||
return _return_or_raise(
|
||||
requested_backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
_backend_activation_key(requested_backend),
|
||||
activation_format,
|
||||
)
|
||||
requested_backends = map_mxfp4_backend(runner_backend)
|
||||
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
requested_backends = [
|
||||
Mxfp4MoeBackend.BATCHED_MARLIN if b == Mxfp4MoeBackend.MARLIN else b
|
||||
for b in requested_backends
|
||||
]
|
||||
last_error: Exception | None = None
|
||||
for requested_backend in requested_backends:
|
||||
try:
|
||||
return _return_or_raise(
|
||||
requested_backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
_backend_activation_key(requested_backend),
|
||||
activation_format,
|
||||
)
|
||||
except ValueError as e:
|
||||
last_error = e
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
# DeepSeek-V4 on ROCm is more accurate with the unfused Triton MXFP4 path
|
||||
# than the default AITER path. Prefer Triton-unfused for this routing mode,
|
||||
|
||||
@@ -35,10 +35,9 @@ QuantizationMethods = Literal[
|
||||
"deepseek_v4_fp8",
|
||||
"cpu_awq",
|
||||
"online",
|
||||
# Below are values of the OnlineQuantScheme enum, specified as strings to
|
||||
# avoid circular import issues. This is here to provide a shortcut where
|
||||
# the user can specify "LLM(..., quantization='fp8_per_tensor')" as
|
||||
# shorthand for creating a more complicated online quant config object
|
||||
# Below are online quant shorthand names (see vllm.config.quantization).
|
||||
# Listed here as strings to avoid a circular import; kept in sync with
|
||||
# _ONLINE_SHORTHANDS by the assertion in get_quantization_config().
|
||||
"fp8_per_tensor",
|
||||
"fp8_per_block",
|
||||
"int8_per_channel_weight_only",
|
||||
@@ -111,7 +110,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
raise ValueError(f"Invalid quantization method: {quantization}")
|
||||
|
||||
# lazy import to avoid triggering `torch.compile` too early
|
||||
from vllm.config.quantization import OnlineQuantScheme
|
||||
from vllm.config.quantization import _ONLINE_SHORTHANDS
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.models.deepseek_v4 import DeepseekV4FP8Config
|
||||
|
||||
@@ -171,16 +170,15 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
"online": OnlineQuantizationConfig,
|
||||
}
|
||||
|
||||
# Below are values of the OnlineQuantScheme enum. This is here to provide
|
||||
# a shortcut where the user can specify
|
||||
# "LLM(..., quantization='fp8_per_tensor')" as shorthand for creating a
|
||||
# more complicated online quant config object
|
||||
for scheme in OnlineQuantScheme:
|
||||
assert scheme.value not in method_to_config, (
|
||||
f"Online quant scheme {scheme.value!r} conflicts with an "
|
||||
# Register online shorthands as quantization methods so the user can
|
||||
# specify "LLM(..., quantization='fp8_per_tensor')" as shorthand for
|
||||
# creating a more complicated online quant config object.
|
||||
for shorthand in _ONLINE_SHORTHANDS:
|
||||
assert shorthand not in method_to_config, (
|
||||
f"Online quant shorthand {shorthand!r} conflicts with an "
|
||||
f"existing quantization method"
|
||||
)
|
||||
method_to_config[scheme.value] = OnlineQuantizationConfig
|
||||
method_to_config[shorthand] = OnlineQuantizationConfig
|
||||
|
||||
# Update the `method_to_config` with customized quantization methods.
|
||||
method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
|
||||
|
||||
@@ -5,10 +5,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.quantization import (
|
||||
OnlineQuantizationConfigArgs,
|
||||
OnlineQuantScheme,
|
||||
)
|
||||
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
@@ -41,31 +38,50 @@ from vllm.model_executor.layers.quantization.online.mxfp8 import (
|
||||
Mxfp8OnlineLinearMethod,
|
||||
Mxfp8OnlineMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Online dispatch tables, keyed by the QuantSpec.weight QuantKey. The
|
||||
# corresponding method class handles the activation choice via its
|
||||
# `supported_activation_quant` set.
|
||||
_ONLINE_LINEAR_METHODS: dict[QuantKey, type] = {
|
||||
kFp8StaticTensorSym: Fp8PerTensorOnlineLinearMethod,
|
||||
kFp8Static128BlockSym: Fp8PerBlockOnlineLinearMethod,
|
||||
kMxfp8Dynamic: Mxfp8OnlineLinearMethod,
|
||||
}
|
||||
|
||||
_ONLINE_MOE_METHODS: dict[QuantKey, type] = {
|
||||
kFp8StaticTensorSym: Fp8PerTensorOnlineMoEMethod,
|
||||
kFp8Static128BlockSym: Fp8PerBlockOnlineMoEMethod,
|
||||
kMxfp8Dynamic: Mxfp8OnlineMoEMethod,
|
||||
kInt8StaticChannelSym: Int8OnlineMoEMethod,
|
||||
}
|
||||
|
||||
|
||||
class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"""Model-level config class for online quantization (quantize fp16/bf16 weights
|
||||
"""Model-level config for online quantization (quantize fp16/bf16 weights
|
||||
during model loading, without requiring a pre-quantized checkpoint)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
args: OnlineQuantizationConfigArgs,
|
||||
args: QuantizationConfigArgs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if (
|
||||
args.global_scheme is None
|
||||
and args.linear_scheme_override is None
|
||||
and args.moe_scheme_override is None
|
||||
):
|
||||
if args.linear is None and args.moe is None:
|
||||
raise ValueError(
|
||||
"OnlineQuantizationConfig requires at least one of "
|
||||
"global_scheme, linear_scheme_override, or "
|
||||
"moe_scheme_override to be set."
|
||||
"quantization_config.linear or quantization_config.moe "
|
||||
"to be set."
|
||||
)
|
||||
self.args = args
|
||||
self.quant_scheme = args.global_scheme
|
||||
self.ignored_layers: list[str] = args.ignore
|
||||
|
||||
@classmethod
|
||||
@@ -94,6 +110,33 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"quantization='fp8_per_tensor'/'fp8_per_block' instead."
|
||||
)
|
||||
|
||||
def _dispatch(
|
||||
self,
|
||||
spec: QuantSpec | None,
|
||||
table: dict[QuantKey, type],
|
||||
layer: torch.nn.Module,
|
||||
) -> "QuantizeMethodBase | None":
|
||||
if spec is None or spec.weight is None:
|
||||
return None
|
||||
cls = table.get(spec.weight)
|
||||
if cls is None:
|
||||
raise ValueError(
|
||||
f"online quantization for {type(layer).__name__} with "
|
||||
f"weight={spec.weight} is not supported; supported weight "
|
||||
f"keys: {sorted(str(k) for k in table)}"
|
||||
)
|
||||
# Online method classes pick their own activation format internally.
|
||||
# Per-class activation overrides are not yet wired through; reject
|
||||
# explicit overrides until the relevant method class opts in.
|
||||
if spec.activation is not None:
|
||||
raise ValueError(
|
||||
f"activation override (activation={spec.activation}) is not "
|
||||
f"yet supported for online {cls.__name__}"
|
||||
)
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return cls(layer=layer)
|
||||
return cls()
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
@@ -104,20 +147,8 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
|
||||
linear_scheme = self.args.linear_scheme_override or self.args.global_scheme
|
||||
if linear_scheme == OnlineQuantScheme.INT8_PER_CHANNEL_WEIGHT_ONLY:
|
||||
logger.warning_once(
|
||||
"INT8 online quantization only quantizes MoE expert "
|
||||
"weights. linear layers remain in full precision."
|
||||
)
|
||||
return UnquantizedLinearMethod()
|
||||
elif linear_scheme == OnlineQuantScheme.FP8_PER_BLOCK:
|
||||
return Fp8PerBlockOnlineLinearMethod()
|
||||
elif linear_scheme == OnlineQuantScheme.MXFP8:
|
||||
return Mxfp8OnlineLinearMethod()
|
||||
else:
|
||||
return Fp8PerTensorOnlineLinearMethod()
|
||||
method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer)
|
||||
return method if method is not None else UnquantizedLinearMethod()
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
@@ -125,14 +156,10 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
|
||||
moe_scheme = self.args.moe_scheme_override or self.args.global_scheme
|
||||
if moe_scheme == OnlineQuantScheme.INT8_PER_CHANNEL_WEIGHT_ONLY:
|
||||
return Int8OnlineMoEMethod(layer=layer)
|
||||
elif moe_scheme == OnlineQuantScheme.FP8_PER_BLOCK:
|
||||
return Fp8PerBlockOnlineMoEMethod(layer=layer)
|
||||
elif moe_scheme == OnlineQuantScheme.MXFP8:
|
||||
return Mxfp8OnlineMoEMethod(layer=layer)
|
||||
else:
|
||||
return Fp8PerTensorOnlineMoEMethod(layer=layer)
|
||||
method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer)
|
||||
return (
|
||||
method
|
||||
if method is not None
|
||||
else UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -300,12 +300,10 @@ def get_quant_config(
|
||||
)
|
||||
|
||||
if hf_quant_config is not None:
|
||||
if model_config.quantization_config is not None:
|
||||
raise ValueError(
|
||||
"Setting `quantization_config` for online "
|
||||
"quantization when the model checkpoint already "
|
||||
"has a `quantization_config` is not supported"
|
||||
)
|
||||
# `model_config.quantization_config` may be set alongside a checkpoint
|
||||
# quant config: the checkpoint determines `quant_cls`, and the user's
|
||||
# QuantizationConfigArgs is consulted by individual quant methods
|
||||
# (e.g. for activation overrides via the MXFP4 oracle).
|
||||
|
||||
# For modelopt_mixed, config.json's quantization_config may or may
|
||||
# not contain the per-layer quantized_layers map. Newer checkpoints
|
||||
@@ -330,12 +328,6 @@ def get_quant_config(
|
||||
quantization_config_file = hf_overrides.get("quantization_config_file", None)
|
||||
if quantization_config_file is not None:
|
||||
if hasattr(quant_cls, "from_config_file"):
|
||||
if model_config.quantization_config is not None:
|
||||
raise ValueError(
|
||||
"Setting `quantization_config` for online "
|
||||
"quantization when the model checkpoint already "
|
||||
"has a `quantization_config` is not supported"
|
||||
)
|
||||
return quant_cls.from_config_file(quantization_config_file)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
@@ -346,12 +338,6 @@ def get_quant_config(
|
||||
quantization_config_json = hf_overrides.get("quantization_config_dict_json", None)
|
||||
if quantization_config_json is not None:
|
||||
if hasattr(quant_cls, "from_config_dict_json"):
|
||||
if model_config.quantization_config is not None:
|
||||
raise ValueError(
|
||||
"Setting `quantization_config` for online "
|
||||
"quantization when the model checkpoint already "
|
||||
"has a `quantization_config` is not supported"
|
||||
)
|
||||
return quant_cls.from_config_dict_json(quantization_config_json)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
@@ -360,17 +346,15 @@ def get_quant_config(
|
||||
f"{quant_cls}"
|
||||
)
|
||||
|
||||
# Online quantization doesn't read from checkpoint configs — it quantizes
|
||||
# Online quantization doesn't read from checkpoint configs - it quantizes
|
||||
# fp16/bf16 weights on the fly during loading.
|
||||
if model_config.quantization_config is not None:
|
||||
from vllm.config.quantization import OnlineQuantizationConfigArgs
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
|
||||
assert isinstance(
|
||||
model_config.quantization_config, OnlineQuantizationConfigArgs
|
||||
)
|
||||
assert isinstance(model_config.quantization_config, QuantizationConfigArgs)
|
||||
return OnlineQuantizationConfig(args=model_config.quantization_config)
|
||||
|
||||
# Inflight BNB quantization
|
||||
|
||||
Reference in New Issue
Block a user