[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
a11dfcff89
commit
5690b02c03
@@ -4,7 +4,10 @@
|
||||
from typing import Literal, get_args
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
resolve_quant_method,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -182,6 +185,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
|
||||
__all__ = [
|
||||
"QuantizationConfig",
|
||||
"resolve_quant_method",
|
||||
"QuantizationMethods",
|
||||
"get_quantization_config",
|
||||
"register_quantization_config",
|
||||
|
||||
@@ -12,6 +12,9 @@ from transformers import PretrainedConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
else:
|
||||
QuantizationMethods = str
|
||||
@@ -105,6 +108,7 @@ class QuantizationConfig(ABC):
|
||||
"""Suffixes of quantization parameters that may be present in the checkpoint but
|
||||
not in the model, and should be ignored if unexpected during loading. These are used
|
||||
after remapping, so should be in vLLM format (e.g. .q_scale, not .q.scale)."""
|
||||
online_quantization_config: "OnlineQuantizationConfig | None" = None
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -187,7 +191,8 @@ class QuantizationConfig(ABC):
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> QuantizeMethodBase | None:
|
||||
"""Get the quantize method to use for the quantized layer.
|
||||
"""Get the quantize method to use for the quantized layer, from the
|
||||
pre-quantized checkpoint quant_method.
|
||||
|
||||
Args:
|
||||
layer: The layer for the quant method.
|
||||
@@ -265,3 +270,46 @@ class QuantizationConfig(ABC):
|
||||
# TODO: revision is never passed currently in vllm.py,
|
||||
# but is used in subclasses, should we remove this parameter?
|
||||
pass
|
||||
|
||||
|
||||
def resolve_quant_method(
|
||||
quant_config: QuantizationConfig, layer: torch.nn.Module, prefix: str
|
||||
) -> QuantizeMethodBase | None:
|
||||
"""Return the checkpoint method with configured online quantization."""
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
|
||||
base_quant_method = quant_config.get_quant_method(layer, prefix)
|
||||
if quant_config.online_quantization_config is None:
|
||||
return base_quant_method
|
||||
# Online quantization currently supports only LinearBase and RoutedExperts.
|
||||
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
|
||||
if not isinstance(layer, (LinearBase, RoutedExperts)):
|
||||
return base_quant_method
|
||||
|
||||
quant_config.online_quantization_config.packed_modules_mapping = (
|
||||
quant_config.packed_modules_mapping
|
||||
)
|
||||
checkpoint_is_quantized = base_quant_method is not None and not isinstance(
|
||||
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
|
||||
)
|
||||
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
|
||||
layer, prefix
|
||||
)
|
||||
if checkpoint_is_quantized:
|
||||
if online_target is not None:
|
||||
raise ValueError(
|
||||
f"Cannot apply requested online quantization {online_target[3]} to "
|
||||
f"pre-quantized layer {prefix}: {base_quant_method} was already "
|
||||
"selected by the checkpoint quantization config."
|
||||
)
|
||||
return base_quant_method
|
||||
if online_target is None:
|
||||
return base_quant_method
|
||||
return quant_config.online_quantization_config.get_quant_method(layer, prefix)
|
||||
|
||||
@@ -56,5 +56,5 @@ class ExpertsInt8Config(QuantizationConfig):
|
||||
if isinstance(layer, LinearBase):
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
return Int8OnlineMoEMethod(layer=layer)
|
||||
return Int8OnlineMoEMethod(moe=layer.moe_config)
|
||||
return None
|
||||
|
||||
@@ -219,7 +219,7 @@ class Fp8Config(QuantizationConfig):
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
)
|
||||
|
||||
return Fp8PerTensorOnlineMoEMethod(layer=layer)
|
||||
return Fp8PerTensorOnlineMoEMethod(moe=layer.moe_config)
|
||||
elif isinstance(layer, Attention):
|
||||
return Fp8KVCacheMethod(self)
|
||||
return None
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
from enum import Enum
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
@@ -14,6 +15,7 @@ from vllm.config.quantization import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEMethodBase,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
@@ -38,6 +40,7 @@ from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
Fp8PtpcOnlineLinearMethod,
|
||||
Fp8PtpcOnlineMoEMethod,
|
||||
OnlineLinearBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.int8 import (
|
||||
Int8OnlineMoEMethod,
|
||||
@@ -71,6 +74,14 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OnlineQuantizationSource(str, Enum):
|
||||
"""Supported online quantization configuration sources."""
|
||||
|
||||
linear = "linear" # LinearBase
|
||||
moe = "moe" # RoutedExperts
|
||||
targets = "targets"
|
||||
|
||||
|
||||
# Online dispatch tables, keyed by the QuantSpec.weight QuantKey. The
|
||||
# corresponding method class handles the activation choice via its
|
||||
# `supported_activation_quant` set.
|
||||
@@ -200,12 +211,23 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"quantization='fp8_per_tensor'/'fp8_per_block' instead."
|
||||
)
|
||||
|
||||
def _dispatch(
|
||||
def _get_method_cls(
|
||||
self,
|
||||
spec: QuantSpec | None,
|
||||
table: dict[QuantKey, type],
|
||||
layer: torch.nn.Module,
|
||||
) -> "QuantizeMethodBase | None":
|
||||
) -> type | None:
|
||||
"""Resolve the online method class for a layer's quantization spec.
|
||||
|
||||
Args:
|
||||
spec: Quantization specification to resolve.
|
||||
table: Mapping from weight quantization keys to method classes.
|
||||
layer: Layer that will use the resolved method.
|
||||
|
||||
Returns:
|
||||
The matching method class, or None when ``spec`` has no weight
|
||||
quantization.
|
||||
"""
|
||||
if spec is None or spec.weight is None:
|
||||
return None
|
||||
cls = table.get(spec.weight)
|
||||
@@ -223,13 +245,75 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
f"activation override (activation={spec.activation}) is not "
|
||||
f"yet supported for online {cls.__name__}"
|
||||
)
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return cls(layer=layer)
|
||||
return cls()
|
||||
return cls
|
||||
|
||||
def _dispatch_target(
|
||||
def resolve_quant_method_cls(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> tuple[OnlineQuantizationSource, str, str | None, QuantSpec, type] | None:
|
||||
"""Resolve quantization metadata and method class without instantiating it.
|
||||
|
||||
Args:
|
||||
layer: Layer for which to resolve online quantization.
|
||||
prefix: Fully qualified layer name.
|
||||
|
||||
Returns:
|
||||
A tuple of source, quantization key string, target pattern, spec,
|
||||
and method class. Returns None when online quantization does not
|
||||
apply to the layer.
|
||||
"""
|
||||
quant_spec: QuantSpec | None
|
||||
if self.args.targets is not None:
|
||||
resolved_pattern = self._resolve_targets_quant_method_metadata(
|
||||
prefix, layer
|
||||
)
|
||||
if resolved_pattern is None:
|
||||
return None
|
||||
source, quant_key_str, target_pattern, quant_spec, table = resolved_pattern
|
||||
else:
|
||||
if isinstance(layer, LinearBase):
|
||||
source = OnlineQuantizationSource.linear
|
||||
quant_spec = self.args.linear
|
||||
table = _ONLINE_LINEAR_METHODS
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
source = OnlineQuantizationSource.moe
|
||||
quant_spec = self.args.moe
|
||||
table = _ONLINE_MOE_METHODS
|
||||
else:
|
||||
return None
|
||||
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
ignore=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
use_fnmatch=True,
|
||||
):
|
||||
return None
|
||||
quant_key_str = str(quant_spec)
|
||||
target_pattern = None
|
||||
|
||||
quant_method_cls = self._get_method_cls(quant_spec, table, layer)
|
||||
if quant_method_cls is None:
|
||||
return None
|
||||
assert quant_spec is not None
|
||||
return source, quant_key_str, target_pattern, quant_spec, quant_method_cls
|
||||
|
||||
def _resolve_targets_quant_method_metadata(
|
||||
self, prefix: str, layer: torch.nn.Module
|
||||
) -> "QuantizeMethodBase | None":
|
||||
) -> (
|
||||
tuple[OnlineQuantizationSource, str, str, QuantSpec, dict[QuantKey, type]]
|
||||
| None
|
||||
):
|
||||
"""Resolve target-pattern quantization metadata and dispatch table.
|
||||
|
||||
Args:
|
||||
prefix: Fully qualified layer name.
|
||||
layer: Layer matched against configured target patterns.
|
||||
|
||||
Returns:
|
||||
A tuple of source, quantization key string, target pattern, spec,
|
||||
and dispatch table. Returns None when no pattern applies or the
|
||||
layer is ignored.
|
||||
"""
|
||||
assert self.args.targets is not None
|
||||
ignored = should_ignore_layer(
|
||||
prefix,
|
||||
@@ -254,7 +338,8 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
f"targets patterns: {matches}. Each layer may match at most "
|
||||
f"one target."
|
||||
)
|
||||
quant_key_str = self.args.targets[matches[0]]
|
||||
target_pattern = matches[0]
|
||||
quant_key_str = self.args.targets[target_pattern]
|
||||
shorthand = _ONLINE_SHORTHANDS[quant_key_str]
|
||||
if isinstance(layer, LinearBase):
|
||||
quant_spec = shorthand.linear
|
||||
@@ -265,63 +350,44 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Layer {prefix} was matched by quantization_config.targets "
|
||||
f"({matches[0]}), but online quantization is not supported for "
|
||||
f"({target_pattern}), but online quantization is not supported for "
|
||||
f"{type(layer).__name__}."
|
||||
)
|
||||
if quant_spec is None:
|
||||
raise ValueError(
|
||||
f"targets pattern {matches[0]} = {quant_key_str} does "
|
||||
f"targets pattern {target_pattern} = {quant_key_str} does "
|
||||
f"not define a QuantSpec for {type(layer).__name__} layers "
|
||||
f"(matched at {prefix})."
|
||||
)
|
||||
method = self._dispatch(quant_spec, table, layer)
|
||||
if method is not None:
|
||||
self.quantized_layers[prefix] = ("targets", quant_key_str, matches[0])
|
||||
return method
|
||||
return (
|
||||
OnlineQuantizationSource.targets,
|
||||
quant_key_str,
|
||||
target_pattern,
|
||||
quant_spec,
|
||||
table,
|
||||
)
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
# `targets` takes precedence over `moe` and `linear` and is exclusive.
|
||||
if self.args.targets is not None:
|
||||
method = self._dispatch_target(prefix, layer)
|
||||
if method is not None:
|
||||
return method
|
||||
|
||||
if isinstance(layer, LinearBase):
|
||||
return UnquantizedLinearMethod()
|
||||
resolved = self.resolve_quant_method_cls(layer, prefix)
|
||||
if resolved is not None:
|
||||
source, quant_key_str, target_pattern, _, quant_method_cls = resolved
|
||||
self.quantized_layers[prefix] = (
|
||||
source.value,
|
||||
quant_key_str,
|
||||
target_pattern,
|
||||
)
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
return None
|
||||
assert issubclass(quant_method_cls, FusedMoEMethodBase)
|
||||
return quant_method_cls(moe=layer.moe_config)
|
||||
|
||||
assert issubclass(quant_method_cls, OnlineLinearBase)
|
||||
return quant_method_cls()
|
||||
|
||||
if isinstance(layer, LinearBase):
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
ignore=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
use_fnmatch=True,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer)
|
||||
if method is not None:
|
||||
self.quantized_layers[prefix] = (
|
||||
"linear",
|
||||
str(self.args.linear),
|
||||
None,
|
||||
)
|
||||
return method
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
ignore=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
use_fnmatch=True,
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer)
|
||||
if method is not None:
|
||||
self.quantized_layers[prefix] = ("moe", str(self.args.moe), None)
|
||||
return method
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
return None
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
select_fp8_moe_backend,
|
||||
)
|
||||
@@ -111,7 +112,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
|
||||
return is_row_parallel or (reduces_output_dim and is_column_parallel)
|
||||
|
||||
|
||||
class _Fp8OnlineLinearBase(LinearMethodBase):
|
||||
class OnlineLinearBase(LinearMethodBase):
|
||||
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
|
||||
weights onto meta device and materializes them just-in-time."""
|
||||
|
||||
@@ -155,7 +156,7 @@ class _Fp8OnlineLinearBase(LinearMethodBase):
|
||||
initialize_online_processing(layer)
|
||||
|
||||
|
||||
class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online tensorwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
|
||||
|
||||
@@ -256,7 +257,7 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
|
||||
class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online blockwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading."""
|
||||
|
||||
@@ -336,7 +337,7 @@ class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
)
|
||||
|
||||
|
||||
class Fp8PtpcOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online PTPC FP8 linear quantization.
|
||||
|
||||
Per-output-channel weight scale + dynamic per-token activation scale. The
|
||||
@@ -443,12 +444,12 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
self,
|
||||
*,
|
||||
weight_block_size: list[int] | None,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
weight_key: "QuantKey | None" = None,
|
||||
activation_key: "QuantKey | None" = None,
|
||||
allow_vllm_cutlass: bool = False,
|
||||
):
|
||||
super().__init__(layer.moe_config)
|
||||
super().__init__(moe)
|
||||
self.weight_block_size = weight_block_size
|
||||
self.block_quant: bool = self.weight_block_size is not None
|
||||
self.weight_scale_name = (
|
||||
@@ -555,11 +556,11 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
super().__init__(
|
||||
weight_block_size=None,
|
||||
layer=layer,
|
||||
moe=moe,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
@@ -612,11 +613,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
super().__init__(
|
||||
weight_block_size=[128, 128],
|
||||
layer=layer,
|
||||
moe=moe,
|
||||
)
|
||||
|
||||
def maybe_roundup_sizes(
|
||||
@@ -716,13 +717,13 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
|
||||
|
||||
super().__init__(
|
||||
weight_block_size=None,
|
||||
layer=layer,
|
||||
moe=moe,
|
||||
weight_key=kFp8StaticChannelSym,
|
||||
activation_key=kFp8DynamicTokenSym,
|
||||
allow_vllm_cutlass=True,
|
||||
|
||||
@@ -12,6 +12,7 @@ if TYPE_CHECKING:
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
|
||||
convert_to_int8_moe_kernel_format,
|
||||
make_int8_moe_kernel,
|
||||
@@ -38,9 +39,9 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
super().__init__(layer.moe_config)
|
||||
super().__init__(moe)
|
||||
self.int8_backend, self.experts_cls = select_int8_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=kInt8StaticChannelSym,
|
||||
|
||||
@@ -17,6 +17,7 @@ if TYPE_CHECKING:
|
||||
)
|
||||
|
||||
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
TRITON_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
@@ -27,7 +28,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
_Fp8OnlineLinearBase,
|
||||
OnlineLinearBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
@@ -73,7 +74,7 @@ def _quantize_mxfp4_moe_weight(
|
||||
return w_quant, w_scales
|
||||
|
||||
|
||||
class Mxfp4OnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Mxfp4OnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online MXFP4 linear method.
|
||||
Loads bf16/fp16 checkpoints and quantizes weights to MXFP4 (microscaling
|
||||
FP4 with block-32 scales) during weight loading.
|
||||
@@ -144,8 +145,8 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
experts_cls: "type[mk.FusedMoEExperts] | None"
|
||||
activation_quant_key = kMxfp4Dynamic
|
||||
|
||||
def __init__(self, *, layer: torch.nn.Module):
|
||||
super().__init__(layer.moe_config)
|
||||
def __init__(self, *, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.weight_block_size: list[int] = [1, MXFP4_BLOCK_SIZE]
|
||||
self.weight_scale_name = "weight_scale"
|
||||
|
||||
|
||||
@@ -8,6 +8,8 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
from torch.nn import Module
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -21,7 +23,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
_Fp8OnlineLinearBase,
|
||||
OnlineLinearBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
@@ -34,7 +36,7 @@ from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
class Mxfp8OnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online MXFP8 linear method.
|
||||
Loads bf16/fp16 checkpoints and quantizes weights to MXFP8 (microscaling
|
||||
FP8 with block-32 scales) during weight loading.
|
||||
@@ -100,8 +102,8 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
fp8_backend: "Fp8MoeBackend"
|
||||
experts_cls: "type[mk.FusedMoEExperts] | None"
|
||||
|
||||
def __init__(self, *, layer: torch.nn.Module):
|
||||
super().__init__(layer.moe_config)
|
||||
def __init__(self, *, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.weight_block_size: list[int] = [1, MXFP8_BLOCK_SIZE]
|
||||
self.weight_scale_name = "weight_scale"
|
||||
|
||||
|
||||
@@ -6,7 +6,10 @@ from torch.nn import Module
|
||||
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
convert_to_nvfp4_moe_kernel_format,
|
||||
make_nvfp4_moe_kernel,
|
||||
@@ -87,13 +90,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
raise ValueError(
|
||||
"nvfp4_per_token online quantization requires a Blackwell (SM100) GPU."
|
||||
)
|
||||
super().__init__(layer.moe_config)
|
||||
super().__init__(moe)
|
||||
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=kNvfp4Static,
|
||||
|
||||
Reference in New Issue
Block a user