[ModelOpt] Redesign the LinearMethod classes using the generic QuantKey-driven method (#49381)

Signed-off-by: Juhi Mittal <juhim@nvidia.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Juhi Mittal
2026-09-01 16:28:56 -04:00
committed by GitHub
co-authored by Claude Opus 4.8 mgoin
parent 4bf06be985
commit 7a977c0699
6 changed files with 1173 additions and 956 deletions
@@ -6,6 +6,8 @@ from dataclasses import dataclass
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
@dataclass
class Mxfp8LinearLayerConfig:
@@ -30,6 +32,12 @@ class Mxfp8LinearKernel(ABC):
assert self.is_supported()[0]
self.config = c
def input_quant_key(self) -> QuantKey | None:
"""Return the input quantization key supported by this kernel. If the kernel
does not support input quantization outside of the kernel, return None.
"""
return None
@classmethod
@abstractmethod
def is_supported(
@@ -377,12 +377,18 @@ def _get_kv_b_proj_input_dtype(
return None
if weight_dtype == current_platform.fp8_dtype():
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptFp8PbWoLinearMethod,
ModelOptLinearMethod,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Static128BlockSym,
)
quant_method = getattr(kv_b_proj, "quant_method", None)
# FP8_PB_WO dynamically quantizes BF16/FP16 inputs in the linear method.
if isinstance(quant_method, ModelOptFp8PbWoLinearMethod):
if (
isinstance(quant_method, ModelOptLinearMethod)
and quant_method.spec.weight is kFp8Static128BlockSym
):
return quant_method.input_dtype
if not use_fp8_prefill:
return None
+2 -5
View File
@@ -55,13 +55,10 @@ WEIGHT_LOADER_V2_SUPPORTED = [
"AutoGPTQLinearMethod",
"Fp8LinearMethod",
"FBGEMMFp8LinearMethod",
"ModelOptFp8LinearMethod",
"ModelOptFp8PcPtLinearMethod",
"ModelOptFp8PbWoLinearMethod",
"QuarkLinearMethod",
"ModelOptNvFp4LinearMethod",
"ModelOptNvFp4W4A16LinearMethod",
"HummingLinearMethod",
# ModelOptLinearMethod self-registers via
# register_weight_loader_v2_supported_method (see modelopt.py).
]
File diff suppressed because it is too large Load Diff
@@ -38,6 +38,11 @@ from vllm.utils.platform_utils import get_device_name_as_file_name
logger = init_logger(__name__)
# Pre-fill value for scale parameters whose shards load independently. The
# shards are combined with .max(), so an unloaded shard must never win; the
# smallest representable float32 guarantees that.
FP8_SCALE_SENTINEL = torch.finfo(torch.float32).min
def is_fp8(x: torch.dtype | torch.Tensor) -> bool:
if isinstance(x, torch.Tensor):