[Quantization] Honor --linear-backend for ModelOpt W4A16 (#50273)
Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
c27b080d26
commit
70bd10930b
@@ -12,8 +12,6 @@ import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MarlinNvFp4LinearKernel,
|
||||
NvFp4LinearLayerConfig,
|
||||
init_fp8_linear_kernel,
|
||||
init_mxfp8_linear_kernel,
|
||||
init_nvfp4_linear_kernel,
|
||||
@@ -1247,16 +1245,16 @@ class ModelOptNvFp4W4A16LinearMethod(LinearMethodBase):
|
||||
"""Linear method for ModelOpt NVFP4 W4A16.
|
||||
|
||||
4-bit NVFP4 weights, fp16/bf16 activations. Loads ModelOpt-style names
|
||||
directly (no on-disk conversion) and dispatches to the FP4 Marlin GEMM:
|
||||
directly (no on-disk conversion) and dispatches to a W4A16 GEMM:
|
||||
|
||||
weight uint8 packed NVFP4 (2 nibbles/byte along input dim)
|
||||
weight_scale fp8-e4m3 per 16-elem group along input dim
|
||||
weight_scale_2 fp32 per-tensor global scale = amax / (6.0 * 448.0)
|
||||
|
||||
No activation quantization. Marlin expects the global scale in the same
|
||||
form ModelOpt stores (amax/2688), so we rename weight_scale_2 ->
|
||||
weight_global_scale **without reciprocation** -- the CT W4A16 path
|
||||
reciprocates only because CT stores the inverse on disk.
|
||||
No activation quantization. ModelOpt stores the global scale as
|
||||
amax/2688, so we rename weight_scale_2 -> weight_global_scale without
|
||||
reciprocation. The selected kernel converts it to its runtime format.
|
||||
The CT W4A16 path reciprocates because CT stores the inverse on disk.
|
||||
|
||||
We also register a placeholder input_scale parameter so that W4A4-shaped
|
||||
checkpoints (which contain *_proj.input_scale tensors) can be loaded
|
||||
@@ -1267,17 +1265,12 @@ class ModelOptNvFp4W4A16LinearMethod(LinearMethodBase):
|
||||
|
||||
def __init__(self, quant_config: ModelOptNvFp4Config) -> None:
|
||||
self.quant_config = quant_config
|
||||
# Vestigial slot mirrored from ModelOptNvFp4LinearMethod: the parent
|
||||
# config's get_quant_method only fills marlin_input_dtype when
|
||||
# backend == "marlin"; we don't set that since we pin the kernel
|
||||
# below, but we keep the attribute for shape parity.
|
||||
self.marlin_input_dtype = None
|
||||
# Direct-instantiate the Marlin NVFP4 adapter rather than going through
|
||||
# init_nvfp4_linear_kernel(): the latter's priority list returns a
|
||||
# cutlass W4A4 kernel as first-pick on this hardware, which would
|
||||
# silently try to quantize activations (we have no input_scale). For
|
||||
# W4A16 there is exactly one valid kernel, so we pin it.
|
||||
self.kernel = MarlinNvFp4LinearKernel(NvFp4LinearLayerConfig())
|
||||
# `init_nvfp4_linear_kernel(use_a16=True)` is best of both worlds:
|
||||
# 1. `use_a16=True` forces `Marlin`: https://github.com/vllm-project/vllm/commit/e68988a#diff-7135ab92aa94dfacb1ad3c77fc13f9c4ffe0b977f8eac5d86c2afe243e5f92a6R842-R889
|
||||
# for `--linear-backend=auto`, avoiding a W4A4 kernel that requires input_scale.
|
||||
# 2. Specifying e.g. `--linear-backend=humming` will override.
|
||||
self.kernel = init_nvfp4_linear_kernel(use_a16=True)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -1300,6 +1293,7 @@ class ModelOptNvFp4W4A16LinearMethod(LinearMethodBase):
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.output_partition_sizes = output_partition_sizes
|
||||
|
||||
if input_size_per_partition % 16 != 0:
|
||||
raise ValueError(
|
||||
@@ -1355,6 +1349,9 @@ class ModelOptNvFp4W4A16LinearMethod(LinearMethodBase):
|
||||
layer.register_parameter("input_scale", input_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if not hasattr(layer, "has_bias"):
|
||||
layer.has_bias = getattr(layer, "bias", None) is not None
|
||||
|
||||
# Discard the input_scale placeholder. Whether it carries values
|
||||
# (W4A4 ckpt loaded as W4A16) or is uninitialized (native W4A16
|
||||
# ckpt), W4A16 mode does not quantize activations, so this is unused.
|
||||
|
||||
Reference in New Issue
Block a user