[Quantization] Honor --linear-backend for ModelOpt W4A16 (#50273)

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Netanel Haber
2026-07-30 13:19:56 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent c27b080d26
commit 70bd10930b
3 changed files with 42 additions and 21 deletions
@@ -12,8 +12,6 @@ import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
MarlinNvFp4LinearKernel,
NvFp4LinearLayerConfig,
init_fp8_linear_kernel,
init_mxfp8_linear_kernel,
init_nvfp4_linear_kernel,
@@ -1247,16 +1245,16 @@ class ModelOptNvFp4W4A16LinearMethod(LinearMethodBase):
"""Linear method for ModelOpt NVFP4 W4A16.
4-bit NVFP4 weights, fp16/bf16 activations. Loads ModelOpt-style names
directly (no on-disk conversion) and dispatches to the FP4 Marlin GEMM:
directly (no on-disk conversion) and dispatches to a W4A16 GEMM:
weight uint8 packed NVFP4 (2 nibbles/byte along input dim)
weight_scale fp8-e4m3 per 16-elem group along input dim
weight_scale_2 fp32 per-tensor global scale = amax / (6.0 * 448.0)
No activation quantization. Marlin expects the global scale in the same
form ModelOpt stores (amax/2688), so we rename weight_scale_2 ->
weight_global_scale **without reciprocation** -- the CT W4A16 path
reciprocates only because CT stores the inverse on disk.
No activation quantization. ModelOpt stores the global scale as
amax/2688, so we rename weight_scale_2 -> weight_global_scale without
reciprocation. The selected kernel converts it to its runtime format.
The CT W4A16 path reciprocates because CT stores the inverse on disk.
We also register a placeholder input_scale parameter so that W4A4-shaped
checkpoints (which contain *_proj.input_scale tensors) can be loaded
@@ -1267,17 +1265,12 @@ class ModelOptNvFp4W4A16LinearMethod(LinearMethodBase):
def __init__(self, quant_config: ModelOptNvFp4Config) -> None:
self.quant_config = quant_config
# Vestigial slot mirrored from ModelOptNvFp4LinearMethod: the parent
# config's get_quant_method only fills marlin_input_dtype when
# backend == "marlin"; we don't set that since we pin the kernel
# below, but we keep the attribute for shape parity.
self.marlin_input_dtype = None
# Direct-instantiate the Marlin NVFP4 adapter rather than going through
# init_nvfp4_linear_kernel(): the latter's priority list returns a
# cutlass W4A4 kernel as first-pick on this hardware, which would
# silently try to quantize activations (we have no input_scale). For
# W4A16 there is exactly one valid kernel, so we pin it.
self.kernel = MarlinNvFp4LinearKernel(NvFp4LinearLayerConfig())
# `init_nvfp4_linear_kernel(use_a16=True)` is best of both worlds:
# 1. `use_a16=True` forces `Marlin`: https://github.com/vllm-project/vllm/commit/e68988a#diff-7135ab92aa94dfacb1ad3c77fc13f9c4ffe0b977f8eac5d86c2afe243e5f92a6R842-R889
# for `--linear-backend=auto`, avoiding a W4A4 kernel that requires input_scale.
# 2. Specifying e.g. `--linear-backend=humming` will override.
self.kernel = init_nvfp4_linear_kernel(use_a16=True)
def create_weights(
self,
@@ -1300,6 +1293,7 @@ class ModelOptNvFp4W4A16LinearMethod(LinearMethodBase):
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.output_partition_sizes = output_partition_sizes
if input_size_per_partition % 16 != 0:
raise ValueError(
@@ -1355,6 +1349,9 @@ class ModelOptNvFp4W4A16LinearMethod(LinearMethodBase):
layer.register_parameter("input_scale", input_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if not hasattr(layer, "has_bias"):
layer.has_bias = getattr(layer, "bias", None) is not None
# Discard the input_scale placeholder. Whether it carries values
# (W4A4 ckpt loaded as W4A16) or is uninitialized (native W4A16
# ckpt), W4A16 mode does not quantize activations, so this is unused.