[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792) (#36205)

Signed-off-by: Carl You <4531192+carlyou@users.noreply.github.com>
Signed-off-by: Carl Y <4531192+carlyou@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Carl Y
2026-04-02 21:16:11 -04:00
committed by GitHub
co-authored by Claude Opus 4.6
parent ee3cf45739
commit 1f5ec2889c
12 changed files with 928 additions and 17 deletions
@@ -449,6 +449,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
group_shape=GroupShape.PER_TENSOR,
compile_native=True,
)
self._quant_fp8_op = QuantFP8(
static=True,
group_shape=GroupShape.PER_TENSOR,
compile_native=True,
)
@property
def chunked_prefill_workspace_size(self) -> int:
@@ -545,9 +550,19 @@ class MLAAttention(nn.Module, AttentionLayerBase):
) -> torch.Tensor:
assert output is not None, "Output tensor must be provided."
if output_scale is not None or output_block_scale is not None:
raise NotImplementedError(
"fused output quantization is not yet supported for MLA"
use_quant = output_scale is not None or output_block_scale is not None
if use_quant:
# The fusion pass has allocated output with quantized dtype
# (FP8 or uint8 for FP4). We can't write into it directly,
# so we swap in a temp buffer for computation, then quantize
# into the real output at the end.
# NOTE(carlyou): this is temporary until kernels support fp8 output
quant_output = output
output = torch.empty(
output.shape[0],
self.num_heads * self.v_head_dim,
dtype=q.dtype,
device=output.device,
)
if attn_metadata is None:
@@ -567,6 +582,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# The zero fill is required when used with DP + EP
# to ensure all ranks within a DP group compute the
# same expert outputs.
if use_quant:
return quant_output.fill_(0)
return output.fill_(0)
if self.impl.dcp_world_size == -1:
@@ -706,6 +723,21 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# v_up projection
self._v_up_proj(attn_out, out=mqa_output_slice)
if use_quant:
# Quantize the BF16 computation result into the quantized output
actual = output[:num_actual_toks]
if output_block_scale is not None:
# NVFP4: two FP4 values packed into one uint8
fp4_data, fp4_scales = ops.scaled_fp4_quant(actual, output_scale)
quant_output[:num_actual_toks].copy_(fp4_data)
output_block_scale.copy_(fp4_scales)
else:
# Static FP8 quantization
fp8_data, _ = self._quant_fp8_op(actual, output_scale)
quant_output[:num_actual_toks].copy_(fp8_data)
return quant_output
return output_padded
def process_weights_after_loading(self, act_dtype: torch.dtype):
@@ -2069,6 +2101,14 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
understand this class
"""
def fused_output_quant_supported(self, quant_key):
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
return quant_key in (kFp8StaticTensorSym, kNvfp4Dynamic)
def __init__(
self,
num_heads: int,
@@ -2513,8 +2553,12 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
if hasattr(self.kv_b_proj, "weight")
else self.kv_b_proj.params_dtype
)
if use_fp8_prefill or _kv_b_proj_w_dtype != current_platform.fp8_dtype():
kv_c_normed = kv_c_normed.to(_kv_b_proj_w_dtype)
# For NVFP4, weights are packed uint8 — keep input in model dtype
# since the NVFP4 linear layer quantizes internally.
if (
use_fp8_prefill or _kv_b_proj_w_dtype != current_platform.fp8_dtype()
) and _kv_b_proj_w_dtype != torch.uint8:
kv_c_normed = kv_c_normed.to(self.kv_b_proj.weight.dtype)
k_pe = workspace[:toks][..., self.kv_lora_rank :].unsqueeze(1)
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(