Signed-off-by: Carl You <4531192+carlyou@users.noreply.github.com> Signed-off-by: Carl Y <4531192+carlyou@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
ee3cf45739
commit
1f5ec2889c
@@ -449,6 +449,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
group_shape=GroupShape.PER_TENSOR,
|
||||
compile_native=True,
|
||||
)
|
||||
self._quant_fp8_op = QuantFP8(
|
||||
static=True,
|
||||
group_shape=GroupShape.PER_TENSOR,
|
||||
compile_native=True,
|
||||
)
|
||||
|
||||
@property
|
||||
def chunked_prefill_workspace_size(self) -> int:
|
||||
@@ -545,9 +550,19 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
) -> torch.Tensor:
|
||||
assert output is not None, "Output tensor must be provided."
|
||||
|
||||
if output_scale is not None or output_block_scale is not None:
|
||||
raise NotImplementedError(
|
||||
"fused output quantization is not yet supported for MLA"
|
||||
use_quant = output_scale is not None or output_block_scale is not None
|
||||
if use_quant:
|
||||
# The fusion pass has allocated output with quantized dtype
|
||||
# (FP8 or uint8 for FP4). We can't write into it directly,
|
||||
# so we swap in a temp buffer for computation, then quantize
|
||||
# into the real output at the end.
|
||||
# NOTE(carlyou): this is temporary until kernels support fp8 output
|
||||
quant_output = output
|
||||
output = torch.empty(
|
||||
output.shape[0],
|
||||
self.num_heads * self.v_head_dim,
|
||||
dtype=q.dtype,
|
||||
device=output.device,
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
@@ -567,6 +582,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
# The zero fill is required when used with DP + EP
|
||||
# to ensure all ranks within a DP group compute the
|
||||
# same expert outputs.
|
||||
if use_quant:
|
||||
return quant_output.fill_(0)
|
||||
return output.fill_(0)
|
||||
|
||||
if self.impl.dcp_world_size == -1:
|
||||
@@ -706,6 +723,21 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
# v_up projection
|
||||
self._v_up_proj(attn_out, out=mqa_output_slice)
|
||||
|
||||
if use_quant:
|
||||
# Quantize the BF16 computation result into the quantized output
|
||||
actual = output[:num_actual_toks]
|
||||
if output_block_scale is not None:
|
||||
# NVFP4: two FP4 values packed into one uint8
|
||||
fp4_data, fp4_scales = ops.scaled_fp4_quant(actual, output_scale)
|
||||
quant_output[:num_actual_toks].copy_(fp4_data)
|
||||
output_block_scale.copy_(fp4_scales)
|
||||
else:
|
||||
# Static FP8 quantization
|
||||
fp8_data, _ = self._quant_fp8_op(actual, output_scale)
|
||||
quant_output[:num_actual_toks].copy_(fp8_data)
|
||||
return quant_output
|
||||
|
||||
return output_padded
|
||||
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype):
|
||||
@@ -2069,6 +2101,14 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
understand this class
|
||||
"""
|
||||
|
||||
def fused_output_quant_supported(self, quant_key):
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
|
||||
return quant_key in (kFp8StaticTensorSym, kNvfp4Dynamic)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
@@ -2513,8 +2553,12 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
if hasattr(self.kv_b_proj, "weight")
|
||||
else self.kv_b_proj.params_dtype
|
||||
)
|
||||
if use_fp8_prefill or _kv_b_proj_w_dtype != current_platform.fp8_dtype():
|
||||
kv_c_normed = kv_c_normed.to(_kv_b_proj_w_dtype)
|
||||
# For NVFP4, weights are packed uint8 — keep input in model dtype
|
||||
# since the NVFP4 linear layer quantizes internally.
|
||||
if (
|
||||
use_fp8_prefill or _kv_b_proj_w_dtype != current_platform.fp8_dtype()
|
||||
) and _kv_b_proj_w_dtype != torch.uint8:
|
||||
kv_c_normed = kv_c_normed.to(self.kv_b_proj.weight.dtype)
|
||||
|
||||
k_pe = workspace[:toks][..., self.kv_lora_rank :].unsqueeze(1)
|
||||
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
|
||||
|
||||
Reference in New Issue
Block a user