feat: MLA prefill enable FA4 fp8 output (#43050)

Signed-off-by: Carl You <4531192+carlyou@users.noreply.github.com>
This commit is contained in:
Carl Y
2026-06-16 07:10:59 -07:00
committed by GitHub
parent ce3ef17bec
commit eb04c769d3
13 changed files with 489 additions and 20 deletions
@@ -687,7 +687,23 @@ class MLAAttention(nn.Module, AttentionLayerBase):
num_mqa_tokens = attn_metadata.num_decode_tokens
num_mha_tokens = q.size(0) - num_mqa_tokens
mha_use_quant_output = (
quant_key is not None
and self.prefill_backend.supports_quant_output(quant_key)
and attn_metadata is not None
and attn_metadata.prefill is not None
and attn_metadata.prefill.chunked_context is None
and self.impl.dcp_world_size <= 1
)
if num_mha_tokens > 0:
if mha_use_quant_output:
mha_output = quant_output
mha_output_scale = output_scale
else:
mha_output = output
mha_output_scale = None
self.impl.forward_mha( # type: ignore[attr-defined]
q[num_mqa_tokens:],
k_c_normed[num_mqa_tokens:],
@@ -695,7 +711,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
kv_cache,
attn_metadata,
self._k_scale,
output=output[num_mqa_tokens:],
output=mha_output[num_mqa_tokens:num_actual_toks],
output_scale=mha_output_scale,
)
if num_mqa_tokens > 0:
@@ -794,13 +811,15 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self._v_up_proj(attn_out, out=mqa_output_slice)
if quant_key is not None:
# Quantize the BF16 computation result into the quantized output
actual = output[:num_actual_toks]
quant_idx = num_mqa_tokens if mha_use_quant_output else num_actual_toks
if quant_idx == 0:
return quant_output
actual = output[:quant_idx]
if quant_key == kNvfp4Dynamic:
# NVFP4: two FP4 values packed into one uint8
assert output_block_scale is not None
fp4_data, fp4_scales = ops.scaled_fp4_quant(actual, output_scale)
quant_output[:num_actual_toks].copy_(fp4_data)
quant_output[:quant_idx].copy_(fp4_data)
output_block_scale[: fp4_scales.shape[0]].copy_(fp4_scales)
elif quant_key in (kFp8Dynamic128Sym, kFp8Dynamic64Sym):
# Per-group FP8
@@ -812,8 +831,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
finfo = torch.finfo(_FP8_DTYPE)
torch.ops._C.per_token_group_fp8_quant(
actual,
quant_output[:num_actual_toks],
output_block_scale[:num_actual_toks],
quant_output[:quant_idx],
output_block_scale[:quant_idx],
quant_group_size,
1e-10, # eps
finfo.min,
@@ -825,7 +844,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
elif quant_key == kFp8StaticTensorSym:
# Static FP8 quantization
fp8_data, _ = self._quant_fp8_op(actual, output_scale)
quant_output[:num_actual_toks].copy_(fp8_data)
quant_output[:quant_idx].copy_(fp8_data)
else:
raise ValueError(f"Unsupported quant_key: {quant_key}")
return quant_output
@@ -2254,6 +2273,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
attn_metadata: MLACommonMetadata,
k_scale: torch.Tensor,
output: torch.Tensor,
output_scale: torch.Tensor | None = None,
) -> None:
assert attn_metadata.prefill is not None
assert self.dcp_world_size != -1
@@ -2267,6 +2287,9 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
q = q.to(prefill_metadata.q_data_type)
has_context = prefill_metadata.chunked_context is not None
assert output_scale is None or not has_context, (
"Fused FP8 output is only wired for the non-chunked-context path"
)
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
@@ -2283,6 +2306,12 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
k=k,
v=v,
return_softmax_lse=has_context,
out=(
output.view(-1, self.num_heads, self.v_head_dim)
if output_scale is not None
else None
),
output_scale=output_scale,
)
if has_context:
@@ -2312,7 +2341,8 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
suffix_lse=suffix_lse,
prefill_tokens_with_context=prefill_metadata.chunked_context.prefill_tokens_with_context,
)
else:
elif output_scale is None:
# With output_scale set, backend already wrote into `output` in place.
assert isinstance(output_prefill, torch.Tensor)
output_prefill = output_prefill.flatten(start_dim=-2)
output.copy_(output_prefill)