feat: MLA prefill enable FA4 fp8 output (#43050)
Signed-off-by: Carl You <4531192+carlyou@users.noreply.github.com>
This commit is contained in:
@@ -687,7 +687,23 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
num_mqa_tokens = attn_metadata.num_decode_tokens
|
||||
num_mha_tokens = q.size(0) - num_mqa_tokens
|
||||
|
||||
mha_use_quant_output = (
|
||||
quant_key is not None
|
||||
and self.prefill_backend.supports_quant_output(quant_key)
|
||||
and attn_metadata is not None
|
||||
and attn_metadata.prefill is not None
|
||||
and attn_metadata.prefill.chunked_context is None
|
||||
and self.impl.dcp_world_size <= 1
|
||||
)
|
||||
|
||||
if num_mha_tokens > 0:
|
||||
if mha_use_quant_output:
|
||||
mha_output = quant_output
|
||||
mha_output_scale = output_scale
|
||||
else:
|
||||
mha_output = output
|
||||
mha_output_scale = None
|
||||
|
||||
self.impl.forward_mha( # type: ignore[attr-defined]
|
||||
q[num_mqa_tokens:],
|
||||
k_c_normed[num_mqa_tokens:],
|
||||
@@ -695,7 +711,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
self._k_scale,
|
||||
output=output[num_mqa_tokens:],
|
||||
output=mha_output[num_mqa_tokens:num_actual_toks],
|
||||
output_scale=mha_output_scale,
|
||||
)
|
||||
|
||||
if num_mqa_tokens > 0:
|
||||
@@ -794,13 +811,15 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self._v_up_proj(attn_out, out=mqa_output_slice)
|
||||
|
||||
if quant_key is not None:
|
||||
# Quantize the BF16 computation result into the quantized output
|
||||
actual = output[:num_actual_toks]
|
||||
quant_idx = num_mqa_tokens if mha_use_quant_output else num_actual_toks
|
||||
if quant_idx == 0:
|
||||
return quant_output
|
||||
actual = output[:quant_idx]
|
||||
if quant_key == kNvfp4Dynamic:
|
||||
# NVFP4: two FP4 values packed into one uint8
|
||||
assert output_block_scale is not None
|
||||
fp4_data, fp4_scales = ops.scaled_fp4_quant(actual, output_scale)
|
||||
quant_output[:num_actual_toks].copy_(fp4_data)
|
||||
quant_output[:quant_idx].copy_(fp4_data)
|
||||
output_block_scale[: fp4_scales.shape[0]].copy_(fp4_scales)
|
||||
elif quant_key in (kFp8Dynamic128Sym, kFp8Dynamic64Sym):
|
||||
# Per-group FP8
|
||||
@@ -812,8 +831,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
finfo = torch.finfo(_FP8_DTYPE)
|
||||
torch.ops._C.per_token_group_fp8_quant(
|
||||
actual,
|
||||
quant_output[:num_actual_toks],
|
||||
output_block_scale[:num_actual_toks],
|
||||
quant_output[:quant_idx],
|
||||
output_block_scale[:quant_idx],
|
||||
quant_group_size,
|
||||
1e-10, # eps
|
||||
finfo.min,
|
||||
@@ -825,7 +844,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
elif quant_key == kFp8StaticTensorSym:
|
||||
# Static FP8 quantization
|
||||
fp8_data, _ = self._quant_fp8_op(actual, output_scale)
|
||||
quant_output[:num_actual_toks].copy_(fp8_data)
|
||||
quant_output[:quant_idx].copy_(fp8_data)
|
||||
else:
|
||||
raise ValueError(f"Unsupported quant_key: {quant_key}")
|
||||
return quant_output
|
||||
@@ -2254,6 +2273,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
attn_metadata: MLACommonMetadata,
|
||||
k_scale: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
assert attn_metadata.prefill is not None
|
||||
assert self.dcp_world_size != -1
|
||||
@@ -2267,6 +2287,9 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
q = q.to(prefill_metadata.q_data_type)
|
||||
|
||||
has_context = prefill_metadata.chunked_context is not None
|
||||
assert output_scale is None or not has_context, (
|
||||
"Fused FP8 output is only wired for the non-chunked-context path"
|
||||
)
|
||||
|
||||
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
|
||||
-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
|
||||
@@ -2283,6 +2306,12 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
k=k,
|
||||
v=v,
|
||||
return_softmax_lse=has_context,
|
||||
out=(
|
||||
output.view(-1, self.num_heads, self.v_head_dim)
|
||||
if output_scale is not None
|
||||
else None
|
||||
),
|
||||
output_scale=output_scale,
|
||||
)
|
||||
|
||||
if has_context:
|
||||
@@ -2312,7 +2341,8 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
suffix_lse=suffix_lse,
|
||||
prefill_tokens_with_context=prefill_metadata.chunked_context.prefill_tokens_with_context,
|
||||
)
|
||||
else:
|
||||
elif output_scale is None:
|
||||
# With output_scale set, backend already wrote into `output` in place.
|
||||
assert isinstance(output_prefill, torch.Tensor)
|
||||
output_prefill = output_prefill.flatten(start_dim=-2)
|
||||
output.copy_(output_prefill)
|
||||
|
||||
Reference in New Issue
Block a user