[Model] Enable FP8 QKV in MoE and refine kernel tuning script (#5039)

This commit is contained in:
Cody Yu
2024-05-31 14:29:19 -07:00
committed by GitHub
parent a377f0bd5e
commit e9899fb7a4
8 changed files with 714 additions and 117 deletions
-9
View File
@@ -278,15 +278,6 @@ class MixtralAttention(nn.Module):
self.scaling = self.head_dim**-0.5
self.rope_theta = rope_theta
if isinstance(
quant_config,
Fp8Config) and not quant_config.is_checkpoint_fp8_serialized:
print_warning_once(
"For Mixtral FP8 quantization, we currently do not quantize "
"the attention layers until their FP8 performance is improved."
)
quant_config = None
self.qkv_proj = QKVParallelLinear(
hidden_size,
self.head_dim,