[Kernel] Optimize FP8 support for MoE kernel / Mixtral via static scales (#4343)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
co-authored by
Woosuk Kwon
parent
258a2c58d0
commit
12628d3c78
+9
-3
@@ -168,10 +168,16 @@ def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
|
||||
|
||||
|
||||
# fp8
|
||||
def scaled_fp8_quant(input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
scale = torch.zeros(1, device=input.device, dtype=torch.float32)
|
||||
def scaled_fp8_quant(
|
||||
input: torch.Tensor,
|
||||
scale: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
|
||||
vllm_ops.scaled_fp8_quant(output, input, scale)
|
||||
if scale is None:
|
||||
scale = torch.zeros(1, device=input.device, dtype=torch.float32)
|
||||
vllm_ops.dynamic_scaled_fp8_quant(output, input, scale)
|
||||
else:
|
||||
vllm_ops.static_scaled_fp8_quant(output, input, scale)
|
||||
return output, scale
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user