[Feature][Hardware][Amd] Add fp8 Linear Layer for Rocm (#7210)

This commit is contained in:
Charlie Fu
2024-08-16 10:06:30 -07:00
committed by GitHub
parent ec724a725e
commit e837b624f2
7 changed files with 164 additions and 49 deletions
+4 -1
View File
@@ -369,9 +369,12 @@ def scaled_fp8_quant(
# This code assumes batch_dim and num_tokens are flattened
assert (input.ndim == 2)
shape: Union[Tuple[int, int], torch.Size] = input.shape
# For rocm, the output fp8 dtype is torch.float_e3m3fnuz
out_dtype: torch.dtype = torch.float8_e4m3fnuz if vllm.utils.is_hip() \
else torch.float8_e4m3fn
if num_token_padding:
shape = (max(num_token_padding, input.shape[0]), shape[1])
output = torch.empty(shape, device=input.device, dtype=torch.float8_e4m3fn)
output = torch.empty(shape, device=input.device, dtype=out_dtype)
if scale is None:
if use_per_token_if_dynamic: