[New Model][ROCm] Add AMD support for DeepSeek V4 (#40871)
Signed-off-by: ganyi <ygan@amd.com> Signed-off-by: whx-sjtu <xiaowang990929@gmail.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: tjtanaavllm <tunjian.tan@amd.com> Co-authored-by: ganyi <ygan@amd.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: tjtanaavllm <tunjian.tan@amd.com>
This commit is contained in:
co-authored by
ganyi
tjtanaa
tjtanaavllm
parent
2228fe6868
commit
628c436301
@@ -312,6 +312,21 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if As.dtype != Bs.dtype:
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
)
|
||||
|
||||
if As.dtype == torch.float8_e8m0fnu:
|
||||
As = _upcast_e8m0_to_fp32(As).contiguous()
|
||||
else:
|
||||
As = As.to(torch.float32)
|
||||
|
||||
if Bs.dtype == torch.float8_e8m0fnu:
|
||||
Bs = _upcast_e8m0_to_fp32(Bs).contiguous()
|
||||
else:
|
||||
Bs = Bs.to(torch.float32)
|
||||
|
||||
out_dtype = self.config.out_dtype
|
||||
if self.use_triton:
|
||||
gemm_a8w8_blockscale_op = rocm_aiter_ops.triton_gemm_a8w8_blockscale
|
||||
|
||||
Reference in New Issue
Block a user