Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
@@ -1058,6 +1058,34 @@ def _upcast_e8m0_to_fp32(scale: torch.Tensor) -> torch.Tensor:
|
||||
return fp32_bits.view(torch.float32)
|
||||
|
||||
|
||||
def deepgemm_post_process_weight_scale_block(
|
||||
ws: torch.Tensor,
|
||||
mn: int,
|
||||
k: int,
|
||||
quant_block_shape: tuple[int, ...],
|
||||
num_groups: int,
|
||||
is_sfa: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
|
||||
# Scales already in E8M0 from checkpoint; upcast to fp32 and let
|
||||
# DeepGEMM pack the layout expected by the target architecture.
|
||||
ws = _upcast_e8m0_to_fp32(ws)
|
||||
else:
|
||||
assert ws.dtype == torch.float32, (
|
||||
f"Expected tensor scales dtype to be torch.float32 or "
|
||||
f"torch.float8_e8m0fnu or torch.uint8, got {ws.dtype} instead"
|
||||
)
|
||||
|
||||
return transform_sf_into_required_layout(
|
||||
sf=ws,
|
||||
mn=mn,
|
||||
k=k,
|
||||
recipe=(1, quant_block_shape[0], quant_block_shape[1]),
|
||||
num_groups=num_groups,
|
||||
is_sfa=is_sfa,
|
||||
)
|
||||
|
||||
|
||||
def deepgemm_post_process_fp8_weight_block(
|
||||
wq: torch.Tensor,
|
||||
ws: torch.Tensor,
|
||||
@@ -1073,13 +1101,13 @@ def deepgemm_post_process_fp8_weight_block(
|
||||
|
||||
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
|
||||
# Scales already in E8M0 from checkpoint (float8_e8m0fnu, or raw E8M0
|
||||
# bits as uint8 for MXFP8) — upcast to fp32 and skip requantization
|
||||
# bits as uint8 for MXFP8) - upcast to fp32 and skip requantization
|
||||
# (weights already have power-of-two scales).
|
||||
ws = _upcast_e8m0_to_fp32(ws)
|
||||
else:
|
||||
assert ws.dtype == torch.float32, (
|
||||
f"Expected tensor scales dtype to be torch.float32 or "
|
||||
f"torch.float8_e8m0fnu, got {ws.dtype} instead"
|
||||
f"torch.float8_e8m0fnu or torch.uint8, got {ws.dtype} instead"
|
||||
)
|
||||
if use_e8m0:
|
||||
requant_weight_ue8m0_inplace(wq, ws, block_size=quant_block_shape)
|
||||
@@ -1094,16 +1122,12 @@ def deepgemm_post_process_fp8_weight_block(
|
||||
r = wq.size(0) // g
|
||||
wq = wq.view(g, r, d)
|
||||
ws = ws.view(g, r // quant_block_shape[0], d // quant_block_shape[1])
|
||||
# Pre-transform scale with recipe=(1, 128, 128) to broadcast + pack
|
||||
# into TMA-aligned UE8M0 (INT32) layout. At runtime fp8_einsum uses
|
||||
# recipe=(1, 1, 128) which sees INT dtype and skips re-transform.
|
||||
dg_ws = transform_sf_into_required_layout(
|
||||
sf=ws,
|
||||
dg_ws = deepgemm_post_process_weight_scale_block(
|
||||
ws=ws,
|
||||
mn=r,
|
||||
k=d,
|
||||
recipe=(1, quant_block_shape[0], quant_block_shape[1]),
|
||||
quant_block_shape=quant_block_shape,
|
||||
num_groups=g,
|
||||
is_sfa=False,
|
||||
)
|
||||
return wq, dg_ws
|
||||
|
||||
@@ -1113,22 +1137,12 @@ def deepgemm_post_process_fp8_weight_block(
|
||||
wq = wq.unsqueeze(0)
|
||||
ws = ws.unsqueeze(0)
|
||||
|
||||
# From https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/utils/layout.hpp#L46
|
||||
# (1, block_n, block_k): (1, 128, 128) for FP8 block, (1, 1, 32) for MXFP8.
|
||||
recipe = (1, quant_block_shape[0], quant_block_shape[1])
|
||||
|
||||
# Ref : https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/apis/gemm.hpp
|
||||
# DeepGemm uses the `transform_sf_into_required_layout` function to
|
||||
# represent scales in the correct format.
|
||||
dg_ws = transform_sf_into_required_layout(
|
||||
sf=ws,
|
||||
dg_ws = deepgemm_post_process_weight_scale_block(
|
||||
ws=ws,
|
||||
mn=wq.size(1),
|
||||
k=wq.size(2),
|
||||
recipe=recipe,
|
||||
quant_block_shape=quant_block_shape,
|
||||
num_groups=wq.size(0),
|
||||
# is the scale factors for A in (Refers to the argument A in A @ B).
|
||||
# Weights are B.
|
||||
is_sfa=False,
|
||||
)
|
||||
|
||||
if original_ndim == 2:
|
||||
|
||||
Reference in New Issue
Block a user