Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Gabriel Wu
2026-06-22 11:54:14 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Yongye Zhu
parent 3ce15fd574
commit 44d95069e9
37 changed files with 2347 additions and 476 deletions
@@ -1058,6 +1058,34 @@ def _upcast_e8m0_to_fp32(scale: torch.Tensor) -> torch.Tensor:
return fp32_bits.view(torch.float32)
def deepgemm_post_process_weight_scale_block(
ws: torch.Tensor,
mn: int,
k: int,
quant_block_shape: tuple[int, ...],
num_groups: int,
is_sfa: bool = False,
) -> torch.Tensor:
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
# Scales already in E8M0 from checkpoint; upcast to fp32 and let
# DeepGEMM pack the layout expected by the target architecture.
ws = _upcast_e8m0_to_fp32(ws)
else:
assert ws.dtype == torch.float32, (
f"Expected tensor scales dtype to be torch.float32 or "
f"torch.float8_e8m0fnu or torch.uint8, got {ws.dtype} instead"
)
return transform_sf_into_required_layout(
sf=ws,
mn=mn,
k=k,
recipe=(1, quant_block_shape[0], quant_block_shape[1]),
num_groups=num_groups,
is_sfa=is_sfa,
)
def deepgemm_post_process_fp8_weight_block(
wq: torch.Tensor,
ws: torch.Tensor,
@@ -1073,13 +1101,13 @@ def deepgemm_post_process_fp8_weight_block(
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
# Scales already in E8M0 from checkpoint (float8_e8m0fnu, or raw E8M0
# bits as uint8 for MXFP8) — upcast to fp32 and skip requantization
# bits as uint8 for MXFP8) - upcast to fp32 and skip requantization
# (weights already have power-of-two scales).
ws = _upcast_e8m0_to_fp32(ws)
else:
assert ws.dtype == torch.float32, (
f"Expected tensor scales dtype to be torch.float32 or "
f"torch.float8_e8m0fnu, got {ws.dtype} instead"
f"torch.float8_e8m0fnu or torch.uint8, got {ws.dtype} instead"
)
if use_e8m0:
requant_weight_ue8m0_inplace(wq, ws, block_size=quant_block_shape)
@@ -1094,16 +1122,12 @@ def deepgemm_post_process_fp8_weight_block(
r = wq.size(0) // g
wq = wq.view(g, r, d)
ws = ws.view(g, r // quant_block_shape[0], d // quant_block_shape[1])
# Pre-transform scale with recipe=(1, 128, 128) to broadcast + pack
# into TMA-aligned UE8M0 (INT32) layout. At runtime fp8_einsum uses
# recipe=(1, 1, 128) which sees INT dtype and skips re-transform.
dg_ws = transform_sf_into_required_layout(
sf=ws,
dg_ws = deepgemm_post_process_weight_scale_block(
ws=ws,
mn=r,
k=d,
recipe=(1, quant_block_shape[0], quant_block_shape[1]),
quant_block_shape=quant_block_shape,
num_groups=g,
is_sfa=False,
)
return wq, dg_ws
@@ -1113,22 +1137,12 @@ def deepgemm_post_process_fp8_weight_block(
wq = wq.unsqueeze(0)
ws = ws.unsqueeze(0)
# From https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/utils/layout.hpp#L46
# (1, block_n, block_k): (1, 128, 128) for FP8 block, (1, 1, 32) for MXFP8.
recipe = (1, quant_block_shape[0], quant_block_shape[1])
# Ref : https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/apis/gemm.hpp
# DeepGemm uses the `transform_sf_into_required_layout` function to
# represent scales in the correct format.
dg_ws = transform_sf_into_required_layout(
sf=ws,
dg_ws = deepgemm_post_process_weight_scale_block(
ws=ws,
mn=wq.size(1),
k=wq.size(2),
recipe=recipe,
quant_block_shape=quant_block_shape,
num_groups=wq.size(0),
# is the scale factors for A in (Refers to the argument A in A @ B).
# Weights are B.
is_sfa=False,
)
if original_ndim == 2: