Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Gabriel Wu
2026-06-22 11:54:14 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Yongye Zhu
parent 3ce15fd574
commit 44d95069e9
37 changed files with 2347 additions and 476 deletions
@@ -208,6 +208,7 @@ from vllm.config import (
get_current_vllm_config,
get_current_vllm_config_or_none,
)
from vllm.config.cache import CacheDType
from vllm.distributed.parallel_state import (
get_dcp_group,
is_global_first_rank,
@@ -319,6 +320,22 @@ def _detect_output_quant_key(
return kFp8StaticTensorSym
def _canonicalize_sparse_mla_kv_cache_dtype(
attn_backend: type[AttentionBackend],
kv_cache_dtype: CacheDType,
) -> CacheDType:
backend_name = attn_backend.get_name()
if backend_name == "FLASHMLA_SPARSE" and is_quantized_kv_cache(kv_cache_dtype):
return "fp8_ds_mla"
if backend_name == "FLASHINFER_MLA_SPARSE_SM120" and kv_cache_dtype in (
"auto",
"fp8",
"fp8_e4m3",
):
return "fp8_ds_mla"
return kv_cache_dtype
class MLAAttention(nn.Module, AttentionLayerBase):
"""Multi-Head Latent Attention layer.
@@ -369,7 +386,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
if cache_config is not None:
kv_cache_dtype = cache_config.cache_dtype
kv_cache_dtype: CacheDType = cache_config.cache_dtype
calculate_kv_scales = cache_config.calculate_kv_scales
else:
kv_cache_dtype = "auto"
@@ -393,24 +410,22 @@ class MLAAttention(nn.Module, AttentionLayerBase):
num_heads=self.num_heads,
)
# FlashMLA Sparse Attention fp8 backend uses "fp8_ds_mla" kv-cache format
# Automatically convert fp8 kv-cache format to "fp8_ds_mla"
if (
self.attn_backend.get_name() == "FLASHMLA_SPARSE"
and is_quantized_kv_cache(kv_cache_dtype)
and kv_cache_dtype != "fp8_ds_mla"
):
assert cache_config is not None
cache_config.cache_dtype = "fp8_ds_mla"
kv_cache_dtype = "fp8_ds_mla"
normalized_kv_cache_dtype = _canonicalize_sparse_mla_kv_cache_dtype(
self.attn_backend, kv_cache_dtype
)
if normalized_kv_cache_dtype != kv_cache_dtype:
if cache_config is not None:
cache_config.cache_dtype = normalized_kv_cache_dtype
kv_cache_dtype = normalized_kv_cache_dtype
logger.info_once(
"Using DeepSeek's fp8_ds_mla KV cache format. To use standard "
"fp8 kv-cache format, please set `--attention-backend "
"FLASHINFER_MLA_SPARSE`"
"Using %s KV cache format for %s backend.",
kv_cache_dtype,
self.attn_backend.get_name(),
)
if (
self.attn_backend.get_name() == "FLASHINFER_MLA_SPARSE"
and kv_cache_dtype != "fp8_ds_mla"
and is_quantized_kv_cache(kv_cache_dtype)
):
logger.info_once(
@@ -23,24 +23,82 @@ def expert_num_tokens_round_up_and_sum(
return torch.sum(ent).item()
def compute_aligned_M_and_alignment(
M: int,
num_topk: int,
local_num_experts: int,
alignment: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
) -> tuple[int, int]:
"""Return (M_sum, alignment_used).
`alignment_used` may be smaller than the caller-supplied `alignment` on
SM100/SM120 when DeepGEMM can JIT a smaller BLOCK_M for the per-call
expected_m. Callers that index by block size (e.g. ``M_sum // block_m``)
or assert workspace alignment must use the returned `alignment_used`,
not their original `alignment` argument.
Prefer this over the int-returning :func:`compute_aligned_M` when the
GEMM call site needs to wrap itself in ``mk_alignment_scope`` or
otherwise reason about the actual per-expert padding.
"""
if (expert_tokens_meta is not None) and (
expert_tokens_meta.expert_num_tokens_cpu is not None
):
return (
expert_num_tokens_round_up_and_sum(
expert_tokens_meta.expert_num_tokens_cpu, alignment=alignment
),
alignment,
)
# expert_num_tokens not on cpu. Cap padding by min(M*num_topk,
# local_num_experts) — at batch=1 decode only `num_topk` experts can be
# active, so the worst-case `local_num_experts*(align-1)` is too loose.
# Also shrink `alignment` to DeepGEMM's per-call theoretical BLOCK_M on
# SM100/SM120 when smaller.
expected_m = M * num_topk
try:
from vllm.utils.deep_gemm import (
get_theoretical_mk_alignment_for_contiguous_layout,
)
# num_groups=local_num_experts so the helper recovers per-expert em;
# omitting it over-picks BLOCK_M on SM120 (heuristic assumes em is
# already per-expert).
per_call_align = get_theoretical_mk_alignment_for_contiguous_layout(
expected_m=expected_m,
num_groups=local_num_experts,
)
if per_call_align and per_call_align <= alignment:
alignment = per_call_align
except Exception:
pass
max_active_experts = min(M * num_topk, local_num_experts)
M_sum = (M * num_topk) + max_active_experts * (alignment - 1)
M_sum = round_up(M_sum, alignment)
return M_sum, alignment
def compute_aligned_M(
M: int,
num_topk: int,
local_num_experts: int,
alignment: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
):
if (expert_tokens_meta is not None) and (
expert_tokens_meta.expert_num_tokens_cpu is not None
):
return expert_num_tokens_round_up_and_sum(
expert_tokens_meta.expert_num_tokens_cpu, alignment=alignment
)
) -> int:
"""Return ``M_sum`` only (backward-compat wrapper).
# expert_num_tokens information is not available on the cpu.
# compute the max required size.
M_sum = (M * num_topk) + local_num_experts * (alignment - 1)
M_sum = round_up(M_sum, alignment)
Equivalent to :func:`compute_aligned_M_and_alignment`'s first return
value. Existing downstream callers and the warmup path that only size
a workspace use this. Call sites that need the actual per-expert
alignment (to wrap GEMMs in ``mk_alignment_scope``) should use
:func:`compute_aligned_M_and_alignment` instead.
"""
M_sum, _ = compute_aligned_M_and_alignment(
M, num_topk, local_num_experts, alignment, expert_tokens_meta
)
return M_sum
@@ -51,12 +109,6 @@ def apply_expert_map(expert_id, expert_map):
return expert_id
@triton.jit
def round_up_128(x: int) -> int:
y = 128
return ((x + y - 1) // y) * y
@triton.jit
def _fwd_kernel_ep_scatter_1(
num_recv_tokens_per_expert,
@@ -65,6 +117,7 @@ def _fwd_kernel_ep_scatter_1(
num_experts: tl.constexpr,
BLOCK_E: tl.constexpr,
BLOCK_EXPERT_NUM: tl.constexpr,
ALIGN_M: tl.constexpr,
):
cur_expert = tl.program_id(0)
@@ -74,7 +127,8 @@ def _fwd_kernel_ep_scatter_1(
mask=offset_cumsum < num_experts,
other=0,
)
tokens_per_expert = round_up_128(tokens_per_expert)
# Round up to ALIGN_M so cumsum matches the workspace's per-expert slices.
tokens_per_expert = ((tokens_per_expert + ALIGN_M - 1) // ALIGN_M) * ALIGN_M
cumsum = tl.cumsum(tokens_per_expert) - tokens_per_expert
# Extract this block's offset from the register vector (warp shuffle,
@@ -227,10 +281,12 @@ def ep_scatter(
output_tensor_scale: torch.Tensor,
m_indices: torch.Tensor,
output_index: torch.Tensor,
align_m: int = 128,
block_size: int = 128,
pack_ue8m0: bool = False,
):
BLOCK_E = 128 # token num of per expert is aligned to 128
# BLOCK_E is the m_indices fill-loop tile (masked), independent of align_m.
BLOCK_E = 128
BLOCK_D = block_size # block size of activation-scale quantization
num_warps = 8
num_experts = num_recv_tokens_per_expert.shape[0]
@@ -238,7 +294,7 @@ def ep_scatter(
# grid = (triton.cdiv(hidden_size, BLOCK_D), num_experts)
grid = num_experts
assert m_indices.shape[0] % BLOCK_E == 0
assert m_indices.shape[0] % align_m == 0
assert expert_start_loc.shape[0] == num_experts
# pack_ue8m0: scatter packs 4 UE8M0 bytes per int32; else copies scales as-is.
@@ -253,6 +309,7 @@ def ep_scatter(
num_warps=num_warps,
BLOCK_E=BLOCK_E,
BLOCK_EXPERT_NUM=triton.next_power_of_2(num_experts),
ALIGN_M=align_m,
)
grid = min(recv_topk.shape[0], 1024 * 8)
@@ -418,7 +475,7 @@ def deepgemm_moe_permute(
if block_size is not None:
block_k = block_size
M_sum = compute_aligned_M(
M_sum, align_used = compute_aligned_M_and_alignment(
M=topk_ids.size(0),
num_topk=topk_ids.size(1),
local_num_experts=local_num_experts,
@@ -482,11 +539,12 @@ def deepgemm_moe_permute(
output_tensor_scale=aq_scale_out,
m_indices=expert_ids,
output_index=inv_perm,
align_m=align_used,
block_size=block_k,
pack_ue8m0=pack_ue8m0,
)
return aq_out, aq_scale_out, expert_ids, inv_perm
return aq_out, aq_scale_out, expert_ids, inv_perm, align_used
def deepgemm_unpermute_and_reduce(
@@ -318,11 +318,12 @@ class BatchedDeepGemmExperts(mk.FusedMoEExpertsModular):
def supports_packed_ue8m0_act_scales(self) -> bool:
"""
DeepGemm supports packed ue8m0 activation scales format in devices == sm100
DeepGemm supports packed ue8m0 activation scales on Blackwell-family
GPUs (SM100 datacenter and SM120 consumer).
"""
return (
is_deep_gemm_e8m0_used()
and current_platform.is_device_capability_family(100)
return is_deep_gemm_e8m0_used() and (
current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
)
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
compute_aligned_M,
compute_aligned_M_and_alignment,
deepgemm_moe_permute,
deepgemm_unpermute_and_reduce,
)
@@ -43,6 +43,7 @@ from vllm.utils.deep_gemm import (
is_deep_gemm_supported,
m_grouped_fp8_fp4_gemm_nt_contiguous,
m_grouped_fp8_gemm_nt_contiguous,
mk_alignment_scope,
)
from vllm.utils.import_utils import has_deep_gemm
@@ -210,10 +211,10 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
# Use the contiguous-layout M alignment (matches apply()); block_shape[0]
# is the quant block (1 for MXFP8) and would under-size the workspace.
block_m = get_mk_alignment_for_contiguous_layout()[0]
M_sum = compute_aligned_M(
M_sum, align_used = compute_aligned_M_and_alignment(
M, topk, local_num_experts, block_m, expert_tokens_meta
)
assert M_sum % block_m == 0
assert M_sum % align_used == 0
activation_out_dim = self.adjust_N_for_activation(N, activation)
workspace1 = (M_sum, max(activation_out_dim, K))
@@ -316,7 +317,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
assert w2.size(1) == K
M_sum = compute_aligned_M(
M_sum, _ = compute_aligned_M_and_alignment(
M=topk_ids.size(0),
num_topk=topk_ids.size(1),
local_num_experts=local_num_experts,
@@ -327,7 +328,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
a1q_perm = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
)
a1q, a1q_scale, expert_ids, inv_perm = deepgemm_moe_permute(
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
aq=a1q,
aq_scale=a1q_scale,
topk_ids=topk_ids,
@@ -349,23 +350,35 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
else {}
)
mm1_out = _resize_cache(workspace2, (M_sum, N))
m_grouped_fp8_gemm_nt_contiguous(
(a1q, a1q_scale), (w1, self.w1_scale), mm1_out, expert_ids, **gemm_kwargs
)
# Cap DG's BLOCK_M heuristic at the workspace's per-expert alignment;
# otherwise the scheduler can pick the wrong expert id from m_indices
# under cudagraph replay.
with mk_alignment_scope(align_used):
mm1_out = _resize_cache(workspace2, (M_sum, N))
m_grouped_fp8_gemm_nt_contiguous(
(a1q, a1q_scale),
(w1, self.w1_scale),
mm1_out,
expert_ids,
**gemm_kwargs,
)
activation_out_dim = self.adjust_N_for_activation(N, activation)
quant_out = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
)
a2q, a2q_scale = self._act_mul_quant(
input=mm1_out.view(-1, N), output=quant_out, activation=activation
)
activation_out_dim = self.adjust_N_for_activation(N, activation)
quant_out = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
)
a2q, a2q_scale = self._act_mul_quant(
input=mm1_out.view(-1, N), output=quant_out, activation=activation
)
mm2_out = _resize_cache(workspace2, (M_sum, K))
m_grouped_fp8_gemm_nt_contiguous(
(a2q, a2q_scale), (w2, self.w2_scale), mm2_out, expert_ids, **gemm_kwargs
)
mm2_out = _resize_cache(workspace2, (M_sum, K))
m_grouped_fp8_gemm_nt_contiguous(
(a2q, a2q_scale),
(w2, self.w2_scale),
mm2_out,
expert_ids,
**gemm_kwargs,
)
if apply_router_weight_on_input:
topk_weights = torch.ones_like(topk_weights)
@@ -384,7 +397,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
"""DeepGemm-based fused MoE expert implementation for FP4 weights.
Uses m_grouped_fp8_fp4_gemm_nt_contiguous with FP8 activations and
MXFP4 (FP4 E2M1 packed as uint8) weights. Requires SM100+ (Blackwell).
MXFP4 (FP4 E2M1 packed as uint8) weights. Requires Blackwell-family
GPUs (SM100 datacenter or SM120 consumer).
"""
# FP8 activation block size (hardcoded since mxfp4_w4a8 quant config
@@ -409,9 +423,9 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
def _supports_current_device() -> bool:
from vllm.platforms import current_platform
return (
is_deep_gemm_supported()
and current_platform.is_device_capability_family(100)
return is_deep_gemm_supported() and (
current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
)
@staticmethod
@@ -454,10 +468,10 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
block_m = get_mk_alignment_for_contiguous_layout()[0]
M_sum = compute_aligned_M(
M_sum, align_used = compute_aligned_M_and_alignment(
M, topk, local_num_experts, block_m, expert_tokens_meta
)
assert M_sum % block_m == 0
assert M_sum % align_used == 0
activation_out_dim = self.adjust_N_for_activation(N, activation)
workspace1 = (M_sum, max(activation_out_dim, K))
@@ -533,7 +547,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
if global_num_experts == -1:
global_num_experts = local_num_experts
M_sum = compute_aligned_M(
M_sum, _ = compute_aligned_M_and_alignment(
M=topk_ids.size(0),
num_topk=topk_ids.size(1),
local_num_experts=local_num_experts,
@@ -544,7 +558,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
a1q_perm = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
)
a1q, a1q_scale, expert_ids, inv_perm = deepgemm_moe_permute(
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
aq=a1q,
aq_scale=a1q_scale,
topk_ids=topk_ids,
@@ -555,37 +569,40 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
)
assert a1q.size(0) == M_sum
# FC1: FP8 activations x FP4 weights
# DeepGEMM 2.4.2 requires FP4-packed weights as int8 (kPackedFP4).
mm1_out = _resize_cache(workspace2, (M_sum, N))
m_grouped_fp8_fp4_gemm_nt_contiguous(
(a1q, a1q_scale),
(w1.view(torch.int8), self.w1_scale),
mm1_out,
expert_ids,
recipe_a=(1, self._ACT_BLOCK_K),
recipe_b=(1, self._WEIGHT_BLOCK_K),
)
# Cap DG's BLOCK_M heuristic at the workspace's per-expert alignment;
# see DeepGemmExperts.apply for rationale.
with mk_alignment_scope(align_used):
# FC1: FP8 activations x FP4 weights
# DeepGEMM 2.4.2 requires FP4-packed weights as int8 (kPackedFP4).
mm1_out = _resize_cache(workspace2, (M_sum, N))
m_grouped_fp8_fp4_gemm_nt_contiguous(
(a1q, a1q_scale),
(w1.view(torch.int8), self.w1_scale),
mm1_out,
expert_ids,
recipe_a=(1, self._ACT_BLOCK_K),
recipe_b=(1, self._WEIGHT_BLOCK_K),
)
# SwiGLU activation + FP8 requant
activation_out_dim = self.adjust_N_for_activation(N, activation)
quant_out = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
)
a2q, a2q_scale = self._act_mul_quant(
input=mm1_out.view(-1, N), output=quant_out, activation=activation
)
# SwiGLU activation + FP8 requant
activation_out_dim = self.adjust_N_for_activation(N, activation)
quant_out = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
)
a2q, a2q_scale = self._act_mul_quant(
input=mm1_out.view(-1, N), output=quant_out, activation=activation
)
# FC2: FP8 activations x FP4 weights
mm2_out = _resize_cache(workspace2, (M_sum, K))
m_grouped_fp8_fp4_gemm_nt_contiguous(
(a2q, a2q_scale),
(w2.view(torch.int8), self.w2_scale),
mm2_out,
expert_ids,
recipe_a=(1, self._ACT_BLOCK_K),
recipe_b=(1, self._WEIGHT_BLOCK_K),
)
# FC2: FP8 activations x FP4 weights
mm2_out = _resize_cache(workspace2, (M_sum, K))
m_grouped_fp8_fp4_gemm_nt_contiguous(
(a2q, a2q_scale),
(w2.view(torch.int8), self.w2_scale),
mm2_out,
expert_ids,
recipe_a=(1, self._ACT_BLOCK_K),
recipe_b=(1, self._WEIGHT_BLOCK_K),
)
if apply_router_weight_on_input:
topk_weights = torch.ones_like(topk_weights)
@@ -57,6 +57,40 @@ if has_triton_kernels():
)
def _pack_deepgemm_mxfp4_scales(
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
w13_weight_scale: torch.Tensor,
w2_weight_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
deepgemm_post_process_weight_scale_block,
)
num_experts = w13_weight.shape[0]
intermediate_size_2 = w13_weight.shape[1] # = intermediate*2
hidden_size = w13_weight.shape[2] * 2 # weight is FP4-packed
intermediate_size = w2_weight.shape[2] * 2 # weight is FP4-packed
block_shape = (1, 32) # MXFP4 block (per-row, K=32)
return (
deepgemm_post_process_weight_scale_block(
ws=w13_weight_scale.data,
mn=intermediate_size_2,
k=hidden_size,
quant_block_shape=block_shape,
num_groups=num_experts,
),
deepgemm_post_process_weight_scale_block(
ws=w2_weight_scale.data,
mn=hidden_size,
k=intermediate_size,
quant_block_shape=block_shape,
num_groups=num_experts,
),
)
class Mxfp4MoeBackend(Enum):
NONE = "None"
# DeepGEMM FP8xFP4 backend (SM100+)
@@ -652,15 +686,18 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
"""Convert loaded weights into backend-specific kernel format."""
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_upcast_e8m0_to_fp32,
w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales(
w13_weight,
w2_weight,
w13_weight_scale,
w2_weight_scale,
)
return (
w13_weight.data,
w2_weight.data,
_upcast_e8m0_to_fp32(w13_weight_scale.data),
_upcast_e8m0_to_fp32(w2_weight_scale.data),
w13_weight_scale,
w2_weight_scale,
w13_bias,
w2_bias,
)
@@ -1195,17 +1232,18 @@ def convert_weight_to_mxfp4_moe_kernel_format(
"""
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_upcast_e8m0_to_fp32,
w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales(
w13_weight,
w2_weight,
w13_weight_scale,
w2_weight_scale,
)
# Weights stay as uint8 packed FP4 — no layout change needed.
# Convert E8M0 uint8 scales to float32.
return (
w13_weight.data,
w2_weight.data,
_upcast_e8m0_to_fp32(w13_weight_scale.data),
_upcast_e8m0_to_fp32(w2_weight_scale.data),
w13_weight_scale,
w2_weight_scale,
w13_bias,
w2_bias,
)
@@ -1058,6 +1058,34 @@ def _upcast_e8m0_to_fp32(scale: torch.Tensor) -> torch.Tensor:
return fp32_bits.view(torch.float32)
def deepgemm_post_process_weight_scale_block(
ws: torch.Tensor,
mn: int,
k: int,
quant_block_shape: tuple[int, ...],
num_groups: int,
is_sfa: bool = False,
) -> torch.Tensor:
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
# Scales already in E8M0 from checkpoint; upcast to fp32 and let
# DeepGEMM pack the layout expected by the target architecture.
ws = _upcast_e8m0_to_fp32(ws)
else:
assert ws.dtype == torch.float32, (
f"Expected tensor scales dtype to be torch.float32 or "
f"torch.float8_e8m0fnu or torch.uint8, got {ws.dtype} instead"
)
return transform_sf_into_required_layout(
sf=ws,
mn=mn,
k=k,
recipe=(1, quant_block_shape[0], quant_block_shape[1]),
num_groups=num_groups,
is_sfa=is_sfa,
)
def deepgemm_post_process_fp8_weight_block(
wq: torch.Tensor,
ws: torch.Tensor,
@@ -1073,13 +1101,13 @@ def deepgemm_post_process_fp8_weight_block(
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
# Scales already in E8M0 from checkpoint (float8_e8m0fnu, or raw E8M0
# bits as uint8 for MXFP8) — upcast to fp32 and skip requantization
# bits as uint8 for MXFP8) - upcast to fp32 and skip requantization
# (weights already have power-of-two scales).
ws = _upcast_e8m0_to_fp32(ws)
else:
assert ws.dtype == torch.float32, (
f"Expected tensor scales dtype to be torch.float32 or "
f"torch.float8_e8m0fnu, got {ws.dtype} instead"
f"torch.float8_e8m0fnu or torch.uint8, got {ws.dtype} instead"
)
if use_e8m0:
requant_weight_ue8m0_inplace(wq, ws, block_size=quant_block_shape)
@@ -1094,16 +1122,12 @@ def deepgemm_post_process_fp8_weight_block(
r = wq.size(0) // g
wq = wq.view(g, r, d)
ws = ws.view(g, r // quant_block_shape[0], d // quant_block_shape[1])
# Pre-transform scale with recipe=(1, 128, 128) to broadcast + pack
# into TMA-aligned UE8M0 (INT32) layout. At runtime fp8_einsum uses
# recipe=(1, 1, 128) which sees INT dtype and skips re-transform.
dg_ws = transform_sf_into_required_layout(
sf=ws,
dg_ws = deepgemm_post_process_weight_scale_block(
ws=ws,
mn=r,
k=d,
recipe=(1, quant_block_shape[0], quant_block_shape[1]),
quant_block_shape=quant_block_shape,
num_groups=g,
is_sfa=False,
)
return wq, dg_ws
@@ -1113,22 +1137,12 @@ def deepgemm_post_process_fp8_weight_block(
wq = wq.unsqueeze(0)
ws = ws.unsqueeze(0)
# From https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/utils/layout.hpp#L46
# (1, block_n, block_k): (1, 128, 128) for FP8 block, (1, 1, 32) for MXFP8.
recipe = (1, quant_block_shape[0], quant_block_shape[1])
# Ref : https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/apis/gemm.hpp
# DeepGemm uses the `transform_sf_into_required_layout` function to
# represent scales in the correct format.
dg_ws = transform_sf_into_required_layout(
sf=ws,
dg_ws = deepgemm_post_process_weight_scale_block(
ws=ws,
mn=wq.size(1),
k=wq.size(2),
recipe=recipe,
quant_block_shape=quant_block_shape,
num_groups=wq.size(0),
# is the scale factors for A in (Refers to the argument A in A @ B).
# Weights are B.
is_sfa=False,
)
if original_ndim == 2: