Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Gabriel Wu
2026-06-22 11:54:14 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Yongye Zhu
parent 3ce15fd574
commit 44d95069e9
37 changed files with 2347 additions and 476 deletions
+58 -25
View File
@@ -12,7 +12,9 @@ from tqdm import tqdm
import vllm.envs as envs
from vllm.distributed.parallel_state import get_dp_group, is_global_first_rank
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import compute_aligned_M
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
compute_aligned_M_and_alignment,
)
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import DeepGemmExperts
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
TritonOrDeepGemmExperts,
@@ -25,6 +27,7 @@ from vllm.utils.deep_gemm import (
fp8_gemm_nt,
get_mk_alignment_for_contiguous_layout,
m_grouped_fp8_gemm_nt_contiguous,
mk_alignment_scope,
)
from vllm.utils.math_utils import cdiv
from vllm.utils.platform_utils import num_compute_units
@@ -238,7 +241,7 @@ def _get_grouped_gemm_params(
w2: torch.Tensor,
num_topk: int,
max_tokens: int,
) -> tuple[int, int, torch.Tensor]:
) -> tuple[int, int, list[tuple[int, int, torch.Tensor]]]:
assert w1.size(0) == w2.size(0), "w1 and w2 must have the same number of experts"
block_m = get_mk_alignment_for_contiguous_layout()[0]
@@ -248,19 +251,46 @@ def _get_grouped_gemm_params(
# Assumes all ranks have the same max_num_batched_tokens
max_tokens = get_dp_group().world_size * max_tokens
# This is the maximum GroupedGemm M size that we expect to run
# the grouped_gemm with.
MAX_M = compute_aligned_M(
max_tokens, num_topk, num_experts, block_m, expert_tokens_meta=None
request_m_values = _generate_optimal_warmup_m_values(
max_tokens,
max(w1.size(1), w2.size(1)),
device,
)
# Distribute expert-ids evenly.
MAX_BLOCKS = MAX_M // block_m
expert_ids_block = torch.randint(
low=0, high=num_experts, size=(MAX_BLOCKS,), device=device, dtype=torch.int32
)
expert_ids = torch.repeat_interleave(expert_ids_block, block_m, dim=0)
request_m_values = sorted({m for m in (*request_m_values, max_tokens) if m > 0})
if not request_m_values:
return 0, block_m, []
return MAX_M, block_m, expert_ids
cases_by_shape: dict[tuple[int, int], torch.Tensor] = {}
for request_m in request_m_values:
M_sum, align_used = compute_aligned_M_and_alignment(
M=request_m,
num_topk=num_topk,
local_num_experts=num_experts,
alignment=block_m,
expert_tokens_meta=None,
)
if (M_sum, align_used) in cases_by_shape:
continue
num_blocks = M_sum // align_used
expert_ids_block = torch.randint(
low=0,
high=num_experts,
size=(num_blocks,),
device=device,
dtype=torch.int32,
)
cases_by_shape[(M_sum, align_used)] = torch.repeat_interleave(
expert_ids_block, align_used, dim=0
)
max_m = max(M_sum for M_sum, _ in cases_by_shape)
warmup_cases = [
(M_sum, align_used, expert_ids)
for (M_sum, align_used), expert_ids in sorted(cases_by_shape.items())
]
return max_m, block_m, warmup_cases
def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
@@ -278,7 +308,11 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
):
return
MAX_M, block_m, expert_ids = _get_grouped_gemm_params(w1, w2, num_topk, max_tokens)
MAX_M, block_m, warmup_cases = _get_grouped_gemm_params(
w1, w2, num_topk, max_tokens
)
if not warmup_cases:
return
device = w1.device
def _warmup(w: torch.Tensor, w_scale: torch.Tensor):
@@ -289,15 +323,14 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
)
out = torch.empty((MAX_M, n), device=device, dtype=torch.bfloat16)
m_values = list(range(block_m, MAX_M + 1, block_m))
for num_tokens in m_values:
m_grouped_fp8_gemm_nt_contiguous(
(a1q[:num_tokens], a1q_scales[:num_tokens]),
(w, w_scale),
out[:num_tokens],
expert_ids[:num_tokens],
)
for num_tokens, align_used, expert_ids in warmup_cases:
with mk_alignment_scope(align_used):
m_grouped_fp8_gemm_nt_contiguous(
(a1q[:num_tokens], a1q_scales[:num_tokens]),
(w, w_scale),
out[:num_tokens],
expert_ids,
)
if pbar is not None:
pbar.update(1)
@@ -350,8 +383,8 @@ def _count_warmup_iterations(model: torch.nn.Module, max_tokens: int) -> int:
w13, _, w2, _, num_topk = _extract_data_from_fused_moe_module(m)
if w13.size() in seen_grouped_sizes and w2.size() in seen_grouped_sizes:
continue
MAX_M, block_m, _ = _get_grouped_gemm_params(w13, w2, num_topk, max_tokens)
n_values = (MAX_M - block_m) // block_m + 1
_, _, warmup_cases = _get_grouped_gemm_params(w13, w2, num_topk, max_tokens)
n_values = len(warmup_cases)
if w13.size() not in seen_grouped_sizes:
total += n_values
seen_grouped_sizes.add(w13.size())