Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
@@ -12,7 +12,9 @@ from tqdm import tqdm
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed.parallel_state import get_dp_group, is_global_first_rank
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import compute_aligned_M
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
|
||||
compute_aligned_M_and_alignment,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import DeepGemmExperts
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
@@ -25,6 +27,7 @@ from vllm.utils.deep_gemm import (
|
||||
fp8_gemm_nt,
|
||||
get_mk_alignment_for_contiguous_layout,
|
||||
m_grouped_fp8_gemm_nt_contiguous,
|
||||
mk_alignment_scope,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
@@ -238,7 +241,7 @@ def _get_grouped_gemm_params(
|
||||
w2: torch.Tensor,
|
||||
num_topk: int,
|
||||
max_tokens: int,
|
||||
) -> tuple[int, int, torch.Tensor]:
|
||||
) -> tuple[int, int, list[tuple[int, int, torch.Tensor]]]:
|
||||
assert w1.size(0) == w2.size(0), "w1 and w2 must have the same number of experts"
|
||||
|
||||
block_m = get_mk_alignment_for_contiguous_layout()[0]
|
||||
@@ -248,19 +251,46 @@ def _get_grouped_gemm_params(
|
||||
# Assumes all ranks have the same max_num_batched_tokens
|
||||
max_tokens = get_dp_group().world_size * max_tokens
|
||||
|
||||
# This is the maximum GroupedGemm M size that we expect to run
|
||||
# the grouped_gemm with.
|
||||
MAX_M = compute_aligned_M(
|
||||
max_tokens, num_topk, num_experts, block_m, expert_tokens_meta=None
|
||||
request_m_values = _generate_optimal_warmup_m_values(
|
||||
max_tokens,
|
||||
max(w1.size(1), w2.size(1)),
|
||||
device,
|
||||
)
|
||||
# Distribute expert-ids evenly.
|
||||
MAX_BLOCKS = MAX_M // block_m
|
||||
expert_ids_block = torch.randint(
|
||||
low=0, high=num_experts, size=(MAX_BLOCKS,), device=device, dtype=torch.int32
|
||||
)
|
||||
expert_ids = torch.repeat_interleave(expert_ids_block, block_m, dim=0)
|
||||
request_m_values = sorted({m for m in (*request_m_values, max_tokens) if m > 0})
|
||||
if not request_m_values:
|
||||
return 0, block_m, []
|
||||
|
||||
return MAX_M, block_m, expert_ids
|
||||
cases_by_shape: dict[tuple[int, int], torch.Tensor] = {}
|
||||
for request_m in request_m_values:
|
||||
M_sum, align_used = compute_aligned_M_and_alignment(
|
||||
M=request_m,
|
||||
num_topk=num_topk,
|
||||
local_num_experts=num_experts,
|
||||
alignment=block_m,
|
||||
expert_tokens_meta=None,
|
||||
)
|
||||
if (M_sum, align_used) in cases_by_shape:
|
||||
continue
|
||||
|
||||
num_blocks = M_sum // align_used
|
||||
expert_ids_block = torch.randint(
|
||||
low=0,
|
||||
high=num_experts,
|
||||
size=(num_blocks,),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
cases_by_shape[(M_sum, align_used)] = torch.repeat_interleave(
|
||||
expert_ids_block, align_used, dim=0
|
||||
)
|
||||
|
||||
max_m = max(M_sum for M_sum, _ in cases_by_shape)
|
||||
warmup_cases = [
|
||||
(M_sum, align_used, expert_ids)
|
||||
for (M_sum, align_used), expert_ids in sorted(cases_by_shape.items())
|
||||
]
|
||||
|
||||
return max_m, block_m, warmup_cases
|
||||
|
||||
|
||||
def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
|
||||
@@ -278,7 +308,11 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
|
||||
):
|
||||
return
|
||||
|
||||
MAX_M, block_m, expert_ids = _get_grouped_gemm_params(w1, w2, num_topk, max_tokens)
|
||||
MAX_M, block_m, warmup_cases = _get_grouped_gemm_params(
|
||||
w1, w2, num_topk, max_tokens
|
||||
)
|
||||
if not warmup_cases:
|
||||
return
|
||||
device = w1.device
|
||||
|
||||
def _warmup(w: torch.Tensor, w_scale: torch.Tensor):
|
||||
@@ -289,15 +323,14 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
|
||||
)
|
||||
out = torch.empty((MAX_M, n), device=device, dtype=torch.bfloat16)
|
||||
|
||||
m_values = list(range(block_m, MAX_M + 1, block_m))
|
||||
|
||||
for num_tokens in m_values:
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(a1q[:num_tokens], a1q_scales[:num_tokens]),
|
||||
(w, w_scale),
|
||||
out[:num_tokens],
|
||||
expert_ids[:num_tokens],
|
||||
)
|
||||
for num_tokens, align_used, expert_ids in warmup_cases:
|
||||
with mk_alignment_scope(align_used):
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(a1q[:num_tokens], a1q_scales[:num_tokens]),
|
||||
(w, w_scale),
|
||||
out[:num_tokens],
|
||||
expert_ids,
|
||||
)
|
||||
if pbar is not None:
|
||||
pbar.update(1)
|
||||
|
||||
@@ -350,8 +383,8 @@ def _count_warmup_iterations(model: torch.nn.Module, max_tokens: int) -> int:
|
||||
w13, _, w2, _, num_topk = _extract_data_from_fused_moe_module(m)
|
||||
if w13.size() in seen_grouped_sizes and w2.size() in seen_grouped_sizes:
|
||||
continue
|
||||
MAX_M, block_m, _ = _get_grouped_gemm_params(w13, w2, num_topk, max_tokens)
|
||||
n_values = (MAX_M - block_m) // block_m + 1
|
||||
_, _, warmup_cases = _get_grouped_gemm_params(w13, w2, num_topk, max_tokens)
|
||||
n_values = len(warmup_cases)
|
||||
if w13.size() not in seen_grouped_sizes:
|
||||
total += n_values
|
||||
seen_grouped_sizes.add(w13.size())
|
||||
|
||||
Reference in New Issue
Block a user