Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
@@ -208,6 +208,7 @@ from vllm.config import (
|
||||
get_current_vllm_config,
|
||||
get_current_vllm_config_or_none,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_dcp_group,
|
||||
is_global_first_rank,
|
||||
@@ -319,6 +320,22 @@ def _detect_output_quant_key(
|
||||
return kFp8StaticTensorSym
|
||||
|
||||
|
||||
def _canonicalize_sparse_mla_kv_cache_dtype(
|
||||
attn_backend: type[AttentionBackend],
|
||||
kv_cache_dtype: CacheDType,
|
||||
) -> CacheDType:
|
||||
backend_name = attn_backend.get_name()
|
||||
if backend_name == "FLASHMLA_SPARSE" and is_quantized_kv_cache(kv_cache_dtype):
|
||||
return "fp8_ds_mla"
|
||||
if backend_name == "FLASHINFER_MLA_SPARSE_SM120" and kv_cache_dtype in (
|
||||
"auto",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
):
|
||||
return "fp8_ds_mla"
|
||||
return kv_cache_dtype
|
||||
|
||||
|
||||
class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
"""Multi-Head Latent Attention layer.
|
||||
|
||||
@@ -369,7 +386,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
|
||||
|
||||
if cache_config is not None:
|
||||
kv_cache_dtype = cache_config.cache_dtype
|
||||
kv_cache_dtype: CacheDType = cache_config.cache_dtype
|
||||
calculate_kv_scales = cache_config.calculate_kv_scales
|
||||
else:
|
||||
kv_cache_dtype = "auto"
|
||||
@@ -393,24 +410,22 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
num_heads=self.num_heads,
|
||||
)
|
||||
|
||||
# FlashMLA Sparse Attention fp8 backend uses "fp8_ds_mla" kv-cache format
|
||||
# Automatically convert fp8 kv-cache format to "fp8_ds_mla"
|
||||
if (
|
||||
self.attn_backend.get_name() == "FLASHMLA_SPARSE"
|
||||
and is_quantized_kv_cache(kv_cache_dtype)
|
||||
and kv_cache_dtype != "fp8_ds_mla"
|
||||
):
|
||||
assert cache_config is not None
|
||||
cache_config.cache_dtype = "fp8_ds_mla"
|
||||
kv_cache_dtype = "fp8_ds_mla"
|
||||
normalized_kv_cache_dtype = _canonicalize_sparse_mla_kv_cache_dtype(
|
||||
self.attn_backend, kv_cache_dtype
|
||||
)
|
||||
if normalized_kv_cache_dtype != kv_cache_dtype:
|
||||
if cache_config is not None:
|
||||
cache_config.cache_dtype = normalized_kv_cache_dtype
|
||||
kv_cache_dtype = normalized_kv_cache_dtype
|
||||
logger.info_once(
|
||||
"Using DeepSeek's fp8_ds_mla KV cache format. To use standard "
|
||||
"fp8 kv-cache format, please set `--attention-backend "
|
||||
"FLASHINFER_MLA_SPARSE`"
|
||||
"Using %s KV cache format for %s backend.",
|
||||
kv_cache_dtype,
|
||||
self.attn_backend.get_name(),
|
||||
)
|
||||
|
||||
if (
|
||||
self.attn_backend.get_name() == "FLASHINFER_MLA_SPARSE"
|
||||
and kv_cache_dtype != "fp8_ds_mla"
|
||||
and is_quantized_kv_cache(kv_cache_dtype)
|
||||
):
|
||||
logger.info_once(
|
||||
|
||||
@@ -23,24 +23,82 @@ def expert_num_tokens_round_up_and_sum(
|
||||
return torch.sum(ent).item()
|
||||
|
||||
|
||||
def compute_aligned_M_and_alignment(
|
||||
M: int,
|
||||
num_topk: int,
|
||||
local_num_experts: int,
|
||||
alignment: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
) -> tuple[int, int]:
|
||||
"""Return (M_sum, alignment_used).
|
||||
|
||||
`alignment_used` may be smaller than the caller-supplied `alignment` on
|
||||
SM100/SM120 when DeepGEMM can JIT a smaller BLOCK_M for the per-call
|
||||
expected_m. Callers that index by block size (e.g. ``M_sum // block_m``)
|
||||
or assert workspace alignment must use the returned `alignment_used`,
|
||||
not their original `alignment` argument.
|
||||
|
||||
Prefer this over the int-returning :func:`compute_aligned_M` when the
|
||||
GEMM call site needs to wrap itself in ``mk_alignment_scope`` or
|
||||
otherwise reason about the actual per-expert padding.
|
||||
"""
|
||||
if (expert_tokens_meta is not None) and (
|
||||
expert_tokens_meta.expert_num_tokens_cpu is not None
|
||||
):
|
||||
return (
|
||||
expert_num_tokens_round_up_and_sum(
|
||||
expert_tokens_meta.expert_num_tokens_cpu, alignment=alignment
|
||||
),
|
||||
alignment,
|
||||
)
|
||||
|
||||
# expert_num_tokens not on cpu. Cap padding by min(M*num_topk,
|
||||
# local_num_experts) — at batch=1 decode only `num_topk` experts can be
|
||||
# active, so the worst-case `local_num_experts*(align-1)` is too loose.
|
||||
# Also shrink `alignment` to DeepGEMM's per-call theoretical BLOCK_M on
|
||||
# SM100/SM120 when smaller.
|
||||
expected_m = M * num_topk
|
||||
try:
|
||||
from vllm.utils.deep_gemm import (
|
||||
get_theoretical_mk_alignment_for_contiguous_layout,
|
||||
)
|
||||
|
||||
# num_groups=local_num_experts so the helper recovers per-expert em;
|
||||
# omitting it over-picks BLOCK_M on SM120 (heuristic assumes em is
|
||||
# already per-expert).
|
||||
per_call_align = get_theoretical_mk_alignment_for_contiguous_layout(
|
||||
expected_m=expected_m,
|
||||
num_groups=local_num_experts,
|
||||
)
|
||||
if per_call_align and per_call_align <= alignment:
|
||||
alignment = per_call_align
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
max_active_experts = min(M * num_topk, local_num_experts)
|
||||
M_sum = (M * num_topk) + max_active_experts * (alignment - 1)
|
||||
M_sum = round_up(M_sum, alignment)
|
||||
return M_sum, alignment
|
||||
|
||||
|
||||
def compute_aligned_M(
|
||||
M: int,
|
||||
num_topk: int,
|
||||
local_num_experts: int,
|
||||
alignment: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
):
|
||||
if (expert_tokens_meta is not None) and (
|
||||
expert_tokens_meta.expert_num_tokens_cpu is not None
|
||||
):
|
||||
return expert_num_tokens_round_up_and_sum(
|
||||
expert_tokens_meta.expert_num_tokens_cpu, alignment=alignment
|
||||
)
|
||||
) -> int:
|
||||
"""Return ``M_sum`` only (backward-compat wrapper).
|
||||
|
||||
# expert_num_tokens information is not available on the cpu.
|
||||
# compute the max required size.
|
||||
M_sum = (M * num_topk) + local_num_experts * (alignment - 1)
|
||||
M_sum = round_up(M_sum, alignment)
|
||||
Equivalent to :func:`compute_aligned_M_and_alignment`'s first return
|
||||
value. Existing downstream callers and the warmup path that only size
|
||||
a workspace use this. Call sites that need the actual per-expert
|
||||
alignment (to wrap GEMMs in ``mk_alignment_scope``) should use
|
||||
:func:`compute_aligned_M_and_alignment` instead.
|
||||
"""
|
||||
M_sum, _ = compute_aligned_M_and_alignment(
|
||||
M, num_topk, local_num_experts, alignment, expert_tokens_meta
|
||||
)
|
||||
return M_sum
|
||||
|
||||
|
||||
@@ -51,12 +109,6 @@ def apply_expert_map(expert_id, expert_map):
|
||||
return expert_id
|
||||
|
||||
|
||||
@triton.jit
|
||||
def round_up_128(x: int) -> int:
|
||||
y = 128
|
||||
return ((x + y - 1) // y) * y
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_kernel_ep_scatter_1(
|
||||
num_recv_tokens_per_expert,
|
||||
@@ -65,6 +117,7 @@ def _fwd_kernel_ep_scatter_1(
|
||||
num_experts: tl.constexpr,
|
||||
BLOCK_E: tl.constexpr,
|
||||
BLOCK_EXPERT_NUM: tl.constexpr,
|
||||
ALIGN_M: tl.constexpr,
|
||||
):
|
||||
cur_expert = tl.program_id(0)
|
||||
|
||||
@@ -74,7 +127,8 @@ def _fwd_kernel_ep_scatter_1(
|
||||
mask=offset_cumsum < num_experts,
|
||||
other=0,
|
||||
)
|
||||
tokens_per_expert = round_up_128(tokens_per_expert)
|
||||
# Round up to ALIGN_M so cumsum matches the workspace's per-expert slices.
|
||||
tokens_per_expert = ((tokens_per_expert + ALIGN_M - 1) // ALIGN_M) * ALIGN_M
|
||||
cumsum = tl.cumsum(tokens_per_expert) - tokens_per_expert
|
||||
|
||||
# Extract this block's offset from the register vector (warp shuffle,
|
||||
@@ -227,10 +281,12 @@ def ep_scatter(
|
||||
output_tensor_scale: torch.Tensor,
|
||||
m_indices: torch.Tensor,
|
||||
output_index: torch.Tensor,
|
||||
align_m: int = 128,
|
||||
block_size: int = 128,
|
||||
pack_ue8m0: bool = False,
|
||||
):
|
||||
BLOCK_E = 128 # token num of per expert is aligned to 128
|
||||
# BLOCK_E is the m_indices fill-loop tile (masked), independent of align_m.
|
||||
BLOCK_E = 128
|
||||
BLOCK_D = block_size # block size of activation-scale quantization
|
||||
num_warps = 8
|
||||
num_experts = num_recv_tokens_per_expert.shape[0]
|
||||
@@ -238,7 +294,7 @@ def ep_scatter(
|
||||
# grid = (triton.cdiv(hidden_size, BLOCK_D), num_experts)
|
||||
grid = num_experts
|
||||
|
||||
assert m_indices.shape[0] % BLOCK_E == 0
|
||||
assert m_indices.shape[0] % align_m == 0
|
||||
assert expert_start_loc.shape[0] == num_experts
|
||||
|
||||
# pack_ue8m0: scatter packs 4 UE8M0 bytes per int32; else copies scales as-is.
|
||||
@@ -253,6 +309,7 @@ def ep_scatter(
|
||||
num_warps=num_warps,
|
||||
BLOCK_E=BLOCK_E,
|
||||
BLOCK_EXPERT_NUM=triton.next_power_of_2(num_experts),
|
||||
ALIGN_M=align_m,
|
||||
)
|
||||
|
||||
grid = min(recv_topk.shape[0], 1024 * 8)
|
||||
@@ -418,7 +475,7 @@ def deepgemm_moe_permute(
|
||||
if block_size is not None:
|
||||
block_k = block_size
|
||||
|
||||
M_sum = compute_aligned_M(
|
||||
M_sum, align_used = compute_aligned_M_and_alignment(
|
||||
M=topk_ids.size(0),
|
||||
num_topk=topk_ids.size(1),
|
||||
local_num_experts=local_num_experts,
|
||||
@@ -482,11 +539,12 @@ def deepgemm_moe_permute(
|
||||
output_tensor_scale=aq_scale_out,
|
||||
m_indices=expert_ids,
|
||||
output_index=inv_perm,
|
||||
align_m=align_used,
|
||||
block_size=block_k,
|
||||
pack_ue8m0=pack_ue8m0,
|
||||
)
|
||||
|
||||
return aq_out, aq_scale_out, expert_ids, inv_perm
|
||||
return aq_out, aq_scale_out, expert_ids, inv_perm, align_used
|
||||
|
||||
|
||||
def deepgemm_unpermute_and_reduce(
|
||||
|
||||
@@ -318,11 +318,12 @@ class BatchedDeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
def supports_packed_ue8m0_act_scales(self) -> bool:
|
||||
"""
|
||||
DeepGemm supports packed ue8m0 activation scales format in devices == sm100
|
||||
DeepGemm supports packed ue8m0 activation scales on Blackwell-family
|
||||
GPUs (SM100 datacenter and SM120 consumer).
|
||||
"""
|
||||
return (
|
||||
is_deep_gemm_e8m0_used()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
return is_deep_gemm_e8m0_used() and (
|
||||
current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
)
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
|
||||
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
|
||||
compute_aligned_M,
|
||||
compute_aligned_M_and_alignment,
|
||||
deepgemm_moe_permute,
|
||||
deepgemm_unpermute_and_reduce,
|
||||
)
|
||||
@@ -43,6 +43,7 @@ from vllm.utils.deep_gemm import (
|
||||
is_deep_gemm_supported,
|
||||
m_grouped_fp8_fp4_gemm_nt_contiguous,
|
||||
m_grouped_fp8_gemm_nt_contiguous,
|
||||
mk_alignment_scope,
|
||||
)
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
|
||||
@@ -210,10 +211,10 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
# Use the contiguous-layout M alignment (matches apply()); block_shape[0]
|
||||
# is the quant block (1 for MXFP8) and would under-size the workspace.
|
||||
block_m = get_mk_alignment_for_contiguous_layout()[0]
|
||||
M_sum = compute_aligned_M(
|
||||
M_sum, align_used = compute_aligned_M_and_alignment(
|
||||
M, topk, local_num_experts, block_m, expert_tokens_meta
|
||||
)
|
||||
assert M_sum % block_m == 0
|
||||
assert M_sum % align_used == 0
|
||||
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
workspace1 = (M_sum, max(activation_out_dim, K))
|
||||
@@ -316,7 +317,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
assert w2.size(1) == K
|
||||
|
||||
M_sum = compute_aligned_M(
|
||||
M_sum, _ = compute_aligned_M_and_alignment(
|
||||
M=topk_ids.size(0),
|
||||
num_topk=topk_ids.size(1),
|
||||
local_num_experts=local_num_experts,
|
||||
@@ -327,7 +328,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
a1q_perm = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
|
||||
)
|
||||
a1q, a1q_scale, expert_ids, inv_perm = deepgemm_moe_permute(
|
||||
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
|
||||
aq=a1q,
|
||||
aq_scale=a1q_scale,
|
||||
topk_ids=topk_ids,
|
||||
@@ -349,23 +350,35 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
else {}
|
||||
)
|
||||
|
||||
mm1_out = _resize_cache(workspace2, (M_sum, N))
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(a1q, a1q_scale), (w1, self.w1_scale), mm1_out, expert_ids, **gemm_kwargs
|
||||
)
|
||||
# Cap DG's BLOCK_M heuristic at the workspace's per-expert alignment;
|
||||
# otherwise the scheduler can pick the wrong expert id from m_indices
|
||||
# under cudagraph replay.
|
||||
with mk_alignment_scope(align_used):
|
||||
mm1_out = _resize_cache(workspace2, (M_sum, N))
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(a1q, a1q_scale),
|
||||
(w1, self.w1_scale),
|
||||
mm1_out,
|
||||
expert_ids,
|
||||
**gemm_kwargs,
|
||||
)
|
||||
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
quant_out = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
||||
)
|
||||
a2q, a2q_scale = self._act_mul_quant(
|
||||
input=mm1_out.view(-1, N), output=quant_out, activation=activation
|
||||
)
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
quant_out = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
||||
)
|
||||
a2q, a2q_scale = self._act_mul_quant(
|
||||
input=mm1_out.view(-1, N), output=quant_out, activation=activation
|
||||
)
|
||||
|
||||
mm2_out = _resize_cache(workspace2, (M_sum, K))
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(a2q, a2q_scale), (w2, self.w2_scale), mm2_out, expert_ids, **gemm_kwargs
|
||||
)
|
||||
mm2_out = _resize_cache(workspace2, (M_sum, K))
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(a2q, a2q_scale),
|
||||
(w2, self.w2_scale),
|
||||
mm2_out,
|
||||
expert_ids,
|
||||
**gemm_kwargs,
|
||||
)
|
||||
|
||||
if apply_router_weight_on_input:
|
||||
topk_weights = torch.ones_like(topk_weights)
|
||||
@@ -384,7 +397,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
"""DeepGemm-based fused MoE expert implementation for FP4 weights.
|
||||
|
||||
Uses m_grouped_fp8_fp4_gemm_nt_contiguous with FP8 activations and
|
||||
MXFP4 (FP4 E2M1 packed as uint8) weights. Requires SM100+ (Blackwell).
|
||||
MXFP4 (FP4 E2M1 packed as uint8) weights. Requires Blackwell-family
|
||||
GPUs (SM100 datacenter or SM120 consumer).
|
||||
"""
|
||||
|
||||
# FP8 activation block size (hardcoded since mxfp4_w4a8 quant config
|
||||
@@ -409,9 +423,9 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
def _supports_current_device() -> bool:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
return (
|
||||
is_deep_gemm_supported()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
return is_deep_gemm_supported() and (
|
||||
current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -454,10 +468,10 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
block_m = get_mk_alignment_for_contiguous_layout()[0]
|
||||
M_sum = compute_aligned_M(
|
||||
M_sum, align_used = compute_aligned_M_and_alignment(
|
||||
M, topk, local_num_experts, block_m, expert_tokens_meta
|
||||
)
|
||||
assert M_sum % block_m == 0
|
||||
assert M_sum % align_used == 0
|
||||
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
workspace1 = (M_sum, max(activation_out_dim, K))
|
||||
@@ -533,7 +547,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
if global_num_experts == -1:
|
||||
global_num_experts = local_num_experts
|
||||
|
||||
M_sum = compute_aligned_M(
|
||||
M_sum, _ = compute_aligned_M_and_alignment(
|
||||
M=topk_ids.size(0),
|
||||
num_topk=topk_ids.size(1),
|
||||
local_num_experts=local_num_experts,
|
||||
@@ -544,7 +558,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
a1q_perm = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
|
||||
)
|
||||
a1q, a1q_scale, expert_ids, inv_perm = deepgemm_moe_permute(
|
||||
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
|
||||
aq=a1q,
|
||||
aq_scale=a1q_scale,
|
||||
topk_ids=topk_ids,
|
||||
@@ -555,37 +569,40 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
)
|
||||
assert a1q.size(0) == M_sum
|
||||
|
||||
# FC1: FP8 activations x FP4 weights
|
||||
# DeepGEMM 2.4.2 requires FP4-packed weights as int8 (kPackedFP4).
|
||||
mm1_out = _resize_cache(workspace2, (M_sum, N))
|
||||
m_grouped_fp8_fp4_gemm_nt_contiguous(
|
||||
(a1q, a1q_scale),
|
||||
(w1.view(torch.int8), self.w1_scale),
|
||||
mm1_out,
|
||||
expert_ids,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
# Cap DG's BLOCK_M heuristic at the workspace's per-expert alignment;
|
||||
# see DeepGemmExperts.apply for rationale.
|
||||
with mk_alignment_scope(align_used):
|
||||
# FC1: FP8 activations x FP4 weights
|
||||
# DeepGEMM 2.4.2 requires FP4-packed weights as int8 (kPackedFP4).
|
||||
mm1_out = _resize_cache(workspace2, (M_sum, N))
|
||||
m_grouped_fp8_fp4_gemm_nt_contiguous(
|
||||
(a1q, a1q_scale),
|
||||
(w1.view(torch.int8), self.w1_scale),
|
||||
mm1_out,
|
||||
expert_ids,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
|
||||
# SwiGLU activation + FP8 requant
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
quant_out = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
||||
)
|
||||
a2q, a2q_scale = self._act_mul_quant(
|
||||
input=mm1_out.view(-1, N), output=quant_out, activation=activation
|
||||
)
|
||||
# SwiGLU activation + FP8 requant
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
quant_out = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
||||
)
|
||||
a2q, a2q_scale = self._act_mul_quant(
|
||||
input=mm1_out.view(-1, N), output=quant_out, activation=activation
|
||||
)
|
||||
|
||||
# FC2: FP8 activations x FP4 weights
|
||||
mm2_out = _resize_cache(workspace2, (M_sum, K))
|
||||
m_grouped_fp8_fp4_gemm_nt_contiguous(
|
||||
(a2q, a2q_scale),
|
||||
(w2.view(torch.int8), self.w2_scale),
|
||||
mm2_out,
|
||||
expert_ids,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
# FC2: FP8 activations x FP4 weights
|
||||
mm2_out = _resize_cache(workspace2, (M_sum, K))
|
||||
m_grouped_fp8_fp4_gemm_nt_contiguous(
|
||||
(a2q, a2q_scale),
|
||||
(w2.view(torch.int8), self.w2_scale),
|
||||
mm2_out,
|
||||
expert_ids,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
|
||||
if apply_router_weight_on_input:
|
||||
topk_weights = torch.ones_like(topk_weights)
|
||||
|
||||
@@ -57,6 +57,40 @@ if has_triton_kernels():
|
||||
)
|
||||
|
||||
|
||||
def _pack_deepgemm_mxfp4_scales(
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
w13_weight_scale: torch.Tensor,
|
||||
w2_weight_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
deepgemm_post_process_weight_scale_block,
|
||||
)
|
||||
|
||||
num_experts = w13_weight.shape[0]
|
||||
intermediate_size_2 = w13_weight.shape[1] # = intermediate*2
|
||||
hidden_size = w13_weight.shape[2] * 2 # weight is FP4-packed
|
||||
intermediate_size = w2_weight.shape[2] * 2 # weight is FP4-packed
|
||||
block_shape = (1, 32) # MXFP4 block (per-row, K=32)
|
||||
|
||||
return (
|
||||
deepgemm_post_process_weight_scale_block(
|
||||
ws=w13_weight_scale.data,
|
||||
mn=intermediate_size_2,
|
||||
k=hidden_size,
|
||||
quant_block_shape=block_shape,
|
||||
num_groups=num_experts,
|
||||
),
|
||||
deepgemm_post_process_weight_scale_block(
|
||||
ws=w2_weight_scale.data,
|
||||
mn=hidden_size,
|
||||
k=intermediate_size,
|
||||
quant_block_shape=block_shape,
|
||||
num_groups=num_experts,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class Mxfp4MoeBackend(Enum):
|
||||
NONE = "None"
|
||||
# DeepGEMM FP8xFP4 backend (SM100+)
|
||||
@@ -652,15 +686,18 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
"""Convert loaded weights into backend-specific kernel format."""
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales(
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
w13_weight_scale,
|
||||
w2_weight_scale,
|
||||
)
|
||||
|
||||
return (
|
||||
w13_weight.data,
|
||||
w2_weight.data,
|
||||
_upcast_e8m0_to_fp32(w13_weight_scale.data),
|
||||
_upcast_e8m0_to_fp32(w2_weight_scale.data),
|
||||
w13_weight_scale,
|
||||
w2_weight_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
@@ -1195,17 +1232,18 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
"""
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales(
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
w13_weight_scale,
|
||||
w2_weight_scale,
|
||||
)
|
||||
|
||||
# Weights stay as uint8 packed FP4 — no layout change needed.
|
||||
# Convert E8M0 uint8 scales to float32.
|
||||
return (
|
||||
w13_weight.data,
|
||||
w2_weight.data,
|
||||
_upcast_e8m0_to_fp32(w13_weight_scale.data),
|
||||
_upcast_e8m0_to_fp32(w2_weight_scale.data),
|
||||
w13_weight_scale,
|
||||
w2_weight_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
@@ -1058,6 +1058,34 @@ def _upcast_e8m0_to_fp32(scale: torch.Tensor) -> torch.Tensor:
|
||||
return fp32_bits.view(torch.float32)
|
||||
|
||||
|
||||
def deepgemm_post_process_weight_scale_block(
|
||||
ws: torch.Tensor,
|
||||
mn: int,
|
||||
k: int,
|
||||
quant_block_shape: tuple[int, ...],
|
||||
num_groups: int,
|
||||
is_sfa: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
|
||||
# Scales already in E8M0 from checkpoint; upcast to fp32 and let
|
||||
# DeepGEMM pack the layout expected by the target architecture.
|
||||
ws = _upcast_e8m0_to_fp32(ws)
|
||||
else:
|
||||
assert ws.dtype == torch.float32, (
|
||||
f"Expected tensor scales dtype to be torch.float32 or "
|
||||
f"torch.float8_e8m0fnu or torch.uint8, got {ws.dtype} instead"
|
||||
)
|
||||
|
||||
return transform_sf_into_required_layout(
|
||||
sf=ws,
|
||||
mn=mn,
|
||||
k=k,
|
||||
recipe=(1, quant_block_shape[0], quant_block_shape[1]),
|
||||
num_groups=num_groups,
|
||||
is_sfa=is_sfa,
|
||||
)
|
||||
|
||||
|
||||
def deepgemm_post_process_fp8_weight_block(
|
||||
wq: torch.Tensor,
|
||||
ws: torch.Tensor,
|
||||
@@ -1073,13 +1101,13 @@ def deepgemm_post_process_fp8_weight_block(
|
||||
|
||||
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
|
||||
# Scales already in E8M0 from checkpoint (float8_e8m0fnu, or raw E8M0
|
||||
# bits as uint8 for MXFP8) — upcast to fp32 and skip requantization
|
||||
# bits as uint8 for MXFP8) - upcast to fp32 and skip requantization
|
||||
# (weights already have power-of-two scales).
|
||||
ws = _upcast_e8m0_to_fp32(ws)
|
||||
else:
|
||||
assert ws.dtype == torch.float32, (
|
||||
f"Expected tensor scales dtype to be torch.float32 or "
|
||||
f"torch.float8_e8m0fnu, got {ws.dtype} instead"
|
||||
f"torch.float8_e8m0fnu or torch.uint8, got {ws.dtype} instead"
|
||||
)
|
||||
if use_e8m0:
|
||||
requant_weight_ue8m0_inplace(wq, ws, block_size=quant_block_shape)
|
||||
@@ -1094,16 +1122,12 @@ def deepgemm_post_process_fp8_weight_block(
|
||||
r = wq.size(0) // g
|
||||
wq = wq.view(g, r, d)
|
||||
ws = ws.view(g, r // quant_block_shape[0], d // quant_block_shape[1])
|
||||
# Pre-transform scale with recipe=(1, 128, 128) to broadcast + pack
|
||||
# into TMA-aligned UE8M0 (INT32) layout. At runtime fp8_einsum uses
|
||||
# recipe=(1, 1, 128) which sees INT dtype and skips re-transform.
|
||||
dg_ws = transform_sf_into_required_layout(
|
||||
sf=ws,
|
||||
dg_ws = deepgemm_post_process_weight_scale_block(
|
||||
ws=ws,
|
||||
mn=r,
|
||||
k=d,
|
||||
recipe=(1, quant_block_shape[0], quant_block_shape[1]),
|
||||
quant_block_shape=quant_block_shape,
|
||||
num_groups=g,
|
||||
is_sfa=False,
|
||||
)
|
||||
return wq, dg_ws
|
||||
|
||||
@@ -1113,22 +1137,12 @@ def deepgemm_post_process_fp8_weight_block(
|
||||
wq = wq.unsqueeze(0)
|
||||
ws = ws.unsqueeze(0)
|
||||
|
||||
# From https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/utils/layout.hpp#L46
|
||||
# (1, block_n, block_k): (1, 128, 128) for FP8 block, (1, 1, 32) for MXFP8.
|
||||
recipe = (1, quant_block_shape[0], quant_block_shape[1])
|
||||
|
||||
# Ref : https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/apis/gemm.hpp
|
||||
# DeepGemm uses the `transform_sf_into_required_layout` function to
|
||||
# represent scales in the correct format.
|
||||
dg_ws = transform_sf_into_required_layout(
|
||||
sf=ws,
|
||||
dg_ws = deepgemm_post_process_weight_scale_block(
|
||||
ws=ws,
|
||||
mn=wq.size(1),
|
||||
k=wq.size(2),
|
||||
recipe=recipe,
|
||||
quant_block_shape=quant_block_shape,
|
||||
num_groups=wq.size(0),
|
||||
# is the scale factors for A in (Refers to the argument A in A @ B).
|
||||
# Weights are B.
|
||||
is_sfa=False,
|
||||
)
|
||||
|
||||
if original_ndim == 2:
|
||||
|
||||
@@ -12,7 +12,9 @@ from tqdm import tqdm
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed.parallel_state import get_dp_group, is_global_first_rank
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import compute_aligned_M
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
|
||||
compute_aligned_M_and_alignment,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import DeepGemmExperts
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
@@ -25,6 +27,7 @@ from vllm.utils.deep_gemm import (
|
||||
fp8_gemm_nt,
|
||||
get_mk_alignment_for_contiguous_layout,
|
||||
m_grouped_fp8_gemm_nt_contiguous,
|
||||
mk_alignment_scope,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
@@ -238,7 +241,7 @@ def _get_grouped_gemm_params(
|
||||
w2: torch.Tensor,
|
||||
num_topk: int,
|
||||
max_tokens: int,
|
||||
) -> tuple[int, int, torch.Tensor]:
|
||||
) -> tuple[int, int, list[tuple[int, int, torch.Tensor]]]:
|
||||
assert w1.size(0) == w2.size(0), "w1 and w2 must have the same number of experts"
|
||||
|
||||
block_m = get_mk_alignment_for_contiguous_layout()[0]
|
||||
@@ -248,19 +251,46 @@ def _get_grouped_gemm_params(
|
||||
# Assumes all ranks have the same max_num_batched_tokens
|
||||
max_tokens = get_dp_group().world_size * max_tokens
|
||||
|
||||
# This is the maximum GroupedGemm M size that we expect to run
|
||||
# the grouped_gemm with.
|
||||
MAX_M = compute_aligned_M(
|
||||
max_tokens, num_topk, num_experts, block_m, expert_tokens_meta=None
|
||||
request_m_values = _generate_optimal_warmup_m_values(
|
||||
max_tokens,
|
||||
max(w1.size(1), w2.size(1)),
|
||||
device,
|
||||
)
|
||||
# Distribute expert-ids evenly.
|
||||
MAX_BLOCKS = MAX_M // block_m
|
||||
expert_ids_block = torch.randint(
|
||||
low=0, high=num_experts, size=(MAX_BLOCKS,), device=device, dtype=torch.int32
|
||||
)
|
||||
expert_ids = torch.repeat_interleave(expert_ids_block, block_m, dim=0)
|
||||
request_m_values = sorted({m for m in (*request_m_values, max_tokens) if m > 0})
|
||||
if not request_m_values:
|
||||
return 0, block_m, []
|
||||
|
||||
return MAX_M, block_m, expert_ids
|
||||
cases_by_shape: dict[tuple[int, int], torch.Tensor] = {}
|
||||
for request_m in request_m_values:
|
||||
M_sum, align_used = compute_aligned_M_and_alignment(
|
||||
M=request_m,
|
||||
num_topk=num_topk,
|
||||
local_num_experts=num_experts,
|
||||
alignment=block_m,
|
||||
expert_tokens_meta=None,
|
||||
)
|
||||
if (M_sum, align_used) in cases_by_shape:
|
||||
continue
|
||||
|
||||
num_blocks = M_sum // align_used
|
||||
expert_ids_block = torch.randint(
|
||||
low=0,
|
||||
high=num_experts,
|
||||
size=(num_blocks,),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
cases_by_shape[(M_sum, align_used)] = torch.repeat_interleave(
|
||||
expert_ids_block, align_used, dim=0
|
||||
)
|
||||
|
||||
max_m = max(M_sum for M_sum, _ in cases_by_shape)
|
||||
warmup_cases = [
|
||||
(M_sum, align_used, expert_ids)
|
||||
for (M_sum, align_used), expert_ids in sorted(cases_by_shape.items())
|
||||
]
|
||||
|
||||
return max_m, block_m, warmup_cases
|
||||
|
||||
|
||||
def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
|
||||
@@ -278,7 +308,11 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
|
||||
):
|
||||
return
|
||||
|
||||
MAX_M, block_m, expert_ids = _get_grouped_gemm_params(w1, w2, num_topk, max_tokens)
|
||||
MAX_M, block_m, warmup_cases = _get_grouped_gemm_params(
|
||||
w1, w2, num_topk, max_tokens
|
||||
)
|
||||
if not warmup_cases:
|
||||
return
|
||||
device = w1.device
|
||||
|
||||
def _warmup(w: torch.Tensor, w_scale: torch.Tensor):
|
||||
@@ -289,15 +323,14 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
|
||||
)
|
||||
out = torch.empty((MAX_M, n), device=device, dtype=torch.bfloat16)
|
||||
|
||||
m_values = list(range(block_m, MAX_M + 1, block_m))
|
||||
|
||||
for num_tokens in m_values:
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(a1q[:num_tokens], a1q_scales[:num_tokens]),
|
||||
(w, w_scale),
|
||||
out[:num_tokens],
|
||||
expert_ids[:num_tokens],
|
||||
)
|
||||
for num_tokens, align_used, expert_ids in warmup_cases:
|
||||
with mk_alignment_scope(align_used):
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(a1q[:num_tokens], a1q_scales[:num_tokens]),
|
||||
(w, w_scale),
|
||||
out[:num_tokens],
|
||||
expert_ids,
|
||||
)
|
||||
if pbar is not None:
|
||||
pbar.update(1)
|
||||
|
||||
@@ -350,8 +383,8 @@ def _count_warmup_iterations(model: torch.nn.Module, max_tokens: int) -> int:
|
||||
w13, _, w2, _, num_topk = _extract_data_from_fused_moe_module(m)
|
||||
if w13.size() in seen_grouped_sizes and w2.size() in seen_grouped_sizes:
|
||||
continue
|
||||
MAX_M, block_m, _ = _get_grouped_gemm_params(w13, w2, num_topk, max_tokens)
|
||||
n_values = (MAX_M - block_m) // block_m + 1
|
||||
_, _, warmup_cases = _get_grouped_gemm_params(w13, w2, num_topk, max_tokens)
|
||||
n_values = len(warmup_cases)
|
||||
if w13.size() not in seen_grouped_sizes:
|
||||
total += n_values
|
||||
seen_grouped_sizes.add(w13.size())
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm up DeepSeek V4 mHC TileLang kernels before serving requests.
|
||||
|
||||
Ported from lucifer1004/vllm-jasl with the two env-var knobs removed
|
||||
(`VLLM_ENABLE_DEEPSEEK_V4_MHC_WARMUP`, `VLLM_DEEPSEEK_V4_MHC_WARMUP_TOKEN_SIZES`).
|
||||
Gating is intrinsic: non-DSv4 models and layers without hc_* attributes
|
||||
return early, so the warmup is a no-op except where it's needed.
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_AUTO_WARMUP_MAX_TOKENS = 16_384
|
||||
_DEFAULT_TOKEN_SIZE_CANDIDATES = (
|
||||
1,
|
||||
2,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128,
|
||||
256,
|
||||
512,
|
||||
1024,
|
||||
2048,
|
||||
4096,
|
||||
8192,
|
||||
16_384,
|
||||
)
|
||||
|
||||
|
||||
def _compute_mhc_pre_num_split(
|
||||
*,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
num_sms: int,
|
||||
) -> int:
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
k = hc_mult * hidden_size
|
||||
grid_size = cdiv(num_tokens, block_m)
|
||||
split_k = num_sms // grid_size
|
||||
num_block_k = cdiv(k, block_k)
|
||||
split_k = min(split_k, num_block_k // 4)
|
||||
return max(split_k, 1)
|
||||
|
||||
|
||||
def _normalize_token_sizes(
|
||||
token_sizes: Iterable[int],
|
||||
*,
|
||||
max_tokens: int,
|
||||
) -> list[int]:
|
||||
return sorted({size for size in token_sizes if 1 <= size <= max_tokens})
|
||||
|
||||
|
||||
def _select_mhc_warmup_token_sizes(
|
||||
*,
|
||||
max_tokens: int,
|
||||
cudagraph_capture_sizes: list[int],
|
||||
) -> list[int]:
|
||||
if max_tokens <= 0:
|
||||
return []
|
||||
|
||||
max_auto_tokens = min(max_tokens, _AUTO_WARMUP_MAX_TOKENS)
|
||||
candidates = list(_DEFAULT_TOKEN_SIZE_CANDIDATES)
|
||||
candidates.extend(cudagraph_capture_sizes)
|
||||
candidates.append(max_auto_tokens)
|
||||
return _normalize_token_sizes(candidates, max_tokens=max_auto_tokens)
|
||||
|
||||
|
||||
def _find_first_mhc_layer(model: torch.nn.Module) -> torch.nn.Module | None:
|
||||
for module in model.modules():
|
||||
if module.__class__.__name__ != "DeepseekV4DecoderLayer":
|
||||
continue
|
||||
if all(
|
||||
hasattr(module, attr)
|
||||
for attr in (
|
||||
"hc_pre",
|
||||
"hc_post",
|
||||
"hc_attn_fn",
|
||||
"hc_attn_scale",
|
||||
"hc_attn_base",
|
||||
"hc_ffn_fn",
|
||||
"hc_ffn_scale",
|
||||
"hc_ffn_base",
|
||||
)
|
||||
):
|
||||
return module
|
||||
return None
|
||||
|
||||
|
||||
def _find_deepseek_v4_model(model: torch.nn.Module) -> torch.nn.Module | None:
|
||||
for module in model.modules():
|
||||
if module.__class__.__name__ != "DeepseekV4Model":
|
||||
continue
|
||||
if all(
|
||||
hasattr(module, attr)
|
||||
for attr in ("hc_head_fn", "hc_head_scale", "hc_head_base")
|
||||
):
|
||||
return module
|
||||
return None
|
||||
|
||||
|
||||
def _warmup_layer_mhc(
|
||||
layer: torch.nn.Module,
|
||||
token_sizes: list[int],
|
||||
) -> None:
|
||||
max_tokens = max(token_sizes)
|
||||
hidden_size = int(layer.hidden_size)
|
||||
hc_mult = int(layer.hc_mult)
|
||||
device = layer.hc_attn_fn.device
|
||||
residual = torch.zeros(
|
||||
max_tokens,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
for size in token_sizes:
|
||||
residual_slice = residual[:size]
|
||||
for fn, scale, base in (
|
||||
(layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base),
|
||||
(layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base),
|
||||
):
|
||||
layer_input, post_mix, comb_mix = layer.hc_pre(
|
||||
residual_slice,
|
||||
fn,
|
||||
scale,
|
||||
base,
|
||||
)
|
||||
layer.hc_post(layer_input, residual_slice, post_mix, comb_mix)
|
||||
|
||||
|
||||
def _warmup_hc_head(
|
||||
model: torch.nn.Module,
|
||||
token_sizes: list[int],
|
||||
) -> None:
|
||||
# Upstream a8887c208 ("[DSV4] aiter mhc support (ROCm)") refactored
|
||||
# ``hc_head`` from a free function into the ``HCHeadOp`` CustomOp
|
||||
# instance attached to the model as ``hc_head_op``. We call through
|
||||
# that instance so the warmup exercises the same dispatched
|
||||
# implementation as the inference path.
|
||||
hc_head_op = getattr(model, "hc_head_op", None)
|
||||
if hc_head_op is None:
|
||||
return
|
||||
|
||||
max_tokens = max(token_sizes)
|
||||
hidden_size = int(model.config.hidden_size)
|
||||
hc_mult = int(model.hc_mult)
|
||||
device = model.hc_head_fn.device
|
||||
hidden_states = torch.zeros(
|
||||
max_tokens,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
for size in token_sizes:
|
||||
hc_head_op(
|
||||
hidden_states[:size],
|
||||
model.hc_head_fn,
|
||||
model.hc_head_scale,
|
||||
model.hc_head_base,
|
||||
model.rms_norm_eps,
|
||||
model.hc_eps,
|
||||
)
|
||||
|
||||
|
||||
@instrument(span_name="DeepSeek V4 mHC warmup")
|
||||
def deepseek_v4_mhc_warmup(
|
||||
model: torch.nn.Module,
|
||||
*,
|
||||
max_tokens: int,
|
||||
cudagraph_capture_sizes: list[int] | None = None,
|
||||
) -> None:
|
||||
# Cheap model-type gate before walking ``model.modules()``. The class
|
||||
# walk below is O(num_layers) and shows up in startup time on very
|
||||
# large checkpoints; bail out for any model that is not DeepSeek V4.
|
||||
config = getattr(model, "config", None)
|
||||
model_type = getattr(config, "model_type", None) if config is not None else None
|
||||
if model_type is not None and model_type != "deepseek_v4":
|
||||
return
|
||||
|
||||
layer = _find_first_mhc_layer(model)
|
||||
if layer is None:
|
||||
return
|
||||
|
||||
device = layer.hc_attn_fn.device
|
||||
if device.type != "cuda":
|
||||
return
|
||||
|
||||
deepseek_model = _find_deepseek_v4_model(model)
|
||||
token_sizes = _select_mhc_warmup_token_sizes(
|
||||
max_tokens=max_tokens,
|
||||
cudagraph_capture_sizes=cudagraph_capture_sizes or [],
|
||||
)
|
||||
if not token_sizes:
|
||||
return
|
||||
|
||||
started = time.perf_counter()
|
||||
logger.info(
|
||||
"Warming up DeepSeek V4 mHC TileLang kernels for token sizes: %s",
|
||||
token_sizes,
|
||||
)
|
||||
with torch.inference_mode():
|
||||
_warmup_layer_mhc(layer, token_sizes)
|
||||
if deepseek_model is not None:
|
||||
_warmup_hc_head(deepseek_model, token_sizes)
|
||||
torch.accelerator.synchronize()
|
||||
logger.info(
|
||||
"DeepSeek V4 mHC TileLang warmup finished in %.2f seconds.",
|
||||
time.perf_counter() - started,
|
||||
)
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FlashInfer autotune cache helpers."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
from contextlib import suppress
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.caching import aot_compile_hash_factors
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
|
||||
def flashinfer_autotune_cache_hash(runner: "GPUModelRunner") -> str:
|
||||
factors = aot_compile_hash_factors(runner.vllm_config)
|
||||
return hashlib.sha256(str(factors).encode()).hexdigest()
|
||||
|
||||
|
||||
def resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
|
||||
override_dir = envs.VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR
|
||||
if override_dir:
|
||||
root = Path(override_dir).expanduser()
|
||||
else:
|
||||
from flashinfer.jit import env as flashinfer_jit_env
|
||||
|
||||
flashinfer_workspace = flashinfer_jit_env.FLASHINFER_WORKSPACE_DIR
|
||||
root = (
|
||||
Path(envs.VLLM_CACHE_ROOT)
|
||||
/ "flashinfer_autotune_cache"
|
||||
/ flashinfer_workspace.parent.name
|
||||
/ flashinfer_workspace.name
|
||||
)
|
||||
|
||||
output_dir = root / flashinfer_autotune_cache_hash(runner)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
return output_dir / "autotune_configs.json"
|
||||
|
||||
|
||||
def write_flashinfer_autotune_cache(cache_path: Path, contents: bytes) -> None:
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp_path = tempfile.mkstemp(
|
||||
dir=cache_path.parent, suffix=".tmp", prefix=f".{cache_path.name}."
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as f:
|
||||
f.write(contents)
|
||||
os.replace(tmp_path, cache_path)
|
||||
except BaseException:
|
||||
with suppress(OSError):
|
||||
os.unlink(tmp_path)
|
||||
raise
|
||||
@@ -0,0 +1,255 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warmup and autotune helpers for FlashInfer sparse MLA backends."""
|
||||
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
write_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import autotune as flashinfer_autotune
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.v1.worker.gpu.warmup import run_mixed_prefill_decode_warmup
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as V2GPUModelRunner
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_DEEPSEEK_V4_SPARSE_MLA_BACKENDS = frozenset(
|
||||
{
|
||||
"FLASHMLA_SPARSE_DSV4",
|
||||
"FLASHINFER_MLA_SPARSE_DSV4",
|
||||
"ROCM_FLASHMLA_SPARSE_DSV4",
|
||||
"DEEPSEEK_SPARSE_SWA",
|
||||
}
|
||||
)
|
||||
_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset({"FLASHINFER_MLA_SPARSE_SM120"})
|
||||
_DEEPSEEK_V4_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset({"FLASHINFER_MLA_SPARSE_DSV4"})
|
||||
|
||||
_FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS = {
|
||||
"FLASHINFER_MLA_SPARSE_SM120": "DSv3.2",
|
||||
"FLASHINFER_MLA_SPARSE_DSV4": "DSv4",
|
||||
}
|
||||
|
||||
_SPARSE_MLA_MIXED_WARMUP_TOKENS = 16
|
||||
|
||||
|
||||
def _attention_backend_name(backend: object) -> str | None:
|
||||
get_name = getattr(backend, "get_name", None)
|
||||
if get_name is None:
|
||||
return None
|
||||
try:
|
||||
return get_name()
|
||||
except NotImplementedError:
|
||||
return None
|
||||
|
||||
|
||||
def _has_deepseek_v4_sparse_mla_backend(runner: "GPUModelRunner") -> bool:
|
||||
for groups in getattr(runner, "attn_groups", []) or ():
|
||||
for group in groups:
|
||||
name = _attention_backend_name(getattr(group, "backend", None))
|
||||
if name in _DEEPSEEK_V4_SPARSE_MLA_BACKENDS:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _flashinfer_sparse_mla_decode_label(
|
||||
runner: "GPUModelRunner",
|
||||
allowed_backends: frozenset[str],
|
||||
) -> str | None:
|
||||
for groups in getattr(runner, "attn_groups", []) or ():
|
||||
for group in groups:
|
||||
name = _attention_backend_name(getattr(group, "backend", None))
|
||||
if name in allowed_backends:
|
||||
return _FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS.get(name)
|
||||
return None
|
||||
|
||||
|
||||
def _clamp_warmup_tokens(num_tokens: int, max_tokens: int) -> int:
|
||||
return max(0, min(num_tokens, max_tokens))
|
||||
|
||||
|
||||
def _uses_v2_model_runner(runner: "GPUModelRunner") -> bool:
|
||||
vllm_config = getattr(runner, "vllm_config", None)
|
||||
return bool(getattr(vllm_config, "use_v2_model_runner", False))
|
||||
|
||||
|
||||
def _run_flashinfer_sparse_mla_decode_autotune(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
allowed_backends: frozenset[str],
|
||||
) -> bool:
|
||||
"""Autotune FlashInfer's SM120 sparse-MLA decode path."""
|
||||
runner = worker.model_runner
|
||||
log_label = _flashinfer_sparse_mla_decode_label(runner, allowed_backends)
|
||||
if log_label is None:
|
||||
return False
|
||||
if worker.vllm_config.kernel_config.enable_flashinfer_autotune is not True:
|
||||
return False
|
||||
if not has_flashinfer() or not current_platform.is_device_capability_family(120):
|
||||
return False
|
||||
|
||||
try:
|
||||
from flashinfer.autotuner import AutoTuner
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"Skipping FlashInfer SM120 sparse MLA decode autotune because "
|
||||
"FlashInfer autotuner is unavailable."
|
||||
)
|
||||
return False
|
||||
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
world = get_world_group()
|
||||
is_leader = world.rank_in_group == 0
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
|
||||
dummy_run_kwargs = dict(
|
||||
num_tokens=num_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
|
||||
if is_leader:
|
||||
logger.info(
|
||||
"Autotuning FlashInfer SM120 sparse MLA %s decode with cache: %s",
|
||||
log_label,
|
||||
cache_path,
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
warmup_executed = True
|
||||
if is_leader:
|
||||
if _uses_v2_model_runner(runner):
|
||||
v2_runner = cast("V2GPUModelRunner", runner)
|
||||
warmup_executed = run_mixed_prefill_decode_warmup(
|
||||
v2_runner,
|
||||
worker.execute_model,
|
||||
worker.sample_tokens,
|
||||
num_tokens,
|
||||
mixed_step_context=flashinfer_autotune(True, cache=str(cache_path)),
|
||||
req_id_prefix="_sparse_mla_v2_warmup",
|
||||
)
|
||||
else:
|
||||
with flashinfer_autotune(True, cache=str(cache_path)):
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
else:
|
||||
if _uses_v2_model_runner(runner):
|
||||
v2_runner = cast("V2GPUModelRunner", runner)
|
||||
warmup_executed = run_mixed_prefill_decode_warmup(
|
||||
v2_runner,
|
||||
worker.execute_model,
|
||||
worker.sample_tokens,
|
||||
num_tokens,
|
||||
req_id_prefix="_sparse_mla_v2_warmup",
|
||||
)
|
||||
else:
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
|
||||
if not warmup_executed:
|
||||
return False
|
||||
|
||||
tune_results: bytes | None = None
|
||||
if is_leader and cache_path.exists():
|
||||
with open(cache_path, "rb") as f:
|
||||
tune_results = f.read()
|
||||
|
||||
tune_results = world.broadcast_object(tune_results, src=0)
|
||||
if tune_results is None:
|
||||
logger.warning(
|
||||
"No FlashInfer SM120 sparse MLA %s decode autotune cache entries found. "
|
||||
"Falling back to FlashInfer's default tactic heuristic.",
|
||||
log_label,
|
||||
)
|
||||
world.barrier()
|
||||
return True
|
||||
|
||||
write_flashinfer_autotune_cache(cache_path, tune_results)
|
||||
world.barrier()
|
||||
|
||||
AutoTuner.get().load_configs(str(cache_path))
|
||||
logger.info(
|
||||
"FlashInfer SM120 sparse MLA %s decode autotune cache loaded on rank %d "
|
||||
"from %s.",
|
||||
log_label,
|
||||
world.rank_in_group,
|
||||
cache_path,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _flashinfer_sparse_mla_decode_autotune(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
) -> bool:
|
||||
return _run_flashinfer_sparse_mla_decode_autotune(
|
||||
worker, num_tokens, _FLASHINFER_MLA_SPARSE_BACKENDS
|
||||
)
|
||||
|
||||
|
||||
def _deepseek_v4_sparse_mla_decode_autotune(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
) -> bool:
|
||||
return _run_flashinfer_sparse_mla_decode_autotune(
|
||||
worker, num_tokens, _DEEPSEEK_V4_FLASHINFER_MLA_SPARSE_BACKENDS
|
||||
)
|
||||
|
||||
|
||||
def flashinfer_sparse_mla_decode_autotune_warmup(worker: "Worker") -> None:
|
||||
"""Autotune generic FlashInfer sparse MLA decode when selected."""
|
||||
runner = worker.model_runner
|
||||
if runner.is_pooling_model:
|
||||
return
|
||||
|
||||
max_tokens = worker.scheduler_config.max_num_batched_tokens
|
||||
mixed_tokens = _clamp_warmup_tokens(_SPARSE_MLA_MIXED_WARMUP_TOKENS, max_tokens)
|
||||
if mixed_tokens <= 0:
|
||||
return
|
||||
_flashinfer_sparse_mla_decode_autotune(worker, mixed_tokens)
|
||||
|
||||
|
||||
def deepseek_v4_sparse_mla_attention_warmup(worker: "Worker") -> None:
|
||||
"""Warm DSv4 sparse-MLA mixed prefill+decode attention."""
|
||||
runner = worker.model_runner
|
||||
if runner.is_pooling_model or not _has_deepseek_v4_sparse_mla_backend(runner):
|
||||
return
|
||||
|
||||
max_tokens = worker.scheduler_config.max_num_batched_tokens
|
||||
mixed_tokens = _clamp_warmup_tokens(_SPARSE_MLA_MIXED_WARMUP_TOKENS, max_tokens)
|
||||
if mixed_tokens <= 0:
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Warming up DeepSeek V4 sparse MLA attention for mixed tokens=%s.",
|
||||
mixed_tokens,
|
||||
)
|
||||
mixed_warmup_done = _deepseek_v4_sparse_mla_decode_autotune(worker, mixed_tokens)
|
||||
if not mixed_warmup_done:
|
||||
if _uses_v2_model_runner(runner):
|
||||
v2_runner = cast("V2GPUModelRunner", runner)
|
||||
run_mixed_prefill_decode_warmup(
|
||||
v2_runner,
|
||||
worker.execute_model,
|
||||
worker.sample_tokens,
|
||||
mixed_tokens,
|
||||
req_id_prefix="_sparse_mla_v2_warmup",
|
||||
)
|
||||
else:
|
||||
runner._dummy_run(
|
||||
num_tokens=mixed_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
@@ -6,16 +6,24 @@ This is useful specifically for JIT'ed kernels as we don't want JIT'ing to
|
||||
happen during model execution.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.caching import aot_compile_hash_factors
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import (
|
||||
deepseek_v4_mhc_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
write_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
deepseek_v4_sparse_mla_attention_warmup,
|
||||
flashinfer_sparse_mla_decode_autotune_warmup,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
@@ -27,36 +35,26 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _flashinfer_autotune_cache_hash(runner: "GPUModelRunner") -> str:
|
||||
factors = aot_compile_hash_factors(runner.vllm_config)
|
||||
return hashlib.sha256(str(factors).encode()).hexdigest()
|
||||
|
||||
|
||||
def _resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
|
||||
override_dir = envs.VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR
|
||||
if override_dir:
|
||||
root = Path(override_dir).expanduser()
|
||||
else:
|
||||
from flashinfer.jit import env as flashinfer_jit_env
|
||||
|
||||
flashinfer_workspace = flashinfer_jit_env.FLASHINFER_WORKSPACE_DIR
|
||||
root = (
|
||||
Path(envs.VLLM_CACHE_ROOT)
|
||||
/ "flashinfer_autotune_cache"
|
||||
/ flashinfer_workspace.parent.name
|
||||
/ flashinfer_workspace.name
|
||||
)
|
||||
|
||||
output_dir = root / _flashinfer_autotune_cache_hash(runner)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
return output_dir / "autotune_configs.json"
|
||||
|
||||
|
||||
def kernel_warmup(worker: "Worker"):
|
||||
from vllm.model_executor.warmup.minimax_m3_msa_warmup import (
|
||||
minimax_m3_msa_warmup,
|
||||
)
|
||||
|
||||
# DSv4 mHC TileLang kernels (hc_pre/hc_post/hc_head_op) run every decoder
|
||||
# layer per token; warm them across token sizes first so the first real
|
||||
# request doesn't pay JIT cost. No-op for non-DSv4 models (gated inside).
|
||||
deepseek_v4_mhc_warmup(
|
||||
worker.get_model(),
|
||||
max_tokens=worker.scheduler_config.max_num_batched_tokens,
|
||||
cudagraph_capture_sizes=(
|
||||
worker.vllm_config.compilation_config.cudagraph_capture_sizes or []
|
||||
),
|
||||
)
|
||||
|
||||
# Run next so input-prep kernels JIT against pristine runner state.
|
||||
flashinfer_sparse_mla_decode_autotune_warmup(worker)
|
||||
deepseek_v4_sparse_mla_attention_warmup(worker)
|
||||
|
||||
# Deep GEMM warmup
|
||||
do_deep_gemm_warmup = (
|
||||
envs.VLLM_USE_DEEP_GEMM
|
||||
@@ -147,7 +145,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
world = get_world_group()
|
||||
is_leader = world.rank_in_group == 0
|
||||
|
||||
cache_path = _resolve_flashinfer_autotune_file(runner)
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
if is_leader:
|
||||
logger.info("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
|
||||
@@ -183,9 +181,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
"Falling back to default tactics."
|
||||
)
|
||||
else:
|
||||
if not is_leader and world.local_rank == 0:
|
||||
with open(cache_path, "wb") as f:
|
||||
f.write(tune_results)
|
||||
write_flashinfer_autotune_cache(cache_path, tune_results)
|
||||
world.barrier()
|
||||
from flashinfer.autotuner import AutoTuner
|
||||
|
||||
|
||||
Reference in New Issue
Block a user