# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Warmup deep_gemm kernels. DeepGEMM JIT's the kernels. The warmup aims to JIT all the kernels that would be used during model execution beforehand. """ import sys from typing import TYPE_CHECKING, TypeGuard import torch from tqdm import tqdm import vllm.envs as envs from vllm.distributed.parallel_state import ( get_dp_group, is_global_first_rank, ) from vllm.forward_context import ForwardContext, override_forward_context from vllm.model_executor.kernels.linear.scaled_mm.deep_gemm import ( DeepGemmFp8BlockScaledMMKernel, ) from vllm.model_executor.layers.fused_moe import MoERunner from vllm.model_executor.layers.fused_moe.deep_gemm_utils import ( compute_aligned_M_and_alignment, ) from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import DeepGemmExperts from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import ( TritonOrDeepGemmExperts, ) from vllm.tracing import instrument from vllm.utils.deep_gemm import ( _import_deep_gemm, fp8_gemm_nt, get_mk_alignment_for_contiguous_layout, m_grouped_fp8_gemm_nt_contiguous, mk_alignment_scope, ) from vllm.utils.math_utils import cdiv from vllm.utils.platform_utils import num_compute_units from vllm.v1.worker.workspace import current_workspace_manager if TYPE_CHECKING: from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MoE from vllm.models.kimi_k3.nvidia.model import KimiMoE def _generate_optimal_warmup_m_values( max_tokens: int, n: int, device: torch.device ) -> list[int]: """Generate M values that cover all possible DeepGEMM kernel configurations. Reference: https://github.com/deepseek-ai/DeepGEMM/blob/79f48ee15a82dd5fad5cd9beaa393c1f755e6b55/csrc/jit_kernels/heuristics/common.hpp Args: max_tokens: Maximum number of tokens to warmup for n: The actual N dimension from the weight tensor device: The torch device to get properties from. """ # DeepGEMM's possible block sizes block_ms = [64, 128, 256] block_ns = list(range(16, min(257, n + 1), 16)) num_sms = num_compute_units(device.index) m_values = set() # Always include small cases m_values.update([1, 2, 4] + [i for i in range(8, 65, 8)]) # Collect M values where different wave patterns occur for block_m in block_ms: for block_n in block_ns: if block_n > n: continue # Add key M boundaries for this block combination for wave in range(1, 11): # Up to 10 waves # M where this block config transitions to next wave target_blocks = wave * num_sms m = target_blocks * block_m // cdiv(n, block_n) if 1 <= m <= max_tokens: m_values.add(m) # Add block_m boundaries for multiple in range(1, max_tokens // block_m + 1): m = multiple * block_m if m <= max_tokens: m_values.add(m) return sorted(m_values) def _extract_data_from_fused_moe_module( m_: torch.nn.Module, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]: """Extract weights, weight scales and num_topk from MoERunner module.""" assert isinstance(m_, MoERunner) m = m_.routed_experts w13 = m.w13_weight w13_s = ( m.w13_weight_scale_inv if hasattr(m, "w13_weight_scale_inv") else m.w13_weight_scale ) w2 = m.w2_weight w2_s = ( m.w2_weight_scale_inv if hasattr(m, "w2_weight_scale_inv") else m.w2_weight_scale ) num_topk = m.top_k assert isinstance(w13, torch.Tensor) assert isinstance(w13_s, torch.Tensor) assert isinstance(w2, torch.Tensor) assert isinstance(w2_s, torch.Tensor) return w13, w13_s, w2, w2_s, num_topk def _deep_gemm_linear_data( module: torch.nn.Module, ) -> tuple[torch.Tensor, torch.Tensor] | None: """Return (weight, weight_scale) for a layer whose kernel stamped itself as ``deep_gemm_warmup_provider`` in ``process_weights_after_loading``, else None. """ provider = getattr(module, "deep_gemm_warmup_provider", None) if not isinstance(provider, DeepGemmFp8BlockScaledMMKernel): return None if module.weight_block_size != get_mk_alignment_for_contiguous_layout(): return None return provider.get_deep_gemm_warmup_weights(module) def _fused_moe_grouped_gemm_may_use_deep_gemm(module: torch.nn.Module) -> bool: if not (envs.VLLM_USE_DEEP_GEMM and envs.VLLM_MOE_USE_DEEP_GEMM): return False if not isinstance(module, MoERunner): return False quant_method = module._quant_method moe_quant_config = quant_method.get_fused_moe_quant_config(module.routed_experts) if ( moe_quant_config is None or moe_quant_config.quant_dtype != torch.float8_e4m3fn or moe_quant_config.block_shape != get_mk_alignment_for_contiguous_layout() ): return False moe_kernel = getattr(quant_method, "moe_kernel", None) if moe_kernel is None: return False fused_experts = moe_kernel.impl.fused_experts return isinstance(fused_experts, (DeepGemmExperts, TritonOrDeepGemmExperts)) # Model construction already imports the model modules. Avoid importing them here. def _mega_moe_uses_mega_gate(module: torch.nn.Module) -> "TypeGuard[DeepseekV4MoE]": # DeepSeek-V4 routes with bf16_mega_gate, Kimi K3 with its GateLinear. dsv4 = sys.modules.get("vllm.models.deepseek_v4.nvidia.model") return dsv4 is not None and isinstance(module, dsv4.DeepseekV4MoE) def _mega_moe_may_use_deep_gemm( module: torch.nn.Module, ) -> "TypeGuard[DeepseekV4MoE | KimiMoE]": if _mega_moe_uses_mega_gate(module): return module.use_native_mega_moe kimi = sys.modules.get("vllm.models.kimi_k3.nvidia.model") return kimi is not None and isinstance(module, kimi.KimiMoE) and module.use_mega_moe FP8_GEMM_NT_WARMUP_CACHE: set[torch.Size] = set() def _get_fp8_gemm_nt_m_values(w: torch.Tensor, max_tokens: int) -> list[int]: """Get the M values to warmup for a given weight tensor.""" n, _ = w.size() device = w.device # Use optimal M values only if VLLM_DEEP_GEMM_WARMUP is set to "relax". # Otherwise warmup all token sizes to avoid JIT compilation in hotpath if envs.VLLM_DEEP_GEMM_WARMUP == "relax": return _generate_optimal_warmup_m_values(max_tokens, n, device) else: assert envs.VLLM_DEEP_GEMM_WARMUP == "full", ( "Expected " 'VLLM_DEEP_GEMM_WARMUP env to be set to "full" but got ' f"{envs.VLLM_DEEP_GEMM_WARMUP}" ) return list(range(1, max_tokens + 1)) def _deepgemm_fp8_gemm_nt_warmup( w: torch.Tensor, ws: torch.Tensor, max_tokens: int, pbar: tqdm | None = None, ): if w.size() in FP8_GEMM_NT_WARMUP_CACHE: return n, k = w.size() block_m = get_mk_alignment_for_contiguous_layout()[0] device = w.device a1q = torch.empty((max_tokens, k), device=device, dtype=torch.float8_e4m3fn) # Must be initialized (UE8M0 packing asserts zero sign/mantissa bits). a1q_scales = torch.zeros( (max_tokens, k // block_m), device=device, dtype=torch.float32 ) out = torch.empty((max_tokens, n), device=device, dtype=torch.bfloat16) m_values = _get_fp8_gemm_nt_m_values(w, max_tokens) for num_tokens in m_values: fp8_gemm_nt( (a1q[:num_tokens], a1q_scales[:num_tokens]), (w, ws), out[:num_tokens] ) if pbar is not None: pbar.update(1) FP8_GEMM_NT_WARMUP_CACHE.add(w.size()) GROUPED_FP8_GEMM_NT_CONTIGUOUS_WARMUP_CACHE: set[torch.Size] = set() def _get_grouped_gemm_params( w1: torch.Tensor, w2: torch.Tensor, num_topk: int, max_tokens: int, ) -> tuple[int, int, list[tuple[int, int, torch.Tensor]]]: assert w1.size(0) == w2.size(0), "w1 and w2 must have the same number of experts" block_m = get_mk_alignment_for_contiguous_layout()[0] num_experts = w1.size(0) device = w1.device # Assumes all ranks have the same max_num_batched_tokens max_tokens = get_dp_group().world_size * max_tokens request_m_values = _generate_optimal_warmup_m_values( max_tokens, max(w1.size(1), w2.size(1)), device, ) request_m_values = sorted({m for m in (*request_m_values, max_tokens) if m > 0}) if not request_m_values: return 0, block_m, [] cases_by_shape: dict[tuple[int, int], torch.Tensor] = {} for request_m in request_m_values: M_sum, align_used = compute_aligned_M_and_alignment( M=request_m, num_topk=num_topk, local_num_experts=num_experts, alignment=block_m, expert_tokens_meta=None, ) if (M_sum, align_used) in cases_by_shape: continue num_blocks = M_sum // align_used expert_ids_block = torch.randint( low=0, high=num_experts, size=(num_blocks,), device=device, dtype=torch.int32, ) cases_by_shape[(M_sum, align_used)] = torch.repeat_interleave( expert_ids_block, align_used, dim=0 ) max_m = max(M_sum for M_sum, _ in cases_by_shape) warmup_cases = [ (M_sum, align_used, expert_ids) for (M_sum, align_used), expert_ids in sorted(cases_by_shape.items()) ] return max_m, block_m, warmup_cases def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup( w1: torch.Tensor, w2: torch.Tensor, w1_scale: torch.Tensor, w2_scale: torch.Tensor, num_topk: int, max_tokens: int, pbar: tqdm | None = None, ): if ( w1.size() in GROUPED_FP8_GEMM_NT_CONTIGUOUS_WARMUP_CACHE and w2.size() in GROUPED_FP8_GEMM_NT_CONTIGUOUS_WARMUP_CACHE ): return MAX_M, block_m, warmup_cases = _get_grouped_gemm_params( w1, w2, num_topk, max_tokens ) if not warmup_cases: return def _warmup(w: torch.Tensor, w_scale: torch.Tensor): _, n, k = w.size() a1q, a1q_scales, out = current_workspace_manager().get_simultaneous( ((MAX_M, k), torch.float8_e4m3fn), ((MAX_M, k // block_m), torch.float32), ((MAX_M, n), torch.bfloat16), ) # Must be initialized (UE8M0 packing asserts zero sign/mantissa bits). a1q_scales.zero_() for num_tokens, align_used, expert_ids in warmup_cases: with mk_alignment_scope(align_used): m_grouped_fp8_gemm_nt_contiguous( (a1q[:num_tokens], a1q_scales[:num_tokens]), (w, w_scale), out[:num_tokens], expert_ids, ) if pbar is not None: pbar.update(1) for w, ws in [(w1, w1_scale), (w2, w2_scale)]: if w.size() not in GROUPED_FP8_GEMM_NT_CONTIGUOUS_WARMUP_CACHE: _warmup(w, ws) GROUPED_FP8_GEMM_NT_CONTIGUOUS_WARMUP_CACHE.add(w.size()) FP8_FP4_MEGA_MOE_WARMUP_CACHE: set[tuple] = set() def _get_fp8_fp4_mega_moe_warmup_key(module: "DeepseekV4MoE | KimiMoE") -> tuple: gate = module.gate return ( gate.output_size, getattr(gate, "tid2eid", None) is not None, gate.e_score_correction_bias is not None, getattr(module, "image_sentinel_lo", 0), module.experts.has_fused_shared_experts, ) def _get_fp8_fp4_mega_moe_m_values( module: "DeepseekV4MoE | KimiMoE", max_tokens: int ) -> list[int]: """Get the largest M per distinct fp8_fp4_mega_moe config and, where used, bf16_mega_gate config.""" dg = _import_deep_gemm() gate, experts = module.gate, module.experts uses_mega_gate = _mega_moe_uses_mega_gate(module) if module.use_sequence_parallel: max_tokens = cdiv(max_tokens, module.tp_size) m_values: dict[tuple, int] = {} for num_tokens in range(1, max_tokens + 1): gate_config = ( dg.get_bf16_mega_gate_config( num_tokens, gate.input_size, gate.output_size, experts.top_k ) if uses_mega_gate and hasattr(dg, "get_bf16_mega_gate_config") else {} ) block_m = experts.get_block_m(num_tokens) # Keep the largest M: DeepSeek-V4 only calls bf16_mega_gate above a # small token threshold, and a config's largest M is always above it. m_values[(tuple(gate_config.items()), block_m)] = num_tokens return sorted(m_values.values()) def _deepgemm_fp8_fp4_mega_moe_warmup( module: "DeepseekV4MoE | KimiMoE", max_tokens: int, pbar: tqdm | None = None, ): from vllm.models.deepseek_v4.nvidia.model import ( prepare_mega_gate_routing_metadata, ) key = _get_fp8_fp4_mega_moe_warmup_key(module) if key in FP8_FP4_MEGA_MOE_WARMUP_CACHE: return m_values = _get_fp8_fp4_mega_moe_m_values(module, max_tokens) max_m = m_values[-1] # sorted device = module.experts.get_symm_buffer().x.device input_ids = torch.zeros(max_m, device=device, dtype=torch.int64) hidden_states = torch.zeros( (max_m, module.gate.input_size), device=device, dtype=torch.bfloat16 ) is_padding = torch.zeros(max_m + 1, device=device, dtype=torch.bool) # With padding skip on, sequence parallel gives ranks > 0 a padding mask # slice at any alignment. sp_masked = module.use_sequence_parallel and envs.VLLM_MOE_SKIP_PADDING offsets = (0, 1) if sp_masked else (0,) # Run the whole layer so the gate sees the same optional inputs as serving. for num_tokens in m_values: inputs: tuple = (hidden_states[:num_tokens],) if _mega_moe_uses_mega_gate(module): ids = input_ids[:num_tokens] inputs += ( ids, prepare_mega_gate_routing_metadata( ids, has_hash_routing=module.gate.tid2eid is not None, image_sentinel_base_id=module.image_sentinel_lo or None, ), ) for offset in offsets: forward_context = ForwardContext( no_compile_layers={}, attn_metadata={}, slot_mapping={}, is_padding=is_padding[offset : offset + num_tokens], ) with override_forward_context(forward_context): module(*inputs) if pbar is not None: pbar.update(1) FP8_FP4_MEGA_MOE_WARMUP_CACHE.add(key) def deepgemm_fp8_gemm_nt_warmup( model: torch.nn.Module, max_tokens: int, pbar: tqdm | None = None ): for m in model.modules(): if (data := _deep_gemm_linear_data(m)) is not None: w, ws = data _deepgemm_fp8_gemm_nt_warmup(w=w, ws=ws, max_tokens=max_tokens, pbar=pbar) def deepgemm_grouped_fp8_gemm_nt_contiguous_warmup( model: torch.nn.Module, max_tokens: int, pbar: tqdm | None = None ): dg_modules = [ m for m in model.modules() if _fused_moe_grouped_gemm_may_use_deep_gemm(m) ] for dgm in dg_modules: w13, w13_scale, w2, w2_scale, num_topk = _extract_data_from_fused_moe_module( dgm ) _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup( w13, w2, w13_scale, w2_scale, num_topk, max_tokens, pbar=pbar ) @torch.inference_mode() def deepgemm_fp8_fp4_mega_moe_warmup( model: torch.nn.Module, max_tokens: int, pbar: tqdm | None = None ): for m in model.modules(): if _mega_moe_may_use_deep_gemm(m): _deepgemm_fp8_fp4_mega_moe_warmup(m, max_tokens, pbar=pbar) def _count_warmup_iterations(model: torch.nn.Module, max_tokens: int) -> int: seen_fp8_sizes: set[torch.Size] = set(FP8_GEMM_NT_WARMUP_CACHE) seen_grouped_sizes: set[torch.Size] = set( GROUPED_FP8_GEMM_NT_CONTIGUOUS_WARMUP_CACHE ) seen_mega_moe_keys: set[tuple] = set(FP8_FP4_MEGA_MOE_WARMUP_CACHE) total = 0 for m in model.modules(): if (data := _deep_gemm_linear_data(m)) is not None: w, _ = data if w.size() not in seen_fp8_sizes: total += len(_get_fp8_gemm_nt_m_values(w, max_tokens)) seen_fp8_sizes.add(w.size()) elif _fused_moe_grouped_gemm_may_use_deep_gemm(m): w13, _, w2, _, num_topk = _extract_data_from_fused_moe_module(m) if w13.size() in seen_grouped_sizes and w2.size() in seen_grouped_sizes: continue _, _, warmup_cases = _get_grouped_gemm_params(w13, w2, num_topk, max_tokens) n_values = len(warmup_cases) if w13.size() not in seen_grouped_sizes: total += n_values seen_grouped_sizes.add(w13.size()) if w2.size() not in seen_grouped_sizes: total += n_values seen_grouped_sizes.add(w2.size()) elif _mega_moe_may_use_deep_gemm(m): key = _get_fp8_fp4_mega_moe_warmup_key(m) if key not in seen_mega_moe_keys: total += len(_get_fp8_fp4_mega_moe_m_values(m, max_tokens)) seen_mega_moe_keys.add(key) return total @instrument(span_name="DeepGemm warmup") def deep_gemm_warmup(model: torch.nn.Module, max_tokens: int): total = _count_warmup_iterations(model, max_tokens) if total == 0: return # Only show progress bar on rank 0 to avoid cluttered output if is_global_first_rank(): with tqdm(total=total, desc="DeepGEMM warmup") as pbar: deepgemm_fp8_gemm_nt_warmup(model, max_tokens, pbar) deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(model, max_tokens, pbar) deepgemm_fp8_fp4_mega_moe_warmup(model, max_tokens, pbar) else: deepgemm_fp8_gemm_nt_warmup(model, max_tokens, None) deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(model, max_tokens, None) deepgemm_fp8_fp4_mega_moe_warmup(model, max_tokens, None)