Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Gabriel Wu
2026-06-22 11:54:14 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Yongye Zhu
parent 3ce15fd574
commit 44d95069e9
37 changed files with 2347 additions and 476 deletions
+58 -25
View File
@@ -12,7 +12,9 @@ from tqdm import tqdm
import vllm.envs as envs
from vllm.distributed.parallel_state import get_dp_group, is_global_first_rank
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import compute_aligned_M
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
compute_aligned_M_and_alignment,
)
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import DeepGemmExperts
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
TritonOrDeepGemmExperts,
@@ -25,6 +27,7 @@ from vllm.utils.deep_gemm import (
fp8_gemm_nt,
get_mk_alignment_for_contiguous_layout,
m_grouped_fp8_gemm_nt_contiguous,
mk_alignment_scope,
)
from vllm.utils.math_utils import cdiv
from vllm.utils.platform_utils import num_compute_units
@@ -238,7 +241,7 @@ def _get_grouped_gemm_params(
w2: torch.Tensor,
num_topk: int,
max_tokens: int,
) -> tuple[int, int, torch.Tensor]:
) -> tuple[int, int, list[tuple[int, int, torch.Tensor]]]:
assert w1.size(0) == w2.size(0), "w1 and w2 must have the same number of experts"
block_m = get_mk_alignment_for_contiguous_layout()[0]
@@ -248,19 +251,46 @@ def _get_grouped_gemm_params(
# Assumes all ranks have the same max_num_batched_tokens
max_tokens = get_dp_group().world_size * max_tokens
# This is the maximum GroupedGemm M size that we expect to run
# the grouped_gemm with.
MAX_M = compute_aligned_M(
max_tokens, num_topk, num_experts, block_m, expert_tokens_meta=None
request_m_values = _generate_optimal_warmup_m_values(
max_tokens,
max(w1.size(1), w2.size(1)),
device,
)
# Distribute expert-ids evenly.
MAX_BLOCKS = MAX_M // block_m
expert_ids_block = torch.randint(
low=0, high=num_experts, size=(MAX_BLOCKS,), device=device, dtype=torch.int32
)
expert_ids = torch.repeat_interleave(expert_ids_block, block_m, dim=0)
request_m_values = sorted({m for m in (*request_m_values, max_tokens) if m > 0})
if not request_m_values:
return 0, block_m, []
return MAX_M, block_m, expert_ids
cases_by_shape: dict[tuple[int, int], torch.Tensor] = {}
for request_m in request_m_values:
M_sum, align_used = compute_aligned_M_and_alignment(
M=request_m,
num_topk=num_topk,
local_num_experts=num_experts,
alignment=block_m,
expert_tokens_meta=None,
)
if (M_sum, align_used) in cases_by_shape:
continue
num_blocks = M_sum // align_used
expert_ids_block = torch.randint(
low=0,
high=num_experts,
size=(num_blocks,),
device=device,
dtype=torch.int32,
)
cases_by_shape[(M_sum, align_used)] = torch.repeat_interleave(
expert_ids_block, align_used, dim=0
)
max_m = max(M_sum for M_sum, _ in cases_by_shape)
warmup_cases = [
(M_sum, align_used, expert_ids)
for (M_sum, align_used), expert_ids in sorted(cases_by_shape.items())
]
return max_m, block_m, warmup_cases
def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
@@ -278,7 +308,11 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
):
return
MAX_M, block_m, expert_ids = _get_grouped_gemm_params(w1, w2, num_topk, max_tokens)
MAX_M, block_m, warmup_cases = _get_grouped_gemm_params(
w1, w2, num_topk, max_tokens
)
if not warmup_cases:
return
device = w1.device
def _warmup(w: torch.Tensor, w_scale: torch.Tensor):
@@ -289,15 +323,14 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
)
out = torch.empty((MAX_M, n), device=device, dtype=torch.bfloat16)
m_values = list(range(block_m, MAX_M + 1, block_m))
for num_tokens in m_values:
m_grouped_fp8_gemm_nt_contiguous(
(a1q[:num_tokens], a1q_scales[:num_tokens]),
(w, w_scale),
out[:num_tokens],
expert_ids[:num_tokens],
)
for num_tokens, align_used, expert_ids in warmup_cases:
with mk_alignment_scope(align_used):
m_grouped_fp8_gemm_nt_contiguous(
(a1q[:num_tokens], a1q_scales[:num_tokens]),
(w, w_scale),
out[:num_tokens],
expert_ids,
)
if pbar is not None:
pbar.update(1)
@@ -350,8 +383,8 @@ def _count_warmup_iterations(model: torch.nn.Module, max_tokens: int) -> int:
w13, _, w2, _, num_topk = _extract_data_from_fused_moe_module(m)
if w13.size() in seen_grouped_sizes and w2.size() in seen_grouped_sizes:
continue
MAX_M, block_m, _ = _get_grouped_gemm_params(w13, w2, num_topk, max_tokens)
n_values = (MAX_M - block_m) // block_m + 1
_, _, warmup_cases = _get_grouped_gemm_params(w13, w2, num_topk, max_tokens)
n_values = len(warmup_cases)
if w13.size() not in seen_grouped_sizes:
total += n_values
seen_grouped_sizes.add(w13.size())
@@ -0,0 +1,226 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warm up DeepSeek V4 mHC TileLang kernels before serving requests.
Ported from lucifer1004/vllm-jasl with the two env-var knobs removed
(`VLLM_ENABLE_DEEPSEEK_V4_MHC_WARMUP`, `VLLM_DEEPSEEK_V4_MHC_WARMUP_TOKEN_SIZES`).
Gating is intrinsic: non-DSv4 models and layers without hc_* attributes
return early, so the warmup is a no-op except where it's needed.
"""
import time
from collections.abc import Iterable
import torch
from vllm.logger import init_logger
from vllm.tracing import instrument
from vllm.utils.math_utils import cdiv
logger = init_logger(__name__)
_AUTO_WARMUP_MAX_TOKENS = 16_384
_DEFAULT_TOKEN_SIZE_CANDIDATES = (
1,
2,
4,
8,
16,
32,
64,
128,
256,
512,
1024,
2048,
4096,
8192,
16_384,
)
def _compute_mhc_pre_num_split(
*,
num_tokens: int,
hidden_size: int,
hc_mult: int,
num_sms: int,
) -> int:
block_k = 64
block_m = 64
k = hc_mult * hidden_size
grid_size = cdiv(num_tokens, block_m)
split_k = num_sms // grid_size
num_block_k = cdiv(k, block_k)
split_k = min(split_k, num_block_k // 4)
return max(split_k, 1)
def _normalize_token_sizes(
token_sizes: Iterable[int],
*,
max_tokens: int,
) -> list[int]:
return sorted({size for size in token_sizes if 1 <= size <= max_tokens})
def _select_mhc_warmup_token_sizes(
*,
max_tokens: int,
cudagraph_capture_sizes: list[int],
) -> list[int]:
if max_tokens <= 0:
return []
max_auto_tokens = min(max_tokens, _AUTO_WARMUP_MAX_TOKENS)
candidates = list(_DEFAULT_TOKEN_SIZE_CANDIDATES)
candidates.extend(cudagraph_capture_sizes)
candidates.append(max_auto_tokens)
return _normalize_token_sizes(candidates, max_tokens=max_auto_tokens)
def _find_first_mhc_layer(model: torch.nn.Module) -> torch.nn.Module | None:
for module in model.modules():
if module.__class__.__name__ != "DeepseekV4DecoderLayer":
continue
if all(
hasattr(module, attr)
for attr in (
"hc_pre",
"hc_post",
"hc_attn_fn",
"hc_attn_scale",
"hc_attn_base",
"hc_ffn_fn",
"hc_ffn_scale",
"hc_ffn_base",
)
):
return module
return None
def _find_deepseek_v4_model(model: torch.nn.Module) -> torch.nn.Module | None:
for module in model.modules():
if module.__class__.__name__ != "DeepseekV4Model":
continue
if all(
hasattr(module, attr)
for attr in ("hc_head_fn", "hc_head_scale", "hc_head_base")
):
return module
return None
def _warmup_layer_mhc(
layer: torch.nn.Module,
token_sizes: list[int],
) -> None:
max_tokens = max(token_sizes)
hidden_size = int(layer.hidden_size)
hc_mult = int(layer.hc_mult)
device = layer.hc_attn_fn.device
residual = torch.zeros(
max_tokens,
hc_mult,
hidden_size,
dtype=torch.bfloat16,
device=device,
)
for size in token_sizes:
residual_slice = residual[:size]
for fn, scale, base in (
(layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base),
(layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base),
):
layer_input, post_mix, comb_mix = layer.hc_pre(
residual_slice,
fn,
scale,
base,
)
layer.hc_post(layer_input, residual_slice, post_mix, comb_mix)
def _warmup_hc_head(
model: torch.nn.Module,
token_sizes: list[int],
) -> None:
# Upstream a8887c208 ("[DSV4] aiter mhc support (ROCm)") refactored
# ``hc_head`` from a free function into the ``HCHeadOp`` CustomOp
# instance attached to the model as ``hc_head_op``. We call through
# that instance so the warmup exercises the same dispatched
# implementation as the inference path.
hc_head_op = getattr(model, "hc_head_op", None)
if hc_head_op is None:
return
max_tokens = max(token_sizes)
hidden_size = int(model.config.hidden_size)
hc_mult = int(model.hc_mult)
device = model.hc_head_fn.device
hidden_states = torch.zeros(
max_tokens,
hc_mult,
hidden_size,
dtype=torch.bfloat16,
device=device,
)
for size in token_sizes:
hc_head_op(
hidden_states[:size],
model.hc_head_fn,
model.hc_head_scale,
model.hc_head_base,
model.rms_norm_eps,
model.hc_eps,
)
@instrument(span_name="DeepSeek V4 mHC warmup")
def deepseek_v4_mhc_warmup(
model: torch.nn.Module,
*,
max_tokens: int,
cudagraph_capture_sizes: list[int] | None = None,
) -> None:
# Cheap model-type gate before walking ``model.modules()``. The class
# walk below is O(num_layers) and shows up in startup time on very
# large checkpoints; bail out for any model that is not DeepSeek V4.
config = getattr(model, "config", None)
model_type = getattr(config, "model_type", None) if config is not None else None
if model_type is not None and model_type != "deepseek_v4":
return
layer = _find_first_mhc_layer(model)
if layer is None:
return
device = layer.hc_attn_fn.device
if device.type != "cuda":
return
deepseek_model = _find_deepseek_v4_model(model)
token_sizes = _select_mhc_warmup_token_sizes(
max_tokens=max_tokens,
cudagraph_capture_sizes=cudagraph_capture_sizes or [],
)
if not token_sizes:
return
started = time.perf_counter()
logger.info(
"Warming up DeepSeek V4 mHC TileLang kernels for token sizes: %s",
token_sizes,
)
with torch.inference_mode():
_warmup_layer_mhc(layer, token_sizes)
if deepseek_model is not None:
_warmup_hc_head(deepseek_model, token_sizes)
torch.accelerator.synchronize()
logger.info(
"DeepSeek V4 mHC TileLang warmup finished in %.2f seconds.",
time.perf_counter() - started,
)
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlashInfer autotune cache helpers."""
import hashlib
import os
import tempfile
from contextlib import suppress
from pathlib import Path
from typing import TYPE_CHECKING
import vllm.envs as envs
from vllm.compilation.caching import aot_compile_hash_factors
if TYPE_CHECKING:
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
def flashinfer_autotune_cache_hash(runner: "GPUModelRunner") -> str:
factors = aot_compile_hash_factors(runner.vllm_config)
return hashlib.sha256(str(factors).encode()).hexdigest()
def resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
override_dir = envs.VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR
if override_dir:
root = Path(override_dir).expanduser()
else:
from flashinfer.jit import env as flashinfer_jit_env
flashinfer_workspace = flashinfer_jit_env.FLASHINFER_WORKSPACE_DIR
root = (
Path(envs.VLLM_CACHE_ROOT)
/ "flashinfer_autotune_cache"
/ flashinfer_workspace.parent.name
/ flashinfer_workspace.name
)
output_dir = root / flashinfer_autotune_cache_hash(runner)
output_dir.mkdir(parents=True, exist_ok=True)
return output_dir / "autotune_configs.json"
def write_flashinfer_autotune_cache(cache_path: Path, contents: bytes) -> None:
cache_path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_path = tempfile.mkstemp(
dir=cache_path.parent, suffix=".tmp", prefix=f".{cache_path.name}."
)
try:
with os.fdopen(fd, "wb") as f:
f.write(contents)
os.replace(tmp_path, cache_path)
except BaseException:
with suppress(OSError):
os.unlink(tmp_path)
raise
@@ -0,0 +1,255 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warmup and autotune helpers for FlashInfer sparse MLA backends."""
from typing import TYPE_CHECKING, cast
import torch
from vllm.logger import init_logger
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
write_flashinfer_autotune_cache,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import autotune as flashinfer_autotune
from vllm.utils.flashinfer import has_flashinfer
from vllm.v1.worker.gpu.warmup import run_mixed_prefill_decode_warmup
if TYPE_CHECKING:
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as V2GPUModelRunner
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
from vllm.v1.worker.gpu_worker import Worker
logger = init_logger(__name__)
_DEEPSEEK_V4_SPARSE_MLA_BACKENDS = frozenset(
{
"FLASHMLA_SPARSE_DSV4",
"FLASHINFER_MLA_SPARSE_DSV4",
"ROCM_FLASHMLA_SPARSE_DSV4",
"DEEPSEEK_SPARSE_SWA",
}
)
_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset({"FLASHINFER_MLA_SPARSE_SM120"})
_DEEPSEEK_V4_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset({"FLASHINFER_MLA_SPARSE_DSV4"})
_FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS = {
"FLASHINFER_MLA_SPARSE_SM120": "DSv3.2",
"FLASHINFER_MLA_SPARSE_DSV4": "DSv4",
}
_SPARSE_MLA_MIXED_WARMUP_TOKENS = 16
def _attention_backend_name(backend: object) -> str | None:
get_name = getattr(backend, "get_name", None)
if get_name is None:
return None
try:
return get_name()
except NotImplementedError:
return None
def _has_deepseek_v4_sparse_mla_backend(runner: "GPUModelRunner") -> bool:
for groups in getattr(runner, "attn_groups", []) or ():
for group in groups:
name = _attention_backend_name(getattr(group, "backend", None))
if name in _DEEPSEEK_V4_SPARSE_MLA_BACKENDS:
return True
return False
def _flashinfer_sparse_mla_decode_label(
runner: "GPUModelRunner",
allowed_backends: frozenset[str],
) -> str | None:
for groups in getattr(runner, "attn_groups", []) or ():
for group in groups:
name = _attention_backend_name(getattr(group, "backend", None))
if name in allowed_backends:
return _FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS.get(name)
return None
def _clamp_warmup_tokens(num_tokens: int, max_tokens: int) -> int:
return max(0, min(num_tokens, max_tokens))
def _uses_v2_model_runner(runner: "GPUModelRunner") -> bool:
vllm_config = getattr(runner, "vllm_config", None)
return bool(getattr(vllm_config, "use_v2_model_runner", False))
def _run_flashinfer_sparse_mla_decode_autotune(
worker: "Worker",
num_tokens: int,
allowed_backends: frozenset[str],
) -> bool:
"""Autotune FlashInfer's SM120 sparse-MLA decode path."""
runner = worker.model_runner
log_label = _flashinfer_sparse_mla_decode_label(runner, allowed_backends)
if log_label is None:
return False
if worker.vllm_config.kernel_config.enable_flashinfer_autotune is not True:
return False
if not has_flashinfer() or not current_platform.is_device_capability_family(120):
return False
try:
from flashinfer.autotuner import AutoTuner
except ImportError:
logger.warning(
"Skipping FlashInfer SM120 sparse MLA decode autotune because "
"FlashInfer autotuner is unavailable."
)
return False
from vllm.distributed.parallel_state import get_world_group
world = get_world_group()
is_leader = world.rank_in_group == 0
cache_path = resolve_flashinfer_autotune_file(runner)
dummy_run_kwargs = dict(
num_tokens=num_tokens,
skip_eplb=True,
is_profile=True,
force_attention=True,
create_mixed_batch=True,
)
if is_leader:
logger.info(
"Autotuning FlashInfer SM120 sparse MLA %s decode with cache: %s",
log_label,
cache_path,
)
with torch.inference_mode():
warmup_executed = True
if is_leader:
if _uses_v2_model_runner(runner):
v2_runner = cast("V2GPUModelRunner", runner)
warmup_executed = run_mixed_prefill_decode_warmup(
v2_runner,
worker.execute_model,
worker.sample_tokens,
num_tokens,
mixed_step_context=flashinfer_autotune(True, cache=str(cache_path)),
req_id_prefix="_sparse_mla_v2_warmup",
)
else:
with flashinfer_autotune(True, cache=str(cache_path)):
runner._dummy_run(**dummy_run_kwargs)
else:
if _uses_v2_model_runner(runner):
v2_runner = cast("V2GPUModelRunner", runner)
warmup_executed = run_mixed_prefill_decode_warmup(
v2_runner,
worker.execute_model,
worker.sample_tokens,
num_tokens,
req_id_prefix="_sparse_mla_v2_warmup",
)
else:
runner._dummy_run(**dummy_run_kwargs)
if not warmup_executed:
return False
tune_results: bytes | None = None
if is_leader and cache_path.exists():
with open(cache_path, "rb") as f:
tune_results = f.read()
tune_results = world.broadcast_object(tune_results, src=0)
if tune_results is None:
logger.warning(
"No FlashInfer SM120 sparse MLA %s decode autotune cache entries found. "
"Falling back to FlashInfer's default tactic heuristic.",
log_label,
)
world.barrier()
return True
write_flashinfer_autotune_cache(cache_path, tune_results)
world.barrier()
AutoTuner.get().load_configs(str(cache_path))
logger.info(
"FlashInfer SM120 sparse MLA %s decode autotune cache loaded on rank %d "
"from %s.",
log_label,
world.rank_in_group,
cache_path,
)
return True
def _flashinfer_sparse_mla_decode_autotune(
worker: "Worker",
num_tokens: int,
) -> bool:
return _run_flashinfer_sparse_mla_decode_autotune(
worker, num_tokens, _FLASHINFER_MLA_SPARSE_BACKENDS
)
def _deepseek_v4_sparse_mla_decode_autotune(
worker: "Worker",
num_tokens: int,
) -> bool:
return _run_flashinfer_sparse_mla_decode_autotune(
worker, num_tokens, _DEEPSEEK_V4_FLASHINFER_MLA_SPARSE_BACKENDS
)
def flashinfer_sparse_mla_decode_autotune_warmup(worker: "Worker") -> None:
"""Autotune generic FlashInfer sparse MLA decode when selected."""
runner = worker.model_runner
if runner.is_pooling_model:
return
max_tokens = worker.scheduler_config.max_num_batched_tokens
mixed_tokens = _clamp_warmup_tokens(_SPARSE_MLA_MIXED_WARMUP_TOKENS, max_tokens)
if mixed_tokens <= 0:
return
_flashinfer_sparse_mla_decode_autotune(worker, mixed_tokens)
def deepseek_v4_sparse_mla_attention_warmup(worker: "Worker") -> None:
"""Warm DSv4 sparse-MLA mixed prefill+decode attention."""
runner = worker.model_runner
if runner.is_pooling_model or not _has_deepseek_v4_sparse_mla_backend(runner):
return
max_tokens = worker.scheduler_config.max_num_batched_tokens
mixed_tokens = _clamp_warmup_tokens(_SPARSE_MLA_MIXED_WARMUP_TOKENS, max_tokens)
if mixed_tokens <= 0:
return
logger.info(
"Warming up DeepSeek V4 sparse MLA attention for mixed tokens=%s.",
mixed_tokens,
)
mixed_warmup_done = _deepseek_v4_sparse_mla_decode_autotune(worker, mixed_tokens)
if not mixed_warmup_done:
if _uses_v2_model_runner(runner):
v2_runner = cast("V2GPUModelRunner", runner)
run_mixed_prefill_decode_warmup(
v2_runner,
worker.execute_model,
worker.sample_tokens,
mixed_tokens,
req_id_prefix="_sparse_mla_v2_warmup",
)
else:
runner._dummy_run(
num_tokens=mixed_tokens,
skip_eplb=True,
is_profile=True,
force_attention=True,
create_mixed_batch=True,
)
+28 -32
View File
@@ -6,16 +6,24 @@ This is useful specifically for JIT'ed kernels as we don't want JIT'ing to
happen during model execution.
"""
import hashlib
from pathlib import Path
from typing import TYPE_CHECKING
import torch
import vllm.envs as envs
from vllm.compilation.caching import aot_compile_hash_factors
from vllm.logger import init_logger
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import (
deepseek_v4_mhc_warmup,
)
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
write_flashinfer_autotune_cache,
)
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
deepseek_v4_sparse_mla_attention_warmup,
flashinfer_sparse_mla_decode_autotune_warmup,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.flashinfer import has_flashinfer
@@ -27,36 +35,26 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _flashinfer_autotune_cache_hash(runner: "GPUModelRunner") -> str:
factors = aot_compile_hash_factors(runner.vllm_config)
return hashlib.sha256(str(factors).encode()).hexdigest()
def _resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
override_dir = envs.VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR
if override_dir:
root = Path(override_dir).expanduser()
else:
from flashinfer.jit import env as flashinfer_jit_env
flashinfer_workspace = flashinfer_jit_env.FLASHINFER_WORKSPACE_DIR
root = (
Path(envs.VLLM_CACHE_ROOT)
/ "flashinfer_autotune_cache"
/ flashinfer_workspace.parent.name
/ flashinfer_workspace.name
)
output_dir = root / _flashinfer_autotune_cache_hash(runner)
output_dir.mkdir(parents=True, exist_ok=True)
return output_dir / "autotune_configs.json"
def kernel_warmup(worker: "Worker"):
from vllm.model_executor.warmup.minimax_m3_msa_warmup import (
minimax_m3_msa_warmup,
)
# DSv4 mHC TileLang kernels (hc_pre/hc_post/hc_head_op) run every decoder
# layer per token; warm them across token sizes first so the first real
# request doesn't pay JIT cost. No-op for non-DSv4 models (gated inside).
deepseek_v4_mhc_warmup(
worker.get_model(),
max_tokens=worker.scheduler_config.max_num_batched_tokens,
cudagraph_capture_sizes=(
worker.vllm_config.compilation_config.cudagraph_capture_sizes or []
),
)
# Run next so input-prep kernels JIT against pristine runner state.
flashinfer_sparse_mla_decode_autotune_warmup(worker)
deepseek_v4_sparse_mla_attention_warmup(worker)
# Deep GEMM warmup
do_deep_gemm_warmup = (
envs.VLLM_USE_DEEP_GEMM
@@ -147,7 +145,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
world = get_world_group()
is_leader = world.rank_in_group == 0
cache_path = _resolve_flashinfer_autotune_file(runner)
cache_path = resolve_flashinfer_autotune_file(runner)
if is_leader:
logger.info("Using FlashInfer autotune cache file: %s", cache_path)
@@ -183,9 +181,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
"Falling back to default tactics."
)
else:
if not is_leader and world.local_rank == 0:
with open(cache_path, "wb") as f:
f.write(tune_results)
write_flashinfer_autotune_cache(cache_path, tune_results)
world.barrier()
from flashinfer.autotuner import AutoTuner