Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Gabriel Wu
2026-06-22 11:54:14 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Yongye Zhu
parent 3ce15fd574
commit 44d95069e9
37 changed files with 2347 additions and 476 deletions
+28 -32
View File
@@ -6,16 +6,24 @@ This is useful specifically for JIT'ed kernels as we don't want JIT'ing to
happen during model execution.
"""
import hashlib
from pathlib import Path
from typing import TYPE_CHECKING
import torch
import vllm.envs as envs
from vllm.compilation.caching import aot_compile_hash_factors
from vllm.logger import init_logger
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import (
deepseek_v4_mhc_warmup,
)
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
write_flashinfer_autotune_cache,
)
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
deepseek_v4_sparse_mla_attention_warmup,
flashinfer_sparse_mla_decode_autotune_warmup,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.flashinfer import has_flashinfer
@@ -27,36 +35,26 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _flashinfer_autotune_cache_hash(runner: "GPUModelRunner") -> str:
factors = aot_compile_hash_factors(runner.vllm_config)
return hashlib.sha256(str(factors).encode()).hexdigest()
def _resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
override_dir = envs.VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR
if override_dir:
root = Path(override_dir).expanduser()
else:
from flashinfer.jit import env as flashinfer_jit_env
flashinfer_workspace = flashinfer_jit_env.FLASHINFER_WORKSPACE_DIR
root = (
Path(envs.VLLM_CACHE_ROOT)
/ "flashinfer_autotune_cache"
/ flashinfer_workspace.parent.name
/ flashinfer_workspace.name
)
output_dir = root / _flashinfer_autotune_cache_hash(runner)
output_dir.mkdir(parents=True, exist_ok=True)
return output_dir / "autotune_configs.json"
def kernel_warmup(worker: "Worker"):
from vllm.model_executor.warmup.minimax_m3_msa_warmup import (
minimax_m3_msa_warmup,
)
# DSv4 mHC TileLang kernels (hc_pre/hc_post/hc_head_op) run every decoder
# layer per token; warm them across token sizes first so the first real
# request doesn't pay JIT cost. No-op for non-DSv4 models (gated inside).
deepseek_v4_mhc_warmup(
worker.get_model(),
max_tokens=worker.scheduler_config.max_num_batched_tokens,
cudagraph_capture_sizes=(
worker.vllm_config.compilation_config.cudagraph_capture_sizes or []
),
)
# Run next so input-prep kernels JIT against pristine runner state.
flashinfer_sparse_mla_decode_autotune_warmup(worker)
deepseek_v4_sparse_mla_attention_warmup(worker)
# Deep GEMM warmup
do_deep_gemm_warmup = (
envs.VLLM_USE_DEEP_GEMM
@@ -147,7 +145,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
world = get_world_group()
is_leader = world.rank_in_group == 0
cache_path = _resolve_flashinfer_autotune_file(runner)
cache_path = resolve_flashinfer_autotune_file(runner)
if is_leader:
logger.info("Using FlashInfer autotune cache file: %s", cache_path)
@@ -183,9 +181,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
"Falling back to default tactics."
)
else:
if not is_leader and world.local_rank == 0:
with open(cache_path, "wb") as f:
f.write(tune_results)
write_flashinfer_autotune_cache(cache_path, tune_results)
world.barrier()
from flashinfer.autotuner import AutoTuner