Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
@@ -6,16 +6,24 @@ This is useful specifically for JIT'ed kernels as we don't want JIT'ing to
|
||||
happen during model execution.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.caching import aot_compile_hash_factors
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import (
|
||||
deepseek_v4_mhc_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
write_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
deepseek_v4_sparse_mla_attention_warmup,
|
||||
flashinfer_sparse_mla_decode_autotune_warmup,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
@@ -27,36 +35,26 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _flashinfer_autotune_cache_hash(runner: "GPUModelRunner") -> str:
|
||||
factors = aot_compile_hash_factors(runner.vllm_config)
|
||||
return hashlib.sha256(str(factors).encode()).hexdigest()
|
||||
|
||||
|
||||
def _resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
|
||||
override_dir = envs.VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR
|
||||
if override_dir:
|
||||
root = Path(override_dir).expanduser()
|
||||
else:
|
||||
from flashinfer.jit import env as flashinfer_jit_env
|
||||
|
||||
flashinfer_workspace = flashinfer_jit_env.FLASHINFER_WORKSPACE_DIR
|
||||
root = (
|
||||
Path(envs.VLLM_CACHE_ROOT)
|
||||
/ "flashinfer_autotune_cache"
|
||||
/ flashinfer_workspace.parent.name
|
||||
/ flashinfer_workspace.name
|
||||
)
|
||||
|
||||
output_dir = root / _flashinfer_autotune_cache_hash(runner)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
return output_dir / "autotune_configs.json"
|
||||
|
||||
|
||||
def kernel_warmup(worker: "Worker"):
|
||||
from vllm.model_executor.warmup.minimax_m3_msa_warmup import (
|
||||
minimax_m3_msa_warmup,
|
||||
)
|
||||
|
||||
# DSv4 mHC TileLang kernels (hc_pre/hc_post/hc_head_op) run every decoder
|
||||
# layer per token; warm them across token sizes first so the first real
|
||||
# request doesn't pay JIT cost. No-op for non-DSv4 models (gated inside).
|
||||
deepseek_v4_mhc_warmup(
|
||||
worker.get_model(),
|
||||
max_tokens=worker.scheduler_config.max_num_batched_tokens,
|
||||
cudagraph_capture_sizes=(
|
||||
worker.vllm_config.compilation_config.cudagraph_capture_sizes or []
|
||||
),
|
||||
)
|
||||
|
||||
# Run next so input-prep kernels JIT against pristine runner state.
|
||||
flashinfer_sparse_mla_decode_autotune_warmup(worker)
|
||||
deepseek_v4_sparse_mla_attention_warmup(worker)
|
||||
|
||||
# Deep GEMM warmup
|
||||
do_deep_gemm_warmup = (
|
||||
envs.VLLM_USE_DEEP_GEMM
|
||||
@@ -147,7 +145,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
world = get_world_group()
|
||||
is_leader = world.rank_in_group == 0
|
||||
|
||||
cache_path = _resolve_flashinfer_autotune_file(runner)
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
if is_leader:
|
||||
logger.info("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
|
||||
@@ -183,9 +181,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
"Falling back to default tactics."
|
||||
)
|
||||
else:
|
||||
if not is_leader and world.local_rank == 0:
|
||||
with open(cache_path, "wb") as f:
|
||||
f.write(tune_results)
|
||||
write_flashinfer_autotune_cache(cache_path, tune_results)
|
||||
world.barrier()
|
||||
from flashinfer.autotuner import AutoTuner
|
||||
|
||||
|
||||
Reference in New Issue
Block a user