Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
@@ -0,0 +1,42 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, cast
from vllm.logger import init_logger
from vllm.v1.worker.gpu.warmup import run_mixed_prefill_decode_warmup
if TYPE_CHECKING:
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
from vllm.v1.worker.gpu_worker import Worker
logger = init_logger(__name__)
def mixed_batch_attention_warmup(worker: "Worker", num_tokens: int = 16) -> None:
"""Run a mixed prefill+decode batch to warm up both attention paths."""
num_tokens = min(
num_tokens,
worker.scheduler_config.max_num_batched_tokens,
worker.model_config.max_model_len,
)
if not worker.use_v2_model_runner:
worker.model_runner._dummy_run(
num_tokens=num_tokens,
skip_eplb=True,
is_profile=True,
force_attention=True,
create_mixed_batch=True,
)
return
if not run_mixed_prefill_decode_warmup(
cast("GPUModelRunner", worker.model_runner),
worker.execute_model,
worker.sample_tokens,
num_tokens,
):
logger.info_once(
"Skipping mixed prefill+decode attention warmup; attention kernels "
"will be loaded on the first request."
)
+151 -1
View File
@@ -5,11 +5,19 @@ DeepGEMM JIT's the kernels. The warmup aims to JIT all the kernels that would
be used during model execution beforehand.
"""
import sys
from typing import TYPE_CHECKING, TypeGuard
import torch
from tqdm import tqdm
import vllm.envs as envs
from vllm.distributed.parallel_state import get_dp_group, is_global_first_rank
from vllm.distributed.parallel_state import (
get_dp_group,
get_ep_group,
is_global_first_rank,
)
from vllm.forward_context import ForwardContext, override_forward_context
from vllm.model_executor.kernels.linear.scaled_mm.deep_gemm import (
DeepGemmFp8BlockScaledMMKernel,
)
@@ -23,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
)
from vllm.tracing import instrument
from vllm.utils.deep_gemm import (
_import_deep_gemm,
fp8_gemm_nt,
get_mk_alignment_for_contiguous_layout,
m_grouped_fp8_gemm_nt_contiguous,
@@ -32,6 +41,10 @@ from vllm.utils.math_utils import cdiv
from vllm.utils.platform_utils import num_compute_units
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MoE
from vllm.models.kimi_k3.nvidia.model import KimiMoE
def _generate_optimal_warmup_m_values(
max_tokens: int, n: int, device: torch.device
@@ -144,6 +157,22 @@ def _fused_moe_grouped_gemm_may_use_deep_gemm(module: torch.nn.Module) -> bool:
return isinstance(fused_experts, (DeepGemmExperts, TritonOrDeepGemmExperts))
# Model construction already imports the model modules. Avoid importing them here.
def _mega_moe_uses_mega_gate(module: torch.nn.Module) -> "TypeGuard[DeepseekV4MoE]":
# DeepSeek-V4 routes with bf16_mega_gate, Kimi K3 with its GateLinear.
dsv4 = sys.modules.get("vllm.models.deepseek_v4.nvidia.model")
return dsv4 is not None and isinstance(module, dsv4.DeepseekV4MoE)
def _mega_moe_may_use_deep_gemm(
module: torch.nn.Module,
) -> "TypeGuard[DeepseekV4MoE | KimiMoE]":
if _mega_moe_uses_mega_gate(module):
return module.use_native_mega_moe
kimi = sys.modules.get("vllm.models.kimi_k3.nvidia.model")
return kimi is not None and isinstance(module, kimi.KimiMoE) and module.use_mega_moe
FP8_GEMM_NT_WARMUP_CACHE: set[torch.Size] = set()
@@ -305,6 +334,110 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
GROUPED_FP8_GEMM_NT_CONTIGUOUS_WARMUP_CACHE.add(w.size())
FP8_FP4_MEGA_MOE_WARMUP_CACHE: set[tuple] = set()
def _get_fp8_fp4_mega_moe_warmup_key(module: "DeepseekV4MoE | KimiMoE") -> tuple:
gate = module.gate
return (
gate.output_size,
getattr(gate, "tid2eid", None) is not None,
gate.e_score_correction_bias is not None,
getattr(module, "image_sentinel_lo", 0),
module.experts.has_fused_shared_experts,
)
def _get_fp8_fp4_mega_moe_m_values(
module: "DeepseekV4MoE | KimiMoE", max_tokens: int
) -> list[int]:
"""Get the largest M per distinct fp8_fp4_mega_moe config and, where used,
bf16_mega_gate config."""
dg = _import_deep_gemm()
gate, experts = module.gate, module.experts
uses_mega_gate = _mega_moe_uses_mega_gate(module)
num_max_tokens_per_rank = experts.get_symm_buffer().num_max_tokens_per_rank
if module.use_sequence_parallel:
max_tokens = cdiv(max_tokens, module.tp_size)
m_values: dict[tuple, int] = {}
for num_tokens in range(1, max_tokens + 1):
gate_config = (
dg.get_bf16_mega_gate_config(
num_tokens, gate.input_size, gate.output_size, experts.top_k
)
if uses_mega_gate and hasattr(dg, "get_bf16_mega_gate_config")
else {}
)
block_m = dg.get_block_m_for_mega_moe(
get_ep_group().world_size,
experts.num_experts,
num_max_tokens_per_rank,
num_tokens,
experts.top_k,
"fp8xfp4",
)
# Keep the largest M: DeepSeek-V4 only calls bf16_mega_gate above a
# small token threshold, and a config's largest M is always above it.
m_values[(tuple(gate_config.items()), block_m)] = num_tokens
return sorted(m_values.values())
def _deepgemm_fp8_fp4_mega_moe_warmup(
module: "DeepseekV4MoE | KimiMoE",
max_tokens: int,
pbar: tqdm | None = None,
):
from vllm.models.deepseek_v4.nvidia.model import (
prepare_mega_gate_routing_metadata,
)
key = _get_fp8_fp4_mega_moe_warmup_key(module)
if key in FP8_FP4_MEGA_MOE_WARMUP_CACHE:
return
m_values = _get_fp8_fp4_mega_moe_m_values(module, max_tokens)
max_m = m_values[-1] # sorted
device = module.experts.get_symm_buffer().x.device
input_ids = torch.zeros(max_m, device=device, dtype=torch.int64)
hidden_states = torch.zeros(
(max_m, module.gate.input_size), device=device, dtype=torch.bfloat16
)
is_padding = torch.zeros(max_m + 1, device=device, dtype=torch.bool)
# With padding skip on, sequence parallel gives ranks > 0 a padding mask
# slice at any alignment.
sp_masked = module.use_sequence_parallel and envs.VLLM_MOE_SKIP_PADDING
offsets = (0, 1) if sp_masked else (0,)
# Run the whole layer so the gate sees the same optional inputs as serving.
for num_tokens in m_values:
inputs: tuple = (hidden_states[:num_tokens],)
if _mega_moe_uses_mega_gate(module):
ids = input_ids[:num_tokens]
inputs += (
ids,
prepare_mega_gate_routing_metadata(
ids,
has_hash_routing=module.gate.tid2eid is not None,
image_sentinel_base_id=module.image_sentinel_lo or None,
),
)
for offset in offsets:
forward_context = ForwardContext(
no_compile_layers={},
attn_metadata={},
slot_mapping={},
is_padding=is_padding[offset : offset + num_tokens],
)
with override_forward_context(forward_context):
module(*inputs)
if pbar is not None:
pbar.update(1)
FP8_FP4_MEGA_MOE_WARMUP_CACHE.add(key)
def deepgemm_fp8_gemm_nt_warmup(
model: torch.nn.Module, max_tokens: int, pbar: tqdm | None = None
):
@@ -330,11 +463,21 @@ def deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
)
@torch.inference_mode()
def deepgemm_fp8_fp4_mega_moe_warmup(
model: torch.nn.Module, max_tokens: int, pbar: tqdm | None = None
):
for m in model.modules():
if _mega_moe_may_use_deep_gemm(m):
_deepgemm_fp8_fp4_mega_moe_warmup(m, max_tokens, pbar=pbar)
def _count_warmup_iterations(model: torch.nn.Module, max_tokens: int) -> int:
seen_fp8_sizes: set[torch.Size] = set(FP8_GEMM_NT_WARMUP_CACHE)
seen_grouped_sizes: set[torch.Size] = set(
GROUPED_FP8_GEMM_NT_CONTIGUOUS_WARMUP_CACHE
)
seen_mega_moe_keys: set[tuple] = set(FP8_FP4_MEGA_MOE_WARMUP_CACHE)
total = 0
for m in model.modules():
@@ -355,6 +498,11 @@ def _count_warmup_iterations(model: torch.nn.Module, max_tokens: int) -> int:
if w2.size() not in seen_grouped_sizes:
total += n_values
seen_grouped_sizes.add(w2.size())
elif _mega_moe_may_use_deep_gemm(m):
key = _get_fp8_fp4_mega_moe_warmup_key(m)
if key not in seen_mega_moe_keys:
total += len(_get_fp8_fp4_mega_moe_m_values(m, max_tokens))
seen_mega_moe_keys.add(key)
return total
@@ -369,6 +517,8 @@ def deep_gemm_warmup(model: torch.nn.Module, max_tokens: int):
with tqdm(total=total, desc="DeepGEMM warmup") as pbar:
deepgemm_fp8_gemm_nt_warmup(model, max_tokens, pbar)
deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(model, max_tokens, pbar)
deepgemm_fp8_fp4_mega_moe_warmup(model, max_tokens, pbar)
else:
deepgemm_fp8_gemm_nt_warmup(model, max_tokens, None)
deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(model, max_tokens, None)
deepgemm_fp8_fp4_mega_moe_warmup(model, max_tokens, None)
@@ -10,14 +10,28 @@ from pathlib import Path
from typing import TYPE_CHECKING
import vllm.envs as envs
from vllm.config import CacheConfig, VllmConfig, replace
if TYPE_CHECKING:
from vllm.distributed.parallel_state import GroupCoordinator
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
def _normalize_cache_config_for_hash(cache_config: CacheConfig) -> CacheConfig:
"""Normalize cache config for hashing"""
if cache_config.mamba_block_size is None and cache_config.kv_cache_layout is None:
return cache_config
normalized = replace(cache_config, mamba_block_size=None)
assert normalized.kv_cache_layout is None
return normalized
def flashinfer_autotune_cache_hash(runner: "GPUModelRunner") -> str:
config_hash = runner.vllm_config.compute_hash(include_version=False)
vllm_config: VllmConfig = runner.vllm_config
cache_config = _normalize_cache_config_for_hash(vllm_config.cache_config)
if cache_config is not vllm_config.cache_config:
vllm_config = replace(vllm_config, cache_config=cache_config)
config_hash = vllm_config.compute_hash(include_version=False)
return hashlib.sha256(config_hash.encode()).hexdigest()
@@ -718,65 +718,6 @@ class TritonPointerInputVariant:
return TritonWarmupTensor(dtype, aligned=self.is_aligned(name), shape=shape)
def _literal_str_refs(node: ast.AST) -> tuple[str | int, ...]:
if isinstance(node, ast.Constant) and isinstance(node.value, str | int):
return (node.value,)
if isinstance(node, ast.List | ast.Tuple):
refs: list[str | int] = []
for elt in node.elts:
if isinstance(elt, ast.Constant) and isinstance(elt.value, str | int):
refs.append(elt.value)
else:
raise ValueError(
f"Unsupported Triton specialization ref: {ast.dump(elt)}"
)
return tuple(refs)
raise ValueError(f"Unsupported Triton specialization refs: {ast.dump(node)}")
def _normalize_arg_refs(
refs: tuple[str | int, ...],
arg_names: tuple[str, ...],
) -> frozenset[str]:
names: set[str] = set()
for ref in refs:
if isinstance(ref, int):
names.add(arg_names[ref])
else:
names.add(ref)
return frozenset(names)
def _decorator_keyword_refs(
function_def: ast.FunctionDef,
keyword_name: str,
) -> tuple[str | int, ...]:
for decorator in function_def.decorator_list:
if not isinstance(decorator, ast.Call):
continue
decorator_name = get_ast_full_name(decorator.func)
if decorator_name not in ("triton.jit", "jit"):
continue
for keyword in decorator.keywords:
if keyword.arg == keyword_name:
return _literal_str_refs(keyword.value)
return ()
def _triton_do_not_specialize_args(
kernel: Callable[..., Any],
function_def: ast.FunctionDef,
arg_names: tuple[str, ...],
) -> frozenset[str]:
refs = getattr(kernel, "do_not_specialize", None)
if refs is not None:
return _normalize_arg_refs(tuple(refs), arg_names)
return _normalize_arg_refs(
_decorator_keyword_refs(function_def, "do_not_specialize"),
arg_names,
)
def _triton_constexpr_arg_names(
kernel: Callable[..., Any],
function_def: ast.FunctionDef,
@@ -821,25 +762,3 @@ def _pointer_arg_names(
if name in candidate_names:
pointer_names.add(name)
return frozenset(pointer_names)
def trace_triton_kernel_specialization_args(
kernel: Callable[..., Any],
) -> tuple[str, ...]:
function_def = get_function_source_node(kernel)
if not isinstance(function_def, ast.FunctionDef):
raise ValueError("Expected Triton kernel to be defined as a function")
source_fn = getattr(kernel, "fn", kernel)
arg_names = tuple(inspect.signature(source_fn).parameters)
constexpr_args = _triton_constexpr_arg_names(kernel, function_def, arg_names)
do_not_specialize_args = _triton_do_not_specialize_args(
kernel, function_def, arg_names
)
pointer_args = _pointer_arg_names(function_def, arg_names)
return tuple(
name
for name in arg_names
if name in constexpr_args
or (name not in pointer_args and name not in do_not_specialize_args)
)
+12 -11
View File
@@ -15,6 +15,9 @@ import torch
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.warmup.attention_warmup import (
mixed_batch_attention_warmup,
)
from vllm.model_executor.warmup.b12x_warmup import b12x_warmup
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
@@ -286,15 +289,7 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
)
):
logger.info_once("Warming up FlashInfer attention.")
# Warmup with mixed batch containing both prefill and decode tokens
# This is to warm up both prefill and decode attention kernels
worker.model_runner._dummy_run(
num_tokens=16,
skip_eplb=True,
is_profile=True,
force_attention=True,
create_mixed_batch=True,
)
mixed_batch_attention_warmup(worker)
def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
@@ -406,7 +401,7 @@ def _all_ranks_have_matching_cache(path: Path, group) -> bool:
return fingerprint is not None and all(f == fingerprint for f in gathered)
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) -> None:
"""Autotune FlashInfer operations.
FlashInfer have many implementations for the same operation,
autotuning runs benchmarks for each implementation and stores
@@ -415,6 +410,10 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
Without autotuning, FlashInfer will rely on heuristics, which may
be significantly slower.
Args:
runner: The model runner to tune.
skip_attn: Additionally skip attention in the tuning dummy runs.
With PP > 1, stages run different layers and may profile different ops,
so each stage's TP group tunes separately with its own cache file;
otherwise the world group tunes together. Per-tactic timings are
@@ -425,6 +424,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
rank. A rank with a cache hit skips the per-tactic reduce the others block
in, so ranks keep loaded configs only if every rank in the tuning group
has a matching file and successfully loads it.
"""
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
@@ -485,11 +485,12 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
hisparse_enabled = (
runner.vllm_config.attention_config.hisparse_config is not None
)
skip_attn = skip_attn or hisparse_enabled
if hisparse_enabled:
# HiSparse hot-buffer attention is bounded by decode batch
# size, not the prefill-sized batch used for the full model.
autotune_hisparse_flashinfer_attention(runner)
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
replayssm_autotune_warmup(runner)
_autotune_kimi_k3_kda_qkvg(runner.get_model())
finally:
@@ -4,6 +4,9 @@
from typing import TYPE_CHECKING
from vllm.logger import init_logger
from vllm.model_executor.warmup.attention_warmup import (
mixed_batch_attention_warmup,
)
from vllm.models.minimax_m3.nvidia.model import MiniMaxM3SparseAttention
from vllm.platforms import current_platform
from vllm.tracing import instrument
@@ -34,10 +37,4 @@ def minimax_m3_msa_warmup(worker: "Worker") -> None:
logger.info("Warming up MiniMax M3 MSA kernels.")
# Cover sparse prefill through the normal model path.
worker.model_runner._dummy_run(
num_tokens=16,
skip_eplb=True,
is_profile=True,
force_attention=True,
create_mixed_batch=True,
)
mixed_batch_attention_warmup(worker)
@@ -72,7 +72,11 @@ def _warm_mrope(runner: "GPUModelRunner", model: torch.nn.Module) -> None:
(module for module in model.modules() if isinstance(module, MRotaryEmbedding)),
None,
)
if rope is None:
if rope is None or all(
getattr(module, "use_fused_qk_norm_rope_gate", False)
for module in model.modules()
if any(child is rope for child in module.children())
):
return
model_config = runner.model_config