Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.worker.gpu.warmup import run_mixed_prefill_decode_warmup
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def mixed_batch_attention_warmup(worker: "Worker", num_tokens: int = 16) -> None:
|
||||
"""Run a mixed prefill+decode batch to warm up both attention paths."""
|
||||
num_tokens = min(
|
||||
num_tokens,
|
||||
worker.scheduler_config.max_num_batched_tokens,
|
||||
worker.model_config.max_model_len,
|
||||
)
|
||||
if not worker.use_v2_model_runner:
|
||||
worker.model_runner._dummy_run(
|
||||
num_tokens=num_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
return
|
||||
|
||||
if not run_mixed_prefill_decode_warmup(
|
||||
cast("GPUModelRunner", worker.model_runner),
|
||||
worker.execute_model,
|
||||
worker.sample_tokens,
|
||||
num_tokens,
|
||||
):
|
||||
logger.info_once(
|
||||
"Skipping mixed prefill+decode attention warmup; attention kernels "
|
||||
"will be loaded on the first request."
|
||||
)
|
||||
@@ -5,11 +5,19 @@ DeepGEMM JIT's the kernels. The warmup aims to JIT all the kernels that would
|
||||
be used during model execution beforehand.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from typing import TYPE_CHECKING, TypeGuard
|
||||
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed.parallel_state import get_dp_group, is_global_first_rank
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_dp_group,
|
||||
get_ep_group,
|
||||
is_global_first_rank,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.deep_gemm import (
|
||||
DeepGemmFp8BlockScaledMMKernel,
|
||||
)
|
||||
@@ -23,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
)
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.deep_gemm import (
|
||||
_import_deep_gemm,
|
||||
fp8_gemm_nt,
|
||||
get_mk_alignment_for_contiguous_layout,
|
||||
m_grouped_fp8_gemm_nt_contiguous,
|
||||
@@ -32,6 +41,10 @@ from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MoE
|
||||
from vllm.models.kimi_k3.nvidia.model import KimiMoE
|
||||
|
||||
|
||||
def _generate_optimal_warmup_m_values(
|
||||
max_tokens: int, n: int, device: torch.device
|
||||
@@ -144,6 +157,22 @@ def _fused_moe_grouped_gemm_may_use_deep_gemm(module: torch.nn.Module) -> bool:
|
||||
return isinstance(fused_experts, (DeepGemmExperts, TritonOrDeepGemmExperts))
|
||||
|
||||
|
||||
# Model construction already imports the model modules. Avoid importing them here.
|
||||
def _mega_moe_uses_mega_gate(module: torch.nn.Module) -> "TypeGuard[DeepseekV4MoE]":
|
||||
# DeepSeek-V4 routes with bf16_mega_gate, Kimi K3 with its GateLinear.
|
||||
dsv4 = sys.modules.get("vllm.models.deepseek_v4.nvidia.model")
|
||||
return dsv4 is not None and isinstance(module, dsv4.DeepseekV4MoE)
|
||||
|
||||
|
||||
def _mega_moe_may_use_deep_gemm(
|
||||
module: torch.nn.Module,
|
||||
) -> "TypeGuard[DeepseekV4MoE | KimiMoE]":
|
||||
if _mega_moe_uses_mega_gate(module):
|
||||
return module.use_native_mega_moe
|
||||
kimi = sys.modules.get("vllm.models.kimi_k3.nvidia.model")
|
||||
return kimi is not None and isinstance(module, kimi.KimiMoE) and module.use_mega_moe
|
||||
|
||||
|
||||
FP8_GEMM_NT_WARMUP_CACHE: set[torch.Size] = set()
|
||||
|
||||
|
||||
@@ -305,6 +334,110 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
|
||||
GROUPED_FP8_GEMM_NT_CONTIGUOUS_WARMUP_CACHE.add(w.size())
|
||||
|
||||
|
||||
FP8_FP4_MEGA_MOE_WARMUP_CACHE: set[tuple] = set()
|
||||
|
||||
|
||||
def _get_fp8_fp4_mega_moe_warmup_key(module: "DeepseekV4MoE | KimiMoE") -> tuple:
|
||||
gate = module.gate
|
||||
return (
|
||||
gate.output_size,
|
||||
getattr(gate, "tid2eid", None) is not None,
|
||||
gate.e_score_correction_bias is not None,
|
||||
getattr(module, "image_sentinel_lo", 0),
|
||||
module.experts.has_fused_shared_experts,
|
||||
)
|
||||
|
||||
|
||||
def _get_fp8_fp4_mega_moe_m_values(
|
||||
module: "DeepseekV4MoE | KimiMoE", max_tokens: int
|
||||
) -> list[int]:
|
||||
"""Get the largest M per distinct fp8_fp4_mega_moe config and, where used,
|
||||
bf16_mega_gate config."""
|
||||
dg = _import_deep_gemm()
|
||||
gate, experts = module.gate, module.experts
|
||||
uses_mega_gate = _mega_moe_uses_mega_gate(module)
|
||||
num_max_tokens_per_rank = experts.get_symm_buffer().num_max_tokens_per_rank
|
||||
if module.use_sequence_parallel:
|
||||
max_tokens = cdiv(max_tokens, module.tp_size)
|
||||
|
||||
m_values: dict[tuple, int] = {}
|
||||
for num_tokens in range(1, max_tokens + 1):
|
||||
gate_config = (
|
||||
dg.get_bf16_mega_gate_config(
|
||||
num_tokens, gate.input_size, gate.output_size, experts.top_k
|
||||
)
|
||||
if uses_mega_gate and hasattr(dg, "get_bf16_mega_gate_config")
|
||||
else {}
|
||||
)
|
||||
block_m = dg.get_block_m_for_mega_moe(
|
||||
get_ep_group().world_size,
|
||||
experts.num_experts,
|
||||
num_max_tokens_per_rank,
|
||||
num_tokens,
|
||||
experts.top_k,
|
||||
"fp8xfp4",
|
||||
)
|
||||
# Keep the largest M: DeepSeek-V4 only calls bf16_mega_gate above a
|
||||
# small token threshold, and a config's largest M is always above it.
|
||||
m_values[(tuple(gate_config.items()), block_m)] = num_tokens
|
||||
return sorted(m_values.values())
|
||||
|
||||
|
||||
def _deepgemm_fp8_fp4_mega_moe_warmup(
|
||||
module: "DeepseekV4MoE | KimiMoE",
|
||||
max_tokens: int,
|
||||
pbar: tqdm | None = None,
|
||||
):
|
||||
from vllm.models.deepseek_v4.nvidia.model import (
|
||||
prepare_mega_gate_routing_metadata,
|
||||
)
|
||||
|
||||
key = _get_fp8_fp4_mega_moe_warmup_key(module)
|
||||
if key in FP8_FP4_MEGA_MOE_WARMUP_CACHE:
|
||||
return
|
||||
|
||||
m_values = _get_fp8_fp4_mega_moe_m_values(module, max_tokens)
|
||||
max_m = m_values[-1] # sorted
|
||||
|
||||
device = module.experts.get_symm_buffer().x.device
|
||||
input_ids = torch.zeros(max_m, device=device, dtype=torch.int64)
|
||||
hidden_states = torch.zeros(
|
||||
(max_m, module.gate.input_size), device=device, dtype=torch.bfloat16
|
||||
)
|
||||
is_padding = torch.zeros(max_m + 1, device=device, dtype=torch.bool)
|
||||
# With padding skip on, sequence parallel gives ranks > 0 a padding mask
|
||||
# slice at any alignment.
|
||||
sp_masked = module.use_sequence_parallel and envs.VLLM_MOE_SKIP_PADDING
|
||||
offsets = (0, 1) if sp_masked else (0,)
|
||||
|
||||
# Run the whole layer so the gate sees the same optional inputs as serving.
|
||||
for num_tokens in m_values:
|
||||
inputs: tuple = (hidden_states[:num_tokens],)
|
||||
if _mega_moe_uses_mega_gate(module):
|
||||
ids = input_ids[:num_tokens]
|
||||
inputs += (
|
||||
ids,
|
||||
prepare_mega_gate_routing_metadata(
|
||||
ids,
|
||||
has_hash_routing=module.gate.tid2eid is not None,
|
||||
image_sentinel_base_id=module.image_sentinel_lo or None,
|
||||
),
|
||||
)
|
||||
for offset in offsets:
|
||||
forward_context = ForwardContext(
|
||||
no_compile_layers={},
|
||||
attn_metadata={},
|
||||
slot_mapping={},
|
||||
is_padding=is_padding[offset : offset + num_tokens],
|
||||
)
|
||||
with override_forward_context(forward_context):
|
||||
module(*inputs)
|
||||
if pbar is not None:
|
||||
pbar.update(1)
|
||||
|
||||
FP8_FP4_MEGA_MOE_WARMUP_CACHE.add(key)
|
||||
|
||||
|
||||
def deepgemm_fp8_gemm_nt_warmup(
|
||||
model: torch.nn.Module, max_tokens: int, pbar: tqdm | None = None
|
||||
):
|
||||
@@ -330,11 +463,21 @@ def deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def deepgemm_fp8_fp4_mega_moe_warmup(
|
||||
model: torch.nn.Module, max_tokens: int, pbar: tqdm | None = None
|
||||
):
|
||||
for m in model.modules():
|
||||
if _mega_moe_may_use_deep_gemm(m):
|
||||
_deepgemm_fp8_fp4_mega_moe_warmup(m, max_tokens, pbar=pbar)
|
||||
|
||||
|
||||
def _count_warmup_iterations(model: torch.nn.Module, max_tokens: int) -> int:
|
||||
seen_fp8_sizes: set[torch.Size] = set(FP8_GEMM_NT_WARMUP_CACHE)
|
||||
seen_grouped_sizes: set[torch.Size] = set(
|
||||
GROUPED_FP8_GEMM_NT_CONTIGUOUS_WARMUP_CACHE
|
||||
)
|
||||
seen_mega_moe_keys: set[tuple] = set(FP8_FP4_MEGA_MOE_WARMUP_CACHE)
|
||||
|
||||
total = 0
|
||||
for m in model.modules():
|
||||
@@ -355,6 +498,11 @@ def _count_warmup_iterations(model: torch.nn.Module, max_tokens: int) -> int:
|
||||
if w2.size() not in seen_grouped_sizes:
|
||||
total += n_values
|
||||
seen_grouped_sizes.add(w2.size())
|
||||
elif _mega_moe_may_use_deep_gemm(m):
|
||||
key = _get_fp8_fp4_mega_moe_warmup_key(m)
|
||||
if key not in seen_mega_moe_keys:
|
||||
total += len(_get_fp8_fp4_mega_moe_m_values(m, max_tokens))
|
||||
seen_mega_moe_keys.add(key)
|
||||
return total
|
||||
|
||||
|
||||
@@ -369,6 +517,8 @@ def deep_gemm_warmup(model: torch.nn.Module, max_tokens: int):
|
||||
with tqdm(total=total, desc="DeepGEMM warmup") as pbar:
|
||||
deepgemm_fp8_gemm_nt_warmup(model, max_tokens, pbar)
|
||||
deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(model, max_tokens, pbar)
|
||||
deepgemm_fp8_fp4_mega_moe_warmup(model, max_tokens, pbar)
|
||||
else:
|
||||
deepgemm_fp8_gemm_nt_warmup(model, max_tokens, None)
|
||||
deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(model, max_tokens, None)
|
||||
deepgemm_fp8_fp4_mega_moe_warmup(model, max_tokens, None)
|
||||
|
||||
@@ -10,14 +10,28 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import CacheConfig, VllmConfig, replace
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
|
||||
def _normalize_cache_config_for_hash(cache_config: CacheConfig) -> CacheConfig:
|
||||
"""Normalize cache config for hashing"""
|
||||
if cache_config.mamba_block_size is None and cache_config.kv_cache_layout is None:
|
||||
return cache_config
|
||||
normalized = replace(cache_config, mamba_block_size=None)
|
||||
assert normalized.kv_cache_layout is None
|
||||
return normalized
|
||||
|
||||
|
||||
def flashinfer_autotune_cache_hash(runner: "GPUModelRunner") -> str:
|
||||
config_hash = runner.vllm_config.compute_hash(include_version=False)
|
||||
vllm_config: VllmConfig = runner.vllm_config
|
||||
cache_config = _normalize_cache_config_for_hash(vllm_config.cache_config)
|
||||
if cache_config is not vllm_config.cache_config:
|
||||
vllm_config = replace(vllm_config, cache_config=cache_config)
|
||||
config_hash = vllm_config.compute_hash(include_version=False)
|
||||
return hashlib.sha256(config_hash.encode()).hexdigest()
|
||||
|
||||
|
||||
|
||||
@@ -718,65 +718,6 @@ class TritonPointerInputVariant:
|
||||
return TritonWarmupTensor(dtype, aligned=self.is_aligned(name), shape=shape)
|
||||
|
||||
|
||||
def _literal_str_refs(node: ast.AST) -> tuple[str | int, ...]:
|
||||
if isinstance(node, ast.Constant) and isinstance(node.value, str | int):
|
||||
return (node.value,)
|
||||
if isinstance(node, ast.List | ast.Tuple):
|
||||
refs: list[str | int] = []
|
||||
for elt in node.elts:
|
||||
if isinstance(elt, ast.Constant) and isinstance(elt.value, str | int):
|
||||
refs.append(elt.value)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported Triton specialization ref: {ast.dump(elt)}"
|
||||
)
|
||||
return tuple(refs)
|
||||
raise ValueError(f"Unsupported Triton specialization refs: {ast.dump(node)}")
|
||||
|
||||
|
||||
def _normalize_arg_refs(
|
||||
refs: tuple[str | int, ...],
|
||||
arg_names: tuple[str, ...],
|
||||
) -> frozenset[str]:
|
||||
names: set[str] = set()
|
||||
for ref in refs:
|
||||
if isinstance(ref, int):
|
||||
names.add(arg_names[ref])
|
||||
else:
|
||||
names.add(ref)
|
||||
return frozenset(names)
|
||||
|
||||
|
||||
def _decorator_keyword_refs(
|
||||
function_def: ast.FunctionDef,
|
||||
keyword_name: str,
|
||||
) -> tuple[str | int, ...]:
|
||||
for decorator in function_def.decorator_list:
|
||||
if not isinstance(decorator, ast.Call):
|
||||
continue
|
||||
decorator_name = get_ast_full_name(decorator.func)
|
||||
if decorator_name not in ("triton.jit", "jit"):
|
||||
continue
|
||||
for keyword in decorator.keywords:
|
||||
if keyword.arg == keyword_name:
|
||||
return _literal_str_refs(keyword.value)
|
||||
return ()
|
||||
|
||||
|
||||
def _triton_do_not_specialize_args(
|
||||
kernel: Callable[..., Any],
|
||||
function_def: ast.FunctionDef,
|
||||
arg_names: tuple[str, ...],
|
||||
) -> frozenset[str]:
|
||||
refs = getattr(kernel, "do_not_specialize", None)
|
||||
if refs is not None:
|
||||
return _normalize_arg_refs(tuple(refs), arg_names)
|
||||
return _normalize_arg_refs(
|
||||
_decorator_keyword_refs(function_def, "do_not_specialize"),
|
||||
arg_names,
|
||||
)
|
||||
|
||||
|
||||
def _triton_constexpr_arg_names(
|
||||
kernel: Callable[..., Any],
|
||||
function_def: ast.FunctionDef,
|
||||
@@ -821,25 +762,3 @@ def _pointer_arg_names(
|
||||
if name in candidate_names:
|
||||
pointer_names.add(name)
|
||||
return frozenset(pointer_names)
|
||||
|
||||
|
||||
def trace_triton_kernel_specialization_args(
|
||||
kernel: Callable[..., Any],
|
||||
) -> tuple[str, ...]:
|
||||
function_def = get_function_source_node(kernel)
|
||||
if not isinstance(function_def, ast.FunctionDef):
|
||||
raise ValueError("Expected Triton kernel to be defined as a function")
|
||||
source_fn = getattr(kernel, "fn", kernel)
|
||||
arg_names = tuple(inspect.signature(source_fn).parameters)
|
||||
constexpr_args = _triton_constexpr_arg_names(kernel, function_def, arg_names)
|
||||
do_not_specialize_args = _triton_do_not_specialize_args(
|
||||
kernel, function_def, arg_names
|
||||
)
|
||||
pointer_args = _pointer_arg_names(function_def, arg_names)
|
||||
|
||||
return tuple(
|
||||
name
|
||||
for name in arg_names
|
||||
if name in constexpr_args
|
||||
or (name not in pointer_args and name not in do_not_specialize_args)
|
||||
)
|
||||
|
||||
@@ -15,6 +15,9 @@ import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.attention_warmup import (
|
||||
mixed_batch_attention_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.b12x_warmup import b12x_warmup
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
@@ -286,15 +289,7 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
)
|
||||
):
|
||||
logger.info_once("Warming up FlashInfer attention.")
|
||||
# Warmup with mixed batch containing both prefill and decode tokens
|
||||
# This is to warm up both prefill and decode attention kernels
|
||||
worker.model_runner._dummy_run(
|
||||
num_tokens=16,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
mixed_batch_attention_warmup(worker)
|
||||
|
||||
|
||||
def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
|
||||
@@ -406,7 +401,7 @@ def _all_ranks_have_matching_cache(path: Path, group) -> bool:
|
||||
return fingerprint is not None and all(f == fingerprint for f in gathered)
|
||||
|
||||
|
||||
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) -> None:
|
||||
"""Autotune FlashInfer operations.
|
||||
FlashInfer have many implementations for the same operation,
|
||||
autotuning runs benchmarks for each implementation and stores
|
||||
@@ -415,6 +410,10 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
Without autotuning, FlashInfer will rely on heuristics, which may
|
||||
be significantly slower.
|
||||
|
||||
Args:
|
||||
runner: The model runner to tune.
|
||||
skip_attn: Additionally skip attention in the tuning dummy runs.
|
||||
|
||||
With PP > 1, stages run different layers and may profile different ops,
|
||||
so each stage's TP group tunes separately with its own cache file;
|
||||
otherwise the world group tunes together. Per-tactic timings are
|
||||
@@ -425,6 +424,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
rank. A rank with a cache hit skips the per-tactic reduce the others block
|
||||
in, so ranks keep loaded configs only if every rank in the tuning group
|
||||
has a matching file and successfully loads it.
|
||||
|
||||
"""
|
||||
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
|
||||
|
||||
@@ -485,11 +485,12 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
hisparse_enabled = (
|
||||
runner.vllm_config.attention_config.hisparse_config is not None
|
||||
)
|
||||
skip_attn = skip_attn or hisparse_enabled
|
||||
if hisparse_enabled:
|
||||
# HiSparse hot-buffer attention is bounded by decode batch
|
||||
# size, not the prefill-sized batch used for the full model.
|
||||
autotune_hisparse_flashinfer_attention(runner)
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
|
||||
replayssm_autotune_warmup(runner)
|
||||
_autotune_kimi_k3_kda_qkvg(runner.get_model())
|
||||
finally:
|
||||
|
||||
@@ -4,6 +4,9 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.attention_warmup import (
|
||||
mixed_batch_attention_warmup,
|
||||
)
|
||||
from vllm.models.minimax_m3.nvidia.model import MiniMaxM3SparseAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tracing import instrument
|
||||
@@ -34,10 +37,4 @@ def minimax_m3_msa_warmup(worker: "Worker") -> None:
|
||||
logger.info("Warming up MiniMax M3 MSA kernels.")
|
||||
|
||||
# Cover sparse prefill through the normal model path.
|
||||
worker.model_runner._dummy_run(
|
||||
num_tokens=16,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
mixed_batch_attention_warmup(worker)
|
||||
|
||||
@@ -72,7 +72,11 @@ def _warm_mrope(runner: "GPUModelRunner", model: torch.nn.Module) -> None:
|
||||
(module for module in model.modules() if isinstance(module, MRotaryEmbedding)),
|
||||
None,
|
||||
)
|
||||
if rope is None:
|
||||
if rope is None or all(
|
||||
getattr(module, "use_fused_qk_norm_rope_gate", False)
|
||||
for module in model.modules()
|
||||
if any(child is rope for child in module.children())
|
||||
):
|
||||
return
|
||||
|
||||
model_config = runner.model_config
|
||||
|
||||
Reference in New Issue
Block a user