Merge branch 'main' into patches

This commit is contained in:
2026-10-17 08:06:56 +00:00
222 changed files with 14823 additions and 3606 deletions
@@ -11,11 +11,14 @@ from typing import TYPE_CHECKING
import vllm.envs as envs
from vllm.config import CacheConfig, VllmConfig, replace
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.distributed.parallel_state import GroupCoordinator
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
logger = init_logger(__name__)
def _normalize_cache_config_for_hash(cache_config: CacheConfig) -> CacheConfig:
"""Normalize cache config for hashing"""
@@ -55,6 +58,53 @@ def resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
return output_dir / "autotune_configs.json"
def use_flashinfer_autotune_v2(runner: "GPUModelRunner") -> bool:
from vllm.distributed.parallel_state import get_node_count, get_world_group
from vllm.utils.flashinfer import has_flashinfer_autotune_v2
# Elastic EP transfers tuning state to joining workers with v1
# save_configs/load_configs, which do not export managed-store winners.
if runner.vllm_config.parallel_config.enable_elastic_ep:
return False
# PP stages profile independently. Keep their group-scoped v1 files until
# managed-cache reads and publishes can be isolated across tuning groups.
if runner.vllm_config.parallel_config.pipeline_parallel_size > 1:
return False
if not has_flashinfer_autotune_v2():
return False
# Use the initialized topology: Ray may span nodes with config.nnodes=1.
# A cache hit skips profiling collectives, so node-local stores with
# different contents cannot safely participate in synchronized tuning.
if get_node_count() != 1:
logger.info_once(
"Using legacy FlashInfer autotune cache synchronization for "
"multi-node deployments; managed-cache synchronization is "
"currently supported only within one node."
)
return False
# Lift this gate after pinning a release with the fix below and validating
# distributed managed-cache tuning: https://github.com/flashinfer-ai/flashinfer/pull/6185
if get_world_group().world_size > 1:
logger.info_once(
"Using legacy FlashInfer autotune cache synchronization for "
"multi-rank deployments until FlashInfer coordinates cache hits."
)
return False
return True
def resolve_flashinfer_autotune_v2_root() -> Path | None:
"""Placement root for FlashInfer's managed autotune store.
Unlike :func:`resolve_flashinfer_autotune_file`, no vLLM-side identity
(config hash, flashinfer version, arch) is folded into the path: the
managed store keys entries per op and hashes the software/hardware
environment itself. ``None`` selects FlashInfer's default root.
"""
override_dir = envs.VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR
return Path(override_dir).expanduser() if override_dir else None
def sync_flashinfer_autotune_cache(
runner: "GPUModelRunner",
group: "GroupCoordinator",
@@ -9,6 +9,8 @@ import torch
from vllm.logger import init_logger
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
resolve_flashinfer_autotune_v2_root,
use_flashinfer_autotune_v2,
write_flashinfer_autotune_cache,
)
from vllm.platforms import current_platform
@@ -145,7 +147,6 @@ def _run_flashinfer_sparse_mla_decode_autotune(
world = get_world_group()
is_leader = world.rank_in_group == 0
cache_path = resolve_flashinfer_autotune_file(runner)
dummy_run_kwargs = dict(
num_tokens=num_tokens,
@@ -155,6 +156,13 @@ def _run_flashinfer_sparse_mla_decode_autotune(
create_mixed_batch=True,
)
if use_flashinfer_autotune_v2(runner):
return _run_sparse_mla_decode_autotune_v2(
worker, num_tokens, log_label, dummy_run_kwargs
)
cache_path = resolve_flashinfer_autotune_file(runner)
if is_leader:
logger.info(
"Autotuning FlashInfer SM120 sparse MLA %s decode with cache: %s",
@@ -223,6 +231,58 @@ def _run_flashinfer_sparse_mla_decode_autotune(
return True
def _run_sparse_mla_decode_autotune_v2(
worker: "Worker",
num_tokens: int,
log_label: str,
dummy_run_kwargs: dict,
) -> bool:
"""Run every rank's sparse warmup in the managed autotune context.
FlashInfer owns persistence; some sparse kernels use a separate calibration
cache instead of managed winner entries.
"""
from flashinfer import autotune_v2, autotune_v2_reload
from vllm.distributed.parallel_state import get_world_group
runner = worker.model_runner
world = get_world_group()
cache_root = resolve_flashinfer_autotune_v2_root()
if world.rank_in_group == 0:
logger.info(
"Autotuning FlashInfer SM120 sparse MLA %s decode with the managed "
"autotune context (root=%s)",
log_label,
cache_root if cache_root is not None else "flashinfer default",
)
with torch.inference_mode():
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
v2_runner = cast("V2GPUModelRunner", runner)
warmup_executed = run_mixed_prefill_decode_warmup(
v2_runner,
worker.execute_model,
worker.sample_tokens,
num_tokens,
mixed_step_context=autotune_v2(mode="tune", cache_root=cache_root),
req_id_prefix="_sparse_mla_v2_warmup",
)
else:
warmup_executed = True
with autotune_v2(mode="tune", cache_root=cache_root):
runner._dummy_run(**dummy_run_kwargs)
if not warmup_executed:
return False
if world.world_size > 1:
world.barrier()
autotune_v2_reload()
world.barrier()
return True
def _flashinfer_sparse_mla_decode_autotune(
worker: "Worker",
num_tokens: int,
+33 -6
View File
@@ -1082,6 +1082,10 @@ class JitWarmupRegistry:
from tqdm import tqdm
from vllm.distributed import is_global_first_rank
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
VllmTritonJitKernel,
compile_triton_warmup,
)
kernel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
for kernel, registrations in self._registrations.items():
@@ -1102,17 +1106,40 @@ class JitWarmupRegistry:
if not kernel_items:
return
parallel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
serial_items = []
for kernel, compile_keys in kernel_items:
if (
isinstance(kernel, VllmTritonJitKernel)
and kernel.supports_parallel_compile
):
parallel_items.append((kernel, compile_keys))
else:
serial_items.append((kernel, compile_keys))
total_keys = sum(len(compile_keys) for _, compile_keys in kernel_items)
with tqdm(
kernel_items,
total=len(kernel_items),
desc=f"JIT kernel warmup ({total_keys} compile keys)",
disable=not is_global_first_rank(),
dynamic_ncols=True,
unit="kernel",
) as progress:
for kernel, compile_keys in progress:
progress.set_postfix_str(
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
refresh=False,
def compile_kernels(
items: list[tuple[VllmJitKernel[Any], dict[Any, None]]],
) -> None:
for kernel, compile_keys in items:
progress.set_postfix_str(
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
refresh=False,
)
kernel.compile_many(compile_keys)
progress.update()
if parallel_items:
compile_triton_warmup(
lambda: compile_kernels(parallel_items),
sum(len(keys) for _, keys in parallel_items),
)
kernel.compile_many(compile_keys)
compile_kernels(serial_items)
@@ -6,7 +6,7 @@ from abc import abstractmethod
from collections.abc import Callable, Hashable, Iterable, Mapping
from concurrent.futures import ThreadPoolExecutor
from contextlib import nullcontext, suppress
from contextvars import copy_context
from contextvars import ContextVar, copy_context
from dataclasses import dataclass, field
from functools import cache, cached_property, update_wrapper
from typing import Any, Generic, ParamSpec, Protocol, TypeVar, cast, overload
@@ -207,6 +207,31 @@ class TritonWarmupTensor:
return result if dim is None else result[dim]
_shared_compile_pool: ContextVar[bool] = ContextVar(
"shared_triton_warmup_compile_pool", default=False
)
def compile_triton_warmup(compile_all: Callable[[], None], num_keys: int) -> None:
"""Share one compiler pool across all eligible warmup kernels."""
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, num_keys)
if num_threads <= 1:
compile_all()
return
def compile_in_context() -> None:
_shared_compile_pool.set(True)
with (
ThreadPoolExecutor(max_workers=num_threads) as executor,
triton.AsyncCompileMode(executor),
):
# Dispatch mutates owner state and must stay on this thread.
compile_all()
# Older Triton versions leave async mode active on compilation failure.
copy_context().run(compile_in_context)
class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
"""Triton owner whose runtime launch specification is reused for warmup."""
@@ -230,32 +255,25 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
self._warming = False
self._warming_compile_key = None
@property
def supports_parallel_compile(self) -> bool:
# AMD LLVM code generation can abort under concurrent compilation.
return (
current_platform.is_cuda()
and not self._run_autotune
and getattr(triton, "AsyncCompileMode", None) is not None
and envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS > 1
)
def compile_many(self, compile_keys: Iterable[CompileKeyT]) -> None:
"""Compile CUDA startup warmup variants in parallel, then wait."""
keys = list(compile_keys)
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, len(keys))
async_compile = getattr(triton, "AsyncCompileMode", None)
# AMD LLVM code generation can abort under concurrent compilation.
if (
not current_platform.is_cuda()
or self._run_autotune
or async_compile is None
or num_threads <= 1
):
if not self.supports_parallel_compile or _shared_compile_pool.get():
return super().compile_many(keys)
def compile_all() -> None:
with (
ThreadPoolExecutor(max_workers=num_threads) as executor,
async_compile(executor),
):
# Dispatch stays on this thread: compile() mutates owner state.
# Only Triton's underlying compiler work runs in the pool.
super(VllmTritonJitKernel, self).compile_many(keys)
# Older Triton versions leave async mode active on compilation failure.
# Isolate that state so it cannot affect later runtime JIT compilation.
copy_context().run(compile_all)
compile_triton_warmup(
lambda: super(VllmTritonJitKernel, self).compile_many(keys), len(keys)
)
@cached_property
def _kernel_arg_names(self) -> tuple[str, ...]:
+42 -30
View File
@@ -23,6 +23,8 @@ from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
resolve_flashinfer_autotune_v2_root,
use_flashinfer_autotune_v2,
)
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
autotune_hisparse_flashinfer_attention,
@@ -419,7 +421,7 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
otherwise the world group tunes together. Per-tactic timings are
averaged over the tuning group so all its ranks select the same tactic.
Results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
Legacy results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
(``MoERunner.get_cache_key_extras``), so one rank's file only hits on that
rank. A rank with a cache hit skips the per-tactic reduce the others block
in, so ranks keep loaded configs only if every rank in the tuning group
@@ -450,38 +452,45 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
)
autotune_kwargs["skip_ops"] = skip_ops
cache_path = resolve_flashinfer_autotune_file(runner)
# The world group only spans DP ranks when vLLM folds DP into it.
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
cache_path = cache_path.with_name(
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
)
if is_leader:
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
use_autotune_v2 = use_flashinfer_autotune_v2(runner)
if use_autotune_v2:
from flashinfer import autotune_v2, autotune_v2_reload
# We skip EPLB here since we don't want to record dummy metrics.
# Randomize inputs to avoid every token pick the same experts,
# which lead to some EP ranks receiving no tokens and skipping their
# MoE kernel entirely, and cause hang due to all-reduce collective
# during synchronized autotuning.
if _all_ranks_have_matching_cache(cache_path, tune_group):
loaded = tuner.load_configs(str(cache_path))
if tune_group.world_size > 1:
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
torch.distributed.all_gather_object(
loaded_by_rank, loaded, group=tune_group.cpu_group
cache_root = resolve_flashinfer_autotune_v2_root()
if is_leader:
logger.info_once(
"Using FlashInfer managed autotune cache (root=%s)",
cache_root if cache_root is not None else "flashinfer default",
)
loaded = all(loaded_by_rank)
if not loaded:
tuner.clear_cache()
autotune_context = autotune_v2(
mode="tune", cache_root=cache_root, **autotune_kwargs
)
else:
cache_path = resolve_flashinfer_autotune_file(runner)
# The world group only spans DP ranks when vLLM folds DP into it.
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
cache_path = cache_path.with_name(
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
)
if is_leader:
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
if _all_ranks_have_matching_cache(cache_path, tune_group):
loaded = tuner.load_configs(str(cache_path))
if tune_group.world_size > 1:
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
torch.distributed.all_gather_object(
loaded_by_rank, loaded, group=tune_group.cpu_group
)
loaded = all(loaded_by_rank)
if not loaded:
tuner.clear_cache()
autotune_context = fi_utils.autotune(tune_mode=True, **autotune_kwargs)
group = tune_group.cpu_group if tune_group.world_size > 1 else None
set_autotune_process_group(group)
try:
with (
torch.inference_mode(),
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
):
with torch.inference_mode(), autotune_context:
hisparse_enabled = (
runner.vllm_config.attention_config.hisparse_config is not None
)
@@ -498,7 +507,10 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
if world.world_size > 1:
world.barrier()
# Skip the rewrite when nothing was tuned this start (every entry came from
# the file). FlashInfer gates its own autotune(cache=...) save the same way.
if tuner._dirty:
if use_autotune_v2:
if world.world_size > 1:
# All ranks serve the shared store's final state after publishing.
autotune_v2_reload()
world.barrier()
elif tuner._dirty:
tuner.save_configs(str(cache_path))
@@ -19,9 +19,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None:
def _get_kda_layer(
worker: Worker, module: str = "vllm.models.kimi_k3.nvidia.kda"
) -> KimiK3DeltaAttention | None:
# Kimi model construction already imports kda. Avoid importing it here.
kda = sys.modules.get("vllm.models.kimi_k3.nvidia.kda")
kda = sys.modules.get(module)
if kda is None:
return None
@@ -172,9 +174,59 @@ def _warm_recurrent_kda(
)
def _warm_gluon_chunk_kda(layer, input_dtype: torch.dtype) -> None:
from vllm.models.kimi_k3.amd.ops.kda_prefill import (
chunk_kda_prefill,
get_gluon_chunk_kda_warmup_profiles,
)
if layer.kda_prefill_backend != "gluon":
return
kv_cache = layer.kv_cache
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
return
state = kv_cache[1]
if not isinstance(state, torch.Tensor) or not state.numel():
return
logger.info("Warming up Kimi-K3 Gluon chunk KDA prefill kernels.")
h, d = int(layer.local_num_heads), int(layer.head_dim)
for num_seqs, seq_len in get_gluon_chunk_kda_warmup_profiles():
num_tokens = num_seqs * seq_len
shape = (1, num_tokens, h, d)
chunk_kda_prefill(
q=torch.zeros(shape, dtype=input_dtype, device=state.device),
k=torch.zeros(shape, dtype=input_dtype, device=state.device),
v=torch.zeros(shape, dtype=input_dtype, device=state.device),
raw_g=torch.zeros(shape, dtype=input_dtype, device=state.device),
raw_beta=torch.zeros(
(1, num_tokens, h), dtype=input_dtype, device=state.device
),
A_log=layer.A_log,
g_bias=layer.dt_bias,
lower_bound=layer.gate_lower_bound,
state_cache=state,
state_indices=torch.zeros(num_seqs, dtype=torch.int32, device=state.device),
has_initial_state=torch.zeros(
num_seqs, dtype=torch.bool, device=state.device
),
use_qk_l2norm_in_kernel=True,
cu_seqlens=torch.arange(
0, num_tokens + 1, seq_len, dtype=torch.int32, device=state.device
),
backend="gluon",
out=torch.empty(shape, dtype=input_dtype, device=state.device),
)
@torch.inference_mode()
def kimi_k3_triton_warmup(worker: Worker) -> None:
"""Warm Kimi-K3 Triton kernels reachable by this server."""
if current_platform.is_rocm():
layer = _get_kda_layer(worker, "vllm.models.kimi_k3.amd.kda")
if layer is not None:
_warm_gluon_chunk_kda(layer, worker.model_config.dtype)
return
if not current_platform.is_cuda():
return