Merge branch 'main' into patches
This commit is contained in:
@@ -11,11 +11,14 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import CacheConfig, VllmConfig, replace
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _normalize_cache_config_for_hash(cache_config: CacheConfig) -> CacheConfig:
|
||||
"""Normalize cache config for hashing"""
|
||||
@@ -55,6 +58,53 @@ def resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
|
||||
return output_dir / "autotune_configs.json"
|
||||
|
||||
|
||||
def use_flashinfer_autotune_v2(runner: "GPUModelRunner") -> bool:
|
||||
from vllm.distributed.parallel_state import get_node_count, get_world_group
|
||||
from vllm.utils.flashinfer import has_flashinfer_autotune_v2
|
||||
|
||||
# Elastic EP transfers tuning state to joining workers with v1
|
||||
# save_configs/load_configs, which do not export managed-store winners.
|
||||
if runner.vllm_config.parallel_config.enable_elastic_ep:
|
||||
return False
|
||||
# PP stages profile independently. Keep their group-scoped v1 files until
|
||||
# managed-cache reads and publishes can be isolated across tuning groups.
|
||||
if runner.vllm_config.parallel_config.pipeline_parallel_size > 1:
|
||||
return False
|
||||
if not has_flashinfer_autotune_v2():
|
||||
return False
|
||||
# Use the initialized topology: Ray may span nodes with config.nnodes=1.
|
||||
# A cache hit skips profiling collectives, so node-local stores with
|
||||
# different contents cannot safely participate in synchronized tuning.
|
||||
if get_node_count() != 1:
|
||||
logger.info_once(
|
||||
"Using legacy FlashInfer autotune cache synchronization for "
|
||||
"multi-node deployments; managed-cache synchronization is "
|
||||
"currently supported only within one node."
|
||||
)
|
||||
return False
|
||||
# Lift this gate after pinning a release with the fix below and validating
|
||||
# distributed managed-cache tuning: https://github.com/flashinfer-ai/flashinfer/pull/6185
|
||||
if get_world_group().world_size > 1:
|
||||
logger.info_once(
|
||||
"Using legacy FlashInfer autotune cache synchronization for "
|
||||
"multi-rank deployments until FlashInfer coordinates cache hits."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def resolve_flashinfer_autotune_v2_root() -> Path | None:
|
||||
"""Placement root for FlashInfer's managed autotune store.
|
||||
|
||||
Unlike :func:`resolve_flashinfer_autotune_file`, no vLLM-side identity
|
||||
(config hash, flashinfer version, arch) is folded into the path: the
|
||||
managed store keys entries per op and hashes the software/hardware
|
||||
environment itself. ``None`` selects FlashInfer's default root.
|
||||
"""
|
||||
override_dir = envs.VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR
|
||||
return Path(override_dir).expanduser() if override_dir else None
|
||||
|
||||
|
||||
def sync_flashinfer_autotune_cache(
|
||||
runner: "GPUModelRunner",
|
||||
group: "GroupCoordinator",
|
||||
|
||||
@@ -9,6 +9,8 @@ import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
resolve_flashinfer_autotune_v2_root,
|
||||
use_flashinfer_autotune_v2,
|
||||
write_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -145,7 +147,6 @@ def _run_flashinfer_sparse_mla_decode_autotune(
|
||||
|
||||
world = get_world_group()
|
||||
is_leader = world.rank_in_group == 0
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
|
||||
dummy_run_kwargs = dict(
|
||||
num_tokens=num_tokens,
|
||||
@@ -155,6 +156,13 @@ def _run_flashinfer_sparse_mla_decode_autotune(
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
|
||||
if use_flashinfer_autotune_v2(runner):
|
||||
return _run_sparse_mla_decode_autotune_v2(
|
||||
worker, num_tokens, log_label, dummy_run_kwargs
|
||||
)
|
||||
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
|
||||
if is_leader:
|
||||
logger.info(
|
||||
"Autotuning FlashInfer SM120 sparse MLA %s decode with cache: %s",
|
||||
@@ -223,6 +231,58 @@ def _run_flashinfer_sparse_mla_decode_autotune(
|
||||
return True
|
||||
|
||||
|
||||
def _run_sparse_mla_decode_autotune_v2(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
log_label: str,
|
||||
dummy_run_kwargs: dict,
|
||||
) -> bool:
|
||||
"""Run every rank's sparse warmup in the managed autotune context.
|
||||
|
||||
FlashInfer owns persistence; some sparse kernels use a separate calibration
|
||||
cache instead of managed winner entries.
|
||||
"""
|
||||
from flashinfer import autotune_v2, autotune_v2_reload
|
||||
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
runner = worker.model_runner
|
||||
world = get_world_group()
|
||||
cache_root = resolve_flashinfer_autotune_v2_root()
|
||||
if world.rank_in_group == 0:
|
||||
logger.info(
|
||||
"Autotuning FlashInfer SM120 sparse MLA %s decode with the managed "
|
||||
"autotune context (root=%s)",
|
||||
log_label,
|
||||
cache_root if cache_root is not None else "flashinfer default",
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
|
||||
v2_runner = cast("V2GPUModelRunner", runner)
|
||||
warmup_executed = run_mixed_prefill_decode_warmup(
|
||||
v2_runner,
|
||||
worker.execute_model,
|
||||
worker.sample_tokens,
|
||||
num_tokens,
|
||||
mixed_step_context=autotune_v2(mode="tune", cache_root=cache_root),
|
||||
req_id_prefix="_sparse_mla_v2_warmup",
|
||||
)
|
||||
else:
|
||||
warmup_executed = True
|
||||
with autotune_v2(mode="tune", cache_root=cache_root):
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
|
||||
if not warmup_executed:
|
||||
return False
|
||||
|
||||
if world.world_size > 1:
|
||||
world.barrier()
|
||||
autotune_v2_reload()
|
||||
world.barrier()
|
||||
return True
|
||||
|
||||
|
||||
def _flashinfer_sparse_mla_decode_autotune(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
|
||||
@@ -1082,6 +1082,10 @@ class JitWarmupRegistry:
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm.distributed import is_global_first_rank
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
VllmTritonJitKernel,
|
||||
compile_triton_warmup,
|
||||
)
|
||||
|
||||
kernel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
|
||||
for kernel, registrations in self._registrations.items():
|
||||
@@ -1102,17 +1106,40 @@ class JitWarmupRegistry:
|
||||
if not kernel_items:
|
||||
return
|
||||
|
||||
parallel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
|
||||
serial_items = []
|
||||
for kernel, compile_keys in kernel_items:
|
||||
if (
|
||||
isinstance(kernel, VllmTritonJitKernel)
|
||||
and kernel.supports_parallel_compile
|
||||
):
|
||||
parallel_items.append((kernel, compile_keys))
|
||||
else:
|
||||
serial_items.append((kernel, compile_keys))
|
||||
|
||||
total_keys = sum(len(compile_keys) for _, compile_keys in kernel_items)
|
||||
with tqdm(
|
||||
kernel_items,
|
||||
total=len(kernel_items),
|
||||
desc=f"JIT kernel warmup ({total_keys} compile keys)",
|
||||
disable=not is_global_first_rank(),
|
||||
dynamic_ncols=True,
|
||||
unit="kernel",
|
||||
) as progress:
|
||||
for kernel, compile_keys in progress:
|
||||
progress.set_postfix_str(
|
||||
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
|
||||
refresh=False,
|
||||
|
||||
def compile_kernels(
|
||||
items: list[tuple[VllmJitKernel[Any], dict[Any, None]]],
|
||||
) -> None:
|
||||
for kernel, compile_keys in items:
|
||||
progress.set_postfix_str(
|
||||
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
|
||||
refresh=False,
|
||||
)
|
||||
kernel.compile_many(compile_keys)
|
||||
progress.update()
|
||||
|
||||
if parallel_items:
|
||||
compile_triton_warmup(
|
||||
lambda: compile_kernels(parallel_items),
|
||||
sum(len(keys) for _, keys in parallel_items),
|
||||
)
|
||||
kernel.compile_many(compile_keys)
|
||||
compile_kernels(serial_items)
|
||||
|
||||
@@ -6,7 +6,7 @@ from abc import abstractmethod
|
||||
from collections.abc import Callable, Hashable, Iterable, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import nullcontext, suppress
|
||||
from contextvars import copy_context
|
||||
from contextvars import ContextVar, copy_context
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cache, cached_property, update_wrapper
|
||||
from typing import Any, Generic, ParamSpec, Protocol, TypeVar, cast, overload
|
||||
@@ -207,6 +207,31 @@ class TritonWarmupTensor:
|
||||
return result if dim is None else result[dim]
|
||||
|
||||
|
||||
_shared_compile_pool: ContextVar[bool] = ContextVar(
|
||||
"shared_triton_warmup_compile_pool", default=False
|
||||
)
|
||||
|
||||
|
||||
def compile_triton_warmup(compile_all: Callable[[], None], num_keys: int) -> None:
|
||||
"""Share one compiler pool across all eligible warmup kernels."""
|
||||
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, num_keys)
|
||||
if num_threads <= 1:
|
||||
compile_all()
|
||||
return
|
||||
|
||||
def compile_in_context() -> None:
|
||||
_shared_compile_pool.set(True)
|
||||
with (
|
||||
ThreadPoolExecutor(max_workers=num_threads) as executor,
|
||||
triton.AsyncCompileMode(executor),
|
||||
):
|
||||
# Dispatch mutates owner state and must stay on this thread.
|
||||
compile_all()
|
||||
|
||||
# Older Triton versions leave async mode active on compilation failure.
|
||||
copy_context().run(compile_in_context)
|
||||
|
||||
|
||||
class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
"""Triton owner whose runtime launch specification is reused for warmup."""
|
||||
|
||||
@@ -230,32 +255,25 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
self._warming = False
|
||||
self._warming_compile_key = None
|
||||
|
||||
@property
|
||||
def supports_parallel_compile(self) -> bool:
|
||||
# AMD LLVM code generation can abort under concurrent compilation.
|
||||
return (
|
||||
current_platform.is_cuda()
|
||||
and not self._run_autotune
|
||||
and getattr(triton, "AsyncCompileMode", None) is not None
|
||||
and envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS > 1
|
||||
)
|
||||
|
||||
def compile_many(self, compile_keys: Iterable[CompileKeyT]) -> None:
|
||||
"""Compile CUDA startup warmup variants in parallel, then wait."""
|
||||
keys = list(compile_keys)
|
||||
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, len(keys))
|
||||
async_compile = getattr(triton, "AsyncCompileMode", None)
|
||||
# AMD LLVM code generation can abort under concurrent compilation.
|
||||
if (
|
||||
not current_platform.is_cuda()
|
||||
or self._run_autotune
|
||||
or async_compile is None
|
||||
or num_threads <= 1
|
||||
):
|
||||
if not self.supports_parallel_compile or _shared_compile_pool.get():
|
||||
return super().compile_many(keys)
|
||||
|
||||
def compile_all() -> None:
|
||||
with (
|
||||
ThreadPoolExecutor(max_workers=num_threads) as executor,
|
||||
async_compile(executor),
|
||||
):
|
||||
# Dispatch stays on this thread: compile() mutates owner state.
|
||||
# Only Triton's underlying compiler work runs in the pool.
|
||||
super(VllmTritonJitKernel, self).compile_many(keys)
|
||||
|
||||
# Older Triton versions leave async mode active on compilation failure.
|
||||
# Isolate that state so it cannot affect later runtime JIT compilation.
|
||||
copy_context().run(compile_all)
|
||||
compile_triton_warmup(
|
||||
lambda: super(VllmTritonJitKernel, self).compile_many(keys), len(keys)
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def _kernel_arg_names(self) -> tuple[str, ...]:
|
||||
|
||||
@@ -23,6 +23,8 @@ from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
resolve_flashinfer_autotune_v2_root,
|
||||
use_flashinfer_autotune_v2,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
autotune_hisparse_flashinfer_attention,
|
||||
@@ -419,7 +421,7 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
otherwise the world group tunes together. Per-tactic timings are
|
||||
averaged over the tuning group so all its ranks select the same tactic.
|
||||
|
||||
Results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
|
||||
Legacy results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
|
||||
(``MoERunner.get_cache_key_extras``), so one rank's file only hits on that
|
||||
rank. A rank with a cache hit skips the per-tactic reduce the others block
|
||||
in, so ranks keep loaded configs only if every rank in the tuning group
|
||||
@@ -450,38 +452,45 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
)
|
||||
autotune_kwargs["skip_ops"] = skip_ops
|
||||
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
# The world group only spans DP ranks when vLLM folds DP into it.
|
||||
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
|
||||
cache_path = cache_path.with_name(
|
||||
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
|
||||
)
|
||||
if is_leader:
|
||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
use_autotune_v2 = use_flashinfer_autotune_v2(runner)
|
||||
if use_autotune_v2:
|
||||
from flashinfer import autotune_v2, autotune_v2_reload
|
||||
|
||||
# We skip EPLB here since we don't want to record dummy metrics.
|
||||
# Randomize inputs to avoid every token pick the same experts,
|
||||
# which lead to some EP ranks receiving no tokens and skipping their
|
||||
# MoE kernel entirely, and cause hang due to all-reduce collective
|
||||
# during synchronized autotuning.
|
||||
if _all_ranks_have_matching_cache(cache_path, tune_group):
|
||||
loaded = tuner.load_configs(str(cache_path))
|
||||
if tune_group.world_size > 1:
|
||||
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
|
||||
torch.distributed.all_gather_object(
|
||||
loaded_by_rank, loaded, group=tune_group.cpu_group
|
||||
cache_root = resolve_flashinfer_autotune_v2_root()
|
||||
if is_leader:
|
||||
logger.info_once(
|
||||
"Using FlashInfer managed autotune cache (root=%s)",
|
||||
cache_root if cache_root is not None else "flashinfer default",
|
||||
)
|
||||
loaded = all(loaded_by_rank)
|
||||
if not loaded:
|
||||
tuner.clear_cache()
|
||||
autotune_context = autotune_v2(
|
||||
mode="tune", cache_root=cache_root, **autotune_kwargs
|
||||
)
|
||||
else:
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
# The world group only spans DP ranks when vLLM folds DP into it.
|
||||
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
|
||||
cache_path = cache_path.with_name(
|
||||
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
|
||||
)
|
||||
if is_leader:
|
||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
|
||||
if _all_ranks_have_matching_cache(cache_path, tune_group):
|
||||
loaded = tuner.load_configs(str(cache_path))
|
||||
if tune_group.world_size > 1:
|
||||
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
|
||||
torch.distributed.all_gather_object(
|
||||
loaded_by_rank, loaded, group=tune_group.cpu_group
|
||||
)
|
||||
loaded = all(loaded_by_rank)
|
||||
if not loaded:
|
||||
tuner.clear_cache()
|
||||
autotune_context = fi_utils.autotune(tune_mode=True, **autotune_kwargs)
|
||||
|
||||
group = tune_group.cpu_group if tune_group.world_size > 1 else None
|
||||
set_autotune_process_group(group)
|
||||
try:
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
|
||||
):
|
||||
with torch.inference_mode(), autotune_context:
|
||||
hisparse_enabled = (
|
||||
runner.vllm_config.attention_config.hisparse_config is not None
|
||||
)
|
||||
@@ -498,7 +507,10 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
|
||||
if world.world_size > 1:
|
||||
world.barrier()
|
||||
# Skip the rewrite when nothing was tuned this start (every entry came from
|
||||
# the file). FlashInfer gates its own autotune(cache=...) save the same way.
|
||||
if tuner._dirty:
|
||||
if use_autotune_v2:
|
||||
if world.world_size > 1:
|
||||
# All ranks serve the shared store's final state after publishing.
|
||||
autotune_v2_reload()
|
||||
world.barrier()
|
||||
elif tuner._dirty:
|
||||
tuner.save_configs(str(cache_path))
|
||||
|
||||
@@ -19,9 +19,11 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None:
|
||||
def _get_kda_layer(
|
||||
worker: Worker, module: str = "vllm.models.kimi_k3.nvidia.kda"
|
||||
) -> KimiK3DeltaAttention | None:
|
||||
# Kimi model construction already imports kda. Avoid importing it here.
|
||||
kda = sys.modules.get("vllm.models.kimi_k3.nvidia.kda")
|
||||
kda = sys.modules.get(module)
|
||||
if kda is None:
|
||||
return None
|
||||
|
||||
@@ -172,9 +174,59 @@ def _warm_recurrent_kda(
|
||||
)
|
||||
|
||||
|
||||
def _warm_gluon_chunk_kda(layer, input_dtype: torch.dtype) -> None:
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import (
|
||||
chunk_kda_prefill,
|
||||
get_gluon_chunk_kda_warmup_profiles,
|
||||
)
|
||||
|
||||
if layer.kda_prefill_backend != "gluon":
|
||||
return
|
||||
kv_cache = layer.kv_cache
|
||||
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
|
||||
return
|
||||
state = kv_cache[1]
|
||||
if not isinstance(state, torch.Tensor) or not state.numel():
|
||||
return
|
||||
|
||||
logger.info("Warming up Kimi-K3 Gluon chunk KDA prefill kernels.")
|
||||
h, d = int(layer.local_num_heads), int(layer.head_dim)
|
||||
for num_seqs, seq_len in get_gluon_chunk_kda_warmup_profiles():
|
||||
num_tokens = num_seqs * seq_len
|
||||
shape = (1, num_tokens, h, d)
|
||||
chunk_kda_prefill(
|
||||
q=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
k=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
v=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
raw_g=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
raw_beta=torch.zeros(
|
||||
(1, num_tokens, h), dtype=input_dtype, device=state.device
|
||||
),
|
||||
A_log=layer.A_log,
|
||||
g_bias=layer.dt_bias,
|
||||
lower_bound=layer.gate_lower_bound,
|
||||
state_cache=state,
|
||||
state_indices=torch.zeros(num_seqs, dtype=torch.int32, device=state.device),
|
||||
has_initial_state=torch.zeros(
|
||||
num_seqs, dtype=torch.bool, device=state.device
|
||||
),
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=torch.arange(
|
||||
0, num_tokens + 1, seq_len, dtype=torch.int32, device=state.device
|
||||
),
|
||||
backend="gluon",
|
||||
out=torch.empty(shape, dtype=input_dtype, device=state.device),
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def kimi_k3_triton_warmup(worker: Worker) -> None:
|
||||
"""Warm Kimi-K3 Triton kernels reachable by this server."""
|
||||
if current_platform.is_rocm():
|
||||
layer = _get_kda_layer(worker, "vllm.models.kimi_k3.amd.kda")
|
||||
if layer is not None:
|
||||
_warm_gluon_chunk_kda(layer, worker.model_config.dtype)
|
||||
return
|
||||
if not current_platform.is_cuda():
|
||||
return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user