Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 17:41:13 +00:00
1122 changed files with 60114 additions and 20956 deletions
@@ -179,6 +179,7 @@ def _deepgemm_fp8_gemm_nt_warmup(
device = w.device
a1q = torch.empty((max_tokens, k), device=device, dtype=torch.float8_e4m3fn)
# Must be initialized (UE8M0 packing asserts zero sign/mantissa bits).
a1q_scales = torch.zeros(
(max_tokens, k // block_m), device=device, dtype=torch.float32
)
@@ -284,6 +285,7 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
((MAX_M, k // block_m), torch.float32),
((MAX_M, n), torch.bfloat16),
)
# Must be initialized (UE8M0 packing asserts zero sign/mantissa bits).
a1q_scales.zero_()
for num_tokens, align_used, expert_ids in warmup_cases:
+72 -63
View File
@@ -5,8 +5,10 @@ This is useful specifically for JIT'ed kernels as we don't want JIT'ing to
happen during model execution.
"""
import json
import sys
import time
from pathlib import Path
from typing import TYPE_CHECKING
import torch
@@ -18,7 +20,6 @@ from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
write_flashinfer_autotune_cache,
)
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
autotune_hisparse_flashinfer_attention,
@@ -220,6 +221,13 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
if process_local_only:
return
if current_platform.is_rocm():
from vllm.model_executor.warmup.rocm_segmented_attn_autotune_warmup import (
rocm_segmented_attn_autotune_warmup,
)
rocm_segmented_attn_autotune_warmup(worker)
flashinfer_sparse_mla_decode_autotune_warmup(worker)
deepseek_v4_sparse_mla_attention_warmup(worker)
@@ -308,9 +316,6 @@ def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
return None
_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS = 32
def _flashinfer_deferred_moe_token_counts(
runner: "GPUModelRunner",
) -> tuple[int, ...]:
@@ -331,19 +336,10 @@ def _flashinfer_deferred_moe_token_counts(
return tuple(dict.fromkeys(token_counts))
def _flashinfer_autotune_token_counts(
runner: "GPUModelRunner", *, include_bf16: bool = True
) -> tuple[int, ...]:
def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ...]:
max_tokens = runner.scheduler_config.max_num_batched_tokens
# Tune the widest bucket set first so bounded passes reuse its configs.
token_counts = [max_tokens]
linear_backend = runner.vllm_config.kernel_config.linear_backend
if (
include_bf16
and linear_backend == "flashinfer_cutedsl"
and max_tokens > _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
):
token_counts.append(_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS)
token_counts.extend(_flashinfer_deferred_moe_token_counts(runner))
return tuple(dict.fromkeys(token_counts))
@@ -354,14 +350,33 @@ def _run_flashinfer_autotune_dummy_runs(
import vllm.utils.flashinfer as fi_utils
dummy_run_kwargs = {"skip_attn": True} if skip_attn else {}
for num_tokens in _flashinfer_autotune_token_counts(runner, include_bf16=False):
tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens)
for num_tokens in _flashinfer_autotune_token_counts(runner):
# The drafter may use more tokens than this pass in the same dummy run.
# Include its max M when building the autotune buckets.
max_tuning_tokens = num_tokens
speculator = getattr(runner, "speculator", None)
if speculator is not None:
num_query_per_req = speculator.num_query_per_req
num_draft_reqs = min(
num_tokens,
runner.max_num_reqs,
runner.max_num_tokens // num_query_per_req,
)
max_tuning_tokens = max(
max_tuning_tokens, num_draft_reqs * num_query_per_req
)
tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(
max_tuning_tokens
)
logger.info(
"Running FlashInfer autotune with %d tokens and token buckets %s.",
num_tokens,
tuning_buckets,
)
with fi_utils.autotune(tuning_buckets=tuning_buckets):
# Round M up to match serving-time bucket selection for non-bucket M.
# See https://github.com/flashinfer-ai/flashinfer/issues/5450
with fi_utils.autotune(tuning_buckets=tuning_buckets, round_up=True):
runner._dummy_run(
num_tokens=num_tokens,
skip_eplb=True,
@@ -371,34 +386,24 @@ def _run_flashinfer_autotune_dummy_runs(
)
def _run_flashinfer_bf16_autotune_dummy_run(
runner: "GPUModelRunner",
*,
skip_ops: set[str] | None = None,
skip_attn: bool = False,
) -> None:
import vllm.utils.flashinfer as fi_utils
def _autotune_cache_fingerprint(path: Path) -> tuple[str, int] | None:
"""Identify a saved autotune file by its FlashInfer metadata and size."""
try:
configs = json.loads(path.read_text())
metadata = configs.pop("_metadata", None)
except (OSError, ValueError, AttributeError):
return None
return json.dumps(metadata, sort_keys=True), len(configs)
if (
runner.vllm_config.kernel_config.linear_backend != "flashinfer_cutedsl"
or runner.scheduler_config.max_num_batched_tokens
<= _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
):
return
num_tokens = _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens)
logger.info("Running FlashInfer BF16-only autotune with %d tokens.", num_tokens)
# Other ops execute from their existing cache or fallback, without the
# BF16 bucket cap. No full-model autotune context may enclose this pass.
with fi_utils.autotune_bf16_only(tuning_buckets, skip_ops=skip_ops):
runner._dummy_run(
num_tokens=num_tokens,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
**({"skip_attn": True} if skip_attn else {}),
)
def _all_ranks_have_matching_cache(path: Path, group) -> bool:
"""True iff every rank in ``group`` has its own, mutually consistent file."""
fingerprint = _autotune_cache_fingerprint(path)
if group.world_size == 1:
return fingerprint is not None
gathered: list[tuple[str, int] | None] = [None] * group.world_size
torch.distributed.all_gather_object(gathered, fingerprint, group=group.cpu_group)
return fingerprint is not None and all(f == fingerprint for f in gathered)
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
@@ -414,6 +419,12 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
so each stage's TP group tunes separately with its own cache file;
otherwise the world group tunes together. Per-tactic timings are
averaged over the tuning group so all its ranks select the same tactic.
Results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
(``MoERunner.get_cache_key_extras``), so one rank's file only hits on that
rank. A rank with a cache hit skips the per-tactic reduce the others block
in, so ranks keep loaded configs only if every rank in the tuning group
has a matching file and successfully loads it.
"""
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
@@ -440,11 +451,11 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
autotune_kwargs["skip_ops"] = skip_ops
cache_path = resolve_flashinfer_autotune_file(runner)
if pp_size > 1:
ranks = "-".join(str(rank) for rank in tune_group.ranks)
cache_path = cache_path.with_name(
f"{cache_path.stem}_tp_{ranks}{cache_path.suffix}"
)
# The world group only spans DP ranks when vLLM folds DP into it.
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
cache_path = cache_path.with_name(
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
)
if is_leader:
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
@@ -453,16 +464,16 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
# which lead to some EP ranks receiving no tokens and skipping their
# MoE kernel entirely, and cause hang due to all-reduce collective
# during synchronized autotuning.
# Read cached autotune results and broadcast within the tuning group.
cached_results: bytes | None = None
if is_leader and cache_path.exists():
with open(cache_path, "rb") as f:
cached_results = f.read()
cached_results = tune_group.broadcast_object(cached_results, src=0)
if cached_results is not None:
write_flashinfer_autotune_cache(cache_path, cached_results)
tune_group.barrier()
tuner.load_configs(str(cache_path))
if _all_ranks_have_matching_cache(cache_path, tune_group):
loaded = tuner.load_configs(str(cache_path))
if tune_group.world_size > 1:
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
torch.distributed.all_gather_object(
loaded_by_rank, loaded, group=tune_group.cpu_group
)
loaded = all(loaded_by_rank)
if not loaded:
tuner.clear_cache()
group = tune_group.cpu_group if tune_group.world_size > 1 else None
set_autotune_process_group(group)
@@ -481,14 +492,12 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
replayssm_autotune_warmup(runner)
_autotune_kimi_k3_kda_qkvg(runner.get_model())
with torch.inference_mode():
_run_flashinfer_bf16_autotune_dummy_run(
runner, skip_ops=skip_ops, skip_attn=hisparse_enabled
)
finally:
set_autotune_process_group(None)
if world.world_size > 1:
world.barrier()
if is_leader:
# Skip the rewrite when nothing was tuned this start (every entry came from
# the file). FlashInfer gates its own autotune(cache=...) save the same way.
if tuner._dirty:
tuner.save_configs(str(cache_path))
File diff suppressed because it is too large Load Diff