[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
@@ -1,350 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warmup and autotune helpers for FlashInfer sparse MLA backends."""
from typing import TYPE_CHECKING, cast
import torch
from vllm.logger import init_logger
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
resolve_flashinfer_autotune_v2_root,
use_flashinfer_autotune_v2,
write_flashinfer_autotune_cache,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import autotune as flashinfer_autotune
from vllm.utils.flashinfer import has_flashinfer
from vllm.v1.worker.gpu.warmup import run_mixed_prefill_decode_warmup
if TYPE_CHECKING:
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as V2GPUModelRunner
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
from vllm.v1.worker.gpu_worker import Worker
logger = init_logger(__name__)
_DEEPSEEK_V4_SPARSE_MLA_BACKENDS = frozenset(
{
"FLASHMLA_SPARSE_DSV4",
"FLASHINFER_MLA_SPARSE_DSV4",
"ROCM_FLASHMLA_SPARSE_DSV4",
"FLASHMLA_SPARSE_DSV41",
"FLASHINFER_MLA_SPARSE_DSV41",
"ROCM_FLASHMLA_SPARSE_DSV41",
"DEEPSEEK_SPARSE_SWA",
}
)
_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset({"FLASHINFER_MLA_SPARSE_SM120"})
_DEEPSEEK_V4_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset(
{"FLASHINFER_MLA_SPARSE_DSV4", "FLASHINFER_MLA_SPARSE_DSV41"}
)
_FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS = {
"FLASHINFER_MLA_SPARSE_SM120": "DSv3.2",
"FLASHINFER_MLA_SPARSE_DSV4": "DSv4",
"FLASHINFER_MLA_SPARSE_DSV41": "DSv4.1",
}
_SPARSE_MLA_MIXED_WARMUP_TOKENS = 16
def autotune_hisparse_flashinfer_attention(runner: "GPUModelRunner") -> None:
"""Autotune each HiSparse FlashInfer sparse-MLA configuration."""
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseImpl,
)
tuned: set[tuple[object, ...]] = set()
for layer in runner.vllm_config.compilation_config.static_forward_context.values():
impl = getattr(layer, "impl", None)
if not isinstance(impl, FlashInferMLASparseImpl):
continue
if getattr(layer, "hisparse_cache", None) is None:
continue
if impl.topk_indices_buffer is None:
continue
key = (
impl.kv_cache_dtype,
impl.num_heads,
impl.qk_nope_head_dim,
impl.qk_rope_head_dim,
impl.kv_lora_rank,
impl.topk_indices_buffer.shape[1],
)
if key in tuned:
continue
impl.autotune_hisparse_decode(layer)
tuned.add(key)
def _attention_backend_name(backend: object) -> str | None:
get_name = getattr(backend, "get_name", None)
if get_name is None:
return None
try:
return get_name()
except NotImplementedError:
return None
def _has_deepseek_v4_sparse_mla_backend(runner: "GPUModelRunner") -> bool:
for groups in getattr(runner, "attn_groups", []) or ():
for group in groups:
name = _attention_backend_name(getattr(group, "backend", None))
if name in _DEEPSEEK_V4_SPARSE_MLA_BACKENDS:
return True
return False
def _flashinfer_sparse_mla_decode_label(
runner: "GPUModelRunner",
allowed_backends: frozenset[str],
) -> str | None:
for groups in getattr(runner, "attn_groups", []) or ():
for group in groups:
name = _attention_backend_name(getattr(group, "backend", None))
if name in allowed_backends:
return _FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS.get(name)
return None
def _clamp_warmup_tokens(num_tokens: int, max_tokens: int) -> int:
return max(0, min(num_tokens, max_tokens))
def _uses_v2_model_runner(runner: "GPUModelRunner") -> bool:
vllm_config = getattr(runner, "vllm_config", None)
return bool(getattr(vllm_config, "use_v2_model_runner", False))
def _run_flashinfer_sparse_mla_decode_autotune(
worker: "Worker",
num_tokens: int,
allowed_backends: frozenset[str],
) -> bool:
"""Autotune FlashInfer's SM120 sparse-MLA decode path."""
runner = worker.model_runner
log_label = _flashinfer_sparse_mla_decode_label(runner, allowed_backends)
if log_label is None:
return False
if worker.vllm_config.kernel_config.enable_flashinfer_autotune is not True:
return False
if not has_flashinfer() or not current_platform.is_device_capability_family(120):
return False
try:
from flashinfer.autotuner import AutoTuner
except ImportError:
logger.warning(
"Skipping FlashInfer SM120 sparse MLA decode autotune because "
"FlashInfer autotuner is unavailable."
)
return False
from vllm.distributed.parallel_state import get_world_group
world = get_world_group()
is_leader = world.rank_in_group == 0
dummy_run_kwargs = dict(
num_tokens=num_tokens,
skip_eplb=True,
is_profile=True,
force_attention=True,
create_mixed_batch=True,
)
if use_flashinfer_autotune_v2(runner):
return _run_sparse_mla_decode_autotune_v2(
worker, num_tokens, log_label, dummy_run_kwargs
)
cache_path = resolve_flashinfer_autotune_file(runner)
if is_leader:
logger.info(
"Autotuning FlashInfer SM120 sparse MLA %s decode with cache: %s",
log_label,
cache_path,
)
with torch.inference_mode():
warmup_executed = True
if is_leader:
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
v2_runner = cast("V2GPUModelRunner", runner)
warmup_executed = run_mixed_prefill_decode_warmup(
v2_runner,
worker.execute_model,
worker.sample_tokens,
num_tokens,
mixed_step_context=flashinfer_autotune(True, cache=str(cache_path)),
req_id_prefix="_sparse_mla_v2_warmup",
)
else:
with flashinfer_autotune(True, cache=str(cache_path)):
runner._dummy_run(**dummy_run_kwargs)
else:
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
v2_runner = cast("V2GPUModelRunner", runner)
warmup_executed = run_mixed_prefill_decode_warmup(
v2_runner,
worker.execute_model,
worker.sample_tokens,
num_tokens,
req_id_prefix="_sparse_mla_v2_warmup",
)
else:
runner._dummy_run(**dummy_run_kwargs)
if not warmup_executed:
return False
tune_results: bytes | None = None
if is_leader and cache_path.exists():
with open(cache_path, "rb") as f:
tune_results = f.read()
tune_results = world.broadcast_object(tune_results, src=0)
if tune_results is None:
logger.warning(
"No FlashInfer SM120 sparse MLA %s decode autotune cache entries found. "
"Falling back to FlashInfer's default tactic heuristic.",
log_label,
)
world.barrier()
return True
write_flashinfer_autotune_cache(cache_path, tune_results)
world.barrier()
AutoTuner.get().load_configs(str(cache_path))
logger.info(
"FlashInfer SM120 sparse MLA %s decode autotune cache loaded on rank %d "
"from %s.",
log_label,
world.rank_in_group,
cache_path,
)
return True
def _run_sparse_mla_decode_autotune_v2(
worker: "Worker",
num_tokens: int,
log_label: str,
dummy_run_kwargs: dict,
) -> bool:
"""Run every rank's sparse warmup in the managed autotune context.
FlashInfer owns persistence; some sparse kernels use a separate calibration
cache instead of managed winner entries.
"""
from flashinfer import autotune_v2, autotune_v2_reload
from vllm.distributed.parallel_state import get_world_group
runner = worker.model_runner
world = get_world_group()
cache_root = resolve_flashinfer_autotune_v2_root()
if world.rank_in_group == 0:
logger.info(
"Autotuning FlashInfer SM120 sparse MLA %s decode with the managed "
"autotune context (root=%s)",
log_label,
cache_root if cache_root is not None else "flashinfer default",
)
with torch.inference_mode():
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
v2_runner = cast("V2GPUModelRunner", runner)
warmup_executed = run_mixed_prefill_decode_warmup(
v2_runner,
worker.execute_model,
worker.sample_tokens,
num_tokens,
mixed_step_context=autotune_v2(mode="tune", cache_root=cache_root),
req_id_prefix="_sparse_mla_v2_warmup",
)
else:
warmup_executed = True
with autotune_v2(mode="tune", cache_root=cache_root):
runner._dummy_run(**dummy_run_kwargs)
if not warmup_executed:
return False
if world.world_size > 1:
world.barrier()
autotune_v2_reload()
world.barrier()
return True
def _flashinfer_sparse_mla_decode_autotune(
worker: "Worker",
num_tokens: int,
) -> bool:
return _run_flashinfer_sparse_mla_decode_autotune(
worker, num_tokens, _FLASHINFER_MLA_SPARSE_BACKENDS
)
def _deepseek_v4_sparse_mla_decode_autotune(
worker: "Worker",
num_tokens: int,
) -> bool:
return _run_flashinfer_sparse_mla_decode_autotune(
worker, num_tokens, _DEEPSEEK_V4_FLASHINFER_MLA_SPARSE_BACKENDS
)
def flashinfer_sparse_mla_decode_autotune_warmup(worker: "Worker") -> None:
"""Autotune generic FlashInfer sparse MLA decode when selected."""
runner = worker.model_runner
if runner.is_pooling_model:
return
max_tokens = worker.scheduler_config.max_num_batched_tokens
mixed_tokens = _clamp_warmup_tokens(_SPARSE_MLA_MIXED_WARMUP_TOKENS, max_tokens)
if mixed_tokens <= 0:
return
_flashinfer_sparse_mla_decode_autotune(worker, mixed_tokens)
def deepseek_v4_sparse_mla_attention_warmup(worker: "Worker") -> None:
"""Warm DSv4 sparse-MLA mixed prefill+decode attention."""
runner = worker.model_runner
if runner.is_pooling_model or not _has_deepseek_v4_sparse_mla_backend(runner):
return
max_tokens = worker.scheduler_config.max_num_batched_tokens
mixed_tokens = _clamp_warmup_tokens(_SPARSE_MLA_MIXED_WARMUP_TOKENS, max_tokens)
if mixed_tokens <= 0:
return
logger.info(
"Warming up DeepSeek V4 sparse MLA attention for mixed tokens=%s.",
mixed_tokens,
)
mixed_warmup_done = _deepseek_v4_sparse_mla_decode_autotune(worker, mixed_tokens)
if not mixed_warmup_done:
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
v2_runner = cast("V2GPUModelRunner", runner)
run_mixed_prefill_decode_warmup(
v2_runner,
worker.execute_model,
worker.sample_tokens,
mixed_tokens,
req_id_prefix="_sparse_mla_v2_warmup",
)
else:
runner._dummy_run(
num_tokens=mixed_tokens,
skip_eplb=True,
is_profile=True,
force_attention=True,
create_mixed_batch=True,
)
+4 -18
View File
@@ -26,14 +26,6 @@ from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_v2_root,
use_flashinfer_autotune_v2,
)
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
autotune_hisparse_flashinfer_attention,
deepseek_v4_sparse_mla_attention_warmup,
flashinfer_sparse_mla_decode_autotune_warmup,
)
from vllm.model_executor.warmup.kimi_k3_triton_warmup import (
kimi_k3_triton_warmup,
)
from vllm.model_executor.warmup.mamba_triton_warmup import mamba_triton_warmup
from vllm.model_executor.warmup.qwen4_exp_qsa_warmup import (
qwen4_exp_qsa_triton_warmup,
@@ -162,10 +154,6 @@ def _autotune_kimi_k3_kda_qkvg(model: torch.nn.Module) -> None:
def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
from vllm.model_executor.warmup.minimax_m3_msa_warmup import (
minimax_m3_msa_warmup,
)
if not worker.use_v2_model_runner:
# The KV-block zeroing kernel is driven by the scheduler's
# `new_block_ids_to_zero`, so no dummy run ever reaches it.
@@ -202,7 +190,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
# Run next so input-prep kernels JIT against pristine runner state.
if enable_jit_warmup:
kimi_k3_triton_warmup(worker)
watermark_sample_warmup(worker)
qwen4_exp_qsa_triton_warmup(worker)
@@ -233,9 +220,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
rocm_segmented_attn_autotune_warmup(worker)
flashinfer_sparse_mla_decode_autotune_warmup(worker)
deepseek_v4_sparse_mla_attention_warmup(worker)
# Deep GEMM warmup
do_deep_gemm_warmup = (
is_deep_gemm_supported() and envs.VLLM_DEEP_GEMM_WARMUP != "skip"
@@ -247,8 +231,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
b12x_warmup(worker, cudagraph_capture_sizes)
minimax_m3_msa_warmup(worker)
# Allocate the exact decode-sized workspace, autotune cache misses, and
# resolve every CUDA Graph bucket before capture begins.
# Lazy import: flashinfer_pcie_ipc_all_reduce imports flashinfer.comm,
@@ -498,6 +480,10 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
if hisparse_enabled:
# HiSparse hot-buffer attention is bounded by decode batch
# size, not the prefill-sized batch used for the full model.
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
autotune_hisparse_flashinfer_attention,
)
autotune_hisparse_flashinfer_attention(runner)
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
replayssm_autotune_warmup(runner)
@@ -1,238 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warm up Kimi-K3 Triton kernels."""
from __future__ import annotations
import sys
from typing import TYPE_CHECKING
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
from vllm.v1.worker.gpu_worker import Worker
logger = init_logger(__name__)
def _get_kda_layer(
worker: Worker, module: str = "vllm.models.kimi_k3.nvidia.kda"
) -> KimiK3DeltaAttention | None:
# Kimi model construction already imports kda. Avoid importing it here.
kda = sys.modules.get(module)
if kda is None:
return None
compilation_config = getattr(
worker.model_runner,
"compilation_config",
None,
)
static_context = getattr(compilation_config, "static_forward_context", None)
if not isinstance(static_context, dict):
return None
return next(
(
layer
for layer in static_context.values()
if isinstance(layer, kda.KimiK3DeltaAttention)
),
None,
)
def _warm_attn_res(worker: Worker) -> None:
from vllm.models.kimi_k3.nvidia.ops.attn_res import (
attn_res,
get_attn_res_triton_warmup_profiles,
)
config = worker.model_config.hf_text_config
block_size = getattr(config, "attn_res_block_size", None)
if block_size is None:
return
hidden_size = int(config.hidden_size)
max_blocks = (int(config.num_hidden_layers) + block_size - 1) // block_size
if max_blocks < 2:
return
dtype = worker.model_config.dtype
device = torch.device("cuda")
eps = float(config.rms_norm_eps)
prefix = torch.zeros((1, hidden_size), dtype=dtype, device=device)
delta = torch.zeros_like(prefix)
blocks = torch.zeros(
(1, max_blocks, hidden_size),
dtype=dtype,
device=device,
)
norm_weight = torch.zeros(hidden_size, dtype=dtype, device=device)
qk_weight = torch.zeros_like(norm_weight)
output_norm_weight = torch.zeros_like(norm_weight)
for (
num_blocks,
has_delta,
block_write_idx,
apply_output_norm,
) in get_attn_res_triton_warmup_profiles(max_blocks):
attn_res(
prefix,
delta if has_delta else None,
blocks,
norm_weight,
qk_weight,
output_norm_weight if apply_output_norm else None,
num_blocks=num_blocks,
block_write_idx=block_write_idx,
eps=eps,
output_norm_eps=eps if apply_output_norm else 0.0,
)
def _warm_recurrent_kda(
layer: KimiK3DeltaAttention,
input_dtype: torch.dtype,
) -> None:
from vllm.models.kimi_k3.nvidia.ops.third_party.kda.fused_recurrent import (
fused_recurrent_kda,
get_fused_recurrent_kda_fwd_warmup_profiles,
)
num_speculative_tokens = int(layer.num_spec)
# fused_recurrent_kda_fwd_kernel is only used by speculative decode.
if num_speculative_tokens <= 0:
return
kv_cache = layer.kv_cache
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
return
state = kv_cache[1]
if not isinstance(state, torch.Tensor) or not state.numel():
return
logger.info("Warming up Kimi-K3 speculative KDA kernels.")
h = int(layer.local_num_heads)
d = int(layer.head_dim)
tokens_per_sequence = num_speculative_tokens + 1
for num_sequences in get_fused_recurrent_kda_fwd_warmup_profiles(h):
num_tokens = num_sequences * tokens_per_sequence
packed_qkv = torch.empty(
(num_tokens, 3 * h * d),
dtype=input_dtype,
device=state.device,
)
q, k, v = (
tensor.view(1, num_tokens, h, d)
for tensor in packed_qkv.split(h * d, dim=-1)
)
fused_recurrent_kda(
q=q,
k=k,
v=v,
raw_g=torch.empty(
(1, num_tokens, h, d),
dtype=input_dtype,
device=state.device,
),
raw_beta=torch.empty(
(1, num_tokens, h),
dtype=input_dtype,
device=state.device,
),
A_log=layer.A_log,
dt_bias=layer.dt_bias,
lower_bound=layer.gate_lower_bound,
initial_state=state[:1],
cu_seqlens=torch.arange(
0,
num_tokens + 1,
tokens_per_sequence,
dtype=torch.int32,
device=state.device,
),
ssm_state_indices=torch.zeros(
(num_sequences, tokens_per_sequence),
dtype=torch.int32,
device=state.device,
),
num_accepted_tokens=torch.ones(
num_sequences,
dtype=torch.int32,
device=state.device,
),
out=torch.empty(
(1, num_tokens, h, d),
dtype=input_dtype,
device=state.device,
),
)
def _warm_gluon_chunk_kda(layer, input_dtype: torch.dtype) -> None:
from vllm.models.kimi_k3.amd.ops.kda_prefill import (
chunk_kda_prefill,
get_gluon_chunk_kda_warmup_profiles,
)
if layer.kda_prefill_backend != "gluon":
return
kv_cache = layer.kv_cache
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
return
state = kv_cache[1]
if not isinstance(state, torch.Tensor) or not state.numel():
return
logger.info("Warming up Kimi-K3 Gluon chunk KDA prefill kernels.")
h, d = int(layer.local_num_heads), int(layer.head_dim)
for num_seqs, seq_len in get_gluon_chunk_kda_warmup_profiles():
num_tokens = num_seqs * seq_len
shape = (1, num_tokens, h, d)
chunk_kda_prefill(
q=torch.zeros(shape, dtype=input_dtype, device=state.device),
k=torch.zeros(shape, dtype=input_dtype, device=state.device),
v=torch.zeros(shape, dtype=input_dtype, device=state.device),
raw_g=torch.zeros(shape, dtype=input_dtype, device=state.device),
raw_beta=torch.zeros(
(1, num_tokens, h), dtype=input_dtype, device=state.device
),
A_log=layer.A_log,
g_bias=layer.dt_bias,
lower_bound=layer.gate_lower_bound,
state_cache=state,
state_indices=torch.zeros(num_seqs, dtype=torch.int32, device=state.device),
has_initial_state=torch.zeros(
num_seqs, dtype=torch.bool, device=state.device
),
use_qk_l2norm_in_kernel=True,
cu_seqlens=torch.arange(
0, num_tokens + 1, seq_len, dtype=torch.int32, device=state.device
),
backend="gluon",
out=torch.empty(shape, dtype=input_dtype, device=state.device),
)
@torch.inference_mode()
def kimi_k3_triton_warmup(worker: Worker) -> None:
"""Warm Kimi-K3 Triton kernels reachable by this server."""
if current_platform.is_rocm():
layer = _get_kda_layer(worker, "vllm.models.kimi_k3.amd.kda")
if layer is not None:
_warm_gluon_chunk_kda(layer, worker.model_config.dtype)
return
if not current_platform.is_cuda():
return
layer = _get_kda_layer(worker)
if layer is None:
return
_warm_attn_res(worker)
_warm_recurrent_kda(layer, worker.model_config.dtype)
@@ -1,40 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.logger import init_logger
from vllm.model_executor.warmup.attention_warmup import (
mixed_batch_attention_warmup,
)
from vllm.models.minimax_m3.nvidia.model import MiniMaxM3SparseAttention
from vllm.platforms import current_platform
from vllm.tracing import instrument
if TYPE_CHECKING:
from vllm.v1.worker.gpu_worker import Worker
logger = init_logger(__name__)
@instrument(span_name="MiniMax M3 MSA warmup")
def minimax_m3_msa_warmup(worker: "Worker") -> None:
sparse_module = next(
(
module
for module in worker.get_model().modules()
if isinstance(module, MiniMaxM3SparseAttention)
),
None,
)
if sparse_module is None:
return
if not (
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
):
return
logger.info("Warming up MiniMax M3 MSA kernels.")
# Cover sparse prefill through the normal model path.
mixed_batch_attention_warmup(worker)
File diff suppressed because it is too large Load Diff