[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,350 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warmup and autotune helpers for FlashInfer sparse MLA backends."""
|
||||
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
resolve_flashinfer_autotune_v2_root,
|
||||
use_flashinfer_autotune_v2,
|
||||
write_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import autotune as flashinfer_autotune
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.v1.worker.gpu.warmup import run_mixed_prefill_decode_warmup
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as V2GPUModelRunner
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_DEEPSEEK_V4_SPARSE_MLA_BACKENDS = frozenset(
|
||||
{
|
||||
"FLASHMLA_SPARSE_DSV4",
|
||||
"FLASHINFER_MLA_SPARSE_DSV4",
|
||||
"ROCM_FLASHMLA_SPARSE_DSV4",
|
||||
"FLASHMLA_SPARSE_DSV41",
|
||||
"FLASHINFER_MLA_SPARSE_DSV41",
|
||||
"ROCM_FLASHMLA_SPARSE_DSV41",
|
||||
"DEEPSEEK_SPARSE_SWA",
|
||||
}
|
||||
)
|
||||
_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset({"FLASHINFER_MLA_SPARSE_SM120"})
|
||||
_DEEPSEEK_V4_FLASHINFER_MLA_SPARSE_BACKENDS = frozenset(
|
||||
{"FLASHINFER_MLA_SPARSE_DSV4", "FLASHINFER_MLA_SPARSE_DSV41"}
|
||||
)
|
||||
|
||||
_FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS = {
|
||||
"FLASHINFER_MLA_SPARSE_SM120": "DSv3.2",
|
||||
"FLASHINFER_MLA_SPARSE_DSV4": "DSv4",
|
||||
"FLASHINFER_MLA_SPARSE_DSV41": "DSv4.1",
|
||||
}
|
||||
|
||||
_SPARSE_MLA_MIXED_WARMUP_TOKENS = 16
|
||||
|
||||
|
||||
def autotune_hisparse_flashinfer_attention(runner: "GPUModelRunner") -> None:
|
||||
"""Autotune each HiSparse FlashInfer sparse-MLA configuration."""
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseImpl,
|
||||
)
|
||||
|
||||
tuned: set[tuple[object, ...]] = set()
|
||||
for layer in runner.vllm_config.compilation_config.static_forward_context.values():
|
||||
impl = getattr(layer, "impl", None)
|
||||
if not isinstance(impl, FlashInferMLASparseImpl):
|
||||
continue
|
||||
if getattr(layer, "hisparse_cache", None) is None:
|
||||
continue
|
||||
if impl.topk_indices_buffer is None:
|
||||
continue
|
||||
key = (
|
||||
impl.kv_cache_dtype,
|
||||
impl.num_heads,
|
||||
impl.qk_nope_head_dim,
|
||||
impl.qk_rope_head_dim,
|
||||
impl.kv_lora_rank,
|
||||
impl.topk_indices_buffer.shape[1],
|
||||
)
|
||||
if key in tuned:
|
||||
continue
|
||||
impl.autotune_hisparse_decode(layer)
|
||||
tuned.add(key)
|
||||
|
||||
|
||||
def _attention_backend_name(backend: object) -> str | None:
|
||||
get_name = getattr(backend, "get_name", None)
|
||||
if get_name is None:
|
||||
return None
|
||||
try:
|
||||
return get_name()
|
||||
except NotImplementedError:
|
||||
return None
|
||||
|
||||
|
||||
def _has_deepseek_v4_sparse_mla_backend(runner: "GPUModelRunner") -> bool:
|
||||
for groups in getattr(runner, "attn_groups", []) or ():
|
||||
for group in groups:
|
||||
name = _attention_backend_name(getattr(group, "backend", None))
|
||||
if name in _DEEPSEEK_V4_SPARSE_MLA_BACKENDS:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _flashinfer_sparse_mla_decode_label(
|
||||
runner: "GPUModelRunner",
|
||||
allowed_backends: frozenset[str],
|
||||
) -> str | None:
|
||||
for groups in getattr(runner, "attn_groups", []) or ():
|
||||
for group in groups:
|
||||
name = _attention_backend_name(getattr(group, "backend", None))
|
||||
if name in allowed_backends:
|
||||
return _FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS.get(name)
|
||||
return None
|
||||
|
||||
|
||||
def _clamp_warmup_tokens(num_tokens: int, max_tokens: int) -> int:
|
||||
return max(0, min(num_tokens, max_tokens))
|
||||
|
||||
|
||||
def _uses_v2_model_runner(runner: "GPUModelRunner") -> bool:
|
||||
vllm_config = getattr(runner, "vllm_config", None)
|
||||
return bool(getattr(vllm_config, "use_v2_model_runner", False))
|
||||
|
||||
|
||||
def _run_flashinfer_sparse_mla_decode_autotune(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
allowed_backends: frozenset[str],
|
||||
) -> bool:
|
||||
"""Autotune FlashInfer's SM120 sparse-MLA decode path."""
|
||||
runner = worker.model_runner
|
||||
log_label = _flashinfer_sparse_mla_decode_label(runner, allowed_backends)
|
||||
if log_label is None:
|
||||
return False
|
||||
if worker.vllm_config.kernel_config.enable_flashinfer_autotune is not True:
|
||||
return False
|
||||
if not has_flashinfer() or not current_platform.is_device_capability_family(120):
|
||||
return False
|
||||
|
||||
try:
|
||||
from flashinfer.autotuner import AutoTuner
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"Skipping FlashInfer SM120 sparse MLA decode autotune because "
|
||||
"FlashInfer autotuner is unavailable."
|
||||
)
|
||||
return False
|
||||
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
world = get_world_group()
|
||||
is_leader = world.rank_in_group == 0
|
||||
|
||||
dummy_run_kwargs = dict(
|
||||
num_tokens=num_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
|
||||
if use_flashinfer_autotune_v2(runner):
|
||||
return _run_sparse_mla_decode_autotune_v2(
|
||||
worker, num_tokens, log_label, dummy_run_kwargs
|
||||
)
|
||||
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
|
||||
if is_leader:
|
||||
logger.info(
|
||||
"Autotuning FlashInfer SM120 sparse MLA %s decode with cache: %s",
|
||||
log_label,
|
||||
cache_path,
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
warmup_executed = True
|
||||
if is_leader:
|
||||
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
|
||||
v2_runner = cast("V2GPUModelRunner", runner)
|
||||
warmup_executed = run_mixed_prefill_decode_warmup(
|
||||
v2_runner,
|
||||
worker.execute_model,
|
||||
worker.sample_tokens,
|
||||
num_tokens,
|
||||
mixed_step_context=flashinfer_autotune(True, cache=str(cache_path)),
|
||||
req_id_prefix="_sparse_mla_v2_warmup",
|
||||
)
|
||||
else:
|
||||
with flashinfer_autotune(True, cache=str(cache_path)):
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
else:
|
||||
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
|
||||
v2_runner = cast("V2GPUModelRunner", runner)
|
||||
warmup_executed = run_mixed_prefill_decode_warmup(
|
||||
v2_runner,
|
||||
worker.execute_model,
|
||||
worker.sample_tokens,
|
||||
num_tokens,
|
||||
req_id_prefix="_sparse_mla_v2_warmup",
|
||||
)
|
||||
else:
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
|
||||
if not warmup_executed:
|
||||
return False
|
||||
|
||||
tune_results: bytes | None = None
|
||||
if is_leader and cache_path.exists():
|
||||
with open(cache_path, "rb") as f:
|
||||
tune_results = f.read()
|
||||
|
||||
tune_results = world.broadcast_object(tune_results, src=0)
|
||||
if tune_results is None:
|
||||
logger.warning(
|
||||
"No FlashInfer SM120 sparse MLA %s decode autotune cache entries found. "
|
||||
"Falling back to FlashInfer's default tactic heuristic.",
|
||||
log_label,
|
||||
)
|
||||
world.barrier()
|
||||
return True
|
||||
|
||||
write_flashinfer_autotune_cache(cache_path, tune_results)
|
||||
world.barrier()
|
||||
|
||||
AutoTuner.get().load_configs(str(cache_path))
|
||||
logger.info(
|
||||
"FlashInfer SM120 sparse MLA %s decode autotune cache loaded on rank %d "
|
||||
"from %s.",
|
||||
log_label,
|
||||
world.rank_in_group,
|
||||
cache_path,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _run_sparse_mla_decode_autotune_v2(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
log_label: str,
|
||||
dummy_run_kwargs: dict,
|
||||
) -> bool:
|
||||
"""Run every rank's sparse warmup in the managed autotune context.
|
||||
|
||||
FlashInfer owns persistence; some sparse kernels use a separate calibration
|
||||
cache instead of managed winner entries.
|
||||
"""
|
||||
from flashinfer import autotune_v2, autotune_v2_reload
|
||||
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
runner = worker.model_runner
|
||||
world = get_world_group()
|
||||
cache_root = resolve_flashinfer_autotune_v2_root()
|
||||
if world.rank_in_group == 0:
|
||||
logger.info(
|
||||
"Autotuning FlashInfer SM120 sparse MLA %s decode with the managed "
|
||||
"autotune context (root=%s)",
|
||||
log_label,
|
||||
cache_root if cache_root is not None else "flashinfer default",
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
|
||||
v2_runner = cast("V2GPUModelRunner", runner)
|
||||
warmup_executed = run_mixed_prefill_decode_warmup(
|
||||
v2_runner,
|
||||
worker.execute_model,
|
||||
worker.sample_tokens,
|
||||
num_tokens,
|
||||
mixed_step_context=autotune_v2(mode="tune", cache_root=cache_root),
|
||||
req_id_prefix="_sparse_mla_v2_warmup",
|
||||
)
|
||||
else:
|
||||
warmup_executed = True
|
||||
with autotune_v2(mode="tune", cache_root=cache_root):
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
|
||||
if not warmup_executed:
|
||||
return False
|
||||
|
||||
if world.world_size > 1:
|
||||
world.barrier()
|
||||
autotune_v2_reload()
|
||||
world.barrier()
|
||||
return True
|
||||
|
||||
|
||||
def _flashinfer_sparse_mla_decode_autotune(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
) -> bool:
|
||||
return _run_flashinfer_sparse_mla_decode_autotune(
|
||||
worker, num_tokens, _FLASHINFER_MLA_SPARSE_BACKENDS
|
||||
)
|
||||
|
||||
|
||||
def _deepseek_v4_sparse_mla_decode_autotune(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
) -> bool:
|
||||
return _run_flashinfer_sparse_mla_decode_autotune(
|
||||
worker, num_tokens, _DEEPSEEK_V4_FLASHINFER_MLA_SPARSE_BACKENDS
|
||||
)
|
||||
|
||||
|
||||
def flashinfer_sparse_mla_decode_autotune_warmup(worker: "Worker") -> None:
|
||||
"""Autotune generic FlashInfer sparse MLA decode when selected."""
|
||||
runner = worker.model_runner
|
||||
if runner.is_pooling_model:
|
||||
return
|
||||
|
||||
max_tokens = worker.scheduler_config.max_num_batched_tokens
|
||||
mixed_tokens = _clamp_warmup_tokens(_SPARSE_MLA_MIXED_WARMUP_TOKENS, max_tokens)
|
||||
if mixed_tokens <= 0:
|
||||
return
|
||||
_flashinfer_sparse_mla_decode_autotune(worker, mixed_tokens)
|
||||
|
||||
|
||||
def deepseek_v4_sparse_mla_attention_warmup(worker: "Worker") -> None:
|
||||
"""Warm DSv4 sparse-MLA mixed prefill+decode attention."""
|
||||
runner = worker.model_runner
|
||||
if runner.is_pooling_model or not _has_deepseek_v4_sparse_mla_backend(runner):
|
||||
return
|
||||
|
||||
max_tokens = worker.scheduler_config.max_num_batched_tokens
|
||||
mixed_tokens = _clamp_warmup_tokens(_SPARSE_MLA_MIXED_WARMUP_TOKENS, max_tokens)
|
||||
if mixed_tokens <= 0:
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Warming up DeepSeek V4 sparse MLA attention for mixed tokens=%s.",
|
||||
mixed_tokens,
|
||||
)
|
||||
mixed_warmup_done = _deepseek_v4_sparse_mla_decode_autotune(worker, mixed_tokens)
|
||||
if not mixed_warmup_done:
|
||||
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
|
||||
v2_runner = cast("V2GPUModelRunner", runner)
|
||||
run_mixed_prefill_decode_warmup(
|
||||
v2_runner,
|
||||
worker.execute_model,
|
||||
worker.sample_tokens,
|
||||
mixed_tokens,
|
||||
req_id_prefix="_sparse_mla_v2_warmup",
|
||||
)
|
||||
else:
|
||||
runner._dummy_run(
|
||||
num_tokens=mixed_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
@@ -26,14 +26,6 @@ from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_v2_root,
|
||||
use_flashinfer_autotune_v2,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
autotune_hisparse_flashinfer_attention,
|
||||
deepseek_v4_sparse_mla_attention_warmup,
|
||||
flashinfer_sparse_mla_decode_autotune_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.kimi_k3_triton_warmup import (
|
||||
kimi_k3_triton_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.mamba_triton_warmup import mamba_triton_warmup
|
||||
from vllm.model_executor.warmup.qwen4_exp_qsa_warmup import (
|
||||
qwen4_exp_qsa_triton_warmup,
|
||||
@@ -162,10 +154,6 @@ def _autotune_kimi_k3_kda_qkvg(model: torch.nn.Module) -> None:
|
||||
|
||||
|
||||
def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
from vllm.model_executor.warmup.minimax_m3_msa_warmup import (
|
||||
minimax_m3_msa_warmup,
|
||||
)
|
||||
|
||||
if not worker.use_v2_model_runner:
|
||||
# The KV-block zeroing kernel is driven by the scheduler's
|
||||
# `new_block_ids_to_zero`, so no dummy run ever reaches it.
|
||||
@@ -202,7 +190,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
|
||||
# Run next so input-prep kernels JIT against pristine runner state.
|
||||
if enable_jit_warmup:
|
||||
kimi_k3_triton_warmup(worker)
|
||||
watermark_sample_warmup(worker)
|
||||
qwen4_exp_qsa_triton_warmup(worker)
|
||||
|
||||
@@ -233,9 +220,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
|
||||
rocm_segmented_attn_autotune_warmup(worker)
|
||||
|
||||
flashinfer_sparse_mla_decode_autotune_warmup(worker)
|
||||
deepseek_v4_sparse_mla_attention_warmup(worker)
|
||||
|
||||
# Deep GEMM warmup
|
||||
do_deep_gemm_warmup = (
|
||||
is_deep_gemm_supported() and envs.VLLM_DEEP_GEMM_WARMUP != "skip"
|
||||
@@ -247,8 +231,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
|
||||
b12x_warmup(worker, cudagraph_capture_sizes)
|
||||
|
||||
minimax_m3_msa_warmup(worker)
|
||||
|
||||
# Allocate the exact decode-sized workspace, autotune cache misses, and
|
||||
# resolve every CUDA Graph bucket before capture begins.
|
||||
# Lazy import: flashinfer_pcie_ipc_all_reduce imports flashinfer.comm,
|
||||
@@ -498,6 +480,10 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
if hisparse_enabled:
|
||||
# HiSparse hot-buffer attention is bounded by decode batch
|
||||
# size, not the prefill-sized batch used for the full model.
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
autotune_hisparse_flashinfer_attention,
|
||||
)
|
||||
|
||||
autotune_hisparse_flashinfer_attention(runner)
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
|
||||
replayssm_autotune_warmup(runner)
|
||||
|
||||
@@ -1,238 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm up Kimi-K3 Triton kernels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_kda_layer(
|
||||
worker: Worker, module: str = "vllm.models.kimi_k3.nvidia.kda"
|
||||
) -> KimiK3DeltaAttention | None:
|
||||
# Kimi model construction already imports kda. Avoid importing it here.
|
||||
kda = sys.modules.get(module)
|
||||
if kda is None:
|
||||
return None
|
||||
|
||||
compilation_config = getattr(
|
||||
worker.model_runner,
|
||||
"compilation_config",
|
||||
None,
|
||||
)
|
||||
static_context = getattr(compilation_config, "static_forward_context", None)
|
||||
if not isinstance(static_context, dict):
|
||||
return None
|
||||
return next(
|
||||
(
|
||||
layer
|
||||
for layer in static_context.values()
|
||||
if isinstance(layer, kda.KimiK3DeltaAttention)
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _warm_attn_res(worker: Worker) -> None:
|
||||
from vllm.models.kimi_k3.nvidia.ops.attn_res import (
|
||||
attn_res,
|
||||
get_attn_res_triton_warmup_profiles,
|
||||
)
|
||||
|
||||
config = worker.model_config.hf_text_config
|
||||
block_size = getattr(config, "attn_res_block_size", None)
|
||||
if block_size is None:
|
||||
return
|
||||
|
||||
hidden_size = int(config.hidden_size)
|
||||
max_blocks = (int(config.num_hidden_layers) + block_size - 1) // block_size
|
||||
if max_blocks < 2:
|
||||
return
|
||||
|
||||
dtype = worker.model_config.dtype
|
||||
device = torch.device("cuda")
|
||||
eps = float(config.rms_norm_eps)
|
||||
prefix = torch.zeros((1, hidden_size), dtype=dtype, device=device)
|
||||
delta = torch.zeros_like(prefix)
|
||||
blocks = torch.zeros(
|
||||
(1, max_blocks, hidden_size),
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
norm_weight = torch.zeros(hidden_size, dtype=dtype, device=device)
|
||||
qk_weight = torch.zeros_like(norm_weight)
|
||||
output_norm_weight = torch.zeros_like(norm_weight)
|
||||
|
||||
for (
|
||||
num_blocks,
|
||||
has_delta,
|
||||
block_write_idx,
|
||||
apply_output_norm,
|
||||
) in get_attn_res_triton_warmup_profiles(max_blocks):
|
||||
attn_res(
|
||||
prefix,
|
||||
delta if has_delta else None,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight if apply_output_norm else None,
|
||||
num_blocks=num_blocks,
|
||||
block_write_idx=block_write_idx,
|
||||
eps=eps,
|
||||
output_norm_eps=eps if apply_output_norm else 0.0,
|
||||
)
|
||||
|
||||
|
||||
def _warm_recurrent_kda(
|
||||
layer: KimiK3DeltaAttention,
|
||||
input_dtype: torch.dtype,
|
||||
) -> None:
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda.fused_recurrent import (
|
||||
fused_recurrent_kda,
|
||||
get_fused_recurrent_kda_fwd_warmup_profiles,
|
||||
)
|
||||
|
||||
num_speculative_tokens = int(layer.num_spec)
|
||||
# fused_recurrent_kda_fwd_kernel is only used by speculative decode.
|
||||
if num_speculative_tokens <= 0:
|
||||
return
|
||||
|
||||
kv_cache = layer.kv_cache
|
||||
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
|
||||
return
|
||||
state = kv_cache[1]
|
||||
if not isinstance(state, torch.Tensor) or not state.numel():
|
||||
return
|
||||
|
||||
logger.info("Warming up Kimi-K3 speculative KDA kernels.")
|
||||
h = int(layer.local_num_heads)
|
||||
d = int(layer.head_dim)
|
||||
tokens_per_sequence = num_speculative_tokens + 1
|
||||
for num_sequences in get_fused_recurrent_kda_fwd_warmup_profiles(h):
|
||||
num_tokens = num_sequences * tokens_per_sequence
|
||||
packed_qkv = torch.empty(
|
||||
(num_tokens, 3 * h * d),
|
||||
dtype=input_dtype,
|
||||
device=state.device,
|
||||
)
|
||||
q, k, v = (
|
||||
tensor.view(1, num_tokens, h, d)
|
||||
for tensor in packed_qkv.split(h * d, dim=-1)
|
||||
)
|
||||
fused_recurrent_kda(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=torch.empty(
|
||||
(1, num_tokens, h, d),
|
||||
dtype=input_dtype,
|
||||
device=state.device,
|
||||
),
|
||||
raw_beta=torch.empty(
|
||||
(1, num_tokens, h),
|
||||
dtype=input_dtype,
|
||||
device=state.device,
|
||||
),
|
||||
A_log=layer.A_log,
|
||||
dt_bias=layer.dt_bias,
|
||||
lower_bound=layer.gate_lower_bound,
|
||||
initial_state=state[:1],
|
||||
cu_seqlens=torch.arange(
|
||||
0,
|
||||
num_tokens + 1,
|
||||
tokens_per_sequence,
|
||||
dtype=torch.int32,
|
||||
device=state.device,
|
||||
),
|
||||
ssm_state_indices=torch.zeros(
|
||||
(num_sequences, tokens_per_sequence),
|
||||
dtype=torch.int32,
|
||||
device=state.device,
|
||||
),
|
||||
num_accepted_tokens=torch.ones(
|
||||
num_sequences,
|
||||
dtype=torch.int32,
|
||||
device=state.device,
|
||||
),
|
||||
out=torch.empty(
|
||||
(1, num_tokens, h, d),
|
||||
dtype=input_dtype,
|
||||
device=state.device,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _warm_gluon_chunk_kda(layer, input_dtype: torch.dtype) -> None:
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import (
|
||||
chunk_kda_prefill,
|
||||
get_gluon_chunk_kda_warmup_profiles,
|
||||
)
|
||||
|
||||
if layer.kda_prefill_backend != "gluon":
|
||||
return
|
||||
kv_cache = layer.kv_cache
|
||||
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
|
||||
return
|
||||
state = kv_cache[1]
|
||||
if not isinstance(state, torch.Tensor) or not state.numel():
|
||||
return
|
||||
|
||||
logger.info("Warming up Kimi-K3 Gluon chunk KDA prefill kernels.")
|
||||
h, d = int(layer.local_num_heads), int(layer.head_dim)
|
||||
for num_seqs, seq_len in get_gluon_chunk_kda_warmup_profiles():
|
||||
num_tokens = num_seqs * seq_len
|
||||
shape = (1, num_tokens, h, d)
|
||||
chunk_kda_prefill(
|
||||
q=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
k=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
v=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
raw_g=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
raw_beta=torch.zeros(
|
||||
(1, num_tokens, h), dtype=input_dtype, device=state.device
|
||||
),
|
||||
A_log=layer.A_log,
|
||||
g_bias=layer.dt_bias,
|
||||
lower_bound=layer.gate_lower_bound,
|
||||
state_cache=state,
|
||||
state_indices=torch.zeros(num_seqs, dtype=torch.int32, device=state.device),
|
||||
has_initial_state=torch.zeros(
|
||||
num_seqs, dtype=torch.bool, device=state.device
|
||||
),
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=torch.arange(
|
||||
0, num_tokens + 1, seq_len, dtype=torch.int32, device=state.device
|
||||
),
|
||||
backend="gluon",
|
||||
out=torch.empty(shape, dtype=input_dtype, device=state.device),
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def kimi_k3_triton_warmup(worker: Worker) -> None:
|
||||
"""Warm Kimi-K3 Triton kernels reachable by this server."""
|
||||
if current_platform.is_rocm():
|
||||
layer = _get_kda_layer(worker, "vllm.models.kimi_k3.amd.kda")
|
||||
if layer is not None:
|
||||
_warm_gluon_chunk_kda(layer, worker.model_config.dtype)
|
||||
return
|
||||
if not current_platform.is_cuda():
|
||||
return
|
||||
|
||||
layer = _get_kda_layer(worker)
|
||||
if layer is None:
|
||||
return
|
||||
|
||||
_warm_attn_res(worker)
|
||||
_warm_recurrent_kda(layer, worker.model_config.dtype)
|
||||
@@ -1,40 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.attention_warmup import (
|
||||
mixed_batch_attention_warmup,
|
||||
)
|
||||
from vllm.models.minimax_m3.nvidia.model import MiniMaxM3SparseAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tracing import instrument
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@instrument(span_name="MiniMax M3 MSA warmup")
|
||||
def minimax_m3_msa_warmup(worker: "Worker") -> None:
|
||||
sparse_module = next(
|
||||
(
|
||||
module
|
||||
for module in worker.get_model().modules()
|
||||
if isinstance(module, MiniMaxM3SparseAttention)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if sparse_module is None:
|
||||
return
|
||||
if not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
|
||||
):
|
||||
return
|
||||
|
||||
logger.info("Warming up MiniMax M3 MSA kernels.")
|
||||
|
||||
# Cover sparse prefill through the normal model path.
|
||||
mixed_batch_attention_warmup(worker)
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user