Merge branch 'main' into patches

This commit is contained in:
2026-10-17 08:06:56 +00:00
222 changed files with 14823 additions and 3606 deletions
@@ -356,10 +356,12 @@ class LLBf16Dotprod:
if tidx == 0:
for m in cutlass.range_constexpr(M):
partials = sm[m, None].load()
gC[m, n_idx] = partials.reduce(
cute.ReductionOp.ADD,
init_val=cutlass.Float32(0.0),
reduction_profile=0,
)
gC[m, n_idx] = cutlass.Float32(
partials.reduce(
cute.ReductionOp.ADD,
init_val=cutlass.Float32(0.0),
reduction_profile=0,
)
).to(gC.element_type)
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_launch_dependents()
@@ -47,7 +47,7 @@ class LLBf16SplitK:
:param ab_dtype: Element type for A and B operands.
:param acc_dtype: Accumulator type used by MMA and reductions.
:param out_dtype: Output element type. The public wrapper uses fp32.
:param out_dtype: Output element type (fp32 or bf16).
:param tile_n: CTA tile size in N. M is fixed to 16 for router batches.
:type tile_n: int
:param tile_k: K tile size staged through shared memory.
@@ -580,6 +580,6 @@ class LLBf16SplitK:
reduction_profile=0,
)
)
gC[local_coord] = acc
gC[local_coord] = cutlass.Float32(acc).to(gC.element_type)
cute.arch.sync_threads()
@@ -76,8 +76,8 @@ def make_fake_gemm_tensors(
a_dtype: torch.dtype,
b_dtype: torch.dtype,
divisibility: int,
output_dtype: torch.dtype = torch.float32,
):
from cutlass import Float32
from quack.compile_utils import make_fake_tensor
hidden_states = make_fake_tensor(
@@ -86,7 +86,7 @@ def make_fake_gemm_tensors(
router_weight = make_fake_tensor(
cutlass_dtype(b_dtype), (N, K), divisibility=divisibility
)
output = make_fake_tensor(Float32, (M, N), divisibility=1)
output = make_fake_tensor(cutlass_dtype(output_dtype), (M, N), divisibility=1)
return hidden_states, router_weight, output
@@ -97,6 +97,7 @@ def validate_common_gemm_inputs(
*,
op_name: str,
k_multiple: int | None = None,
output_dtypes: tuple[torch.dtype, ...] = (torch.float32,),
) -> None:
if hidden_states.dim() != 2 or router_weight.dim() != 2:
raise ValueError("hidden_states and router_weight must be 2D tensors")
@@ -106,8 +107,11 @@ def validate_common_gemm_inputs(
raise ValueError(
"hidden_states and router_weight must be on the same CUDA device"
)
if output_dtype != torch.float32:
raise ValueError(f"{op_name} only supports output_dtype=torch.float32")
if output_dtype not in output_dtypes:
raise ValueError(
f"{op_name} only supports output_dtype="
+ " or ".join(str(dtype) for dtype in output_dtypes)
)
if hidden_states.shape[1] != router_weight.shape[1]:
raise ValueError(
"hidden_states and router_weight must have matching K dimensions"
@@ -47,6 +47,8 @@ _SM100F_TUNED_SPLITK_CONFIGS: dict[tuple[int, int], dict[int, tuple[int, int]]]
**{M: (8, 5) for M in (5, 8)},
9: (8, 2),
},
(5120, 1792): {M: (2, 4) for M in range(5, 17)},
(5120, 3584): {M: (2, 4) for M in range(5, 17)},
(7168, 256): {14: (8, 2)},
(6144, 256): {M: (8, 2) for M in (9, 12, 16)},
(7168, 384): {M: (7, 5) for M in (13, 16)},
@@ -101,39 +103,52 @@ class LLBf16Gemm:
bs: int = 0
split_k: int = 0
num_stages: int = 0
output_dtype: torch.dtype = torch.float32
def __init__(self, *, prefetch_pdl_weights: bool = False) -> None:
self._prefetch_pdl_weights = prefetch_pdl_weights
# Dot-prod: keyed on (M, K, bs), because M and K are Constexpr.
self._compiled_cache: dict[tuple[int, int, int], Any] = {}
# Split-K: keyed on (split_k, num_stages), fully shape-dynamic.
self._splitk_cache: dict[tuple[int, int], Any] = {}
# M and K are compile-time constants for dot-product.
self._compiled_cache: dict[tuple[int, int, int, torch.dtype], Any] = {}
# Split-K is shape-dynamic; both caches distinguish output dtype.
self._splitk_cache: dict[tuple[int, int, torch.dtype], Any] = {}
def dispatch(self, *, M: int, K: int, N: int) -> CompileKey:
def dispatch(
self, *, M: int, K: int, N: int, output_dtype: torch.dtype = torch.float32
) -> CompileKey:
tuned_bs, tuned_splitk = _arch_tuned_configs()
if M <= _DEFAULT_DOTPROD_MAX_M or K < 2048:
bs = tuned_bs.get((K, N), {}).get(M, _DEFAULT_DOTPROD_BS)
return self.CompileKey(backend="dotprod", M=M, K=K, bs=bs)
return self.CompileKey(
backend="dotprod", M=M, K=K, bs=bs, output_dtype=output_dtype
)
split_k, num_stages = tuned_splitk.get((K, N), {}).get(
M, _DEFAULT_SPLITK_CONFIG
)
return self.CompileKey(backend="splitk", split_k=split_k, num_stages=num_stages)
return self.CompileKey(
backend="splitk",
split_k=split_k,
num_stages=num_stages,
output_dtype=output_dtype,
)
def get_warmup_keys(
self,
*,
shapes: Iterable[tuple[int, int]],
m_values: Iterable[int],
output_dtype: torch.dtype = torch.float32,
) -> list[CompileKey]:
return list(
dict.fromkeys(
self.dispatch(M=M, K=K, N=N) for K, N in shapes for M in m_values
self.dispatch(M=M, K=K, N=N, output_dtype=output_dtype)
for K, N in shapes
for M in m_values
)
)
@staticmethod
def _fake_gemm_tensors(*, M, K, N, divisibility: int):
def _fake_gemm_tensors(*, M, K, N, divisibility: int, output_dtype: torch.dtype):
return make_fake_gemm_tensors(
M=M,
K=K,
@@ -141,10 +156,13 @@ class LLBf16Gemm:
a_dtype=torch.bfloat16,
b_dtype=torch.bfloat16,
divisibility=divisibility,
output_dtype=output_dtype,
)
def _compile_splitk(self, compile_key: CompileKey) -> None:
cute, _ = cute_context()
from cutlass import BFloat16, Float32
from ._ll_bf16_splitk import LLBf16SplitK
hidden_states, router_weight, output = self._fake_gemm_tensors(
@@ -152,8 +170,12 @@ class LLBf16Gemm:
K=cute.sym_int(),
N=cute.sym_int(),
divisibility=8,
output_dtype=compile_key.output_dtype,
)
gemm = LLBf16SplitK(
out_dtype=BFloat16
if compile_key.output_dtype == torch.bfloat16
else Float32,
tile_n=16,
tile_k=256,
num_stages=compile_key.num_stages,
@@ -169,7 +191,9 @@ class LLBf16Gemm:
current_cuda_stream(),
options="--enable-tvm-ffi",
)
self._splitk_cache[(compile_key.split_k, compile_key.num_stages)] = compiled
self._splitk_cache[
(compile_key.split_k, compile_key.num_stages, compile_key.output_dtype)
] = compiled
logger.debug(
"Compiled ll_bf16_splitk: sk=%d ns=%d",
compile_key.split_k,
@@ -187,6 +211,7 @@ class LLBf16Gemm:
K=compile_key.K,
N=N,
divisibility=stride_divisibility,
output_dtype=compile_key.output_dtype,
)
gemm = LLBf16Dotprod(
k=compile_key.K,
@@ -205,7 +230,9 @@ class LLBf16Gemm:
current_cuda_stream(),
options="--enable-tvm-ffi --ptxas-options -maxrregcount=64",
)
self._compiled_cache[(compile_key.M, compile_key.K, compile_key.bs)] = compiled
self._compiled_cache[
(compile_key.M, compile_key.K, compile_key.bs, compile_key.output_dtype)
] = compiled
logger.debug(
"Compiled ll_bf16_dotprod: M=%d, K=%d, bs=%d",
compile_key.M,
@@ -215,12 +242,21 @@ class LLBf16Gemm:
def compile(self, compile_key: CompileKey) -> None:
if compile_key.backend == "splitk":
splitk_cache_key = (compile_key.split_k, compile_key.num_stages)
splitk_cache_key = (
compile_key.split_k,
compile_key.num_stages,
compile_key.output_dtype,
)
if splitk_cache_key not in self._splitk_cache:
self._compile_splitk(compile_key)
return
dotprod_cache_key = (compile_key.M, compile_key.K, compile_key.bs)
dotprod_cache_key = (
compile_key.M,
compile_key.K,
compile_key.bs,
compile_key.output_dtype,
)
if dotprod_cache_key not in self._compiled_cache:
self._compile_dotprod(compile_key)
@@ -229,8 +265,11 @@ class LLBf16Gemm:
*,
shapes: Iterable[tuple[int, int]],
m_values: Iterable[int],
output_dtype: torch.dtype = torch.float32,
) -> None:
for compile_key in self.get_warmup_keys(shapes=shapes, m_values=m_values):
for compile_key in self.get_warmup_keys(
shapes=shapes, m_values=m_values, output_dtype=output_dtype
):
self.compile(compile_key)
@staticmethod
@@ -251,6 +290,7 @@ class LLBf16Gemm:
output_dtype,
op_name="ll_bf16_gemm",
k_multiple=8,
output_dtypes=(torch.float32, torch.bfloat16),
)
def __call__(
@@ -258,19 +298,28 @@ class LLBf16Gemm:
hidden_states: torch.Tensor, # [M, K] bf16
router_weight: torch.Tensor, # [N, K] bf16
output_dtype: torch.dtype = torch.float32,
) -> torch.Tensor: # [M, N] fp32
) -> torch.Tensor: # [M, N]
self._validate_inputs(hidden_states, router_weight, output_dtype)
M, K = hidden_states.shape
N = router_weight.shape[0]
compile_key = self.dispatch(M=M, K=K, N=N)
compile_key = self.dispatch(M=M, K=K, N=N, output_dtype=output_dtype)
if compile_key.backend == "splitk":
splitk_cache_key = (compile_key.split_k, compile_key.num_stages)
splitk_cache_key = (
compile_key.split_k,
compile_key.num_stages,
compile_key.output_dtype,
)
if splitk_cache_key not in self._splitk_cache:
self.compile(compile_key)
kernel = self._splitk_cache[splitk_cache_key]
else:
dotprod_cache_key = (compile_key.M, compile_key.K, compile_key.bs)
dotprod_cache_key = (
compile_key.M,
compile_key.K,
compile_key.bs,
compile_key.output_dtype,
)
if dotprod_cache_key not in self._compiled_cache:
self.compile(compile_key)
kernel = self._compiled_cache[dotprod_cache_key]
@@ -6,19 +6,6 @@ from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import direct_register_custom_op
def _torch_hc_prenorm_gemm(
x: torch.Tensor,
fn: torch.Tensor,
out: torch.Tensor,
sqrsum: torch.Tensor,
) -> None:
assert out.shape[0] == 1
assert sqrsum.shape[0] == 1
x_float = x.float()
out[0].copy_(x_float @ fn.t())
sqrsum[0].copy_(x_float.square().sum(dim=-1))
def _hc_prenorm_gemm_outputs(
x: torch.Tensor,
fn: torch.Tensor,
@@ -2024,6 +2024,7 @@ def build_mla_chunked_context_metadata(
dcp_local_block_size: int,
dcp_virtual_block_size: int,
dcp_manager: MLADCPManager | None = None,
align_indptr: bool = False,
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
"""Build chunked-context metadata for an MLA prefill.
@@ -2043,6 +2044,7 @@ def build_mla_chunked_context_metadata(
dcp_local_block_size: Per-rank interleave block size for DCP.
dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``.
dcp_manager: Shared MLA DCP collective manager.
align_indptr: Align packed int32 indptr slices to 16 bytes for CuTe DSL.
Returns:
The chunked-context metadata, or None when no prefill has any context.
@@ -2123,6 +2125,15 @@ def build_mla_chunked_context_metadata(
num_tokens = sum(plan.seq_lens)
num_local_tokens = num_tokens
if align_indptr:
# CuTe DSL requires 16-byte-aligned int32 indptrs.
padding = -boundary_offset % 4
cu_seq_lens_flat.extend([0] * padding)
cu_seqlens_q_flat.extend([0] * padding)
if use_dcp:
padded_local_cu_seq_lens_flat.extend([0] * padding)
boundary_offset += padding
seq_lens_flat.extend(plan.seq_lens)
cu_seq_lens_flat.extend(itertools.accumulate(plan.seq_lens, initial=0))
query_base = query_start_loc[plan.request_start]
@@ -2616,6 +2627,14 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
query_start_loc_cpu[reqs_start:] - query_start_loc_cpu[reqs_start]
)
align_indptr = False
if (
self._prefill_backend.get_name() == "TRTLLM_RAGGED"
and self.vllm_config.attention_config.trtllm_ragged_backend
== "cute-dsl"
):
align_indptr = True
chunked_context_metadata = build_mla_chunked_context_metadata(
context_lens_cpu=context_lens_cpu,
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
@@ -2628,6 +2647,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
dcp_local_block_size=self.dcp_local_block_size,
dcp_virtual_block_size=self.dcp_virtual_block_size,
dcp_manager=self.dcp_manager,
align_indptr=align_indptr,
)
prefill_metadata = MLACommonPrefillMetadata(
@@ -359,13 +359,6 @@ class ExpertMapManager:
return True
return self._expert_map[global_id] != -1
def get_local_expert_ids(self) -> list[int]:
"""Get list of global IDs for experts on this rank."""
if self._expert_map is None:
return list(range(self.global_num_experts))
return torch.where(self._expert_map != -1)[0].tolist()
def update(
self,
moe_parallel_config: FusedMoEParallelConfig,
@@ -545,7 +545,12 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
return (workspace1, workspace2, output)
def _act_mul_quant(
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
self,
input: torch.Tensor,
output: torch.Tensor,
activation: MoEActivation,
expert_ends: torch.Tensor | None = None,
expert_alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
block_k = self._ACT_BLOCK_K
scale_fmt = DeepGemmQuantScaleFMT.from_oracle()
@@ -561,6 +566,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
output_q=output,
group_size=block_k,
clamp_limit=self.gemm1_clamp_limit,
expert_ends=expert_ends,
expert_alignment=expert_alignment,
)
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
return silu_mul_per_token_group_quant_fp8_colmajor(
@@ -634,7 +641,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
a1q_perm = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
)
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
use_psum_layout = current_platform.is_device_capability_family(100)
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
aq=a1q,
aq_scale=a1q_scale,
topk_ids=topk_ids,
@@ -642,6 +650,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
expert_map=expert_map,
expert_tokens_meta=expert_tokens_meta,
aq_out=a1q_perm,
use_psum_layout=use_psum_layout,
)
assert a1q.size(0) == M_sum
@@ -655,18 +664,23 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
(a1q, a1q_scale),
(w1.view(torch.int8), self.w1_scale),
mm1_out,
expert_ids,
grouped_layout,
use_psum_layout=use_psum_layout,
recipe_a=(1, self._ACT_BLOCK_K),
recipe_b=(1, self._WEIGHT_BLOCK_K),
)
# SwiGLU activation + FP8 requant
# SiLU+mul and FP8 requant use the same live expert ranges as GEMM.
activation_out_dim = self.adjust_N_for_activation(N, activation)
quant_out = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
)
a2q, a2q_scale = self._act_mul_quant(
input=mm1_out.view(-1, N), output=quant_out, activation=activation
input=mm1_out.view(-1, N),
output=quant_out,
activation=activation,
expert_ends=grouped_layout if use_psum_layout else None,
expert_alignment=align_used if use_psum_layout else 0,
)
# FC2: FP8 activations x FP4 weights
@@ -675,7 +689,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
(a2q, a2q_scale),
(w2.view(torch.int8), self.w2_scale),
mm2_out,
expert_ids,
grouped_layout,
use_psum_layout=use_psum_layout,
recipe_a=(1, self._ACT_BLOCK_K),
recipe_b=(1, self._WEIGHT_BLOCK_K),
)
@@ -517,6 +517,7 @@ _TILE_CONFIG_OVERRIDES = {
256: (128, 32),
512: (128, 19),
},
(5120, 256): {64: (64, 64)}, # GB200
(6144, 128): { # MiniMax-M3
64: (32, 64),
128: (64, 64),
@@ -533,7 +534,7 @@ def _pick_tile_config(N: int, K: int, M: int, num_sms: int) -> tuple[int, int]:
if 32 < N <= 512:
token_bucket = triton.next_power_of_2(N)
overrides = _TILE_CONFIG_OVERRIDES.get((K, M))
if overrides is not None:
if overrides is not None and token_bucket in overrides:
BN, split_k = overrides[token_bucket]
return BN, min(split_k, k_tiles)
@@ -609,6 +610,8 @@ def _bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
N, K = X.shape
M, _ = W.shape
if N == 0:
return X.new_empty((N, M), dtype=torch.float32)
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
BN, split_k = _pick_tile_config(N, K, M, num_sms)
@@ -24,8 +24,8 @@ class GateLinear(ReplicatedLinear):
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
any shape w/ K divisible by 8)
2. fp32 specialized kernel (bf16/fp32 in, fp32 out, M<=32; SM90+
model-specific shapes, gfx950 any shape)
2. fp32 specialized kernel (bf16/fp32 in, fp32 out; SM90+
model-specific shapes and batch limits, gfx950 any shape with M<=32)
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
any shape)
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
@@ -46,9 +46,14 @@ class GateLinear(ReplicatedLinear):
split-K from the token count and skip the batch-invariant linear path.
"""
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
# (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128)}
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel.
FP32_SUPPORTED_SHAPES = {
(3072, 256),
(6144, 128),
(5120, 256),
(4096, 352),
(8192, 256),
}
FP32_MAX_TOKENS = 32
# Largest batch for which tier 1 beats the cuBLAS epilogue. Subclasses
# whose router shape stays ahead further up the range may raise it.
@@ -454,9 +459,9 @@ def fp32_router_gemm_dispatch_impl(
weight: torch.Tensor,
allow_bf16x3_router_gemm: bool,
) -> torch.Tensor:
"""Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS,
otherwise optionally run the experimental BF16x3 kernel for medium/large
SM100 router batches, then fall back to F.linear.
"""Run the FP32 kernel for small batches, with shape-specific limits.
Larger SM100 BF16 batches use BF16x3; other inputs fall back to F.linear.
This must be wrapped in a custom op because our torch.compile integration
does not support runtime dispatching on num_tokens.
"""
@@ -473,7 +478,15 @@ def fp32_router_gemm_dispatch_impl(
if weight.dtype == torch.bfloat16:
return torch.mm(x, weight.T, out_dtype=torch.float32)
return torch.nn.functional.linear(x.float(), weight)
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
max_tokens = _FP32_ROUTER_GEMM_MAX_TOKENS
if (
allow_bf16x3_router_gemm
and x.dtype == torch.bfloat16
and weight.shape == (256, 5120)
):
# GB200: four token groups win through 48 for multiples of eight.
max_tokens = 48 if x.shape[0] % 8 == 0 else 16
if x.shape[0] <= max_tokens:
return ops.fp32_router_gemm(x, weight)
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
@@ -9,139 +9,162 @@
from packaging import version
from vllm.model_executor.layers.mamba.ops.triton_helpers import fast_exp
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
TRITON_22 = version.parse(triton.__version__) >= version.parse("2.2.0")
def _drop_racy_wgmma_configs(configs: list[triton.Config]) -> list[triton.Config]:
"""Drop the configs that Triton miscompiles on sm90.
With whole warpgroups (num_warps % 4 == 0) and BLOCK_SIZE_M % 64 == 0,
`tl.dot(cb, x)` lowers to wgmma with `cb` in registers. With
num_stages > 1, Triton leaves that wgmma running across the loop
back-edge while the next iteration overwrites the registers of `cb`,
which writes NaN/Inf into whole output rows.
"""
if not (current_platform.is_cuda() and current_platform.is_device_capability(90)):
return configs
return [
c
for c in configs
if c.num_stages == 1
or c.num_warps % 4 != 0
or c.kwargs["BLOCK_SIZE_M"] % 64 != 0
]
@triton.autotune(
configs=[
# =================================================================
# Higher warp count configs for better latency hiding
# More warps = more instructions in flight = better memory latency hiding
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
# Smaller tiles with more stages for software pipelining
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
# =================================================================
# Low register pressure configs (num_stages=1) for large dstate
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
# num_stages=2 configs - moderate register pressure
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
# Original configs for larger dstate values
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
num_stages=3,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=2,
),
],
configs=_drop_racy_wgmma_configs(
[
# =================================================================
# Higher warp count configs for better latency hiding
# More warps = more instructions in flight = better memory latency hiding
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
# Smaller tiles with more stages for software pipelining
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
# =================================================================
# Low register pressure configs (num_stages=1) for large dstate
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
# num_stages=2 configs - moderate register pressure
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
# Original configs for larger dstate values
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
num_stages=3,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=2,
),
]
),
key=["chunk_size", "hdim", "dstate", "IS_CAUSAL"],
)
@triton.jit
@@ -12,35 +12,6 @@ from vllm.triton_utils import tl, triton
logger = logging.getLogger(__name__)
def block_dequant(
x_q_block: torch.Tensor,
x_s: torch.Tensor,
block_size: list[int],
) -> torch.Tensor:
"""This function conducts block-wise dequantization.
The inputs are block-wise quantization tensor `x_q_block`,
block-wise quantization scale and the block size.
The outputs are dequantized tensor.
"""
block_n, block_k = block_size[0], block_size[1]
n, k = x_q_block.shape
n_tiles = (n + block_n - 1) // block_n
k_tiles = (k + block_k - 1) // block_k
assert n_tiles == x_s.shape[0]
assert k_tiles == x_s.shape[1]
x_dq_block = x_q_block.to(torch.float32)
for i in range(k_tiles):
for j in range(n_tiles):
x_dq_block[
j * block_n : min((j + 1) * block_n, n),
i * block_k : min((i + 1) * block_k, k),
] *= x_s[j][i]
return x_dq_block
if current_platform.is_rocm():
@triton.jit
@@ -1404,14 +1404,17 @@ class DiffusionSampler:
# unaffected; only the canvas exploration is constrained. Applied
# before canvas padding so phantom positions stay uniform.
if num_decode > 0:
top_k, top_p = self.sampling_states.get_top_k_top_p(
top_k, top_p, has_topp_only_rows = self.sampling_states.get_top_k_top_p(
decode_slots.repeat_interleave(
valid_canvas_len, output_size=int(valid_canvas_len_np.sum())
),
decode_slots_np,
logits.shape[-1],
)
if top_k is not None or top_p is not None:
logits = apply_top_k_top_p(logits.float(), top_k, top_p)
logits = apply_top_k_top_p(
logits.float(), top_k, top_p, has_topp_only_rows=has_topp_only_rows
)
# Where each decode request's rows start in the flat logits. Tiles
# below gather and pad a request to the tile's width.
+42
View File
@@ -64,6 +64,7 @@ if TYPE_CHECKING:
from vllm.tasks import ScoreType
from vllm.tokenizers import TokenizerLike
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
EncoderCudaGraphConfig,
@@ -670,6 +671,47 @@ def supports_score_template(
return getattr(model, "supports_score_template", False)
@runtime_checkable
class SupportsKVCacheGrouping(Protocol):
"""The interface required for models that own their KV cache grouping.
.. note::
Experimental. The classmethod contract may change as model-owned
grouping stabilizes.
Implementing it lets a model replace the default KV cache grouping (which
layers share a block table) while Core keeps owning placement, byte
accounting, and admission.
"""
@classmethod
def get_kv_cache_groups(
cls,
vllm_config: "VllmConfig",
kv_cache_spec: dict[str, "KVCacheSpec"],
) -> list["KVCacheGroupSpec"]:
"""Split the model's layers into logical KV cache groups."""
...
@overload
def supports_kv_cache_grouping(
model: type[object],
) -> TypeIs[type[SupportsKVCacheGrouping]]: ...
@overload
def supports_kv_cache_grouping(
model: object,
) -> TypeIs[SupportsKVCacheGrouping]: ...
def supports_kv_cache_grouping(
model: type[object] | object,
) -> TypeIs[type[SupportsKVCacheGrouping]] | TypeIs[SupportsKVCacheGrouping]:
return isinstance(model, SupportsKVCacheGrouping)
@runtime_checkable
class SupportsLoRA(Protocol):
"""The interface required for all models that support LoRA."""
+3
View File
@@ -50,6 +50,7 @@ from .interfaces import (
is_hybrid,
requires_raw_input_tokens,
supports_hc_sp,
supports_kv_cache_grouping,
supports_multimodal,
supports_multimodal_encoder_tp_data,
supports_multimodal_raw_input_only,
@@ -882,6 +883,7 @@ class _ModelInfo:
supports_replayssm: bool
supports_transcription: bool
supports_transcription_only: bool
supports_kv_cache_grouping: bool
supported_video_pruning_methods: tuple[str, ...]
supports_mm_device_do_normalize: bool
@@ -913,6 +915,7 @@ class _ModelInfo:
supports_transcription_only=(
supports_transcription(model) and model.supports_transcription_only
),
supports_kv_cache_grouping=supports_kv_cache_grouping(model),
has_noops=has_noops(model),
supported_video_pruning_methods=getattr(
model, "supported_video_pruning_methods", ()
@@ -11,11 +11,14 @@ from typing import TYPE_CHECKING
import vllm.envs as envs
from vllm.config import CacheConfig, VllmConfig, replace
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.distributed.parallel_state import GroupCoordinator
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
logger = init_logger(__name__)
def _normalize_cache_config_for_hash(cache_config: CacheConfig) -> CacheConfig:
"""Normalize cache config for hashing"""
@@ -55,6 +58,53 @@ def resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
return output_dir / "autotune_configs.json"
def use_flashinfer_autotune_v2(runner: "GPUModelRunner") -> bool:
from vllm.distributed.parallel_state import get_node_count, get_world_group
from vllm.utils.flashinfer import has_flashinfer_autotune_v2
# Elastic EP transfers tuning state to joining workers with v1
# save_configs/load_configs, which do not export managed-store winners.
if runner.vllm_config.parallel_config.enable_elastic_ep:
return False
# PP stages profile independently. Keep their group-scoped v1 files until
# managed-cache reads and publishes can be isolated across tuning groups.
if runner.vllm_config.parallel_config.pipeline_parallel_size > 1:
return False
if not has_flashinfer_autotune_v2():
return False
# Use the initialized topology: Ray may span nodes with config.nnodes=1.
# A cache hit skips profiling collectives, so node-local stores with
# different contents cannot safely participate in synchronized tuning.
if get_node_count() != 1:
logger.info_once(
"Using legacy FlashInfer autotune cache synchronization for "
"multi-node deployments; managed-cache synchronization is "
"currently supported only within one node."
)
return False
# Lift this gate after pinning a release with the fix below and validating
# distributed managed-cache tuning: https://github.com/flashinfer-ai/flashinfer/pull/6185
if get_world_group().world_size > 1:
logger.info_once(
"Using legacy FlashInfer autotune cache synchronization for "
"multi-rank deployments until FlashInfer coordinates cache hits."
)
return False
return True
def resolve_flashinfer_autotune_v2_root() -> Path | None:
"""Placement root for FlashInfer's managed autotune store.
Unlike :func:`resolve_flashinfer_autotune_file`, no vLLM-side identity
(config hash, flashinfer version, arch) is folded into the path: the
managed store keys entries per op and hashes the software/hardware
environment itself. ``None`` selects FlashInfer's default root.
"""
override_dir = envs.VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR
return Path(override_dir).expanduser() if override_dir else None
def sync_flashinfer_autotune_cache(
runner: "GPUModelRunner",
group: "GroupCoordinator",
@@ -9,6 +9,8 @@ import torch
from vllm.logger import init_logger
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
resolve_flashinfer_autotune_v2_root,
use_flashinfer_autotune_v2,
write_flashinfer_autotune_cache,
)
from vllm.platforms import current_platform
@@ -145,7 +147,6 @@ def _run_flashinfer_sparse_mla_decode_autotune(
world = get_world_group()
is_leader = world.rank_in_group == 0
cache_path = resolve_flashinfer_autotune_file(runner)
dummy_run_kwargs = dict(
num_tokens=num_tokens,
@@ -155,6 +156,13 @@ def _run_flashinfer_sparse_mla_decode_autotune(
create_mixed_batch=True,
)
if use_flashinfer_autotune_v2(runner):
return _run_sparse_mla_decode_autotune_v2(
worker, num_tokens, log_label, dummy_run_kwargs
)
cache_path = resolve_flashinfer_autotune_file(runner)
if is_leader:
logger.info(
"Autotuning FlashInfer SM120 sparse MLA %s decode with cache: %s",
@@ -223,6 +231,58 @@ def _run_flashinfer_sparse_mla_decode_autotune(
return True
def _run_sparse_mla_decode_autotune_v2(
worker: "Worker",
num_tokens: int,
log_label: str,
dummy_run_kwargs: dict,
) -> bool:
"""Run every rank's sparse warmup in the managed autotune context.
FlashInfer owns persistence; some sparse kernels use a separate calibration
cache instead of managed winner entries.
"""
from flashinfer import autotune_v2, autotune_v2_reload
from vllm.distributed.parallel_state import get_world_group
runner = worker.model_runner
world = get_world_group()
cache_root = resolve_flashinfer_autotune_v2_root()
if world.rank_in_group == 0:
logger.info(
"Autotuning FlashInfer SM120 sparse MLA %s decode with the managed "
"autotune context (root=%s)",
log_label,
cache_root if cache_root is not None else "flashinfer default",
)
with torch.inference_mode():
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
v2_runner = cast("V2GPUModelRunner", runner)
warmup_executed = run_mixed_prefill_decode_warmup(
v2_runner,
worker.execute_model,
worker.sample_tokens,
num_tokens,
mixed_step_context=autotune_v2(mode="tune", cache_root=cache_root),
req_id_prefix="_sparse_mla_v2_warmup",
)
else:
warmup_executed = True
with autotune_v2(mode="tune", cache_root=cache_root):
runner._dummy_run(**dummy_run_kwargs)
if not warmup_executed:
return False
if world.world_size > 1:
world.barrier()
autotune_v2_reload()
world.barrier()
return True
def _flashinfer_sparse_mla_decode_autotune(
worker: "Worker",
num_tokens: int,
+33 -6
View File
@@ -1082,6 +1082,10 @@ class JitWarmupRegistry:
from tqdm import tqdm
from vllm.distributed import is_global_first_rank
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
VllmTritonJitKernel,
compile_triton_warmup,
)
kernel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
for kernel, registrations in self._registrations.items():
@@ -1102,17 +1106,40 @@ class JitWarmupRegistry:
if not kernel_items:
return
parallel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
serial_items = []
for kernel, compile_keys in kernel_items:
if (
isinstance(kernel, VllmTritonJitKernel)
and kernel.supports_parallel_compile
):
parallel_items.append((kernel, compile_keys))
else:
serial_items.append((kernel, compile_keys))
total_keys = sum(len(compile_keys) for _, compile_keys in kernel_items)
with tqdm(
kernel_items,
total=len(kernel_items),
desc=f"JIT kernel warmup ({total_keys} compile keys)",
disable=not is_global_first_rank(),
dynamic_ncols=True,
unit="kernel",
) as progress:
for kernel, compile_keys in progress:
progress.set_postfix_str(
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
refresh=False,
def compile_kernels(
items: list[tuple[VllmJitKernel[Any], dict[Any, None]]],
) -> None:
for kernel, compile_keys in items:
progress.set_postfix_str(
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
refresh=False,
)
kernel.compile_many(compile_keys)
progress.update()
if parallel_items:
compile_triton_warmup(
lambda: compile_kernels(parallel_items),
sum(len(keys) for _, keys in parallel_items),
)
kernel.compile_many(compile_keys)
compile_kernels(serial_items)
@@ -6,7 +6,7 @@ from abc import abstractmethod
from collections.abc import Callable, Hashable, Iterable, Mapping
from concurrent.futures import ThreadPoolExecutor
from contextlib import nullcontext, suppress
from contextvars import copy_context
from contextvars import ContextVar, copy_context
from dataclasses import dataclass, field
from functools import cache, cached_property, update_wrapper
from typing import Any, Generic, ParamSpec, Protocol, TypeVar, cast, overload
@@ -207,6 +207,31 @@ class TritonWarmupTensor:
return result if dim is None else result[dim]
_shared_compile_pool: ContextVar[bool] = ContextVar(
"shared_triton_warmup_compile_pool", default=False
)
def compile_triton_warmup(compile_all: Callable[[], None], num_keys: int) -> None:
"""Share one compiler pool across all eligible warmup kernels."""
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, num_keys)
if num_threads <= 1:
compile_all()
return
def compile_in_context() -> None:
_shared_compile_pool.set(True)
with (
ThreadPoolExecutor(max_workers=num_threads) as executor,
triton.AsyncCompileMode(executor),
):
# Dispatch mutates owner state and must stay on this thread.
compile_all()
# Older Triton versions leave async mode active on compilation failure.
copy_context().run(compile_in_context)
class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
"""Triton owner whose runtime launch specification is reused for warmup."""
@@ -230,32 +255,25 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
self._warming = False
self._warming_compile_key = None
@property
def supports_parallel_compile(self) -> bool:
# AMD LLVM code generation can abort under concurrent compilation.
return (
current_platform.is_cuda()
and not self._run_autotune
and getattr(triton, "AsyncCompileMode", None) is not None
and envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS > 1
)
def compile_many(self, compile_keys: Iterable[CompileKeyT]) -> None:
"""Compile CUDA startup warmup variants in parallel, then wait."""
keys = list(compile_keys)
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, len(keys))
async_compile = getattr(triton, "AsyncCompileMode", None)
# AMD LLVM code generation can abort under concurrent compilation.
if (
not current_platform.is_cuda()
or self._run_autotune
or async_compile is None
or num_threads <= 1
):
if not self.supports_parallel_compile or _shared_compile_pool.get():
return super().compile_many(keys)
def compile_all() -> None:
with (
ThreadPoolExecutor(max_workers=num_threads) as executor,
async_compile(executor),
):
# Dispatch stays on this thread: compile() mutates owner state.
# Only Triton's underlying compiler work runs in the pool.
super(VllmTritonJitKernel, self).compile_many(keys)
# Older Triton versions leave async mode active on compilation failure.
# Isolate that state so it cannot affect later runtime JIT compilation.
copy_context().run(compile_all)
compile_triton_warmup(
lambda: super(VllmTritonJitKernel, self).compile_many(keys), len(keys)
)
@cached_property
def _kernel_arg_names(self) -> tuple[str, ...]:
+42 -30
View File
@@ -23,6 +23,8 @@ from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
resolve_flashinfer_autotune_v2_root,
use_flashinfer_autotune_v2,
)
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
autotune_hisparse_flashinfer_attention,
@@ -419,7 +421,7 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
otherwise the world group tunes together. Per-tactic timings are
averaged over the tuning group so all its ranks select the same tactic.
Results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
Legacy results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
(``MoERunner.get_cache_key_extras``), so one rank's file only hits on that
rank. A rank with a cache hit skips the per-tactic reduce the others block
in, so ranks keep loaded configs only if every rank in the tuning group
@@ -450,38 +452,45 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
)
autotune_kwargs["skip_ops"] = skip_ops
cache_path = resolve_flashinfer_autotune_file(runner)
# The world group only spans DP ranks when vLLM folds DP into it.
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
cache_path = cache_path.with_name(
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
)
if is_leader:
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
use_autotune_v2 = use_flashinfer_autotune_v2(runner)
if use_autotune_v2:
from flashinfer import autotune_v2, autotune_v2_reload
# We skip EPLB here since we don't want to record dummy metrics.
# Randomize inputs to avoid every token pick the same experts,
# which lead to some EP ranks receiving no tokens and skipping their
# MoE kernel entirely, and cause hang due to all-reduce collective
# during synchronized autotuning.
if _all_ranks_have_matching_cache(cache_path, tune_group):
loaded = tuner.load_configs(str(cache_path))
if tune_group.world_size > 1:
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
torch.distributed.all_gather_object(
loaded_by_rank, loaded, group=tune_group.cpu_group
cache_root = resolve_flashinfer_autotune_v2_root()
if is_leader:
logger.info_once(
"Using FlashInfer managed autotune cache (root=%s)",
cache_root if cache_root is not None else "flashinfer default",
)
loaded = all(loaded_by_rank)
if not loaded:
tuner.clear_cache()
autotune_context = autotune_v2(
mode="tune", cache_root=cache_root, **autotune_kwargs
)
else:
cache_path = resolve_flashinfer_autotune_file(runner)
# The world group only spans DP ranks when vLLM folds DP into it.
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
cache_path = cache_path.with_name(
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
)
if is_leader:
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
if _all_ranks_have_matching_cache(cache_path, tune_group):
loaded = tuner.load_configs(str(cache_path))
if tune_group.world_size > 1:
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
torch.distributed.all_gather_object(
loaded_by_rank, loaded, group=tune_group.cpu_group
)
loaded = all(loaded_by_rank)
if not loaded:
tuner.clear_cache()
autotune_context = fi_utils.autotune(tune_mode=True, **autotune_kwargs)
group = tune_group.cpu_group if tune_group.world_size > 1 else None
set_autotune_process_group(group)
try:
with (
torch.inference_mode(),
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
):
with torch.inference_mode(), autotune_context:
hisparse_enabled = (
runner.vllm_config.attention_config.hisparse_config is not None
)
@@ -498,7 +507,10 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
if world.world_size > 1:
world.barrier()
# Skip the rewrite when nothing was tuned this start (every entry came from
# the file). FlashInfer gates its own autotune(cache=...) save the same way.
if tuner._dirty:
if use_autotune_v2:
if world.world_size > 1:
# All ranks serve the shared store's final state after publishing.
autotune_v2_reload()
world.barrier()
elif tuner._dirty:
tuner.save_configs(str(cache_path))
@@ -19,9 +19,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None:
def _get_kda_layer(
worker: Worker, module: str = "vllm.models.kimi_k3.nvidia.kda"
) -> KimiK3DeltaAttention | None:
# Kimi model construction already imports kda. Avoid importing it here.
kda = sys.modules.get("vllm.models.kimi_k3.nvidia.kda")
kda = sys.modules.get(module)
if kda is None:
return None
@@ -172,9 +174,59 @@ def _warm_recurrent_kda(
)
def _warm_gluon_chunk_kda(layer, input_dtype: torch.dtype) -> None:
from vllm.models.kimi_k3.amd.ops.kda_prefill import (
chunk_kda_prefill,
get_gluon_chunk_kda_warmup_profiles,
)
if layer.kda_prefill_backend != "gluon":
return
kv_cache = layer.kv_cache
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
return
state = kv_cache[1]
if not isinstance(state, torch.Tensor) or not state.numel():
return
logger.info("Warming up Kimi-K3 Gluon chunk KDA prefill kernels.")
h, d = int(layer.local_num_heads), int(layer.head_dim)
for num_seqs, seq_len in get_gluon_chunk_kda_warmup_profiles():
num_tokens = num_seqs * seq_len
shape = (1, num_tokens, h, d)
chunk_kda_prefill(
q=torch.zeros(shape, dtype=input_dtype, device=state.device),
k=torch.zeros(shape, dtype=input_dtype, device=state.device),
v=torch.zeros(shape, dtype=input_dtype, device=state.device),
raw_g=torch.zeros(shape, dtype=input_dtype, device=state.device),
raw_beta=torch.zeros(
(1, num_tokens, h), dtype=input_dtype, device=state.device
),
A_log=layer.A_log,
g_bias=layer.dt_bias,
lower_bound=layer.gate_lower_bound,
state_cache=state,
state_indices=torch.zeros(num_seqs, dtype=torch.int32, device=state.device),
has_initial_state=torch.zeros(
num_seqs, dtype=torch.bool, device=state.device
),
use_qk_l2norm_in_kernel=True,
cu_seqlens=torch.arange(
0, num_tokens + 1, seq_len, dtype=torch.int32, device=state.device
),
backend="gluon",
out=torch.empty(shape, dtype=input_dtype, device=state.device),
)
@torch.inference_mode()
def kimi_k3_triton_warmup(worker: Worker) -> None:
"""Warm Kimi-K3 Triton kernels reachable by this server."""
if current_platform.is_rocm():
layer = _get_kda_layer(worker, "vllm.models.kimi_k3.amd.kda")
if layer is not None:
_warm_gluon_chunk_kda(layer, worker.model_config.dtype)
return
if not current_platform.is_cuda():
return