Merge branch 'main' into patches
This commit is contained in:
@@ -356,10 +356,12 @@ class LLBf16Dotprod:
|
||||
if tidx == 0:
|
||||
for m in cutlass.range_constexpr(M):
|
||||
partials = sm[m, None].load()
|
||||
gC[m, n_idx] = partials.reduce(
|
||||
cute.ReductionOp.ADD,
|
||||
init_val=cutlass.Float32(0.0),
|
||||
reduction_profile=0,
|
||||
)
|
||||
gC[m, n_idx] = cutlass.Float32(
|
||||
partials.reduce(
|
||||
cute.ReductionOp.ADD,
|
||||
init_val=cutlass.Float32(0.0),
|
||||
reduction_profile=0,
|
||||
)
|
||||
).to(gC.element_type)
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
@@ -47,7 +47,7 @@ class LLBf16SplitK:
|
||||
|
||||
:param ab_dtype: Element type for A and B operands.
|
||||
:param acc_dtype: Accumulator type used by MMA and reductions.
|
||||
:param out_dtype: Output element type. The public wrapper uses fp32.
|
||||
:param out_dtype: Output element type (fp32 or bf16).
|
||||
:param tile_n: CTA tile size in N. M is fixed to 16 for router batches.
|
||||
:type tile_n: int
|
||||
:param tile_k: K tile size staged through shared memory.
|
||||
@@ -580,6 +580,6 @@ class LLBf16SplitK:
|
||||
reduction_profile=0,
|
||||
)
|
||||
)
|
||||
gC[local_coord] = acc
|
||||
gC[local_coord] = cutlass.Float32(acc).to(gC.element_type)
|
||||
|
||||
cute.arch.sync_threads()
|
||||
|
||||
@@ -76,8 +76,8 @@ def make_fake_gemm_tensors(
|
||||
a_dtype: torch.dtype,
|
||||
b_dtype: torch.dtype,
|
||||
divisibility: int,
|
||||
output_dtype: torch.dtype = torch.float32,
|
||||
):
|
||||
from cutlass import Float32
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
hidden_states = make_fake_tensor(
|
||||
@@ -86,7 +86,7 @@ def make_fake_gemm_tensors(
|
||||
router_weight = make_fake_tensor(
|
||||
cutlass_dtype(b_dtype), (N, K), divisibility=divisibility
|
||||
)
|
||||
output = make_fake_tensor(Float32, (M, N), divisibility=1)
|
||||
output = make_fake_tensor(cutlass_dtype(output_dtype), (M, N), divisibility=1)
|
||||
return hidden_states, router_weight, output
|
||||
|
||||
|
||||
@@ -97,6 +97,7 @@ def validate_common_gemm_inputs(
|
||||
*,
|
||||
op_name: str,
|
||||
k_multiple: int | None = None,
|
||||
output_dtypes: tuple[torch.dtype, ...] = (torch.float32,),
|
||||
) -> None:
|
||||
if hidden_states.dim() != 2 or router_weight.dim() != 2:
|
||||
raise ValueError("hidden_states and router_weight must be 2D tensors")
|
||||
@@ -106,8 +107,11 @@ def validate_common_gemm_inputs(
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must be on the same CUDA device"
|
||||
)
|
||||
if output_dtype != torch.float32:
|
||||
raise ValueError(f"{op_name} only supports output_dtype=torch.float32")
|
||||
if output_dtype not in output_dtypes:
|
||||
raise ValueError(
|
||||
f"{op_name} only supports output_dtype="
|
||||
+ " or ".join(str(dtype) for dtype in output_dtypes)
|
||||
)
|
||||
if hidden_states.shape[1] != router_weight.shape[1]:
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must have matching K dimensions"
|
||||
|
||||
@@ -47,6 +47,8 @@ _SM100F_TUNED_SPLITK_CONFIGS: dict[tuple[int, int], dict[int, tuple[int, int]]]
|
||||
**{M: (8, 5) for M in (5, 8)},
|
||||
9: (8, 2),
|
||||
},
|
||||
(5120, 1792): {M: (2, 4) for M in range(5, 17)},
|
||||
(5120, 3584): {M: (2, 4) for M in range(5, 17)},
|
||||
(7168, 256): {14: (8, 2)},
|
||||
(6144, 256): {M: (8, 2) for M in (9, 12, 16)},
|
||||
(7168, 384): {M: (7, 5) for M in (13, 16)},
|
||||
@@ -101,39 +103,52 @@ class LLBf16Gemm:
|
||||
bs: int = 0
|
||||
split_k: int = 0
|
||||
num_stages: int = 0
|
||||
output_dtype: torch.dtype = torch.float32
|
||||
|
||||
def __init__(self, *, prefetch_pdl_weights: bool = False) -> None:
|
||||
self._prefetch_pdl_weights = prefetch_pdl_weights
|
||||
# Dot-prod: keyed on (M, K, bs), because M and K are Constexpr.
|
||||
self._compiled_cache: dict[tuple[int, int, int], Any] = {}
|
||||
# Split-K: keyed on (split_k, num_stages), fully shape-dynamic.
|
||||
self._splitk_cache: dict[tuple[int, int], Any] = {}
|
||||
# M and K are compile-time constants for dot-product.
|
||||
self._compiled_cache: dict[tuple[int, int, int, torch.dtype], Any] = {}
|
||||
# Split-K is shape-dynamic; both caches distinguish output dtype.
|
||||
self._splitk_cache: dict[tuple[int, int, torch.dtype], Any] = {}
|
||||
|
||||
def dispatch(self, *, M: int, K: int, N: int) -> CompileKey:
|
||||
def dispatch(
|
||||
self, *, M: int, K: int, N: int, output_dtype: torch.dtype = torch.float32
|
||||
) -> CompileKey:
|
||||
tuned_bs, tuned_splitk = _arch_tuned_configs()
|
||||
if M <= _DEFAULT_DOTPROD_MAX_M or K < 2048:
|
||||
bs = tuned_bs.get((K, N), {}).get(M, _DEFAULT_DOTPROD_BS)
|
||||
return self.CompileKey(backend="dotprod", M=M, K=K, bs=bs)
|
||||
return self.CompileKey(
|
||||
backend="dotprod", M=M, K=K, bs=bs, output_dtype=output_dtype
|
||||
)
|
||||
|
||||
split_k, num_stages = tuned_splitk.get((K, N), {}).get(
|
||||
M, _DEFAULT_SPLITK_CONFIG
|
||||
)
|
||||
return self.CompileKey(backend="splitk", split_k=split_k, num_stages=num_stages)
|
||||
return self.CompileKey(
|
||||
backend="splitk",
|
||||
split_k=split_k,
|
||||
num_stages=num_stages,
|
||||
output_dtype=output_dtype,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
shapes: Iterable[tuple[int, int]],
|
||||
m_values: Iterable[int],
|
||||
output_dtype: torch.dtype = torch.float32,
|
||||
) -> list[CompileKey]:
|
||||
return list(
|
||||
dict.fromkeys(
|
||||
self.dispatch(M=M, K=K, N=N) for K, N in shapes for M in m_values
|
||||
self.dispatch(M=M, K=K, N=N, output_dtype=output_dtype)
|
||||
for K, N in shapes
|
||||
for M in m_values
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _fake_gemm_tensors(*, M, K, N, divisibility: int):
|
||||
def _fake_gemm_tensors(*, M, K, N, divisibility: int, output_dtype: torch.dtype):
|
||||
return make_fake_gemm_tensors(
|
||||
M=M,
|
||||
K=K,
|
||||
@@ -141,10 +156,13 @@ class LLBf16Gemm:
|
||||
a_dtype=torch.bfloat16,
|
||||
b_dtype=torch.bfloat16,
|
||||
divisibility=divisibility,
|
||||
output_dtype=output_dtype,
|
||||
)
|
||||
|
||||
def _compile_splitk(self, compile_key: CompileKey) -> None:
|
||||
cute, _ = cute_context()
|
||||
from cutlass import BFloat16, Float32
|
||||
|
||||
from ._ll_bf16_splitk import LLBf16SplitK
|
||||
|
||||
hidden_states, router_weight, output = self._fake_gemm_tensors(
|
||||
@@ -152,8 +170,12 @@ class LLBf16Gemm:
|
||||
K=cute.sym_int(),
|
||||
N=cute.sym_int(),
|
||||
divisibility=8,
|
||||
output_dtype=compile_key.output_dtype,
|
||||
)
|
||||
gemm = LLBf16SplitK(
|
||||
out_dtype=BFloat16
|
||||
if compile_key.output_dtype == torch.bfloat16
|
||||
else Float32,
|
||||
tile_n=16,
|
||||
tile_k=256,
|
||||
num_stages=compile_key.num_stages,
|
||||
@@ -169,7 +191,9 @@ class LLBf16Gemm:
|
||||
current_cuda_stream(),
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
self._splitk_cache[(compile_key.split_k, compile_key.num_stages)] = compiled
|
||||
self._splitk_cache[
|
||||
(compile_key.split_k, compile_key.num_stages, compile_key.output_dtype)
|
||||
] = compiled
|
||||
logger.debug(
|
||||
"Compiled ll_bf16_splitk: sk=%d ns=%d",
|
||||
compile_key.split_k,
|
||||
@@ -187,6 +211,7 @@ class LLBf16Gemm:
|
||||
K=compile_key.K,
|
||||
N=N,
|
||||
divisibility=stride_divisibility,
|
||||
output_dtype=compile_key.output_dtype,
|
||||
)
|
||||
gemm = LLBf16Dotprod(
|
||||
k=compile_key.K,
|
||||
@@ -205,7 +230,9 @@ class LLBf16Gemm:
|
||||
current_cuda_stream(),
|
||||
options="--enable-tvm-ffi --ptxas-options -maxrregcount=64",
|
||||
)
|
||||
self._compiled_cache[(compile_key.M, compile_key.K, compile_key.bs)] = compiled
|
||||
self._compiled_cache[
|
||||
(compile_key.M, compile_key.K, compile_key.bs, compile_key.output_dtype)
|
||||
] = compiled
|
||||
logger.debug(
|
||||
"Compiled ll_bf16_dotprod: M=%d, K=%d, bs=%d",
|
||||
compile_key.M,
|
||||
@@ -215,12 +242,21 @@ class LLBf16Gemm:
|
||||
|
||||
def compile(self, compile_key: CompileKey) -> None:
|
||||
if compile_key.backend == "splitk":
|
||||
splitk_cache_key = (compile_key.split_k, compile_key.num_stages)
|
||||
splitk_cache_key = (
|
||||
compile_key.split_k,
|
||||
compile_key.num_stages,
|
||||
compile_key.output_dtype,
|
||||
)
|
||||
if splitk_cache_key not in self._splitk_cache:
|
||||
self._compile_splitk(compile_key)
|
||||
return
|
||||
|
||||
dotprod_cache_key = (compile_key.M, compile_key.K, compile_key.bs)
|
||||
dotprod_cache_key = (
|
||||
compile_key.M,
|
||||
compile_key.K,
|
||||
compile_key.bs,
|
||||
compile_key.output_dtype,
|
||||
)
|
||||
if dotprod_cache_key not in self._compiled_cache:
|
||||
self._compile_dotprod(compile_key)
|
||||
|
||||
@@ -229,8 +265,11 @@ class LLBf16Gemm:
|
||||
*,
|
||||
shapes: Iterable[tuple[int, int]],
|
||||
m_values: Iterable[int],
|
||||
output_dtype: torch.dtype = torch.float32,
|
||||
) -> None:
|
||||
for compile_key in self.get_warmup_keys(shapes=shapes, m_values=m_values):
|
||||
for compile_key in self.get_warmup_keys(
|
||||
shapes=shapes, m_values=m_values, output_dtype=output_dtype
|
||||
):
|
||||
self.compile(compile_key)
|
||||
|
||||
@staticmethod
|
||||
@@ -251,6 +290,7 @@ class LLBf16Gemm:
|
||||
output_dtype,
|
||||
op_name="ll_bf16_gemm",
|
||||
k_multiple=8,
|
||||
output_dtypes=(torch.float32, torch.bfloat16),
|
||||
)
|
||||
|
||||
def __call__(
|
||||
@@ -258,19 +298,28 @@ class LLBf16Gemm:
|
||||
hidden_states: torch.Tensor, # [M, K] bf16
|
||||
router_weight: torch.Tensor, # [N, K] bf16
|
||||
output_dtype: torch.dtype = torch.float32,
|
||||
) -> torch.Tensor: # [M, N] fp32
|
||||
) -> torch.Tensor: # [M, N]
|
||||
self._validate_inputs(hidden_states, router_weight, output_dtype)
|
||||
|
||||
M, K = hidden_states.shape
|
||||
N = router_weight.shape[0]
|
||||
compile_key = self.dispatch(M=M, K=K, N=N)
|
||||
compile_key = self.dispatch(M=M, K=K, N=N, output_dtype=output_dtype)
|
||||
if compile_key.backend == "splitk":
|
||||
splitk_cache_key = (compile_key.split_k, compile_key.num_stages)
|
||||
splitk_cache_key = (
|
||||
compile_key.split_k,
|
||||
compile_key.num_stages,
|
||||
compile_key.output_dtype,
|
||||
)
|
||||
if splitk_cache_key not in self._splitk_cache:
|
||||
self.compile(compile_key)
|
||||
kernel = self._splitk_cache[splitk_cache_key]
|
||||
else:
|
||||
dotprod_cache_key = (compile_key.M, compile_key.K, compile_key.bs)
|
||||
dotprod_cache_key = (
|
||||
compile_key.M,
|
||||
compile_key.K,
|
||||
compile_key.bs,
|
||||
compile_key.output_dtype,
|
||||
)
|
||||
if dotprod_cache_key not in self._compiled_cache:
|
||||
self.compile(compile_key)
|
||||
kernel = self._compiled_cache[dotprod_cache_key]
|
||||
|
||||
@@ -6,19 +6,6 @@ from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
def _torch_hc_prenorm_gemm(
|
||||
x: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
sqrsum: torch.Tensor,
|
||||
) -> None:
|
||||
assert out.shape[0] == 1
|
||||
assert sqrsum.shape[0] == 1
|
||||
x_float = x.float()
|
||||
out[0].copy_(x_float @ fn.t())
|
||||
sqrsum[0].copy_(x_float.square().sum(dim=-1))
|
||||
|
||||
|
||||
def _hc_prenorm_gemm_outputs(
|
||||
x: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
|
||||
@@ -2024,6 +2024,7 @@ def build_mla_chunked_context_metadata(
|
||||
dcp_local_block_size: int,
|
||||
dcp_virtual_block_size: int,
|
||||
dcp_manager: MLADCPManager | None = None,
|
||||
align_indptr: bool = False,
|
||||
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
|
||||
"""Build chunked-context metadata for an MLA prefill.
|
||||
|
||||
@@ -2043,6 +2044,7 @@ def build_mla_chunked_context_metadata(
|
||||
dcp_local_block_size: Per-rank interleave block size for DCP.
|
||||
dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``.
|
||||
dcp_manager: Shared MLA DCP collective manager.
|
||||
align_indptr: Align packed int32 indptr slices to 16 bytes for CuTe DSL.
|
||||
|
||||
Returns:
|
||||
The chunked-context metadata, or None when no prefill has any context.
|
||||
@@ -2123,6 +2125,15 @@ def build_mla_chunked_context_metadata(
|
||||
num_tokens = sum(plan.seq_lens)
|
||||
num_local_tokens = num_tokens
|
||||
|
||||
if align_indptr:
|
||||
# CuTe DSL requires 16-byte-aligned int32 indptrs.
|
||||
padding = -boundary_offset % 4
|
||||
cu_seq_lens_flat.extend([0] * padding)
|
||||
cu_seqlens_q_flat.extend([0] * padding)
|
||||
if use_dcp:
|
||||
padded_local_cu_seq_lens_flat.extend([0] * padding)
|
||||
boundary_offset += padding
|
||||
|
||||
seq_lens_flat.extend(plan.seq_lens)
|
||||
cu_seq_lens_flat.extend(itertools.accumulate(plan.seq_lens, initial=0))
|
||||
query_base = query_start_loc[plan.request_start]
|
||||
@@ -2616,6 +2627,14 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
query_start_loc_cpu[reqs_start:] - query_start_loc_cpu[reqs_start]
|
||||
)
|
||||
|
||||
align_indptr = False
|
||||
if (
|
||||
self._prefill_backend.get_name() == "TRTLLM_RAGGED"
|
||||
and self.vllm_config.attention_config.trtllm_ragged_backend
|
||||
== "cute-dsl"
|
||||
):
|
||||
align_indptr = True
|
||||
|
||||
chunked_context_metadata = build_mla_chunked_context_metadata(
|
||||
context_lens_cpu=context_lens_cpu,
|
||||
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
|
||||
@@ -2628,6 +2647,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
dcp_local_block_size=self.dcp_local_block_size,
|
||||
dcp_virtual_block_size=self.dcp_virtual_block_size,
|
||||
dcp_manager=self.dcp_manager,
|
||||
align_indptr=align_indptr,
|
||||
)
|
||||
|
||||
prefill_metadata = MLACommonPrefillMetadata(
|
||||
|
||||
@@ -359,13 +359,6 @@ class ExpertMapManager:
|
||||
return True
|
||||
return self._expert_map[global_id] != -1
|
||||
|
||||
def get_local_expert_ids(self) -> list[int]:
|
||||
"""Get list of global IDs for experts on this rank."""
|
||||
if self._expert_map is None:
|
||||
return list(range(self.global_num_experts))
|
||||
|
||||
return torch.where(self._expert_map != -1)[0].tolist()
|
||||
|
||||
def update(
|
||||
self,
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
|
||||
@@ -545,7 +545,12 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def _act_mul_quant(
|
||||
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
expert_ends: torch.Tensor | None = None,
|
||||
expert_alignment: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
block_k = self._ACT_BLOCK_K
|
||||
scale_fmt = DeepGemmQuantScaleFMT.from_oracle()
|
||||
@@ -561,6 +566,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
output_q=output,
|
||||
group_size=block_k,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
expert_ends=expert_ends,
|
||||
expert_alignment=expert_alignment,
|
||||
)
|
||||
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
|
||||
return silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
@@ -634,7 +641,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
a1q_perm = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
|
||||
)
|
||||
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
|
||||
use_psum_layout = current_platform.is_device_capability_family(100)
|
||||
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
|
||||
aq=a1q,
|
||||
aq_scale=a1q_scale,
|
||||
topk_ids=topk_ids,
|
||||
@@ -642,6 +650,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
expert_map=expert_map,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
aq_out=a1q_perm,
|
||||
use_psum_layout=use_psum_layout,
|
||||
)
|
||||
assert a1q.size(0) == M_sum
|
||||
|
||||
@@ -655,18 +664,23 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
(a1q, a1q_scale),
|
||||
(w1.view(torch.int8), self.w1_scale),
|
||||
mm1_out,
|
||||
expert_ids,
|
||||
grouped_layout,
|
||||
use_psum_layout=use_psum_layout,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
|
||||
# SwiGLU activation + FP8 requant
|
||||
# SiLU+mul and FP8 requant use the same live expert ranges as GEMM.
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
quant_out = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
||||
)
|
||||
a2q, a2q_scale = self._act_mul_quant(
|
||||
input=mm1_out.view(-1, N), output=quant_out, activation=activation
|
||||
input=mm1_out.view(-1, N),
|
||||
output=quant_out,
|
||||
activation=activation,
|
||||
expert_ends=grouped_layout if use_psum_layout else None,
|
||||
expert_alignment=align_used if use_psum_layout else 0,
|
||||
)
|
||||
|
||||
# FC2: FP8 activations x FP4 weights
|
||||
@@ -675,7 +689,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
(a2q, a2q_scale),
|
||||
(w2.view(torch.int8), self.w2_scale),
|
||||
mm2_out,
|
||||
expert_ids,
|
||||
grouped_layout,
|
||||
use_psum_layout=use_psum_layout,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
|
||||
@@ -517,6 +517,7 @@ _TILE_CONFIG_OVERRIDES = {
|
||||
256: (128, 32),
|
||||
512: (128, 19),
|
||||
},
|
||||
(5120, 256): {64: (64, 64)}, # GB200
|
||||
(6144, 128): { # MiniMax-M3
|
||||
64: (32, 64),
|
||||
128: (64, 64),
|
||||
@@ -533,7 +534,7 @@ def _pick_tile_config(N: int, K: int, M: int, num_sms: int) -> tuple[int, int]:
|
||||
if 32 < N <= 512:
|
||||
token_bucket = triton.next_power_of_2(N)
|
||||
overrides = _TILE_CONFIG_OVERRIDES.get((K, M))
|
||||
if overrides is not None:
|
||||
if overrides is not None and token_bucket in overrides:
|
||||
BN, split_k = overrides[token_bucket]
|
||||
return BN, min(split_k, k_tiles)
|
||||
|
||||
@@ -609,6 +610,8 @@ def _bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
|
||||
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
|
||||
N, K = X.shape
|
||||
M, _ = W.shape
|
||||
if N == 0:
|
||||
return X.new_empty((N, M), dtype=torch.float32)
|
||||
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
|
||||
|
||||
BN, split_k = _pick_tile_config(N, K, M, num_sms)
|
||||
|
||||
@@ -24,8 +24,8 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
|
||||
any shape w/ K divisible by 8)
|
||||
2. fp32 specialized kernel (bf16/fp32 in, fp32 out, M<=32; SM90+
|
||||
model-specific shapes, gfx950 any shape)
|
||||
2. fp32 specialized kernel (bf16/fp32 in, fp32 out; SM90+
|
||||
model-specific shapes and batch limits, gfx950 any shape with M<=32)
|
||||
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
|
||||
any shape)
|
||||
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||
@@ -46,9 +46,14 @@ class GateLinear(ReplicatedLinear):
|
||||
split-K from the token count and skip the batch-invariant linear path.
|
||||
"""
|
||||
|
||||
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
|
||||
# (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
|
||||
FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128)}
|
||||
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel.
|
||||
FP32_SUPPORTED_SHAPES = {
|
||||
(3072, 256),
|
||||
(6144, 128),
|
||||
(5120, 256),
|
||||
(4096, 352),
|
||||
(8192, 256),
|
||||
}
|
||||
FP32_MAX_TOKENS = 32
|
||||
# Largest batch for which tier 1 beats the cuBLAS epilogue. Subclasses
|
||||
# whose router shape stays ahead further up the range may raise it.
|
||||
@@ -454,9 +459,9 @@ def fp32_router_gemm_dispatch_impl(
|
||||
weight: torch.Tensor,
|
||||
allow_bf16x3_router_gemm: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS,
|
||||
otherwise optionally run the experimental BF16x3 kernel for medium/large
|
||||
SM100 router batches, then fall back to F.linear.
|
||||
"""Run the FP32 kernel for small batches, with shape-specific limits.
|
||||
|
||||
Larger SM100 BF16 batches use BF16x3; other inputs fall back to F.linear.
|
||||
This must be wrapped in a custom op because our torch.compile integration
|
||||
does not support runtime dispatching on num_tokens.
|
||||
"""
|
||||
@@ -473,7 +478,15 @@ def fp32_router_gemm_dispatch_impl(
|
||||
if weight.dtype == torch.bfloat16:
|
||||
return torch.mm(x, weight.T, out_dtype=torch.float32)
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
max_tokens = _FP32_ROUTER_GEMM_MAX_TOKENS
|
||||
if (
|
||||
allow_bf16x3_router_gemm
|
||||
and x.dtype == torch.bfloat16
|
||||
and weight.shape == (256, 5120)
|
||||
):
|
||||
# GB200: four token groups win through 48 for multiples of eight.
|
||||
max_tokens = 48 if x.shape[0] % 8 == 0 else 16
|
||||
if x.shape[0] <= max_tokens:
|
||||
return ops.fp32_router_gemm(x, weight)
|
||||
|
||||
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
|
||||
@@ -9,139 +9,162 @@
|
||||
from packaging import version
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.triton_helpers import fast_exp
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
TRITON_22 = version.parse(triton.__version__) >= version.parse("2.2.0")
|
||||
|
||||
|
||||
def _drop_racy_wgmma_configs(configs: list[triton.Config]) -> list[triton.Config]:
|
||||
"""Drop the configs that Triton miscompiles on sm90.
|
||||
|
||||
With whole warpgroups (num_warps % 4 == 0) and BLOCK_SIZE_M % 64 == 0,
|
||||
`tl.dot(cb, x)` lowers to wgmma with `cb` in registers. With
|
||||
num_stages > 1, Triton leaves that wgmma running across the loop
|
||||
back-edge while the next iteration overwrites the registers of `cb`,
|
||||
which writes NaN/Inf into whole output rows.
|
||||
"""
|
||||
if not (current_platform.is_cuda() and current_platform.is_device_capability(90)):
|
||||
return configs
|
||||
return [
|
||||
c
|
||||
for c in configs
|
||||
if c.num_stages == 1
|
||||
or c.num_warps % 4 != 0
|
||||
or c.kwargs["BLOCK_SIZE_M"] % 64 != 0
|
||||
]
|
||||
|
||||
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
# =================================================================
|
||||
# Higher warp count configs for better latency hiding
|
||||
# More warps = more instructions in flight = better memory latency hiding
|
||||
# =================================================================
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
# Smaller tiles with more stages for software pipelining
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=3,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
# =================================================================
|
||||
# Low register pressure configs (num_stages=1) for large dstate
|
||||
# =================================================================
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
# num_stages=2 configs - moderate register pressure
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
# Original configs for larger dstate values
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
|
||||
num_stages=3,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=5,
|
||||
num_warps=2,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=5,
|
||||
num_warps=2,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=2,
|
||||
),
|
||||
],
|
||||
configs=_drop_racy_wgmma_configs(
|
||||
[
|
||||
# =================================================================
|
||||
# Higher warp count configs for better latency hiding
|
||||
# More warps = more instructions in flight = better memory latency hiding
|
||||
# =================================================================
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
# Smaller tiles with more stages for software pipelining
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=3,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
# =================================================================
|
||||
# Low register pressure configs (num_stages=1) for large dstate
|
||||
# =================================================================
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
# num_stages=2 configs - moderate register pressure
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
# Original configs for larger dstate values
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
|
||||
num_stages=3,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=5,
|
||||
num_warps=2,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=5,
|
||||
num_warps=2,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=2,
|
||||
),
|
||||
]
|
||||
),
|
||||
key=["chunk_size", "hdim", "dstate", "IS_CAUSAL"],
|
||||
)
|
||||
@triton.jit
|
||||
|
||||
@@ -12,35 +12,6 @@ from vllm.triton_utils import tl, triton
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def block_dequant(
|
||||
x_q_block: torch.Tensor,
|
||||
x_s: torch.Tensor,
|
||||
block_size: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""This function conducts block-wise dequantization.
|
||||
The inputs are block-wise quantization tensor `x_q_block`,
|
||||
block-wise quantization scale and the block size.
|
||||
The outputs are dequantized tensor.
|
||||
"""
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
n, k = x_q_block.shape
|
||||
n_tiles = (n + block_n - 1) // block_n
|
||||
k_tiles = (k + block_k - 1) // block_k
|
||||
assert n_tiles == x_s.shape[0]
|
||||
assert k_tiles == x_s.shape[1]
|
||||
|
||||
x_dq_block = x_q_block.to(torch.float32)
|
||||
|
||||
for i in range(k_tiles):
|
||||
for j in range(n_tiles):
|
||||
x_dq_block[
|
||||
j * block_n : min((j + 1) * block_n, n),
|
||||
i * block_k : min((i + 1) * block_k, k),
|
||||
] *= x_s[j][i]
|
||||
|
||||
return x_dq_block
|
||||
|
||||
|
||||
if current_platform.is_rocm():
|
||||
|
||||
@triton.jit
|
||||
|
||||
@@ -1404,14 +1404,17 @@ class DiffusionSampler:
|
||||
# unaffected; only the canvas exploration is constrained. Applied
|
||||
# before canvas padding so phantom positions stay uniform.
|
||||
if num_decode > 0:
|
||||
top_k, top_p = self.sampling_states.get_top_k_top_p(
|
||||
top_k, top_p, has_topp_only_rows = self.sampling_states.get_top_k_top_p(
|
||||
decode_slots.repeat_interleave(
|
||||
valid_canvas_len, output_size=int(valid_canvas_len_np.sum())
|
||||
),
|
||||
decode_slots_np,
|
||||
logits.shape[-1],
|
||||
)
|
||||
if top_k is not None or top_p is not None:
|
||||
logits = apply_top_k_top_p(logits.float(), top_k, top_p)
|
||||
logits = apply_top_k_top_p(
|
||||
logits.float(), top_k, top_p, has_topp_only_rows=has_topp_only_rows
|
||||
)
|
||||
|
||||
# Where each decode request's rows start in the flat logits. Tiles
|
||||
# below gather and pad a request to the tile's width.
|
||||
|
||||
@@ -64,6 +64,7 @@ if TYPE_CHECKING:
|
||||
from vllm.tasks import ScoreType
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
EncoderCudaGraphConfig,
|
||||
@@ -670,6 +671,47 @@ def supports_score_template(
|
||||
return getattr(model, "supports_score_template", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsKVCacheGrouping(Protocol):
|
||||
"""The interface required for models that own their KV cache grouping.
|
||||
|
||||
.. note::
|
||||
Experimental. The classmethod contract may change as model-owned
|
||||
grouping stabilizes.
|
||||
|
||||
Implementing it lets a model replace the default KV cache grouping (which
|
||||
layers share a block table) while Core keeps owning placement, byte
|
||||
accounting, and admission.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def get_kv_cache_groups(
|
||||
cls,
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_spec: dict[str, "KVCacheSpec"],
|
||||
) -> list["KVCacheGroupSpec"]:
|
||||
"""Split the model's layers into logical KV cache groups."""
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_kv_cache_grouping(
|
||||
model: type[object],
|
||||
) -> TypeIs[type[SupportsKVCacheGrouping]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_kv_cache_grouping(
|
||||
model: object,
|
||||
) -> TypeIs[SupportsKVCacheGrouping]: ...
|
||||
|
||||
|
||||
def supports_kv_cache_grouping(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsKVCacheGrouping]] | TypeIs[SupportsKVCacheGrouping]:
|
||||
return isinstance(model, SupportsKVCacheGrouping)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsLoRA(Protocol):
|
||||
"""The interface required for all models that support LoRA."""
|
||||
|
||||
@@ -50,6 +50,7 @@ from .interfaces import (
|
||||
is_hybrid,
|
||||
requires_raw_input_tokens,
|
||||
supports_hc_sp,
|
||||
supports_kv_cache_grouping,
|
||||
supports_multimodal,
|
||||
supports_multimodal_encoder_tp_data,
|
||||
supports_multimodal_raw_input_only,
|
||||
@@ -882,6 +883,7 @@ class _ModelInfo:
|
||||
supports_replayssm: bool
|
||||
supports_transcription: bool
|
||||
supports_transcription_only: bool
|
||||
supports_kv_cache_grouping: bool
|
||||
supported_video_pruning_methods: tuple[str, ...]
|
||||
supports_mm_device_do_normalize: bool
|
||||
|
||||
@@ -913,6 +915,7 @@ class _ModelInfo:
|
||||
supports_transcription_only=(
|
||||
supports_transcription(model) and model.supports_transcription_only
|
||||
),
|
||||
supports_kv_cache_grouping=supports_kv_cache_grouping(model),
|
||||
has_noops=has_noops(model),
|
||||
supported_video_pruning_methods=getattr(
|
||||
model, "supported_video_pruning_methods", ()
|
||||
|
||||
@@ -11,11 +11,14 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import CacheConfig, VllmConfig, replace
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _normalize_cache_config_for_hash(cache_config: CacheConfig) -> CacheConfig:
|
||||
"""Normalize cache config for hashing"""
|
||||
@@ -55,6 +58,53 @@ def resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
|
||||
return output_dir / "autotune_configs.json"
|
||||
|
||||
|
||||
def use_flashinfer_autotune_v2(runner: "GPUModelRunner") -> bool:
|
||||
from vllm.distributed.parallel_state import get_node_count, get_world_group
|
||||
from vllm.utils.flashinfer import has_flashinfer_autotune_v2
|
||||
|
||||
# Elastic EP transfers tuning state to joining workers with v1
|
||||
# save_configs/load_configs, which do not export managed-store winners.
|
||||
if runner.vllm_config.parallel_config.enable_elastic_ep:
|
||||
return False
|
||||
# PP stages profile independently. Keep their group-scoped v1 files until
|
||||
# managed-cache reads and publishes can be isolated across tuning groups.
|
||||
if runner.vllm_config.parallel_config.pipeline_parallel_size > 1:
|
||||
return False
|
||||
if not has_flashinfer_autotune_v2():
|
||||
return False
|
||||
# Use the initialized topology: Ray may span nodes with config.nnodes=1.
|
||||
# A cache hit skips profiling collectives, so node-local stores with
|
||||
# different contents cannot safely participate in synchronized tuning.
|
||||
if get_node_count() != 1:
|
||||
logger.info_once(
|
||||
"Using legacy FlashInfer autotune cache synchronization for "
|
||||
"multi-node deployments; managed-cache synchronization is "
|
||||
"currently supported only within one node."
|
||||
)
|
||||
return False
|
||||
# Lift this gate after pinning a release with the fix below and validating
|
||||
# distributed managed-cache tuning: https://github.com/flashinfer-ai/flashinfer/pull/6185
|
||||
if get_world_group().world_size > 1:
|
||||
logger.info_once(
|
||||
"Using legacy FlashInfer autotune cache synchronization for "
|
||||
"multi-rank deployments until FlashInfer coordinates cache hits."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def resolve_flashinfer_autotune_v2_root() -> Path | None:
|
||||
"""Placement root for FlashInfer's managed autotune store.
|
||||
|
||||
Unlike :func:`resolve_flashinfer_autotune_file`, no vLLM-side identity
|
||||
(config hash, flashinfer version, arch) is folded into the path: the
|
||||
managed store keys entries per op and hashes the software/hardware
|
||||
environment itself. ``None`` selects FlashInfer's default root.
|
||||
"""
|
||||
override_dir = envs.VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR
|
||||
return Path(override_dir).expanduser() if override_dir else None
|
||||
|
||||
|
||||
def sync_flashinfer_autotune_cache(
|
||||
runner: "GPUModelRunner",
|
||||
group: "GroupCoordinator",
|
||||
|
||||
@@ -9,6 +9,8 @@ import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
resolve_flashinfer_autotune_v2_root,
|
||||
use_flashinfer_autotune_v2,
|
||||
write_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -145,7 +147,6 @@ def _run_flashinfer_sparse_mla_decode_autotune(
|
||||
|
||||
world = get_world_group()
|
||||
is_leader = world.rank_in_group == 0
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
|
||||
dummy_run_kwargs = dict(
|
||||
num_tokens=num_tokens,
|
||||
@@ -155,6 +156,13 @@ def _run_flashinfer_sparse_mla_decode_autotune(
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
|
||||
if use_flashinfer_autotune_v2(runner):
|
||||
return _run_sparse_mla_decode_autotune_v2(
|
||||
worker, num_tokens, log_label, dummy_run_kwargs
|
||||
)
|
||||
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
|
||||
if is_leader:
|
||||
logger.info(
|
||||
"Autotuning FlashInfer SM120 sparse MLA %s decode with cache: %s",
|
||||
@@ -223,6 +231,58 @@ def _run_flashinfer_sparse_mla_decode_autotune(
|
||||
return True
|
||||
|
||||
|
||||
def _run_sparse_mla_decode_autotune_v2(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
log_label: str,
|
||||
dummy_run_kwargs: dict,
|
||||
) -> bool:
|
||||
"""Run every rank's sparse warmup in the managed autotune context.
|
||||
|
||||
FlashInfer owns persistence; some sparse kernels use a separate calibration
|
||||
cache instead of managed winner entries.
|
||||
"""
|
||||
from flashinfer import autotune_v2, autotune_v2_reload
|
||||
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
runner = worker.model_runner
|
||||
world = get_world_group()
|
||||
cache_root = resolve_flashinfer_autotune_v2_root()
|
||||
if world.rank_in_group == 0:
|
||||
logger.info(
|
||||
"Autotuning FlashInfer SM120 sparse MLA %s decode with the managed "
|
||||
"autotune context (root=%s)",
|
||||
log_label,
|
||||
cache_root if cache_root is not None else "flashinfer default",
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
|
||||
v2_runner = cast("V2GPUModelRunner", runner)
|
||||
warmup_executed = run_mixed_prefill_decode_warmup(
|
||||
v2_runner,
|
||||
worker.execute_model,
|
||||
worker.sample_tokens,
|
||||
num_tokens,
|
||||
mixed_step_context=autotune_v2(mode="tune", cache_root=cache_root),
|
||||
req_id_prefix="_sparse_mla_v2_warmup",
|
||||
)
|
||||
else:
|
||||
warmup_executed = True
|
||||
with autotune_v2(mode="tune", cache_root=cache_root):
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
|
||||
if not warmup_executed:
|
||||
return False
|
||||
|
||||
if world.world_size > 1:
|
||||
world.barrier()
|
||||
autotune_v2_reload()
|
||||
world.barrier()
|
||||
return True
|
||||
|
||||
|
||||
def _flashinfer_sparse_mla_decode_autotune(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
|
||||
@@ -1082,6 +1082,10 @@ class JitWarmupRegistry:
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm.distributed import is_global_first_rank
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
VllmTritonJitKernel,
|
||||
compile_triton_warmup,
|
||||
)
|
||||
|
||||
kernel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
|
||||
for kernel, registrations in self._registrations.items():
|
||||
@@ -1102,17 +1106,40 @@ class JitWarmupRegistry:
|
||||
if not kernel_items:
|
||||
return
|
||||
|
||||
parallel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
|
||||
serial_items = []
|
||||
for kernel, compile_keys in kernel_items:
|
||||
if (
|
||||
isinstance(kernel, VllmTritonJitKernel)
|
||||
and kernel.supports_parallel_compile
|
||||
):
|
||||
parallel_items.append((kernel, compile_keys))
|
||||
else:
|
||||
serial_items.append((kernel, compile_keys))
|
||||
|
||||
total_keys = sum(len(compile_keys) for _, compile_keys in kernel_items)
|
||||
with tqdm(
|
||||
kernel_items,
|
||||
total=len(kernel_items),
|
||||
desc=f"JIT kernel warmup ({total_keys} compile keys)",
|
||||
disable=not is_global_first_rank(),
|
||||
dynamic_ncols=True,
|
||||
unit="kernel",
|
||||
) as progress:
|
||||
for kernel, compile_keys in progress:
|
||||
progress.set_postfix_str(
|
||||
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
|
||||
refresh=False,
|
||||
|
||||
def compile_kernels(
|
||||
items: list[tuple[VllmJitKernel[Any], dict[Any, None]]],
|
||||
) -> None:
|
||||
for kernel, compile_keys in items:
|
||||
progress.set_postfix_str(
|
||||
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
|
||||
refresh=False,
|
||||
)
|
||||
kernel.compile_many(compile_keys)
|
||||
progress.update()
|
||||
|
||||
if parallel_items:
|
||||
compile_triton_warmup(
|
||||
lambda: compile_kernels(parallel_items),
|
||||
sum(len(keys) for _, keys in parallel_items),
|
||||
)
|
||||
kernel.compile_many(compile_keys)
|
||||
compile_kernels(serial_items)
|
||||
|
||||
@@ -6,7 +6,7 @@ from abc import abstractmethod
|
||||
from collections.abc import Callable, Hashable, Iterable, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import nullcontext, suppress
|
||||
from contextvars import copy_context
|
||||
from contextvars import ContextVar, copy_context
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cache, cached_property, update_wrapper
|
||||
from typing import Any, Generic, ParamSpec, Protocol, TypeVar, cast, overload
|
||||
@@ -207,6 +207,31 @@ class TritonWarmupTensor:
|
||||
return result if dim is None else result[dim]
|
||||
|
||||
|
||||
_shared_compile_pool: ContextVar[bool] = ContextVar(
|
||||
"shared_triton_warmup_compile_pool", default=False
|
||||
)
|
||||
|
||||
|
||||
def compile_triton_warmup(compile_all: Callable[[], None], num_keys: int) -> None:
|
||||
"""Share one compiler pool across all eligible warmup kernels."""
|
||||
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, num_keys)
|
||||
if num_threads <= 1:
|
||||
compile_all()
|
||||
return
|
||||
|
||||
def compile_in_context() -> None:
|
||||
_shared_compile_pool.set(True)
|
||||
with (
|
||||
ThreadPoolExecutor(max_workers=num_threads) as executor,
|
||||
triton.AsyncCompileMode(executor),
|
||||
):
|
||||
# Dispatch mutates owner state and must stay on this thread.
|
||||
compile_all()
|
||||
|
||||
# Older Triton versions leave async mode active on compilation failure.
|
||||
copy_context().run(compile_in_context)
|
||||
|
||||
|
||||
class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
"""Triton owner whose runtime launch specification is reused for warmup."""
|
||||
|
||||
@@ -230,32 +255,25 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
self._warming = False
|
||||
self._warming_compile_key = None
|
||||
|
||||
@property
|
||||
def supports_parallel_compile(self) -> bool:
|
||||
# AMD LLVM code generation can abort under concurrent compilation.
|
||||
return (
|
||||
current_platform.is_cuda()
|
||||
and not self._run_autotune
|
||||
and getattr(triton, "AsyncCompileMode", None) is not None
|
||||
and envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS > 1
|
||||
)
|
||||
|
||||
def compile_many(self, compile_keys: Iterable[CompileKeyT]) -> None:
|
||||
"""Compile CUDA startup warmup variants in parallel, then wait."""
|
||||
keys = list(compile_keys)
|
||||
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, len(keys))
|
||||
async_compile = getattr(triton, "AsyncCompileMode", None)
|
||||
# AMD LLVM code generation can abort under concurrent compilation.
|
||||
if (
|
||||
not current_platform.is_cuda()
|
||||
or self._run_autotune
|
||||
or async_compile is None
|
||||
or num_threads <= 1
|
||||
):
|
||||
if not self.supports_parallel_compile or _shared_compile_pool.get():
|
||||
return super().compile_many(keys)
|
||||
|
||||
def compile_all() -> None:
|
||||
with (
|
||||
ThreadPoolExecutor(max_workers=num_threads) as executor,
|
||||
async_compile(executor),
|
||||
):
|
||||
# Dispatch stays on this thread: compile() mutates owner state.
|
||||
# Only Triton's underlying compiler work runs in the pool.
|
||||
super(VllmTritonJitKernel, self).compile_many(keys)
|
||||
|
||||
# Older Triton versions leave async mode active on compilation failure.
|
||||
# Isolate that state so it cannot affect later runtime JIT compilation.
|
||||
copy_context().run(compile_all)
|
||||
compile_triton_warmup(
|
||||
lambda: super(VllmTritonJitKernel, self).compile_many(keys), len(keys)
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def _kernel_arg_names(self) -> tuple[str, ...]:
|
||||
|
||||
@@ -23,6 +23,8 @@ from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
resolve_flashinfer_autotune_v2_root,
|
||||
use_flashinfer_autotune_v2,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
autotune_hisparse_flashinfer_attention,
|
||||
@@ -419,7 +421,7 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
otherwise the world group tunes together. Per-tactic timings are
|
||||
averaged over the tuning group so all its ranks select the same tactic.
|
||||
|
||||
Results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
|
||||
Legacy results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
|
||||
(``MoERunner.get_cache_key_extras``), so one rank's file only hits on that
|
||||
rank. A rank with a cache hit skips the per-tactic reduce the others block
|
||||
in, so ranks keep loaded configs only if every rank in the tuning group
|
||||
@@ -450,38 +452,45 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
)
|
||||
autotune_kwargs["skip_ops"] = skip_ops
|
||||
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
# The world group only spans DP ranks when vLLM folds DP into it.
|
||||
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
|
||||
cache_path = cache_path.with_name(
|
||||
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
|
||||
)
|
||||
if is_leader:
|
||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
use_autotune_v2 = use_flashinfer_autotune_v2(runner)
|
||||
if use_autotune_v2:
|
||||
from flashinfer import autotune_v2, autotune_v2_reload
|
||||
|
||||
# We skip EPLB here since we don't want to record dummy metrics.
|
||||
# Randomize inputs to avoid every token pick the same experts,
|
||||
# which lead to some EP ranks receiving no tokens and skipping their
|
||||
# MoE kernel entirely, and cause hang due to all-reduce collective
|
||||
# during synchronized autotuning.
|
||||
if _all_ranks_have_matching_cache(cache_path, tune_group):
|
||||
loaded = tuner.load_configs(str(cache_path))
|
||||
if tune_group.world_size > 1:
|
||||
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
|
||||
torch.distributed.all_gather_object(
|
||||
loaded_by_rank, loaded, group=tune_group.cpu_group
|
||||
cache_root = resolve_flashinfer_autotune_v2_root()
|
||||
if is_leader:
|
||||
logger.info_once(
|
||||
"Using FlashInfer managed autotune cache (root=%s)",
|
||||
cache_root if cache_root is not None else "flashinfer default",
|
||||
)
|
||||
loaded = all(loaded_by_rank)
|
||||
if not loaded:
|
||||
tuner.clear_cache()
|
||||
autotune_context = autotune_v2(
|
||||
mode="tune", cache_root=cache_root, **autotune_kwargs
|
||||
)
|
||||
else:
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
# The world group only spans DP ranks when vLLM folds DP into it.
|
||||
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
|
||||
cache_path = cache_path.with_name(
|
||||
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
|
||||
)
|
||||
if is_leader:
|
||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
|
||||
if _all_ranks_have_matching_cache(cache_path, tune_group):
|
||||
loaded = tuner.load_configs(str(cache_path))
|
||||
if tune_group.world_size > 1:
|
||||
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
|
||||
torch.distributed.all_gather_object(
|
||||
loaded_by_rank, loaded, group=tune_group.cpu_group
|
||||
)
|
||||
loaded = all(loaded_by_rank)
|
||||
if not loaded:
|
||||
tuner.clear_cache()
|
||||
autotune_context = fi_utils.autotune(tune_mode=True, **autotune_kwargs)
|
||||
|
||||
group = tune_group.cpu_group if tune_group.world_size > 1 else None
|
||||
set_autotune_process_group(group)
|
||||
try:
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
|
||||
):
|
||||
with torch.inference_mode(), autotune_context:
|
||||
hisparse_enabled = (
|
||||
runner.vllm_config.attention_config.hisparse_config is not None
|
||||
)
|
||||
@@ -498,7 +507,10 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
|
||||
if world.world_size > 1:
|
||||
world.barrier()
|
||||
# Skip the rewrite when nothing was tuned this start (every entry came from
|
||||
# the file). FlashInfer gates its own autotune(cache=...) save the same way.
|
||||
if tuner._dirty:
|
||||
if use_autotune_v2:
|
||||
if world.world_size > 1:
|
||||
# All ranks serve the shared store's final state after publishing.
|
||||
autotune_v2_reload()
|
||||
world.barrier()
|
||||
elif tuner._dirty:
|
||||
tuner.save_configs(str(cache_path))
|
||||
|
||||
@@ -19,9 +19,11 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None:
|
||||
def _get_kda_layer(
|
||||
worker: Worker, module: str = "vllm.models.kimi_k3.nvidia.kda"
|
||||
) -> KimiK3DeltaAttention | None:
|
||||
# Kimi model construction already imports kda. Avoid importing it here.
|
||||
kda = sys.modules.get("vllm.models.kimi_k3.nvidia.kda")
|
||||
kda = sys.modules.get(module)
|
||||
if kda is None:
|
||||
return None
|
||||
|
||||
@@ -172,9 +174,59 @@ def _warm_recurrent_kda(
|
||||
)
|
||||
|
||||
|
||||
def _warm_gluon_chunk_kda(layer, input_dtype: torch.dtype) -> None:
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import (
|
||||
chunk_kda_prefill,
|
||||
get_gluon_chunk_kda_warmup_profiles,
|
||||
)
|
||||
|
||||
if layer.kda_prefill_backend != "gluon":
|
||||
return
|
||||
kv_cache = layer.kv_cache
|
||||
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
|
||||
return
|
||||
state = kv_cache[1]
|
||||
if not isinstance(state, torch.Tensor) or not state.numel():
|
||||
return
|
||||
|
||||
logger.info("Warming up Kimi-K3 Gluon chunk KDA prefill kernels.")
|
||||
h, d = int(layer.local_num_heads), int(layer.head_dim)
|
||||
for num_seqs, seq_len in get_gluon_chunk_kda_warmup_profiles():
|
||||
num_tokens = num_seqs * seq_len
|
||||
shape = (1, num_tokens, h, d)
|
||||
chunk_kda_prefill(
|
||||
q=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
k=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
v=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
raw_g=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
raw_beta=torch.zeros(
|
||||
(1, num_tokens, h), dtype=input_dtype, device=state.device
|
||||
),
|
||||
A_log=layer.A_log,
|
||||
g_bias=layer.dt_bias,
|
||||
lower_bound=layer.gate_lower_bound,
|
||||
state_cache=state,
|
||||
state_indices=torch.zeros(num_seqs, dtype=torch.int32, device=state.device),
|
||||
has_initial_state=torch.zeros(
|
||||
num_seqs, dtype=torch.bool, device=state.device
|
||||
),
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=torch.arange(
|
||||
0, num_tokens + 1, seq_len, dtype=torch.int32, device=state.device
|
||||
),
|
||||
backend="gluon",
|
||||
out=torch.empty(shape, dtype=input_dtype, device=state.device),
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def kimi_k3_triton_warmup(worker: Worker) -> None:
|
||||
"""Warm Kimi-K3 Triton kernels reachable by this server."""
|
||||
if current_platform.is_rocm():
|
||||
layer = _get_kda_layer(worker, "vllm.models.kimi_k3.amd.kda")
|
||||
if layer is not None:
|
||||
_warm_gluon_chunk_kda(layer, worker.model_config.dtype)
|
||||
return
|
||||
if not current_platform.is_cuda():
|
||||
return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user