Merge branch 'main' into patches

This commit is contained in:
2026-10-17 08:06:56 +00:00
222 changed files with 14823 additions and 3606 deletions
@@ -2024,6 +2024,7 @@ def build_mla_chunked_context_metadata(
dcp_local_block_size: int,
dcp_virtual_block_size: int,
dcp_manager: MLADCPManager | None = None,
align_indptr: bool = False,
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
"""Build chunked-context metadata for an MLA prefill.
@@ -2043,6 +2044,7 @@ def build_mla_chunked_context_metadata(
dcp_local_block_size: Per-rank interleave block size for DCP.
dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``.
dcp_manager: Shared MLA DCP collective manager.
align_indptr: Align packed int32 indptr slices to 16 bytes for CuTe DSL.
Returns:
The chunked-context metadata, or None when no prefill has any context.
@@ -2123,6 +2125,15 @@ def build_mla_chunked_context_metadata(
num_tokens = sum(plan.seq_lens)
num_local_tokens = num_tokens
if align_indptr:
# CuTe DSL requires 16-byte-aligned int32 indptrs.
padding = -boundary_offset % 4
cu_seq_lens_flat.extend([0] * padding)
cu_seqlens_q_flat.extend([0] * padding)
if use_dcp:
padded_local_cu_seq_lens_flat.extend([0] * padding)
boundary_offset += padding
seq_lens_flat.extend(plan.seq_lens)
cu_seq_lens_flat.extend(itertools.accumulate(plan.seq_lens, initial=0))
query_base = query_start_loc[plan.request_start]
@@ -2616,6 +2627,14 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
query_start_loc_cpu[reqs_start:] - query_start_loc_cpu[reqs_start]
)
align_indptr = False
if (
self._prefill_backend.get_name() == "TRTLLM_RAGGED"
and self.vllm_config.attention_config.trtllm_ragged_backend
== "cute-dsl"
):
align_indptr = True
chunked_context_metadata = build_mla_chunked_context_metadata(
context_lens_cpu=context_lens_cpu,
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
@@ -2628,6 +2647,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
dcp_local_block_size=self.dcp_local_block_size,
dcp_virtual_block_size=self.dcp_virtual_block_size,
dcp_manager=self.dcp_manager,
align_indptr=align_indptr,
)
prefill_metadata = MLACommonPrefillMetadata(
@@ -359,13 +359,6 @@ class ExpertMapManager:
return True
return self._expert_map[global_id] != -1
def get_local_expert_ids(self) -> list[int]:
"""Get list of global IDs for experts on this rank."""
if self._expert_map is None:
return list(range(self.global_num_experts))
return torch.where(self._expert_map != -1)[0].tolist()
def update(
self,
moe_parallel_config: FusedMoEParallelConfig,
@@ -545,7 +545,12 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
return (workspace1, workspace2, output)
def _act_mul_quant(
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
self,
input: torch.Tensor,
output: torch.Tensor,
activation: MoEActivation,
expert_ends: torch.Tensor | None = None,
expert_alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
block_k = self._ACT_BLOCK_K
scale_fmt = DeepGemmQuantScaleFMT.from_oracle()
@@ -561,6 +566,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
output_q=output,
group_size=block_k,
clamp_limit=self.gemm1_clamp_limit,
expert_ends=expert_ends,
expert_alignment=expert_alignment,
)
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
return silu_mul_per_token_group_quant_fp8_colmajor(
@@ -634,7 +641,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
a1q_perm = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
)
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
use_psum_layout = current_platform.is_device_capability_family(100)
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
aq=a1q,
aq_scale=a1q_scale,
topk_ids=topk_ids,
@@ -642,6 +650,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
expert_map=expert_map,
expert_tokens_meta=expert_tokens_meta,
aq_out=a1q_perm,
use_psum_layout=use_psum_layout,
)
assert a1q.size(0) == M_sum
@@ -655,18 +664,23 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
(a1q, a1q_scale),
(w1.view(torch.int8), self.w1_scale),
mm1_out,
expert_ids,
grouped_layout,
use_psum_layout=use_psum_layout,
recipe_a=(1, self._ACT_BLOCK_K),
recipe_b=(1, self._WEIGHT_BLOCK_K),
)
# SwiGLU activation + FP8 requant
# SiLU+mul and FP8 requant use the same live expert ranges as GEMM.
activation_out_dim = self.adjust_N_for_activation(N, activation)
quant_out = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
)
a2q, a2q_scale = self._act_mul_quant(
input=mm1_out.view(-1, N), output=quant_out, activation=activation
input=mm1_out.view(-1, N),
output=quant_out,
activation=activation,
expert_ends=grouped_layout if use_psum_layout else None,
expert_alignment=align_used if use_psum_layout else 0,
)
# FC2: FP8 activations x FP4 weights
@@ -675,7 +689,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
(a2q, a2q_scale),
(w2.view(torch.int8), self.w2_scale),
mm2_out,
expert_ids,
grouped_layout,
use_psum_layout=use_psum_layout,
recipe_a=(1, self._ACT_BLOCK_K),
recipe_b=(1, self._WEIGHT_BLOCK_K),
)
@@ -517,6 +517,7 @@ _TILE_CONFIG_OVERRIDES = {
256: (128, 32),
512: (128, 19),
},
(5120, 256): {64: (64, 64)}, # GB200
(6144, 128): { # MiniMax-M3
64: (32, 64),
128: (64, 64),
@@ -533,7 +534,7 @@ def _pick_tile_config(N: int, K: int, M: int, num_sms: int) -> tuple[int, int]:
if 32 < N <= 512:
token_bucket = triton.next_power_of_2(N)
overrides = _TILE_CONFIG_OVERRIDES.get((K, M))
if overrides is not None:
if overrides is not None and token_bucket in overrides:
BN, split_k = overrides[token_bucket]
return BN, min(split_k, k_tiles)
@@ -609,6 +610,8 @@ def _bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
N, K = X.shape
M, _ = W.shape
if N == 0:
return X.new_empty((N, M), dtype=torch.float32)
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
BN, split_k = _pick_tile_config(N, K, M, num_sms)
@@ -24,8 +24,8 @@ class GateLinear(ReplicatedLinear):
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
any shape w/ K divisible by 8)
2. fp32 specialized kernel (bf16/fp32 in, fp32 out, M<=32; SM90+
model-specific shapes, gfx950 any shape)
2. fp32 specialized kernel (bf16/fp32 in, fp32 out; SM90+
model-specific shapes and batch limits, gfx950 any shape with M<=32)
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
any shape)
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
@@ -46,9 +46,14 @@ class GateLinear(ReplicatedLinear):
split-K from the token count and skip the batch-invariant linear path.
"""
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
# (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128)}
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel.
FP32_SUPPORTED_SHAPES = {
(3072, 256),
(6144, 128),
(5120, 256),
(4096, 352),
(8192, 256),
}
FP32_MAX_TOKENS = 32
# Largest batch for which tier 1 beats the cuBLAS epilogue. Subclasses
# whose router shape stays ahead further up the range may raise it.
@@ -454,9 +459,9 @@ def fp32_router_gemm_dispatch_impl(
weight: torch.Tensor,
allow_bf16x3_router_gemm: bool,
) -> torch.Tensor:
"""Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS,
otherwise optionally run the experimental BF16x3 kernel for medium/large
SM100 router batches, then fall back to F.linear.
"""Run the FP32 kernel for small batches, with shape-specific limits.
Larger SM100 BF16 batches use BF16x3; other inputs fall back to F.linear.
This must be wrapped in a custom op because our torch.compile integration
does not support runtime dispatching on num_tokens.
"""
@@ -473,7 +478,15 @@ def fp32_router_gemm_dispatch_impl(
if weight.dtype == torch.bfloat16:
return torch.mm(x, weight.T, out_dtype=torch.float32)
return torch.nn.functional.linear(x.float(), weight)
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
max_tokens = _FP32_ROUTER_GEMM_MAX_TOKENS
if (
allow_bf16x3_router_gemm
and x.dtype == torch.bfloat16
and weight.shape == (256, 5120)
):
# GB200: four token groups win through 48 for multiples of eight.
max_tokens = 48 if x.shape[0] % 8 == 0 else 16
if x.shape[0] <= max_tokens:
return ops.fp32_router_gemm(x, weight)
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
@@ -9,139 +9,162 @@
from packaging import version
from vllm.model_executor.layers.mamba.ops.triton_helpers import fast_exp
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
TRITON_22 = version.parse(triton.__version__) >= version.parse("2.2.0")
def _drop_racy_wgmma_configs(configs: list[triton.Config]) -> list[triton.Config]:
"""Drop the configs that Triton miscompiles on sm90.
With whole warpgroups (num_warps % 4 == 0) and BLOCK_SIZE_M % 64 == 0,
`tl.dot(cb, x)` lowers to wgmma with `cb` in registers. With
num_stages > 1, Triton leaves that wgmma running across the loop
back-edge while the next iteration overwrites the registers of `cb`,
which writes NaN/Inf into whole output rows.
"""
if not (current_platform.is_cuda() and current_platform.is_device_capability(90)):
return configs
return [
c
for c in configs
if c.num_stages == 1
or c.num_warps % 4 != 0
or c.kwargs["BLOCK_SIZE_M"] % 64 != 0
]
@triton.autotune(
configs=[
# =================================================================
# Higher warp count configs for better latency hiding
# More warps = more instructions in flight = better memory latency hiding
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
# Smaller tiles with more stages for software pipelining
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
# =================================================================
# Low register pressure configs (num_stages=1) for large dstate
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
# num_stages=2 configs - moderate register pressure
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
# Original configs for larger dstate values
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
num_stages=3,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=2,
),
],
configs=_drop_racy_wgmma_configs(
[
# =================================================================
# Higher warp count configs for better latency hiding
# More warps = more instructions in flight = better memory latency hiding
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
# Smaller tiles with more stages for software pipelining
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
# =================================================================
# Low register pressure configs (num_stages=1) for large dstate
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
# num_stages=2 configs - moderate register pressure
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
# Original configs for larger dstate values
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
num_stages=3,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=2,
),
]
),
key=["chunk_size", "hdim", "dstate", "IS_CAUSAL"],
)
@triton.jit
@@ -12,35 +12,6 @@ from vllm.triton_utils import tl, triton
logger = logging.getLogger(__name__)
def block_dequant(
x_q_block: torch.Tensor,
x_s: torch.Tensor,
block_size: list[int],
) -> torch.Tensor:
"""This function conducts block-wise dequantization.
The inputs are block-wise quantization tensor `x_q_block`,
block-wise quantization scale and the block size.
The outputs are dequantized tensor.
"""
block_n, block_k = block_size[0], block_size[1]
n, k = x_q_block.shape
n_tiles = (n + block_n - 1) // block_n
k_tiles = (k + block_k - 1) // block_k
assert n_tiles == x_s.shape[0]
assert k_tiles == x_s.shape[1]
x_dq_block = x_q_block.to(torch.float32)
for i in range(k_tiles):
for j in range(n_tiles):
x_dq_block[
j * block_n : min((j + 1) * block_n, n),
i * block_k : min((i + 1) * block_k, k),
] *= x_s[j][i]
return x_dq_block
if current_platform.is_rocm():
@triton.jit