[Bugfix] Support non-power-of-2 top_k in legacy triton_kernels routing (#46406)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Woosuk Kwon
2026-06-24 13:52:09 -07:00
committed by GitHub
co-authored by Claude
parent d6696e2385
commit 6a1570711c
@@ -220,6 +220,241 @@ def _patch_make_bitmatrix_metadata() -> None:
_bm.make_bitmatrix_metadata = _make_bitmatrix_metadata_pow2_safe
def _patch_legacy_routing_for_nonpow2_topk() -> None:
"""Monkey-patch the legacy (v3.5.1) triton_kernels routing path to support
non-power-of-2 top_k (e.g. DeepSeek-V4 top_k=6).
The bundled ``_routing_compute_indx`` does ``tl.arange(0, N_EXPTS_ACT *
BLOCK_M)``, which fails to compile when ``N_EXPTS_ACT`` (top_k) is not a
power of 2 (6 * 32 = 192). This installs a pow2-safe variant that pads the
``tl.arange`` to the next power of 2, strides by the real per-block size,
and masks the padded tail so it neither loads the next block's gates nor
writes any output. For power-of-2 top_k it is identical to the original.
A matching ``sort_tokens`` is installed that threads the padded size into
the patched kernel. Only needed on the legacy path; the v3.6+ SparseMatrix
path is handled by ``_patch_make_bitmatrix_metadata``.
"""
import triton
import triton.language as tl
# Import via the `triton_kernels` alias (set up by has_triton_kernels) so
# we patch the SAME module object that `make_routing_data` consumes. The
# `vllm.third_party.triton_kernels.routing` path is a *different* module
# object under the import alias, so patching it would have no effect.
try:
import triton_kernels.routing as _routing
from triton_kernels.routing_details import _routing_compute as _rc
except ImportError:
return
_keyed_add = _rc._keyed_add
_expt_data_compute = _rc._expt_data_compute
@triton.jit
def _routing_compute_indx_pow2(
pid_m,
GatherIndx,
ScatterIndx,
GateScal,
ExptScal,
ExptIndx,
PartialOffs,
stride_pm,
stride_pn,
TokensStart,
n_tokens,
BLOCK_M: tl.constexpr,
N_EXPTS_ACT: tl.constexpr,
BLOCK_SIZE_PADDED: tl.constexpr,
):
if isinstance(n_tokens, tl.tensor) and n_tokens.dtype.is_ptr():
n_tokens = tl.load(n_tokens)
n_gates = n_tokens * N_EXPTS_ACT
BLOCK_SIZE: tl.constexpr = N_EXPTS_ACT * BLOCK_M
tl.static_assert(BLOCK_SIZE_PADDED <= 32768)
local_offs = tl.arange(0, BLOCK_SIZE_PADDED)
offs = pid_m * BLOCK_SIZE + local_offs
expert = tl.load(
ExptIndx + offs,
mask=(local_offs < BLOCK_SIZE) & (offs < n_gates),
other=-1,
).to(tl.uint32)
kv_pairs = ((expert << 16) | local_offs).to(tl.uint32)
kv_pairs = tl.sort(kv_pairs, 0)
expert = kv_pairs >> 16
offs = pid_m * BLOCK_SIZE + (kv_pairs & 0xFFFF)
mask = expert != 0xFFFF
gate_scal = tl.load(ExptScal + offs, mask=mask)
x = kv_pairs & 0xFFFF0000 | 0x00000001
run_lengths = tl.associative_scan(x, 0, _keyed_add)
exclusive_run_lengths = (run_lengths - 1) & 0xFFFF
gates = tl.load(PartialOffs + pid_m * stride_pm + expert * stride_pn, mask=mask)
gates += tl.load(TokensStart + expert, mask=mask)
gates += exclusive_run_lengths
tl.store(ScatterIndx + offs, gates, mask=mask)
tl.store(GatherIndx + gates, offs, mask=mask)
tl.store(GateScal + gates, gate_scal, mask=mask)
@triton.jit
def _combined_routing_compute_pow2(
GatherIndx,
ScatterIndx,
GateScal,
ExptScal,
ExptIndx,
PartialOffs,
stride_pm,
stride_pn,
TokensStart,
n_tokens,
BLOCK_M: tl.constexpr,
N_EXPTS_ACT: tl.constexpr,
Hist,
MDTileStarts,
tile_starts_stridem,
MDTileInfo,
tile_info_stridem,
first_tile_dim_log2,
SIZES: tl.constexpr,
BLOCK: tl.constexpr,
blocks2a,
BLOCK_SIZE_PADDED: tl.constexpr,
):
pid = tl.program_id(0)
if pid < blocks2a:
_expt_data_compute(
Hist,
MDTileStarts,
tile_starts_stridem,
MDTileInfo,
tile_info_stridem,
first_tile_dim_log2,
SIZES,
BLOCK,
)
else:
pid -= blocks2a
_routing_compute_indx_pow2(
pid,
GatherIndx,
ScatterIndx,
GateScal,
ExptScal,
ExptIndx,
PartialOffs,
stride_pm,
stride_pn,
TokensStart,
n_tokens,
BLOCK_M,
N_EXPTS_ACT,
BLOCK_SIZE_PADDED,
)
def _sort_tokens_pow2(expt_scal, expt_indx, n_expts_tot, bitmatrix):
import torch
HIST_BLOCK_M = 32
INDX_OFFS_BLOCK_M = 512
MEMSET_BLOCK = 1024
cdiv = triton.cdiv
device = expt_scal.device
dtype = expt_scal.dtype
n_tokens_raw, _ = bitmatrix.shape
n_tokens_pad, n_expts_act = expt_scal.shape
n_gates_pad = n_tokens_pad * n_expts_act
# pad per-block gate count (HIST_BLOCK_M * top_k) up to a pow2.
block_size_padded = triton.next_power_of_2(HIST_BLOCK_M * n_expts_act)
hist, partial_hist = bitmatrix.sum(partials_block_size=HIST_BLOCK_M)
hist = hist[:n_expts_tot]
expt_offs = torch.empty(n_expts_tot, dtype=torch.int32, device=device)
combined_indx = torch.empty(n_gates_pad * 2, dtype=torch.int32, device=device)
topk_indx = combined_indx[:n_gates_pad]
gate_indx = combined_indx[n_gates_pad:]
gate_scal = torch.empty(n_gates_pad, dtype=dtype, device=device)
(
token_offs_combined,
token_offs_raw,
token_offs_pad,
block_pid_map,
blocks1a,
blocks2a,
MEMSET_BLOCK_A,
HIST2_BLOCK_M,
block_m_log2_start,
block_m_num,
) = _routing._compute_expt_data_internal(hist, n_expts_tot, n_gates_pad)
blocks1b = cdiv(n_gates_pad * 2, MEMSET_BLOCK) + n_expts_tot + 1
blocks2b = cdiv(n_tokens_pad, HIST_BLOCK_M)
_rc._combined_routing_memset[(blocks1a + blocks1b,)](
combined_indx,
n_gates_pad * 2,
-1,
MEMSET_BLOCK,
hist,
expt_offs,
hist.shape[0],
n_expts_tot,
partial_hist,
partial_hist.shape[0],
partial_hist.stride(0),
partial_hist.stride(1),
token_offs_combined,
token_offs_combined.stride(0),
blocks1a,
block_pid_map,
block_m_log2_start,
SIZES=block_m_num,
BLOCK_A=MEMSET_BLOCK_A,
BLOCK_N=512,
BLOCK_M=INDX_OFFS_BLOCK_M,
)
indx_offs = partial_hist
_combined_routing_compute_pow2[(blocks2a + blocks2b,)](
topk_indx,
gate_indx,
gate_scal,
expt_scal,
expt_indx,
indx_offs,
indx_offs.stride(0),
indx_offs.stride(1),
expt_offs,
n_tokens_raw,
HIST_BLOCK_M,
n_expts_act,
hist,
token_offs_pad,
token_offs_pad.stride(0),
block_pid_map,
block_pid_map.stride(0),
block_m_log2_start,
block_m_num,
HIST2_BLOCK_M,
blocks2a,
block_size_padded,
)
return (
hist,
topk_indx,
gate_indx,
gate_scal,
token_offs_raw,
token_offs_pad,
block_pid_map,
)
# `routing_from_bitmatrix` looks up `sort_tokens` via the routing module
# global, so replacing it here redirects the legacy path to the pow2 kernel.
_routing.sort_tokens = _sort_tokens_pow2
# Two API generations of triton_kernels are supported:
# - v3.5.1 (the version bundled with vLLM): exposes `routing()` and
# `routing_from_bitmatrix()` in triton_kernels.routing; the `Bitmatrix`
@@ -260,6 +495,9 @@ if has_triton_kernels():
use_legacy_triton_kernels = True
if not use_legacy_triton_kernels:
_patch_make_bitmatrix_metadata()
else:
# Legacy routing fails to compile for non-pow2 top_k (DeepSeek-V4).
_patch_legacy_routing_for_nonpow2_topk()
except (AttributeError, ImportError) as e:
logger.error(
"Failed to import Triton kernels. Please make sure your triton "