[Qwen3.8-Flash-Next] Fuse Qwen4Exp PLE kernels (#54517)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Thien Tran
2026-09-02 12:32:16 +08:00
committed by GitHub
co-authored by Codex
parent 05201d8ecf
commit f870b92976
6 changed files with 1836 additions and 594 deletions
File diff suppressed because it is too large Load Diff
+9 -5
View File
@@ -141,9 +141,9 @@ _QWEN4_EXP_IGNORED_MISSING_SUFFIXES = [
"_input_scale",
]
# The checkpoint keeps down and injection projections separate; runtime packs
# them into adjacent logical shards of one MergedColumnParallelLinear.
_HC_WEIGHTS_MAPPER = WeightsMapper(
# The checkpoint stores these projections separately; runtime packs each group
# into adjacent logical shards of a MergedColumnParallelLinear.
_EXTRA_WEIGHTS_MAPPER = WeightsMapper(
orig_to_new_stacked={
"hyper_connection.input_mix_weight_down.weight": (
"hyper_connection.input_mix_weight_down_block_inject.weight",
@@ -153,6 +153,8 @@ _HC_WEIGHTS_MAPPER = WeightsMapper(
"hyper_connection.input_mix_weight_down_block_inject.weight",
1,
),
"ple.key_proj": ("ple.kv_proj", 0),
"ple.value_proj": ("ple.kv_proj", 1),
}
)
@@ -389,7 +391,7 @@ class Qwen4ExpMixtureOfExperts(MixtureOfExperts):
}
)
class Qwen4ExpModel(nn.Module):
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
@@ -612,6 +614,7 @@ class Qwen4ExpForCausalLM(
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
"kv_proj": ["key_proj", "value_proj"],
"in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
"in_proj_ba": ["in_proj_b", "in_proj_a"],
"input_mix_weight_down_block_inject": [
@@ -848,11 +851,12 @@ class Qwen4ExpForConditionalGeneration(
requires_raw_input_tokens = True
packed_modules_mapping = Qwen3_5ForConditionalGeneration.packed_modules_mapping | {
"kv_proj": ["key_proj", "value_proj"],
"input_mix_weight_down_block_inject": [
"input_mix_weight_down",
"block_inject_weight",
"_input_mix_padding",
]
],
}
@staticmethod
+2 -2
View File
@@ -52,7 +52,7 @@ from vllm.transformers_utils.configs.qwen4_exp import (
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .model import (
_HC_WEIGHTS_MAPPER,
_EXTRA_WEIGHTS_MAPPER,
_QWEN4_EXP_IGNORED_MISSING_SUFFIXES,
Qwen4ExpDecoderLayer,
Qwen4ExpMixtureOfExperts,
@@ -157,7 +157,7 @@ def _make_draft_vllm_config(
}
)
class Qwen4ExpMultiTokenPredictor(nn.Module):
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
+669
View File
@@ -0,0 +1,669 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused Qwen4Exp PLE kernels.
N-gram IDs hash each suffix with
``offset[h] + (xor_i(token[t-i] * multiplier[i]) % size[h])``. The gate computes
``d = dot(RMSNorm(key), RMSNorm(query)) / sqrt(H)`` and
``g = sigmoid(sign(d) * sqrt(max(abs(d), 1e-6)))``. Short convolution adds
``silu(sum_k weight[k] * history[t + k * dilation])`` to the gated output and
updates its persistent history.
"""
from typing import Literal
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
# N-gram ID generation
@triton.jit(do_not_specialize=["num_tokens", "num_reqs", "binary_search_iters"])
def _ple_ngram_ids_kernel(
input_ids_ptr,
qsl_ptr,
ctx_ptr,
multipliers_ptr,
sizes_ptr,
offsets_ptr,
out_ptr,
num_tokens,
num_reqs,
eos_token_id,
binary_search_iters,
NGRAM_CONTEXT_LEN: tl.constexpr,
HEADS_PER_NGRAM: tl.constexpr,
BLOCK_T: tl.constexpr,
launch_pdl: tl.constexpr,
):
NGRAM_HEADS: tl.constexpr = NGRAM_CONTEXT_LEN * HEADS_PER_NGRAM
BLOCK_H: tl.constexpr = triton.next_power_of_2(NGRAM_HEADS)
pid = tl.program_id(0)
token_offsets = pid * BLOCK_T + tl.arange(0, BLOCK_T)
token_mask = token_offsets < num_tokens
if launch_pdl:
tl.extra.cuda.gdc_wait()
# A flattened token offset does not identify its request when query chunks
# have different lengths. Binary-search the request boundaries instead.
p_lo = tl.full([BLOCK_T], 1, tl.int32)
p_hi = tl.full([BLOCK_T], num_reqs + 1, tl.int32)
for _ in range(binary_search_iters):
mid = (p_lo + p_hi) // 2
qmid = tl.load(qsl_ptr + mid, mask=token_mask & (mid <= num_reqs), other=0)
pred = qmid <= token_offsets
p_lo = tl.where(pred, mid + 1, p_lo)
p_hi = tl.where(pred, p_hi, mid)
req = tl.minimum(p_lo - 1, num_reqs - 1).to(tl.int64)
request_start = tl.load(qsl_ptr + req, mask=token_mask, other=0)
chunk_pos = token_offsets - request_start
# Initialize every head with the current-token term.
current_token = tl.load(input_ids_ptr + token_offsets, mask=token_mask, other=0).to(
tl.int64
)
current_multiplier = tl.load(multipliers_ptr)
mixed = current_token[:, None] * current_multiplier
g = tl.arange(0, BLOCK_H)
head_mask = g < NGRAM_HEADS
ngram_order = g // HEADS_PER_NGRAM + 2
# Walk predecessors from newest to oldest. At chunk boundaries they come
# from ngram_context; otherwise they come from this step's input_ids.
crossed = tl.zeros([BLOCK_T], tl.int1)
for shift in tl.static_range(1, NGRAM_CONTEXT_LEN + 1):
in_step = chunk_pos >= shift
ctx_col = NGRAM_CONTEXT_LEN - shift + chunk_pos
step_token = tl.load(
input_ids_ptr + token_offsets - shift,
mask=token_mask & in_step,
other=0,
)
context_token = tl.load(
ctx_ptr + req * NGRAM_CONTEXT_LEN + ctx_col,
mask=token_mask & (~in_step),
other=0,
)
candidate = tl.where(in_step, step_token, context_token).to(tl.int64)
# Older positions remain behind the first EOS boundary.
candidate = tl.where(crossed, eos_token_id, candidate)
crossed = crossed | (candidate == eos_token_id)
multiplier = tl.load(multipliers_ptr + shift)
term = candidate[:, None] * multiplier
mixed = mixed ^ tl.where((ngram_order > shift)[None, :], term, 0)
# Map each hash into its embedding-table partition.
sizes = tl.load(sizes_ptr + g, mask=head_mask, other=1)[None, :]
head_offsets = tl.load(offsets_ptr + g, mask=head_mask, other=0)[None, :]
# Hash products may overflow int64; preserve torch.remainder semantics.
remainders = mixed % sizes
remainders = tl.where(remainders < 0, remainders + sizes, remainders)
ids = remainders + head_offsets
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(
out_ptr + token_offsets[:, None] * NGRAM_HEADS + g[None, :],
ids,
mask=token_mask[:, None] & head_mask[None, :],
)
def _ple_ngram_ids(
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
layer_multipliers: torch.Tensor,
ngram_heads_vocab_sizes: torch.Tensor,
ngram_heads_offsets: torch.Tensor,
output: torch.Tensor,
eos_token_id: int,
heads_per_ngram: int,
) -> None:
input_ids = input_ids.reshape(-1)
num_tokens = input_ids.shape[0]
num_reqs = query_start_loc.numel() - 1
ctx_len = ngram_context.shape[1]
BLOCK_T = 8
launch_pdl = current_platform.is_arch_support_pdl()
_ple_ngram_ids_kernel[(triton.cdiv(num_tokens, BLOCK_T),)](
input_ids,
query_start_loc,
ngram_context,
layer_multipliers,
ngram_heads_vocab_sizes,
ngram_heads_offsets,
output,
num_tokens,
num_reqs,
eos_token_id,
binary_search_iters=num_reqs.bit_length(),
NGRAM_CONTEXT_LEN=ctx_len,
HEADS_PER_NGRAM=heads_per_ngram,
BLOCK_T=BLOCK_T,
launch_pdl=launch_pdl,
num_warps=4,
)
def ple_ngram_ids(
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
layer_multipliers: torch.Tensor,
ngram_heads_vocab_sizes: torch.Tensor,
ngram_heads_offsets: torch.Tensor,
eos_token_id: int,
heads_per_ngram: int,
output: torch.Tensor | None = None,
) -> torch.Tensor:
if output is None:
output = torch.empty(
(input_ids.numel(), ngram_context.shape[1] * heads_per_ngram),
dtype=torch.int64,
device=input_ids.device,
)
_ple_ngram_ids(
input_ids,
query_start_loc,
ngram_context,
layer_multipliers,
ngram_heads_vocab_sizes,
ngram_heads_offsets,
output,
eos_token_id,
heads_per_ngram,
)
return output
# Gating and normalization
@triton.jit
def _ple_gate_kernel(
key_ptr, # [T, HC*H] with row stride key_rs
value_ptr, # [T, H] with row stride value_rs
hidden_ptr, # [T, HC*H]
nk_ptr, # [HC*H] norm_key weight
nq_ptr, # [HC*H] norm_query weight
ncw_ptr, # [HC*H] norm_conv weight
gated_ptr, # [T, HC*H] out
normed_ptr, # [T, HC*H] out
key_rs,
value_rs,
eps,
H: tl.constexpr,
HC: tl.constexpr,
BLOCK_H: tl.constexpr,
launch_pdl: tl.constexpr,
):
t = tl.program_id(0)
s = tl.program_id(1)
lanes = tl.arange(0, BLOCK_H)
mask = lanes < H
offs = s * H + lanes
dtype: tl.constexpr = key_ptr.dtype.element_ty
if launch_pdl:
tl.extra.cuda.gdc_wait()
k = tl.load(key_ptr + t * key_rs + offs, mask=mask, other=0.0).to(tl.float32)
q = tl.load(hidden_ptr + t * HC * H + offs, mask=mask, other=0.0).to(tl.float32)
nk = tl.load(nk_ptr + offs, mask=mask, other=0.0).to(tl.float32)
nq = tl.load(nq_ptr + offs, mask=mask, other=0.0).to(tl.float32)
# Match eager materialization at each intermediate tensor boundary.
k_n = (k * tl.rsqrt(tl.sum(k * k) / H + eps) * (1.0 + nk)).to(dtype)
q_n = (q * tl.rsqrt(tl.sum(q * q) / H + eps) * (1.0 + nq)).to(dtype)
products = (k_n.to(tl.float32) * q_n.to(tl.float32)).to(dtype)
dot = tl.sum(products.to(tl.float32)).to(dtype).to(tl.float32)
d = dot / tl.sqrt(float(H))
d = d.to(dtype).to(tl.float32)
sign = tl.where(d < 0, -1.0, 0.0)
sign = tl.where(d > 0, 1.0, sign)
magnitude = tl.sqrt(tl.maximum(tl.abs(d), 1e-6)).to(dtype)
g = tl.sigmoid(sign * magnitude)
g = g.to(dtype).to(tl.float32)
v = tl.load(value_ptr + t * value_rs + lanes, mask=mask, other=0.0).to(tl.float32)
gated = (g * v).to(dtype)
gf = gated.to(tl.float32)
ncw = tl.load(ncw_ptr + offs, mask=mask, other=0.0).to(tl.float32)
normed = gf * tl.rsqrt(tl.sum(gf * gf) / H + eps) * (1.0 + ncw)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(gated_ptr + t * HC * H + offs, gated, mask=mask)
tl.store(normed_ptr + t * HC * H + offs, normed, mask=mask)
def _ple_gate(
key: torch.Tensor,
value: torch.Tensor,
hidden: torch.Tensor,
norm_key_w: torch.Tensor,
norm_query_w: torch.Tensor,
norm_conv_w: torch.Tensor,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens = hidden.shape[0]
h = value.shape[-1]
hc = hidden.shape[-1] // h
assert key.stride(1) == 1 and value.stride(1) == 1
assert hidden.is_contiguous()
if key.dtype != value.dtype or key.dtype != hidden.dtype:
raise ValueError("key, value, and hidden must have the same dtype")
if key.dtype not in (torch.bfloat16, torch.float16):
raise ValueError("PLE gate supports BF16 and FP16 inputs")
gated = torch.empty_like(hidden)
normed = torch.empty_like(hidden)
_ple_gate_kernel[(num_tokens, hc)](
key,
value,
hidden,
norm_key_w,
norm_query_w,
norm_conv_w,
gated,
normed,
key.stride(0),
value.stride(0),
eps,
H=h,
HC=hc,
BLOCK_H=triton.next_power_of_2(h),
num_warps=4,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return gated, normed
def _ple_gate_fake(
key: torch.Tensor,
value: torch.Tensor,
hidden: torch.Tensor,
norm_key_w: torch.Tensor,
norm_query_w: torch.Tensor,
norm_conv_w: torch.Tensor,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.empty_like(hidden), torch.empty_like(hidden)
direct_register_custom_op(
op_name="qwen4_exp_ple_gate",
op_func=_ple_gate,
mutates_args=[],
fake_impl=_ple_gate_fake,
)
def ple_gate(
key: torch.Tensor,
value: torch.Tensor,
hidden: torch.Tensor,
norm_key_w: torch.Tensor,
norm_query_w: torch.Tensor,
norm_conv_w: torch.Tensor,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops.vllm.qwen4_exp_ple_gate(
key, value, hidden, norm_key_w, norm_query_w, norm_conv_w, eps
)
# Dilated short convolution
@triton.jit(do_not_specialize=["num_reqs", "bs_iters", "has_token_map"])
def _ple_conv_kernel(
x_ptr,
state_ptr,
w_ptr,
residual_ptr,
state_idx_ptr,
qsl_ptr,
num_acc_ptr,
has_init_ptr,
token_idx_ptr,
has_token_map,
num_reqs,
bs_iters,
state_bs,
state_ws,
state_cs,
C: tl.constexpr,
BLOCK_C: tl.constexpr,
STATE_LEN: tl.constexpr,
DILATION: tl.constexpr,
KERNEL_SIZE: tl.constexpr,
SPEC_QUERY_LEN: tl.constexpr,
MODE: tl.constexpr,
HAS_INIT: tl.constexpr,
NULL_STATE_ID: tl.constexpr,
launch_pdl: tl.constexpr,
):
t = tl.program_id(0)
pid_c = tl.program_id(1)
c_offs = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
c_mask = c_offs < C
if launch_pdl:
tl.extra.cuda.gdc_wait()
# Mode-specific launches use logical row indices. Mixed batches map them
# back to the original token order without gathering inputs or residuals.
output_t = tl.load(token_idx_ptr + t, mask=has_token_map, other=t).to(tl.int64)
if MODE == "decode":
r = t
q_start = t
j = tl.full([], 0, tl.int32)
slot_off = tl.full([], 0, tl.int32)
in_range = True
else:
# Locate the request containing this token.
lo = tl.full([], 1, tl.int32)
hi = tl.full([], num_reqs + 1, tl.int32)
for _ in range(bs_iters):
mid = (lo + hi) // 2
qmid = tl.load(qsl_ptr + mid, mask=mid <= num_reqs, other=0)
pred = qmid <= t
lo = tl.where(pred, mid + 1, lo)
hi = tl.where(pred, hi, mid)
r = tl.minimum(lo - 1, num_reqs - 1)
q_start = tl.load(qsl_ptr + r)
j = (t - q_start).to(tl.int32)
total_real = tl.load(qsl_ptr + num_reqs)
in_range = t < total_real
if MODE == "spec":
num_acc = tl.load(num_acc_ptr + r)
# Roll back draft state to the last accepted token.
slot_off = tl.minimum(tl.maximum(num_acc - 1, 0), SPEC_QUERY_LEN - 1).to(
tl.int32
)
else:
slot_off = tl.full([], 0, tl.int32)
sid = tl.load(state_idx_ptr + r).to(tl.int64)
state_ok = sid != NULL_STATE_ID
sid_safe = tl.where(state_ok, sid, 0)
if HAS_INIT:
has_init = tl.load(has_init_ptr + r, mask=state_ok, other=0) != 0
else:
has_init = state_ok
if MODE == "spec":
# A new spec chunk can produce output from its own tokens without a
# prior state; null slots in other modes denote inactive rows.
read_state = state_ok
out_ok = in_range
else:
read_state = state_ok & has_init
out_ok = in_range & state_ok
# state[sid, w, c] may be a strided view into a shared pool.
base_state = state_ptr + sid_safe * state_bs
acc = tl.zeros([BLOCK_C], tl.float32)
for k in tl.static_range(0, KERNEL_SIZE):
h = j + DILATION * k
from_state = h <= STATE_LEN - 1
state_tap = tl.load(
base_state + (h + slot_off) * state_ws + c_offs * state_cs,
mask=c_mask & read_state & from_state,
other=0.0,
)
input_t = q_start + h - STATE_LEN
input_t = tl.load(
token_idx_ptr + input_t,
mask=has_token_map & out_ok & (~from_state),
other=input_t,
).to(tl.int64)
input_tap = tl.load(
x_ptr + input_t * C + c_offs,
mask=c_mask & out_ok & (~from_state),
other=0.0,
)
tap = tl.where(from_state, state_tap, input_tap).to(tl.float32)
weight = tl.load(
w_ptr + c_offs * KERNEL_SIZE + k,
mask=c_mask,
other=0.0,
).to(tl.float32)
acc += weight * tap
# F.conv1d materializes its output dtype before SiLU.
conv = acc.to(residual_ptr.dtype.element_ty).to(tl.float32)
y = conv * tl.sigmoid(conv)
conv_output = tl.where(out_ok, y, 0.0).to(residual_ptr.dtype.element_ty)
residual = tl.load(
residual_ptr + output_t * C + c_offs,
mask=c_mask,
other=0.0,
)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(
residual_ptr + output_t * C + c_offs,
residual + conv_output,
mask=c_mask,
)
if MODE == "decode":
# Decode has one program per request and channel block, so no peer can
# still be reading this state slice when it is updated.
decode_input = tl.load(
x_ptr + output_t * C + c_offs,
mask=c_mask & state_ok,
other=0.0,
)
for i in tl.static_range(0, STATE_LEN):
if i < STATE_LEN - 1:
next_state = tl.load(
base_state + (i + 1) * state_ws + c_offs * state_cs,
mask=c_mask & state_ok & has_init,
other=0.0,
)
else:
next_state = decode_input
tl.store(
base_state + i * state_ws + c_offs * state_cs,
next_state,
mask=c_mask & state_ok,
)
# Prefill and spec write back separately so every token reads the old state.
@triton.jit(do_not_specialize=["has_token_map"])
def _ple_conv_writeback_kernel(
x_ptr,
state_ptr,
state_idx_ptr,
qsl_ptr,
num_acc_ptr,
has_init_ptr,
token_idx_ptr,
has_token_map,
state_bs,
state_ws,
state_cs,
C: tl.constexpr,
BLOCK_C: tl.constexpr,
STATE_LEN: tl.constexpr,
SPEC_QUERY_LEN: tl.constexpr,
STATE_WIDTH: tl.constexpr,
MODE: tl.constexpr,
HAS_INIT: tl.constexpr,
NULL_STATE_ID: tl.constexpr,
):
r = tl.program_id(0)
pid_c = tl.program_id(1)
c_offs = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
c_mask = c_offs < C
sid = tl.load(state_idx_ptr + r).to(tl.int64)
state_ok = sid != NULL_STATE_ID
if not state_ok:
return
q_start = tl.load(qsl_ptr + r)
q_end = tl.load(qsl_ptr + r + 1)
qlen = (q_end - q_start).to(tl.int32)
if MODE == "spec":
num_acc = tl.load(num_acc_ptr + r)
# Use the same rollback window as the output kernel.
slot_off = tl.minimum(tl.maximum(num_acc - 1, 0), SPEC_QUERY_LEN - 1).to(
tl.int32
)
shift = 1
else:
slot_off = tl.full([], 0, tl.int32)
shift = qlen
if qlen <= 0:
return
has_init = tl.load(has_init_ptr + r) != 0 if HAS_INIT else True
src_ok = state_ok if MODE == "spec" else state_ok & has_init
WRITE_W: tl.constexpr = STATE_WIDTH if MODE == "spec" else STATE_LEN
base_state = state_ptr + sid * state_bs
for i in tl.static_range(0, WRITE_W):
m = shift + i
from_state = m <= STATE_LEN - 1
# Short or graph-padded spec chunks preserve the unused state tail.
do_write = i < STATE_LEN + qlen - 1 if MODE == "spec" else True
state_value = tl.load(
base_state + (slot_off + m) * state_ws + c_offs * state_cs,
mask=c_mask & from_state & src_ok,
other=0.0,
)
input_t = q_start + m - STATE_LEN
input_t = tl.load(
token_idx_ptr + input_t,
mask=has_token_map & (~from_state) & do_write,
other=input_t,
).to(tl.int64)
input_value = tl.load(
x_ptr + input_t * C + c_offs,
mask=c_mask & (~from_state) & do_write,
other=0.0,
)
value = tl.where(from_state, state_value, input_value)
tl.store(
base_state + i * state_ws + c_offs * state_cs,
value,
mask=c_mask & do_write,
)
def ple_conv(
inputs: torch.Tensor,
residual: torch.Tensor,
conv_state: torch.Tensor,
conv_weights: torch.Tensor,
state_indices: torch.Tensor,
*,
mode: Literal["decode", "spec", "prefill"],
dilation: int,
query_start_loc: torch.Tensor | None = None,
num_accepted_tokens: torch.Tensor | None = None,
has_initial_states: torch.Tensor | None = None,
spec_query_len: int = 1,
token_indices: torch.Tensor | None = None,
) -> None:
"""Add short-convolution output to ``residual`` and update its state."""
BLOCK_C = 512
kernel_spec_query_len = spec_query_len if mode == "spec" else 1
T, C = inputs.shape
K = conv_weights.shape[1]
state_len = (K - 1) * dilation
state_width = state_len + kernel_spec_query_len - 1
if token_indices is not None:
T = token_indices.numel()
if conv_state.shape[1] != C or conv_state.shape[2] < state_width:
raise ValueError(
"conv_state must have shape [slots, channels, window], with "
f"channels={C} and window >= {state_width}"
)
state_bs, state_cs, state_ws = conv_state.stride()
if mode == "decode":
num_reqs = T
binary_search_iters = 1
has_initial_states_arg = has_initial_states is not None
elif mode == "spec":
if query_start_loc is None or num_accepted_tokens is None:
raise ValueError(
"query_start_loc and num_accepted_tokens are required for spec decode"
)
num_reqs = state_indices.numel()
binary_search_iters = max(num_reqs, 1).bit_length()
has_initial_states_arg = False
elif mode == "prefill":
if query_start_loc is None or has_initial_states is None:
raise ValueError(
"query_start_loc and has_initial_states are required for prefill"
)
num_reqs = state_indices.numel()
binary_search_iters = max(num_reqs, 1).bit_length()
has_initial_states_arg = True
else:
raise ValueError(f"Unsupported short-conv mode: {mode}")
num_warps = 4 if mode == "prefill" else 8
launch_pdl = current_platform.is_arch_support_pdl()
# Constexpr flags eliminate accesses to optional None arguments. Without a
# token map, state_indices is an unused but device-resident placeholder.
_ple_conv_kernel[(T, triton.cdiv(C, BLOCK_C))](
inputs,
conv_state,
conv_weights,
residual,
state_indices,
query_start_loc,
num_accepted_tokens,
has_initial_states,
token_indices if token_indices is not None else state_indices,
token_indices is not None,
num_reqs,
binary_search_iters,
state_bs,
state_ws,
state_cs,
C=C,
BLOCK_C=BLOCK_C,
STATE_LEN=state_len,
DILATION=dilation,
KERNEL_SIZE=K,
SPEC_QUERY_LEN=kernel_spec_query_len,
MODE=mode,
HAS_INIT=has_initial_states_arg,
NULL_STATE_ID=NULL_BLOCK_ID,
launch_pdl=launch_pdl,
num_warps=num_warps,
)
# conv state update is fused with the kernel above for decode
if mode != "decode":
_ple_conv_writeback_kernel[(num_reqs, triton.cdiv(C, BLOCK_C))](
inputs,
conv_state,
state_indices,
query_start_loc,
num_accepted_tokens,
has_initial_states,
token_indices if token_indices is not None else state_indices,
token_indices is not None,
state_bs,
state_ws,
state_cs,
C=C,
BLOCK_C=BLOCK_C,
STATE_LEN=state_len,
SPEC_QUERY_LEN=kernel_spec_query_len,
STATE_WIDTH=state_width,
MODE=mode,
HAS_INIT=has_initial_states_arg,
NULL_STATE_ID=NULL_BLOCK_ID,
num_warps=num_warps,
)
+171 -514
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GPU-resident Qwen4Exp position-learning enhancement layers."""
import math
from collections.abc import Iterable, Sequence
import torch
@@ -11,7 +10,7 @@ from torch import nn
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
from vllm.model_executor.layers.mamba.abstract import MambaBase
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateDtypeCalculator,
@@ -42,9 +41,9 @@ from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionBackend,
PleShortConvAttentionMetadata,
)
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from ..common.ple import PLEVocabParallelEmbedding
from .ops.ple import ple_conv, ple_gate, ple_ngram_ids
class Qwen4ExpPLEGroupedNorm(nn.Module):
@@ -256,8 +255,6 @@ class Qwen4ExpNGramEmbedding(nn.Module):
config: Qwen4ExpTextConfig,
embedding_dim: int,
ple_dense_layer_id: int,
max_total_tokens: int,
max_num_reqs: int,
prefix: str,
layer_name: str,
quant_config: QuantizationConfig | None = None,
@@ -324,20 +321,6 @@ class Qwen4ExpNGramEmbedding(nn.Module):
quant_config, f"{prefix}.ngram_embedding"
),
)
self.register_buffer(
"positions_buffer",
torch.arange(max_total_tokens, dtype=torch.int64),
persistent=False,
)
self.register_buffer(
"padded_buffer",
torch.full(
(max_num_reqs, max_total_tokens),
self.eos_token_id,
dtype=torch.int64,
),
persistent=False,
)
@staticmethod
def _shift_precompute(
@@ -379,26 +362,34 @@ class Qwen4ExpNGramEmbedding(nn.Module):
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
output: torch.Tensor | None = None,
) -> torch.Tensor:
"""Compute n-gram embedding indices for the current request layout."""
input_ids = input_ids.reshape(-1).long()
query_start_loc = query_start_loc.long()
input_ids = input_ids.reshape(-1)
num_reqs = query_start_loc.numel() - 1
num_tokens = input_ids.shape[0]
if num_tokens > self.positions_buffer.numel():
raise ValueError(
f"PLE received {num_tokens} tokens, but its workspace supports "
f"at most {self.positions_buffer.numel()}"
)
if num_reqs > self.padded_buffer.shape[0]:
raise ValueError(
f"PLE received {num_reqs} requests, but its workspace supports "
f"at most {self.padded_buffer.shape[0]}"
)
positions = self.positions_buffer[:num_tokens]
packed = self.padded_buffer[:num_reqs, :num_tokens]
packed.fill_(self.eos_token_id)
if input_ids.is_cuda:
return ple_ngram_ids(
input_ids=input_ids,
query_start_loc=query_start_loc,
ngram_context=ngram_context,
layer_multipliers=self.layer_multipliers,
ngram_heads_vocab_sizes=self.ngram_heads_vocab_sizes,
ngram_heads_offsets=self.ngram_heads_offsets,
eos_token_id=self.eos_token_id,
heads_per_ngram=self.heads_per_ngram,
output=output,
)
input_ids = input_ids.long()
query_start_loc = query_start_loc.long()
positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64)
packed = torch.full(
(num_reqs, num_tokens),
self.eos_token_id,
device=input_ids.device,
dtype=torch.int64,
)
request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1
request_indices.clamp_(max=num_reqs - 1)
columns = (positions - query_start_loc[request_indices]).clamp(
@@ -446,12 +437,13 @@ class Qwen4ExpNGramEmbedding(nn.Module):
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
ngram_ids = input_ids.new_empty(
(input_ids.shape[0], self.ngram_heads),
dtype=torch.long,
)
# Keep num_reqs-dependent ID generation outside PIECEWISE CUDA graphs,
# which dispatch only on the padded token count.
# torch.compile requires the splitting op to write graph-owned storage.
# Once compilation is removed, the op can return the IDs directly.
ngram_ids = input_ids.new_empty(
(input_ids.numel(), self.ngram_heads), dtype=torch.long
)
torch.ops.vllm.qwen4_exp_compute_ple_ngram_ids(
input_ids,
query_start_loc,
@@ -562,26 +554,20 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
config,
int(config.ple_embed_dim),
self.ple_dense_layer_id,
vllm_config.scheduler_config.max_num_batched_tokens,
vllm_config.scheduler_config.max_num_seqs,
prefix=f"{prefix}.ple_embedding",
layer_name=prefix,
quant_config=quant_config,
params_dtype=model_config.dtype,
)
self.key_proj = ReplicatedLinear(
# The PLE cache is TP-replicated, so this merged projection is too.
self.kv_proj = MergedColumnParallelLinear(
int(config.ple_embed_dim),
self.hc_hidden_size,
[self.hc_hidden_size, self.hidden_size],
bias=False,
params_dtype=model_config.dtype,
quant_config=quant_config,
prefix=f"{prefix}.key_proj",
)
self.value_proj = ReplicatedLinear(
int(config.ple_embed_dim),
self.hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.value_proj",
prefix=f"{prefix}.kv_proj",
disable_tp=True,
)
norm_args = (
self.hc_hidden_size,
@@ -654,360 +640,14 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
num_spec=self.num_spec_tokens,
)
def _apply_norm(
self, norm: Qwen4ExpPLEGroupedNorm, hidden_states: torch.Tensor
) -> torch.Tensor:
shape = hidden_states.shape
return norm(hidden_states.flatten(-2)).reshape(shape)
def _short_conv_fallback(self, inputs: torch.Tensor) -> torch.Tensor:
# Profiling / CUDA graph capture only; conv state is not updated.
inputs_t = inputs.transpose(0, 1).unsqueeze(0)
output = self.conv1d(inputs_t)[..., : inputs_t.size(-1)]
return F.silu(output).squeeze(0).transpose(0, 1)
def _short_conv_dilated_decode_batched(
self,
x_d: torch.Tensor,
conv_state: torch.Tensor,
conv_weights: torch.Tensor,
state_indices_tensor_d: torch.Tensor,
has_initial_states_d: torch.Tensor | None,
) -> torch.Tensor:
state_indices = state_indices_tensor_d.to(
device=conv_state.device, dtype=torch.int64
)
# FULL cudagraph padded decode rows use NULL_BLOCK_ID. Remap them to
# slot 0 for a safe gather, then zero output and skip write-back.
valid_state = state_indices != NULL_BLOCK_ID
state_indices = torch.where(
valid_state, state_indices, torch.zeros_like(state_indices)
)
if has_initial_states_d is None:
has_initial_state = valid_state
else:
if has_initial_states_d.numel() < state_indices_tensor_d.numel():
raise ValueError(
"has_initial_states_d size mismatch: "
f"got {has_initial_states_d.numel()}, "
f"need >= {state_indices_tensor_d.numel()}."
)
has_initial_state = has_initial_states_d[
: state_indices_tensor_d.numel()
].to(device=conv_state.device, dtype=torch.bool)
has_initial_state = has_initial_state & valid_state
cached_state = conv_state.index_select(0, state_indices)
state = cached_state[..., : self.conv_state_len].to(x_d.dtype)
if self.conv_state_len > 0:
initial_state = torch.where(
has_initial_state.view(-1, 1, 1),
state,
torch.zeros_like(state),
)
history = torch.cat((initial_state, x_d.unsqueeze(-1)), dim=-1)
else:
history = x_d.unsqueeze(-1)
conv_output = F.conv1d(
history,
conv_weights.unsqueeze(1).contiguous(),
groups=history.size(1),
dilation=self.short_conv_dilation,
).squeeze(-1)
output = F.silu(conv_output)
output = output * valid_state.view(-1, 1).to(output.dtype)
if self.conv_state_len > 0:
next_state = history[..., -self.conv_state_len :]
# Padded rows are remapped to the reserved null slot. Preserve its
# existing value while writing the new states for valid rows.
existing_base_state = cached_state[..., : self.conv_state_len]
safe_next_state = torch.where(
valid_state.view(-1, 1, 1),
next_state.to(conv_state.dtype),
existing_base_state,
)
cached_state[..., : self.conv_state_len] = safe_next_state
conv_state.index_copy_(0, state_indices, cached_state)
return output
def _short_conv_dilated_prefill_batched(
self,
x_p: torch.Tensor,
metadata: PleShortConvAttentionMetadata,
conv_state: torch.Tensor,
conv_weights: torch.Tensor,
state_indices_tensor_p: torch.Tensor,
num_prefills: int,
num_decode_tokens: int,
num_prefill_tokens: int,
) -> torch.Tensor:
# ``non_spec_query_start_loc`` covers the non-spec (decode + prefill)
# requests and equals ``query_start_loc`` when spec-decode is inactive.
non_spec_query_start_loc = metadata.non_spec_query_start_loc
if non_spec_query_start_loc is None:
raise ValueError("query_start_loc is required for prefill short-conv")
query_start_loc_p = (
non_spec_query_start_loc[-num_prefills - 1 :] - num_decode_tokens
)
# The metadata builder guarantees that the prefill query offsets start
# at 0 and end at num_prefill_tokens. Avoid reading those values here,
# since doing so would force a device-to-host synchronization.
has_initial_states_p = metadata.has_initial_states_p
if has_initial_states_p is None:
raise ValueError("has_initial_states_p is required for prefill short-conv")
output = torch.empty_like(x_p)
q_starts = query_start_loc_p.to(torch.int64)
if state_indices_tensor_p.numel() < num_prefills:
raise ValueError(
"state_indices_tensor_p size mismatch: "
f"got {state_indices_tensor_p.numel()}, "
f"need >= {num_prefills}."
)
if has_initial_states_p.numel() < num_prefills:
raise ValueError(
"has_initial_states_p size mismatch: "
f"got {has_initial_states_p.numel()}, "
f"need >= {num_prefills}."
)
if num_prefills == 0 or x_p.numel() == 0:
return output
lengths = q_starts[1:] - q_starts[:-1]
# Use the CPU-computed packing width from the metadata builder instead
# of synchronizing on lengths.max().
max_len = metadata.max_prefill_query_len
if max_len <= 0:
return output
hidden_size = x_p.shape[1]
positions = torch.arange(
num_prefill_tokens, device=x_p.device, dtype=torch.int64
)
req_indices = torch.searchsorted(q_starts[1:], positions, right=True)
col_indices = positions - q_starts[req_indices]
packed_tokens = x_p.new_zeros((num_prefills, max_len, hidden_size))
packed_tokens[req_indices, col_indices] = x_p
packed_tokens = packed_tokens.transpose(1, 2).contiguous()
state_indices = state_indices_tensor_p[:num_prefills].to(
device=conv_state.device, dtype=torch.int64
)
valid_state = state_indices != NULL_BLOCK_ID
state_indices = torch.where(
valid_state, state_indices, torch.zeros_like(state_indices)
)
has_initial = has_initial_states_p[:num_prefills].to(
device=conv_state.device, dtype=torch.bool
)
if self.conv_state_len > 0:
if conv_state.shape[0] == 0:
state = conv_state.new_zeros(
(num_prefills, hidden_size, self.conv_state_len),
dtype=x_p.dtype,
)
else:
state = conv_state.index_select(0, state_indices)[
..., : self.conv_state_len
].to(x_p.dtype)
use_initial_mask = (valid_state & has_initial).view(num_prefills, 1, 1)
initial_state = torch.where(
use_initial_mask,
state,
torch.zeros_like(state),
)
history = torch.cat((initial_state, packed_tokens), dim=-1)
else:
history = packed_tokens
conv_output = F.conv1d(
history,
conv_weights.unsqueeze(1).contiguous(),
groups=history.size(1),
dilation=self.short_conv_dilation,
)
conv_output = F.silu(conv_output).transpose(1, 2).contiguous()
token_positions = torch.arange(max_len, device=x_p.device, dtype=torch.int64)
valid_tokens = token_positions.view(1, max_len) < lengths.view(num_prefills, 1)
valid_output_mask = valid_tokens & valid_state.to(device=x_p.device).view(
num_prefills, 1
)
conv_output.masked_fill_(~valid_output_mask.unsqueeze(-1), 0)
output.copy_(conv_output[req_indices, col_indices])
if self.conv_state_len > 0 and conv_state.shape[0] > 0:
state_starts = lengths.to(device=history.device, dtype=torch.int64).view(
num_prefills, 1, 1
)
state_offsets = torch.arange(
self.conv_state_len, device=history.device, dtype=torch.int64
).view(1, 1, self.conv_state_len)
next_state = history.gather(
dim=2,
index=(state_starts + state_offsets).expand(-1, history.size(1), -1),
)
# Write back without a host synchronization. Valid, non-empty rows
# receive their new state; padding and zero-length rows keep the
# current cache value.
existing_state = conv_state.index_select(0, state_indices)
existing_base_state = existing_state[..., : self.conv_state_len]
update_mask = valid_state & (lengths.to(device=conv_state.device) > 0)
safe_next_state = torch.where(
update_mask.view(num_prefills, 1, 1),
next_state.to(conv_state.dtype),
existing_base_state,
)
existing_state[..., : self.conv_state_len] = safe_next_state
conv_state.index_copy_(0, state_indices, existing_state)
return output
def _short_conv_dilated_spec_batched(
self,
x_spec: torch.Tensor,
conv_state: torch.Tensor,
conv_weights: torch.Tensor,
spec_state_indices_tensor: torch.Tensor,
spec_query_start_loc: torch.Tensor,
num_accepted_tokens: torch.Tensor,
spec_query_len: int,
) -> torch.Tensor:
"""Dilated short-conv for speculative-decode (MTP) requests.
Each spec request feeds multiple (draft + 1) query tokens. The conv
outputs are computed causally after rolling back the previous draft
state by ``num_accepted_tokens - 1``. The current candidate inputs stay
in the extended cache for the next forward, matching
``causal_conv1d_update``.
``spec_query_len`` (== num_speculative_tokens + 1) is the maximum query
length and is a Python int, so no host synchronization is needed; this
keeps the path safe for full CUDA-graph capture/replay where the buffers
are padded at the request level.
"""
num_reqs = spec_state_indices_tensor.numel()
hidden_size = x_spec.size(-1)
# Use a fixed packing width instead of synchronizing on lengths.max().
max_len = spec_query_len
# Full CUDA graphs can pad these buffers. Only the first num_reqs
# accepted-token counts belong to actual speculative requests.
num_accepted_tokens = num_accepted_tokens[:num_reqs]
q_starts = spec_query_start_loc[: num_reqs + 1].to(torch.int64)
# Keep the number of real speculative tokens on the device.
total_real_tokens = q_starts[num_reqs]
state_indices = spec_state_indices_tensor.to(
device=conv_state.device, dtype=torch.int64
)
valid_state = state_indices != NULL_BLOCK_ID
state_indices = torch.where(
valid_state, state_indices, torch.zeros_like(state_indices)
)
positions = torch.arange(
x_spec.size(0), device=x_spec.device, dtype=torch.int64
)
# Route graph-padded token rows to the discarded dummy request so that
# they cannot overwrite real packed data.
req_indices = torch.searchsorted(q_starts[1:], positions, right=True)
valid_tokens = (positions < total_real_tokens) & (req_indices < num_reqs)
clamped_req_indices = req_indices.clamp_max(max(num_reqs - 1, 0))
col_indices = (positions - q_starts[clamped_req_indices]).clamp_(0, max_len - 1)
pack_req_indices = torch.where(
valid_tokens,
clamped_req_indices,
torch.full_like(req_indices, num_reqs),
)
pack_col_indices = torch.where(
valid_tokens, col_indices, torch.zeros_like(col_indices)
)
# The last request row is the dummy sink for graph padding.
packed = x_spec.new_zeros((num_reqs + 1, max_len, hidden_size))
packed[pack_req_indices, pack_col_indices] = x_spec
packed = packed.transpose(1, 2).contiguous()
if self.conv_state_len > 0:
cached_state = conv_state.index_select(0, state_indices)
rollback_offsets = num_accepted_tokens.to(
device=conv_state.device, dtype=torch.int64
).sub(1)
rollback_offsets = torch.where(
valid_state,
rollback_offsets.clamp_(0, max_len - 1),
torch.zeros_like(rollback_offsets),
)
state_offsets = torch.arange(
self.conv_state_len, device=conv_state.device, dtype=torch.int64
).view(1, 1, self.conv_state_len)
rollback_indices = rollback_offsets.view(-1, 1, 1) + state_offsets
state = cached_state.gather(
2, rollback_indices.expand(-1, hidden_size, -1)
).to(x_spec.dtype)
state = torch.where(
valid_state.view(num_reqs, 1, 1),
state,
torch.zeros_like(state),
)
# Append a zeroed dummy-row state to match the [num_reqs + 1] pack.
dummy_state = state.new_zeros((1, hidden_size, self.conv_state_len))
state_full = torch.cat((state, dummy_state), dim=0)
history = torch.cat((state_full, packed), dim=-1)
else:
history = packed
conv_output = F.conv1d(
history,
conv_weights.unsqueeze(1).contiguous(),
groups=history.size(1),
dilation=self.short_conv_dilation,
)
conv_output = F.silu(conv_output).transpose(1, 2).contiguous()
output = conv_output[pack_req_indices, pack_col_indices]
output = output * valid_tokens.view(-1, 1).to(output.dtype)
# Keep all current candidate inputs in the extended state. On the next
# target forward, ``num_accepted_tokens - 1`` selects the rollback
# window before processing the newly scheduled tokens.
if self.conv_state_len > 0:
state_capacity = self.conv_state_len + max_len - 1
if conv_state.size(-1) < state_capacity:
raise RuntimeError(
"PLE short-conv cache cannot retain speculative tokens: "
f"got {conv_state.size(-1)}, need {state_capacity}."
)
candidate_state = history[:num_reqs, :, 1 : state_capacity + 1]
query_lengths = q_starts[1:] - q_starts[:-1]
state_positions = torch.arange(
state_capacity, device=history.device, dtype=torch.int64
).view(1, 1, state_capacity)
update_lengths = (self.conv_state_len + query_lengths - 1).view(
num_reqs, 1, 1
)
update_mask = valid_state.view(num_reqs, 1, 1) & (
state_positions < update_lengths
)
existing_state = cached_state[..., :state_capacity]
next_state = torch.where(
update_mask,
candidate_state.to(conv_state.dtype),
existing_state,
)
cached_state[..., :state_capacity] = next_state
conv_state.index_copy_(0, state_indices, cached_state)
return output
def _short_conv_dilated_dispatch(
self,
inputs: torch.Tensor,
residual: torch.Tensor,
metadata: PleShortConvAttentionMetadata,
conv_state: torch.Tensor,
conv_weights: torch.Tensor,
) -> torch.Tensor:
) -> None:
num_prefills = metadata.num_prefills
num_decodes = metadata.num_decodes
num_decode_tokens = metadata.num_decode_tokens
@@ -1015,110 +655,129 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
has_prefill = num_prefills > 0
has_decode = num_decodes > 0
has_spec = metadata.spec_sequence_masks is not None
x = inputs[: metadata.num_actual_tokens]
has_non_spec = has_prefill or has_decode
inputs = inputs[: metadata.num_actual_tokens]
residual = residual[: metadata.num_actual_tokens]
# Split spec / non-spec tokens.
if has_spec:
if has_prefill or has_decode:
assert metadata.spec_token_indx is not None
assert metadata.non_spec_token_indx is not None
x_spec = x.index_select(0, metadata.spec_token_indx.long())
x_non_spec = x.index_select(0, metadata.non_spec_token_indx.long())
else:
x_spec = x
x_non_spec = None
else:
x_spec = None
x_non_spec = x
spec_token_indices = None
non_spec_token_indices = None
if has_spec and has_non_spec:
assert metadata.spec_token_indx is not None
assert metadata.non_spec_token_indx is not None
spec_token_indices = metadata.spec_token_indx
non_spec_token_indices = metadata.non_spec_token_indx
spec_output = None
# 1. Run the multi-query speculative-decode part.
if has_spec:
assert metadata.spec_state_indices_tensor is not None
assert metadata.spec_query_start_loc is not None
assert metadata.num_accepted_tokens is not None
spec_output = self._short_conv_dilated_spec_batched(
x_spec=x_spec,
query_start_loc = metadata.spec_query_start_loc
num_accepted_tokens = metadata.num_accepted_tokens
assert query_start_loc is not None
assert num_accepted_tokens is not None
spec_state_indices = metadata.spec_state_indices_tensor[
: metadata.num_spec_decodes
]
# Mixed batches stay in their original row order; the kernels map
# logical spec/non-spec rows instead of materializing both groups.
ple_conv(
inputs=inputs,
residual=residual,
conv_state=conv_state,
conv_weights=conv_weights,
spec_state_indices_tensor=metadata.spec_state_indices_tensor[
: metadata.num_spec_decodes
],
spec_query_start_loc=metadata.spec_query_start_loc,
num_accepted_tokens=metadata.num_accepted_tokens,
state_indices=spec_state_indices,
mode="spec",
dilation=self.short_conv_dilation,
query_start_loc=query_start_loc,
num_accepted_tokens=num_accepted_tokens,
spec_query_len=metadata.spec_query_len,
token_indices=spec_token_indices,
)
# 2. Run regular decode and prefill requests.
conv_out_non_spec = None
state_indices_tensor = metadata.state_indices_tensor
if x_non_spec is not None:
assert state_indices_tensor is not None
if has_prefill:
state_indices_tensor_d, state_indices_tensor_p = torch.split(
state_indices_tensor,
[num_decodes, num_prefills],
dim=0,
if not has_non_spec:
return
state_indices = metadata.state_indices_tensor
assert state_indices is not None
if has_prefill:
state_indices_d, state_indices_p = torch.split(
state_indices, [num_decodes, num_prefills], dim=0
)
if non_spec_token_indices is None:
inputs_d, inputs_p = torch.split(
inputs, [num_decode_tokens, num_prefill_tokens], dim=0
)
x_d, x_p = torch.split(
x_non_spec,
residual_d, residual_p = torch.split(
residual, [num_decode_tokens, num_prefill_tokens], dim=0
)
token_indices_d = None
token_indices_p = None
else:
inputs_d = inputs_p = inputs
residual_d = residual_p = residual
token_indices_d, token_indices_p = torch.split(
non_spec_token_indices,
[num_decode_tokens, num_prefill_tokens],
dim=0,
)
non_spec_parts: list[torch.Tensor] = []
if has_decode:
non_spec_parts.append(
self._short_conv_dilated_decode_batched(
x_d=x_d,
conv_state=conv_state,
conv_weights=conv_weights,
state_indices_tensor_d=state_indices_tensor_d,
has_initial_states_d=metadata.has_initial_states_d,
)
)
non_spec_parts.append(
self._short_conv_dilated_prefill_batched(
x_p=x_p,
metadata=metadata,
conv_state=conv_state,
conv_weights=conv_weights,
state_indices_tensor_p=state_indices_tensor_p,
num_prefills=num_prefills,
num_decode_tokens=num_decode_tokens,
num_prefill_tokens=num_prefill_tokens,
)
)
conv_out_non_spec = torch.vstack(non_spec_parts)
else:
conv_out_non_spec = self._short_conv_dilated_decode_batched(
x_d=x_non_spec,
if has_decode:
ple_conv(
inputs=inputs_d,
residual=residual_d,
conv_state=conv_state,
conv_weights=conv_weights,
state_indices_tensor_d=state_indices_tensor[: x_non_spec.size(0)],
has_initial_states_d=metadata.has_initial_states_d,
state_indices=state_indices_d,
mode="decode",
dilation=self.short_conv_dilation,
has_initial_states=metadata.has_initial_states_d,
token_indices=token_indices_d,
)
# 3. Merge both parts back into the original token order.
if has_spec and conv_out_non_spec is not None:
assert metadata.spec_token_indx is not None
assert metadata.non_spec_token_indx is not None
assert spec_output is not None
output = x.new_empty((metadata.num_actual_tokens, x.size(-1)))
output.index_copy_(0, metadata.spec_token_indx, spec_output)
output.index_copy_(0, metadata.non_spec_token_indx, conv_out_non_spec)
return output
elif has_spec:
assert spec_output is not None
return spec_output
if conv_out_non_spec is None:
return x
return conv_out_non_spec
query_start_loc = metadata.non_spec_query_start_loc
if query_start_loc is None:
raise ValueError("query_start_loc is required for prefill short-conv")
query_start_loc = query_start_loc[-num_prefills - 1 :] - num_decode_tokens
has_initial_states = metadata.has_initial_states_p
if has_initial_states is None:
raise ValueError(
"has_initial_states_p is required for prefill short-conv"
)
ple_conv(
inputs=inputs_p,
residual=residual_p,
conv_state=conv_state,
conv_weights=conv_weights,
state_indices=state_indices_p,
mode="prefill",
dilation=self.short_conv_dilation,
query_start_loc=query_start_loc,
has_initial_states=has_initial_states,
token_indices=token_indices_p,
)
else:
num_decode_rows = (
non_spec_token_indices.numel()
if non_spec_token_indices is not None
else inputs.size(0)
)
ple_conv(
inputs=inputs,
residual=residual,
conv_state=conv_state,
conv_weights=conv_weights,
state_indices=state_indices[:num_decode_rows],
mode="decode",
dilation=self.short_conv_dilation,
has_initial_states=metadata.has_initial_states_d,
token_indices=non_spec_token_indices,
)
def _short_conv(self, inputs: torch.Tensor) -> torch.Tensor:
def _short_conv(self, inputs: torch.Tensor, residual: torch.Tensor) -> None:
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
# Profiling omits all metadata or this Mamba entry. The residual
# already contains the gated output, so short convolution is a no-op.
if attn_metadata is None:
return self._short_conv_fallback(inputs)
return
if not isinstance(attn_metadata, dict):
raise RuntimeError(
@@ -1128,8 +787,7 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
layer_attn_metadata = attn_metadata.get(self.prefix)
if layer_attn_metadata is None:
# MRV2 omits Mamba-family metadata during profile warmup.
return self._short_conv_fallback(inputs)
return
if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata):
raise TypeError(
"Expected PleShortConvAttentionMetadata for layer "
@@ -1138,24 +796,27 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
)
conv_state = self.kv_cache[0]
# Canonicalize both backend cache layouts to [slot, channel, window].
if not is_conv_state_dim_first():
conv_state = conv_state.transpose(-1, -2)
conv_weights = self.conv1d.weight.squeeze(1)
state_capacity = self.conv_state_len + self.num_spec_tokens
if state_capacity > 0:
if conv_state.size(-1) < state_capacity:
state_size = conv_state.size(-1)
if state_size < state_capacity:
raise RuntimeError(
"PLE short-conv cache is smaller than expected for "
f"dilated convolution: got {conv_state.size(-1)}, "
f"dilated convolution: got {state_size}, "
f"expect at least {state_capacity}."
)
conv_state = conv_state[..., -state_capacity:]
return self._short_conv_dilated_dispatch(
inputs,
layer_attn_metadata,
conv_state,
conv_weights.to(dtype=inputs.dtype),
self._short_conv_dilated_dispatch(
inputs=inputs,
residual=residual,
metadata=layer_attn_metadata,
conv_state=conv_state,
conv_weights=conv_weights.to(dtype=inputs.dtype),
)
def forward(
@@ -1174,24 +835,21 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
)
embeddings = self.ple_embedding(input_ids, query_start_loc, ngram_context)
embeddings = self._dequantize_embeddings(embeddings, hidden_states.dtype)
key, _ = self.key_proj(embeddings)
value, _ = self.value_proj(embeddings)
token_count = hidden_states.shape[0]
key = key.reshape(token_count, self.hc_count, self.hidden_size)
query = hidden_states.reshape(token_count, self.hc_count, self.hidden_size)
key = self._apply_norm(self.norm_key, key)
query = self._apply_norm(self.norm_query, query)
gate = (key * query).sum(dim=-1, keepdim=True) / math.sqrt(self.hidden_size)
gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt())
gated_value = gate * value.unsqueeze(-2)
normalized = self._apply_norm(self.norm_conv, gated_value).flatten(-2)
conv_output = torch.zeros_like(normalized)
torch.ops.vllm.qwen4_exp_ple_short_conv(
normalized,
conv_output,
self.prefix,
kv, _ = self.kv_proj(embeddings)
key, value = kv.split(self.kv_proj.output_sizes, dim=-1)
gated_output, conv_input = ple_gate(
key,
value,
hidden_states,
self.norm_key.weight,
self.norm_query.weight,
self.norm_conv.weight,
self.norm_key.eps,
)
return gated_value.flatten(-2) + conv_output
# State routing depends on runtime request metadata and remains outside
# the piecewise graph; short convolution accumulates into gated_output.
torch.ops.vllm.qwen4_exp_ple_short_conv(conv_input, gated_output, self.prefix)
return gated_output
def qwen4_exp_compute_ple_ngram_ids(
@@ -1203,12 +861,12 @@ def qwen4_exp_compute_ple_ngram_ids(
) -> None:
"""Compute request-dependent PLE n-gram IDs outside piecewise graphs."""
layer = get_forward_context().no_compile_layers[layer_name]
ngram_ids = layer.ple_embedding.compute_ngram_ids(
layer.ple_embedding.compute_ngram_ids(
input_ids,
query_start_loc,
ngram_context,
output,
)
output.copy_(ngram_ids)
def qwen4_exp_compute_ple_ngram_ids_fake(
@@ -1223,17 +881,16 @@ def qwen4_exp_compute_ple_ngram_ids_fake(
def qwen4_exp_ple_short_conv(
inputs: torch.Tensor,
output: torch.Tensor,
residual_output: torch.Tensor,
layer_name: str,
) -> None:
layer = get_forward_context().no_compile_layers[layer_name]
result = layer._short_conv(inputs)
output[: result.shape[0]].copy_(result)
layer._short_conv(inputs, residual_output)
def qwen4_exp_ple_short_conv_fake(
inputs: torch.Tensor,
output: torch.Tensor,
residual_output: torch.Tensor,
layer_name: str,
) -> None:
return
@@ -1250,7 +907,7 @@ direct_register_custom_op(
direct_register_custom_op(
op_name="qwen4_exp_ple_short_conv",
op_func=qwen4_exp_ple_short_conv,
mutates_args=["output"],
mutates_args=["residual_output"],
fake_impl=qwen4_exp_ple_short_conv_fake,
)
@@ -141,12 +141,6 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder):
self.spec_sequence_masks = torch.empty(
(self.decode_cudagraph_max_bs,), dtype=torch.bool, device=device
)
self.spec_token_indx = torch.empty(
(self.decode_cudagraph_max_tokens,), dtype=torch.int32, device=device
)
self.non_spec_token_indx = torch.empty(
(self.decode_cudagraph_max_tokens,), dtype=torch.int32, device=device
)
self.spec_query_start_loc = torch.empty(
(self.decode_cudagraph_max_bs + 1,), dtype=torch.int32, device=device
)