[Qwen3.8-Flash-Next] Fuse Qwen4Exp PLE kernels (#54517)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -141,9 +141,9 @@ _QWEN4_EXP_IGNORED_MISSING_SUFFIXES = [
|
||||
"_input_scale",
|
||||
]
|
||||
|
||||
# The checkpoint keeps down and injection projections separate; runtime packs
|
||||
# them into adjacent logical shards of one MergedColumnParallelLinear.
|
||||
_HC_WEIGHTS_MAPPER = WeightsMapper(
|
||||
# The checkpoint stores these projections separately; runtime packs each group
|
||||
# into adjacent logical shards of a MergedColumnParallelLinear.
|
||||
_EXTRA_WEIGHTS_MAPPER = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
"hyper_connection.input_mix_weight_down.weight": (
|
||||
"hyper_connection.input_mix_weight_down_block_inject.weight",
|
||||
@@ -153,6 +153,8 @@ _HC_WEIGHTS_MAPPER = WeightsMapper(
|
||||
"hyper_connection.input_mix_weight_down_block_inject.weight",
|
||||
1,
|
||||
),
|
||||
"ple.key_proj": ("ple.kv_proj", 0),
|
||||
"ple.value_proj": ("ple.kv_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -389,7 +391,7 @@ class Qwen4ExpMixtureOfExperts(MixtureOfExperts):
|
||||
}
|
||||
)
|
||||
class Qwen4ExpModel(nn.Module):
|
||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER
|
||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
@@ -612,6 +614,7 @@ class Qwen4ExpForCausalLM(
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
"kv_proj": ["key_proj", "value_proj"],
|
||||
"in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
|
||||
"in_proj_ba": ["in_proj_b", "in_proj_a"],
|
||||
"input_mix_weight_down_block_inject": [
|
||||
@@ -848,11 +851,12 @@ class Qwen4ExpForConditionalGeneration(
|
||||
requires_raw_input_tokens = True
|
||||
|
||||
packed_modules_mapping = Qwen3_5ForConditionalGeneration.packed_modules_mapping | {
|
||||
"kv_proj": ["key_proj", "value_proj"],
|
||||
"input_mix_weight_down_block_inject": [
|
||||
"input_mix_weight_down",
|
||||
"block_inject_weight",
|
||||
"_input_mix_padding",
|
||||
]
|
||||
],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -52,7 +52,7 @@ from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .model import (
|
||||
_HC_WEIGHTS_MAPPER,
|
||||
_EXTRA_WEIGHTS_MAPPER,
|
||||
_QWEN4_EXP_IGNORED_MISSING_SUFFIXES,
|
||||
Qwen4ExpDecoderLayer,
|
||||
Qwen4ExpMixtureOfExperts,
|
||||
@@ -157,7 +157,7 @@ def _make_draft_vllm_config(
|
||||
}
|
||||
)
|
||||
class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER
|
||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -0,0 +1,669 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused Qwen4Exp PLE kernels.
|
||||
|
||||
N-gram IDs hash each suffix with
|
||||
``offset[h] + (xor_i(token[t-i] * multiplier[i]) % size[h])``. The gate computes
|
||||
``d = dot(RMSNorm(key), RMSNorm(query)) / sqrt(H)`` and
|
||||
``g = sigmoid(sign(d) * sqrt(max(abs(d), 1e-6)))``. Short convolution adds
|
||||
``silu(sum_k weight[k] * history[t + k * dilation])`` to the gated output and
|
||||
updates its persistent history.
|
||||
"""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
# N-gram ID generation
|
||||
@triton.jit(do_not_specialize=["num_tokens", "num_reqs", "binary_search_iters"])
|
||||
def _ple_ngram_ids_kernel(
|
||||
input_ids_ptr,
|
||||
qsl_ptr,
|
||||
ctx_ptr,
|
||||
multipliers_ptr,
|
||||
sizes_ptr,
|
||||
offsets_ptr,
|
||||
out_ptr,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
eos_token_id,
|
||||
binary_search_iters,
|
||||
NGRAM_CONTEXT_LEN: tl.constexpr,
|
||||
HEADS_PER_NGRAM: tl.constexpr,
|
||||
BLOCK_T: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
NGRAM_HEADS: tl.constexpr = NGRAM_CONTEXT_LEN * HEADS_PER_NGRAM
|
||||
BLOCK_H: tl.constexpr = triton.next_power_of_2(NGRAM_HEADS)
|
||||
pid = tl.program_id(0)
|
||||
token_offsets = pid * BLOCK_T + tl.arange(0, BLOCK_T)
|
||||
token_mask = token_offsets < num_tokens
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
# A flattened token offset does not identify its request when query chunks
|
||||
# have different lengths. Binary-search the request boundaries instead.
|
||||
p_lo = tl.full([BLOCK_T], 1, tl.int32)
|
||||
p_hi = tl.full([BLOCK_T], num_reqs + 1, tl.int32)
|
||||
for _ in range(binary_search_iters):
|
||||
mid = (p_lo + p_hi) // 2
|
||||
qmid = tl.load(qsl_ptr + mid, mask=token_mask & (mid <= num_reqs), other=0)
|
||||
pred = qmid <= token_offsets
|
||||
p_lo = tl.where(pred, mid + 1, p_lo)
|
||||
p_hi = tl.where(pred, p_hi, mid)
|
||||
req = tl.minimum(p_lo - 1, num_reqs - 1).to(tl.int64)
|
||||
request_start = tl.load(qsl_ptr + req, mask=token_mask, other=0)
|
||||
chunk_pos = token_offsets - request_start
|
||||
|
||||
# Initialize every head with the current-token term.
|
||||
current_token = tl.load(input_ids_ptr + token_offsets, mask=token_mask, other=0).to(
|
||||
tl.int64
|
||||
)
|
||||
current_multiplier = tl.load(multipliers_ptr)
|
||||
mixed = current_token[:, None] * current_multiplier
|
||||
|
||||
g = tl.arange(0, BLOCK_H)
|
||||
head_mask = g < NGRAM_HEADS
|
||||
ngram_order = g // HEADS_PER_NGRAM + 2
|
||||
|
||||
# Walk predecessors from newest to oldest. At chunk boundaries they come
|
||||
# from ngram_context; otherwise they come from this step's input_ids.
|
||||
crossed = tl.zeros([BLOCK_T], tl.int1)
|
||||
for shift in tl.static_range(1, NGRAM_CONTEXT_LEN + 1):
|
||||
in_step = chunk_pos >= shift
|
||||
ctx_col = NGRAM_CONTEXT_LEN - shift + chunk_pos
|
||||
step_token = tl.load(
|
||||
input_ids_ptr + token_offsets - shift,
|
||||
mask=token_mask & in_step,
|
||||
other=0,
|
||||
)
|
||||
context_token = tl.load(
|
||||
ctx_ptr + req * NGRAM_CONTEXT_LEN + ctx_col,
|
||||
mask=token_mask & (~in_step),
|
||||
other=0,
|
||||
)
|
||||
candidate = tl.where(in_step, step_token, context_token).to(tl.int64)
|
||||
# Older positions remain behind the first EOS boundary.
|
||||
candidate = tl.where(crossed, eos_token_id, candidate)
|
||||
crossed = crossed | (candidate == eos_token_id)
|
||||
multiplier = tl.load(multipliers_ptr + shift)
|
||||
term = candidate[:, None] * multiplier
|
||||
mixed = mixed ^ tl.where((ngram_order > shift)[None, :], term, 0)
|
||||
|
||||
# Map each hash into its embedding-table partition.
|
||||
sizes = tl.load(sizes_ptr + g, mask=head_mask, other=1)[None, :]
|
||||
head_offsets = tl.load(offsets_ptr + g, mask=head_mask, other=0)[None, :]
|
||||
# Hash products may overflow int64; preserve torch.remainder semantics.
|
||||
remainders = mixed % sizes
|
||||
remainders = tl.where(remainders < 0, remainders + sizes, remainders)
|
||||
ids = remainders + head_offsets
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(
|
||||
out_ptr + token_offsets[:, None] * NGRAM_HEADS + g[None, :],
|
||||
ids,
|
||||
mask=token_mask[:, None] & head_mask[None, :],
|
||||
)
|
||||
|
||||
|
||||
def _ple_ngram_ids(
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
layer_multipliers: torch.Tensor,
|
||||
ngram_heads_vocab_sizes: torch.Tensor,
|
||||
ngram_heads_offsets: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
eos_token_id: int,
|
||||
heads_per_ngram: int,
|
||||
) -> None:
|
||||
input_ids = input_ids.reshape(-1)
|
||||
num_tokens = input_ids.shape[0]
|
||||
num_reqs = query_start_loc.numel() - 1
|
||||
ctx_len = ngram_context.shape[1]
|
||||
BLOCK_T = 8
|
||||
launch_pdl = current_platform.is_arch_support_pdl()
|
||||
_ple_ngram_ids_kernel[(triton.cdiv(num_tokens, BLOCK_T),)](
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
layer_multipliers,
|
||||
ngram_heads_vocab_sizes,
|
||||
ngram_heads_offsets,
|
||||
output,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
eos_token_id,
|
||||
binary_search_iters=num_reqs.bit_length(),
|
||||
NGRAM_CONTEXT_LEN=ctx_len,
|
||||
HEADS_PER_NGRAM=heads_per_ngram,
|
||||
BLOCK_T=BLOCK_T,
|
||||
launch_pdl=launch_pdl,
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
|
||||
def ple_ngram_ids(
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
layer_multipliers: torch.Tensor,
|
||||
ngram_heads_vocab_sizes: torch.Tensor,
|
||||
ngram_heads_offsets: torch.Tensor,
|
||||
eos_token_id: int,
|
||||
heads_per_ngram: int,
|
||||
output: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if output is None:
|
||||
output = torch.empty(
|
||||
(input_ids.numel(), ngram_context.shape[1] * heads_per_ngram),
|
||||
dtype=torch.int64,
|
||||
device=input_ids.device,
|
||||
)
|
||||
_ple_ngram_ids(
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
layer_multipliers,
|
||||
ngram_heads_vocab_sizes,
|
||||
ngram_heads_offsets,
|
||||
output,
|
||||
eos_token_id,
|
||||
heads_per_ngram,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
# Gating and normalization
|
||||
@triton.jit
|
||||
def _ple_gate_kernel(
|
||||
key_ptr, # [T, HC*H] with row stride key_rs
|
||||
value_ptr, # [T, H] with row stride value_rs
|
||||
hidden_ptr, # [T, HC*H]
|
||||
nk_ptr, # [HC*H] norm_key weight
|
||||
nq_ptr, # [HC*H] norm_query weight
|
||||
ncw_ptr, # [HC*H] norm_conv weight
|
||||
gated_ptr, # [T, HC*H] out
|
||||
normed_ptr, # [T, HC*H] out
|
||||
key_rs,
|
||||
value_rs,
|
||||
eps,
|
||||
H: tl.constexpr,
|
||||
HC: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
t = tl.program_id(0)
|
||||
s = tl.program_id(1)
|
||||
lanes = tl.arange(0, BLOCK_H)
|
||||
mask = lanes < H
|
||||
offs = s * H + lanes
|
||||
dtype: tl.constexpr = key_ptr.dtype.element_ty
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
k = tl.load(key_ptr + t * key_rs + offs, mask=mask, other=0.0).to(tl.float32)
|
||||
q = tl.load(hidden_ptr + t * HC * H + offs, mask=mask, other=0.0).to(tl.float32)
|
||||
nk = tl.load(nk_ptr + offs, mask=mask, other=0.0).to(tl.float32)
|
||||
nq = tl.load(nq_ptr + offs, mask=mask, other=0.0).to(tl.float32)
|
||||
|
||||
# Match eager materialization at each intermediate tensor boundary.
|
||||
k_n = (k * tl.rsqrt(tl.sum(k * k) / H + eps) * (1.0 + nk)).to(dtype)
|
||||
q_n = (q * tl.rsqrt(tl.sum(q * q) / H + eps) * (1.0 + nq)).to(dtype)
|
||||
products = (k_n.to(tl.float32) * q_n.to(tl.float32)).to(dtype)
|
||||
dot = tl.sum(products.to(tl.float32)).to(dtype).to(tl.float32)
|
||||
d = dot / tl.sqrt(float(H))
|
||||
d = d.to(dtype).to(tl.float32)
|
||||
sign = tl.where(d < 0, -1.0, 0.0)
|
||||
sign = tl.where(d > 0, 1.0, sign)
|
||||
magnitude = tl.sqrt(tl.maximum(tl.abs(d), 1e-6)).to(dtype)
|
||||
g = tl.sigmoid(sign * magnitude)
|
||||
g = g.to(dtype).to(tl.float32)
|
||||
|
||||
v = tl.load(value_ptr + t * value_rs + lanes, mask=mask, other=0.0).to(tl.float32)
|
||||
gated = (g * v).to(dtype)
|
||||
gf = gated.to(tl.float32)
|
||||
ncw = tl.load(ncw_ptr + offs, mask=mask, other=0.0).to(tl.float32)
|
||||
normed = gf * tl.rsqrt(tl.sum(gf * gf) / H + eps) * (1.0 + ncw)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(gated_ptr + t * HC * H + offs, gated, mask=mask)
|
||||
tl.store(normed_ptr + t * HC * H + offs, normed, mask=mask)
|
||||
|
||||
|
||||
def _ple_gate(
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
hidden: torch.Tensor,
|
||||
norm_key_w: torch.Tensor,
|
||||
norm_query_w: torch.Tensor,
|
||||
norm_conv_w: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens = hidden.shape[0]
|
||||
h = value.shape[-1]
|
||||
hc = hidden.shape[-1] // h
|
||||
assert key.stride(1) == 1 and value.stride(1) == 1
|
||||
assert hidden.is_contiguous()
|
||||
if key.dtype != value.dtype or key.dtype != hidden.dtype:
|
||||
raise ValueError("key, value, and hidden must have the same dtype")
|
||||
if key.dtype not in (torch.bfloat16, torch.float16):
|
||||
raise ValueError("PLE gate supports BF16 and FP16 inputs")
|
||||
gated = torch.empty_like(hidden)
|
||||
normed = torch.empty_like(hidden)
|
||||
_ple_gate_kernel[(num_tokens, hc)](
|
||||
key,
|
||||
value,
|
||||
hidden,
|
||||
norm_key_w,
|
||||
norm_query_w,
|
||||
norm_conv_w,
|
||||
gated,
|
||||
normed,
|
||||
key.stride(0),
|
||||
value.stride(0),
|
||||
eps,
|
||||
H=h,
|
||||
HC=hc,
|
||||
BLOCK_H=triton.next_power_of_2(h),
|
||||
num_warps=4,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return gated, normed
|
||||
|
||||
|
||||
def _ple_gate_fake(
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
hidden: torch.Tensor,
|
||||
norm_key_w: torch.Tensor,
|
||||
norm_query_w: torch.Tensor,
|
||||
norm_conv_w: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.empty_like(hidden), torch.empty_like(hidden)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_ple_gate",
|
||||
op_func=_ple_gate,
|
||||
mutates_args=[],
|
||||
fake_impl=_ple_gate_fake,
|
||||
)
|
||||
|
||||
|
||||
def ple_gate(
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
hidden: torch.Tensor,
|
||||
norm_key_w: torch.Tensor,
|
||||
norm_query_w: torch.Tensor,
|
||||
norm_conv_w: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.ops.vllm.qwen4_exp_ple_gate(
|
||||
key, value, hidden, norm_key_w, norm_query_w, norm_conv_w, eps
|
||||
)
|
||||
|
||||
|
||||
# Dilated short convolution
|
||||
@triton.jit(do_not_specialize=["num_reqs", "bs_iters", "has_token_map"])
|
||||
def _ple_conv_kernel(
|
||||
x_ptr,
|
||||
state_ptr,
|
||||
w_ptr,
|
||||
residual_ptr,
|
||||
state_idx_ptr,
|
||||
qsl_ptr,
|
||||
num_acc_ptr,
|
||||
has_init_ptr,
|
||||
token_idx_ptr,
|
||||
has_token_map,
|
||||
num_reqs,
|
||||
bs_iters,
|
||||
state_bs,
|
||||
state_ws,
|
||||
state_cs,
|
||||
C: tl.constexpr,
|
||||
BLOCK_C: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
DILATION: tl.constexpr,
|
||||
KERNEL_SIZE: tl.constexpr,
|
||||
SPEC_QUERY_LEN: tl.constexpr,
|
||||
MODE: tl.constexpr,
|
||||
HAS_INIT: tl.constexpr,
|
||||
NULL_STATE_ID: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
t = tl.program_id(0)
|
||||
pid_c = tl.program_id(1)
|
||||
c_offs = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
|
||||
c_mask = c_offs < C
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
# Mode-specific launches use logical row indices. Mixed batches map them
|
||||
# back to the original token order without gathering inputs or residuals.
|
||||
output_t = tl.load(token_idx_ptr + t, mask=has_token_map, other=t).to(tl.int64)
|
||||
|
||||
if MODE == "decode":
|
||||
r = t
|
||||
q_start = t
|
||||
j = tl.full([], 0, tl.int32)
|
||||
slot_off = tl.full([], 0, tl.int32)
|
||||
in_range = True
|
||||
else:
|
||||
# Locate the request containing this token.
|
||||
lo = tl.full([], 1, tl.int32)
|
||||
hi = tl.full([], num_reqs + 1, tl.int32)
|
||||
for _ in range(bs_iters):
|
||||
mid = (lo + hi) // 2
|
||||
qmid = tl.load(qsl_ptr + mid, mask=mid <= num_reqs, other=0)
|
||||
pred = qmid <= t
|
||||
lo = tl.where(pred, mid + 1, lo)
|
||||
hi = tl.where(pred, hi, mid)
|
||||
r = tl.minimum(lo - 1, num_reqs - 1)
|
||||
q_start = tl.load(qsl_ptr + r)
|
||||
j = (t - q_start).to(tl.int32)
|
||||
total_real = tl.load(qsl_ptr + num_reqs)
|
||||
in_range = t < total_real
|
||||
if MODE == "spec":
|
||||
num_acc = tl.load(num_acc_ptr + r)
|
||||
# Roll back draft state to the last accepted token.
|
||||
slot_off = tl.minimum(tl.maximum(num_acc - 1, 0), SPEC_QUERY_LEN - 1).to(
|
||||
tl.int32
|
||||
)
|
||||
else:
|
||||
slot_off = tl.full([], 0, tl.int32)
|
||||
|
||||
sid = tl.load(state_idx_ptr + r).to(tl.int64)
|
||||
state_ok = sid != NULL_STATE_ID
|
||||
sid_safe = tl.where(state_ok, sid, 0)
|
||||
if HAS_INIT:
|
||||
has_init = tl.load(has_init_ptr + r, mask=state_ok, other=0) != 0
|
||||
else:
|
||||
has_init = state_ok
|
||||
if MODE == "spec":
|
||||
# A new spec chunk can produce output from its own tokens without a
|
||||
# prior state; null slots in other modes denote inactive rows.
|
||||
read_state = state_ok
|
||||
out_ok = in_range
|
||||
else:
|
||||
read_state = state_ok & has_init
|
||||
out_ok = in_range & state_ok
|
||||
|
||||
# state[sid, w, c] may be a strided view into a shared pool.
|
||||
base_state = state_ptr + sid_safe * state_bs
|
||||
acc = tl.zeros([BLOCK_C], tl.float32)
|
||||
for k in tl.static_range(0, KERNEL_SIZE):
|
||||
h = j + DILATION * k
|
||||
from_state = h <= STATE_LEN - 1
|
||||
state_tap = tl.load(
|
||||
base_state + (h + slot_off) * state_ws + c_offs * state_cs,
|
||||
mask=c_mask & read_state & from_state,
|
||||
other=0.0,
|
||||
)
|
||||
input_t = q_start + h - STATE_LEN
|
||||
input_t = tl.load(
|
||||
token_idx_ptr + input_t,
|
||||
mask=has_token_map & out_ok & (~from_state),
|
||||
other=input_t,
|
||||
).to(tl.int64)
|
||||
input_tap = tl.load(
|
||||
x_ptr + input_t * C + c_offs,
|
||||
mask=c_mask & out_ok & (~from_state),
|
||||
other=0.0,
|
||||
)
|
||||
tap = tl.where(from_state, state_tap, input_tap).to(tl.float32)
|
||||
weight = tl.load(
|
||||
w_ptr + c_offs * KERNEL_SIZE + k,
|
||||
mask=c_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
acc += weight * tap
|
||||
|
||||
# F.conv1d materializes its output dtype before SiLU.
|
||||
conv = acc.to(residual_ptr.dtype.element_ty).to(tl.float32)
|
||||
y = conv * tl.sigmoid(conv)
|
||||
conv_output = tl.where(out_ok, y, 0.0).to(residual_ptr.dtype.element_ty)
|
||||
residual = tl.load(
|
||||
residual_ptr + output_t * C + c_offs,
|
||||
mask=c_mask,
|
||||
other=0.0,
|
||||
)
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(
|
||||
residual_ptr + output_t * C + c_offs,
|
||||
residual + conv_output,
|
||||
mask=c_mask,
|
||||
)
|
||||
|
||||
if MODE == "decode":
|
||||
# Decode has one program per request and channel block, so no peer can
|
||||
# still be reading this state slice when it is updated.
|
||||
decode_input = tl.load(
|
||||
x_ptr + output_t * C + c_offs,
|
||||
mask=c_mask & state_ok,
|
||||
other=0.0,
|
||||
)
|
||||
for i in tl.static_range(0, STATE_LEN):
|
||||
if i < STATE_LEN - 1:
|
||||
next_state = tl.load(
|
||||
base_state + (i + 1) * state_ws + c_offs * state_cs,
|
||||
mask=c_mask & state_ok & has_init,
|
||||
other=0.0,
|
||||
)
|
||||
else:
|
||||
next_state = decode_input
|
||||
tl.store(
|
||||
base_state + i * state_ws + c_offs * state_cs,
|
||||
next_state,
|
||||
mask=c_mask & state_ok,
|
||||
)
|
||||
|
||||
|
||||
# Prefill and spec write back separately so every token reads the old state.
|
||||
@triton.jit(do_not_specialize=["has_token_map"])
|
||||
def _ple_conv_writeback_kernel(
|
||||
x_ptr,
|
||||
state_ptr,
|
||||
state_idx_ptr,
|
||||
qsl_ptr,
|
||||
num_acc_ptr,
|
||||
has_init_ptr,
|
||||
token_idx_ptr,
|
||||
has_token_map,
|
||||
state_bs,
|
||||
state_ws,
|
||||
state_cs,
|
||||
C: tl.constexpr,
|
||||
BLOCK_C: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
SPEC_QUERY_LEN: tl.constexpr,
|
||||
STATE_WIDTH: tl.constexpr,
|
||||
MODE: tl.constexpr,
|
||||
HAS_INIT: tl.constexpr,
|
||||
NULL_STATE_ID: tl.constexpr,
|
||||
):
|
||||
r = tl.program_id(0)
|
||||
pid_c = tl.program_id(1)
|
||||
c_offs = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
|
||||
c_mask = c_offs < C
|
||||
|
||||
sid = tl.load(state_idx_ptr + r).to(tl.int64)
|
||||
state_ok = sid != NULL_STATE_ID
|
||||
if not state_ok:
|
||||
return
|
||||
|
||||
q_start = tl.load(qsl_ptr + r)
|
||||
q_end = tl.load(qsl_ptr + r + 1)
|
||||
qlen = (q_end - q_start).to(tl.int32)
|
||||
if MODE == "spec":
|
||||
num_acc = tl.load(num_acc_ptr + r)
|
||||
# Use the same rollback window as the output kernel.
|
||||
slot_off = tl.minimum(tl.maximum(num_acc - 1, 0), SPEC_QUERY_LEN - 1).to(
|
||||
tl.int32
|
||||
)
|
||||
shift = 1
|
||||
else:
|
||||
slot_off = tl.full([], 0, tl.int32)
|
||||
shift = qlen
|
||||
if qlen <= 0:
|
||||
return
|
||||
|
||||
has_init = tl.load(has_init_ptr + r) != 0 if HAS_INIT else True
|
||||
src_ok = state_ok if MODE == "spec" else state_ok & has_init
|
||||
|
||||
WRITE_W: tl.constexpr = STATE_WIDTH if MODE == "spec" else STATE_LEN
|
||||
base_state = state_ptr + sid * state_bs
|
||||
for i in tl.static_range(0, WRITE_W):
|
||||
m = shift + i
|
||||
from_state = m <= STATE_LEN - 1
|
||||
# Short or graph-padded spec chunks preserve the unused state tail.
|
||||
do_write = i < STATE_LEN + qlen - 1 if MODE == "spec" else True
|
||||
state_value = tl.load(
|
||||
base_state + (slot_off + m) * state_ws + c_offs * state_cs,
|
||||
mask=c_mask & from_state & src_ok,
|
||||
other=0.0,
|
||||
)
|
||||
input_t = q_start + m - STATE_LEN
|
||||
input_t = tl.load(
|
||||
token_idx_ptr + input_t,
|
||||
mask=has_token_map & (~from_state) & do_write,
|
||||
other=input_t,
|
||||
).to(tl.int64)
|
||||
input_value = tl.load(
|
||||
x_ptr + input_t * C + c_offs,
|
||||
mask=c_mask & (~from_state) & do_write,
|
||||
other=0.0,
|
||||
)
|
||||
value = tl.where(from_state, state_value, input_value)
|
||||
tl.store(
|
||||
base_state + i * state_ws + c_offs * state_cs,
|
||||
value,
|
||||
mask=c_mask & do_write,
|
||||
)
|
||||
|
||||
|
||||
def ple_conv(
|
||||
inputs: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
conv_weights: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
*,
|
||||
mode: Literal["decode", "spec", "prefill"],
|
||||
dilation: int,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
has_initial_states: torch.Tensor | None = None,
|
||||
spec_query_len: int = 1,
|
||||
token_indices: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Add short-convolution output to ``residual`` and update its state."""
|
||||
BLOCK_C = 512
|
||||
kernel_spec_query_len = spec_query_len if mode == "spec" else 1
|
||||
T, C = inputs.shape
|
||||
K = conv_weights.shape[1]
|
||||
state_len = (K - 1) * dilation
|
||||
state_width = state_len + kernel_spec_query_len - 1
|
||||
if token_indices is not None:
|
||||
T = token_indices.numel()
|
||||
if conv_state.shape[1] != C or conv_state.shape[2] < state_width:
|
||||
raise ValueError(
|
||||
"conv_state must have shape [slots, channels, window], with "
|
||||
f"channels={C} and window >= {state_width}"
|
||||
)
|
||||
state_bs, state_cs, state_ws = conv_state.stride()
|
||||
|
||||
if mode == "decode":
|
||||
num_reqs = T
|
||||
binary_search_iters = 1
|
||||
has_initial_states_arg = has_initial_states is not None
|
||||
elif mode == "spec":
|
||||
if query_start_loc is None or num_accepted_tokens is None:
|
||||
raise ValueError(
|
||||
"query_start_loc and num_accepted_tokens are required for spec decode"
|
||||
)
|
||||
num_reqs = state_indices.numel()
|
||||
binary_search_iters = max(num_reqs, 1).bit_length()
|
||||
has_initial_states_arg = False
|
||||
elif mode == "prefill":
|
||||
if query_start_loc is None or has_initial_states is None:
|
||||
raise ValueError(
|
||||
"query_start_loc and has_initial_states are required for prefill"
|
||||
)
|
||||
num_reqs = state_indices.numel()
|
||||
binary_search_iters = max(num_reqs, 1).bit_length()
|
||||
has_initial_states_arg = True
|
||||
else:
|
||||
raise ValueError(f"Unsupported short-conv mode: {mode}")
|
||||
|
||||
num_warps = 4 if mode == "prefill" else 8
|
||||
launch_pdl = current_platform.is_arch_support_pdl()
|
||||
|
||||
# Constexpr flags eliminate accesses to optional None arguments. Without a
|
||||
# token map, state_indices is an unused but device-resident placeholder.
|
||||
_ple_conv_kernel[(T, triton.cdiv(C, BLOCK_C))](
|
||||
inputs,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
residual,
|
||||
state_indices,
|
||||
query_start_loc,
|
||||
num_accepted_tokens,
|
||||
has_initial_states,
|
||||
token_indices if token_indices is not None else state_indices,
|
||||
token_indices is not None,
|
||||
num_reqs,
|
||||
binary_search_iters,
|
||||
state_bs,
|
||||
state_ws,
|
||||
state_cs,
|
||||
C=C,
|
||||
BLOCK_C=BLOCK_C,
|
||||
STATE_LEN=state_len,
|
||||
DILATION=dilation,
|
||||
KERNEL_SIZE=K,
|
||||
SPEC_QUERY_LEN=kernel_spec_query_len,
|
||||
MODE=mode,
|
||||
HAS_INIT=has_initial_states_arg,
|
||||
NULL_STATE_ID=NULL_BLOCK_ID,
|
||||
launch_pdl=launch_pdl,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
# conv state update is fused with the kernel above for decode
|
||||
if mode != "decode":
|
||||
_ple_conv_writeback_kernel[(num_reqs, triton.cdiv(C, BLOCK_C))](
|
||||
inputs,
|
||||
conv_state,
|
||||
state_indices,
|
||||
query_start_loc,
|
||||
num_accepted_tokens,
|
||||
has_initial_states,
|
||||
token_indices if token_indices is not None else state_indices,
|
||||
token_indices is not None,
|
||||
state_bs,
|
||||
state_ws,
|
||||
state_cs,
|
||||
C=C,
|
||||
BLOCK_C=BLOCK_C,
|
||||
STATE_LEN=state_len,
|
||||
SPEC_QUERY_LEN=kernel_spec_query_len,
|
||||
STATE_WIDTH=state_width,
|
||||
MODE=mode,
|
||||
HAS_INIT=has_initial_states_arg,
|
||||
NULL_STATE_ID=NULL_BLOCK_ID,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GPU-resident Qwen4Exp position-learning enhancement layers."""
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Sequence
|
||||
|
||||
import torch
|
||||
@@ -11,7 +10,7 @@ from torch import nn
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.model_executor.layers.mamba.abstract import MambaBase
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
@@ -42,9 +41,9 @@ from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionBackend,
|
||||
PleShortConvAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
from ..common.ple import PLEVocabParallelEmbedding
|
||||
from .ops.ple import ple_conv, ple_gate, ple_ngram_ids
|
||||
|
||||
|
||||
class Qwen4ExpPLEGroupedNorm(nn.Module):
|
||||
@@ -256,8 +255,6 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
config: Qwen4ExpTextConfig,
|
||||
embedding_dim: int,
|
||||
ple_dense_layer_id: int,
|
||||
max_total_tokens: int,
|
||||
max_num_reqs: int,
|
||||
prefix: str,
|
||||
layer_name: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
@@ -324,20 +321,6 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
quant_config, f"{prefix}.ngram_embedding"
|
||||
),
|
||||
)
|
||||
self.register_buffer(
|
||||
"positions_buffer",
|
||||
torch.arange(max_total_tokens, dtype=torch.int64),
|
||||
persistent=False,
|
||||
)
|
||||
self.register_buffer(
|
||||
"padded_buffer",
|
||||
torch.full(
|
||||
(max_num_reqs, max_total_tokens),
|
||||
self.eos_token_id,
|
||||
dtype=torch.int64,
|
||||
),
|
||||
persistent=False,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _shift_precompute(
|
||||
@@ -379,26 +362,34 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
output: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute n-gram embedding indices for the current request layout."""
|
||||
input_ids = input_ids.reshape(-1).long()
|
||||
query_start_loc = query_start_loc.long()
|
||||
input_ids = input_ids.reshape(-1)
|
||||
num_reqs = query_start_loc.numel() - 1
|
||||
num_tokens = input_ids.shape[0]
|
||||
if num_tokens > self.positions_buffer.numel():
|
||||
raise ValueError(
|
||||
f"PLE received {num_tokens} tokens, but its workspace supports "
|
||||
f"at most {self.positions_buffer.numel()}"
|
||||
)
|
||||
if num_reqs > self.padded_buffer.shape[0]:
|
||||
raise ValueError(
|
||||
f"PLE received {num_reqs} requests, but its workspace supports "
|
||||
f"at most {self.padded_buffer.shape[0]}"
|
||||
)
|
||||
|
||||
positions = self.positions_buffer[:num_tokens]
|
||||
packed = self.padded_buffer[:num_reqs, :num_tokens]
|
||||
packed.fill_(self.eos_token_id)
|
||||
if input_ids.is_cuda:
|
||||
return ple_ngram_ids(
|
||||
input_ids=input_ids,
|
||||
query_start_loc=query_start_loc,
|
||||
ngram_context=ngram_context,
|
||||
layer_multipliers=self.layer_multipliers,
|
||||
ngram_heads_vocab_sizes=self.ngram_heads_vocab_sizes,
|
||||
ngram_heads_offsets=self.ngram_heads_offsets,
|
||||
eos_token_id=self.eos_token_id,
|
||||
heads_per_ngram=self.heads_per_ngram,
|
||||
output=output,
|
||||
)
|
||||
input_ids = input_ids.long()
|
||||
query_start_loc = query_start_loc.long()
|
||||
positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64)
|
||||
packed = torch.full(
|
||||
(num_reqs, num_tokens),
|
||||
self.eos_token_id,
|
||||
device=input_ids.device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1
|
||||
request_indices.clamp_(max=num_reqs - 1)
|
||||
columns = (positions - query_start_loc[request_indices]).clamp(
|
||||
@@ -446,12 +437,13 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
ngram_ids = input_ids.new_empty(
|
||||
(input_ids.shape[0], self.ngram_heads),
|
||||
dtype=torch.long,
|
||||
)
|
||||
# Keep num_reqs-dependent ID generation outside PIECEWISE CUDA graphs,
|
||||
# which dispatch only on the padded token count.
|
||||
# torch.compile requires the splitting op to write graph-owned storage.
|
||||
# Once compilation is removed, the op can return the IDs directly.
|
||||
ngram_ids = input_ids.new_empty(
|
||||
(input_ids.numel(), self.ngram_heads), dtype=torch.long
|
||||
)
|
||||
torch.ops.vllm.qwen4_exp_compute_ple_ngram_ids(
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
@@ -562,26 +554,20 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
config,
|
||||
int(config.ple_embed_dim),
|
||||
self.ple_dense_layer_id,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
vllm_config.scheduler_config.max_num_seqs,
|
||||
prefix=f"{prefix}.ple_embedding",
|
||||
layer_name=prefix,
|
||||
quant_config=quant_config,
|
||||
params_dtype=model_config.dtype,
|
||||
)
|
||||
self.key_proj = ReplicatedLinear(
|
||||
# The PLE cache is TP-replicated, so this merged projection is too.
|
||||
self.kv_proj = MergedColumnParallelLinear(
|
||||
int(config.ple_embed_dim),
|
||||
self.hc_hidden_size,
|
||||
[self.hc_hidden_size, self.hidden_size],
|
||||
bias=False,
|
||||
params_dtype=model_config.dtype,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.key_proj",
|
||||
)
|
||||
self.value_proj = ReplicatedLinear(
|
||||
int(config.ple_embed_dim),
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.value_proj",
|
||||
prefix=f"{prefix}.kv_proj",
|
||||
disable_tp=True,
|
||||
)
|
||||
norm_args = (
|
||||
self.hc_hidden_size,
|
||||
@@ -654,360 +640,14 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
num_spec=self.num_spec_tokens,
|
||||
)
|
||||
|
||||
def _apply_norm(
|
||||
self, norm: Qwen4ExpPLEGroupedNorm, hidden_states: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
shape = hidden_states.shape
|
||||
return norm(hidden_states.flatten(-2)).reshape(shape)
|
||||
|
||||
def _short_conv_fallback(self, inputs: torch.Tensor) -> torch.Tensor:
|
||||
# Profiling / CUDA graph capture only; conv state is not updated.
|
||||
inputs_t = inputs.transpose(0, 1).unsqueeze(0)
|
||||
output = self.conv1d(inputs_t)[..., : inputs_t.size(-1)]
|
||||
return F.silu(output).squeeze(0).transpose(0, 1)
|
||||
|
||||
def _short_conv_dilated_decode_batched(
|
||||
self,
|
||||
x_d: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
conv_weights: torch.Tensor,
|
||||
state_indices_tensor_d: torch.Tensor,
|
||||
has_initial_states_d: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
state_indices = state_indices_tensor_d.to(
|
||||
device=conv_state.device, dtype=torch.int64
|
||||
)
|
||||
# FULL cudagraph padded decode rows use NULL_BLOCK_ID. Remap them to
|
||||
# slot 0 for a safe gather, then zero output and skip write-back.
|
||||
valid_state = state_indices != NULL_BLOCK_ID
|
||||
state_indices = torch.where(
|
||||
valid_state, state_indices, torch.zeros_like(state_indices)
|
||||
)
|
||||
if has_initial_states_d is None:
|
||||
has_initial_state = valid_state
|
||||
else:
|
||||
if has_initial_states_d.numel() < state_indices_tensor_d.numel():
|
||||
raise ValueError(
|
||||
"has_initial_states_d size mismatch: "
|
||||
f"got {has_initial_states_d.numel()}, "
|
||||
f"need >= {state_indices_tensor_d.numel()}."
|
||||
)
|
||||
has_initial_state = has_initial_states_d[
|
||||
: state_indices_tensor_d.numel()
|
||||
].to(device=conv_state.device, dtype=torch.bool)
|
||||
has_initial_state = has_initial_state & valid_state
|
||||
|
||||
cached_state = conv_state.index_select(0, state_indices)
|
||||
state = cached_state[..., : self.conv_state_len].to(x_d.dtype)
|
||||
if self.conv_state_len > 0:
|
||||
initial_state = torch.where(
|
||||
has_initial_state.view(-1, 1, 1),
|
||||
state,
|
||||
torch.zeros_like(state),
|
||||
)
|
||||
history = torch.cat((initial_state, x_d.unsqueeze(-1)), dim=-1)
|
||||
else:
|
||||
history = x_d.unsqueeze(-1)
|
||||
|
||||
conv_output = F.conv1d(
|
||||
history,
|
||||
conv_weights.unsqueeze(1).contiguous(),
|
||||
groups=history.size(1),
|
||||
dilation=self.short_conv_dilation,
|
||||
).squeeze(-1)
|
||||
output = F.silu(conv_output)
|
||||
output = output * valid_state.view(-1, 1).to(output.dtype)
|
||||
|
||||
if self.conv_state_len > 0:
|
||||
next_state = history[..., -self.conv_state_len :]
|
||||
# Padded rows are remapped to the reserved null slot. Preserve its
|
||||
# existing value while writing the new states for valid rows.
|
||||
existing_base_state = cached_state[..., : self.conv_state_len]
|
||||
safe_next_state = torch.where(
|
||||
valid_state.view(-1, 1, 1),
|
||||
next_state.to(conv_state.dtype),
|
||||
existing_base_state,
|
||||
)
|
||||
cached_state[..., : self.conv_state_len] = safe_next_state
|
||||
conv_state.index_copy_(0, state_indices, cached_state)
|
||||
|
||||
return output
|
||||
|
||||
def _short_conv_dilated_prefill_batched(
|
||||
self,
|
||||
x_p: torch.Tensor,
|
||||
metadata: PleShortConvAttentionMetadata,
|
||||
conv_state: torch.Tensor,
|
||||
conv_weights: torch.Tensor,
|
||||
state_indices_tensor_p: torch.Tensor,
|
||||
num_prefills: int,
|
||||
num_decode_tokens: int,
|
||||
num_prefill_tokens: int,
|
||||
) -> torch.Tensor:
|
||||
# ``non_spec_query_start_loc`` covers the non-spec (decode + prefill)
|
||||
# requests and equals ``query_start_loc`` when spec-decode is inactive.
|
||||
non_spec_query_start_loc = metadata.non_spec_query_start_loc
|
||||
if non_spec_query_start_loc is None:
|
||||
raise ValueError("query_start_loc is required for prefill short-conv")
|
||||
query_start_loc_p = (
|
||||
non_spec_query_start_loc[-num_prefills - 1 :] - num_decode_tokens
|
||||
)
|
||||
# The metadata builder guarantees that the prefill query offsets start
|
||||
# at 0 and end at num_prefill_tokens. Avoid reading those values here,
|
||||
# since doing so would force a device-to-host synchronization.
|
||||
has_initial_states_p = metadata.has_initial_states_p
|
||||
if has_initial_states_p is None:
|
||||
raise ValueError("has_initial_states_p is required for prefill short-conv")
|
||||
|
||||
output = torch.empty_like(x_p)
|
||||
q_starts = query_start_loc_p.to(torch.int64)
|
||||
if state_indices_tensor_p.numel() < num_prefills:
|
||||
raise ValueError(
|
||||
"state_indices_tensor_p size mismatch: "
|
||||
f"got {state_indices_tensor_p.numel()}, "
|
||||
f"need >= {num_prefills}."
|
||||
)
|
||||
if has_initial_states_p.numel() < num_prefills:
|
||||
raise ValueError(
|
||||
"has_initial_states_p size mismatch: "
|
||||
f"got {has_initial_states_p.numel()}, "
|
||||
f"need >= {num_prefills}."
|
||||
)
|
||||
if num_prefills == 0 or x_p.numel() == 0:
|
||||
return output
|
||||
lengths = q_starts[1:] - q_starts[:-1]
|
||||
# Use the CPU-computed packing width from the metadata builder instead
|
||||
# of synchronizing on lengths.max().
|
||||
max_len = metadata.max_prefill_query_len
|
||||
if max_len <= 0:
|
||||
return output
|
||||
|
||||
hidden_size = x_p.shape[1]
|
||||
positions = torch.arange(
|
||||
num_prefill_tokens, device=x_p.device, dtype=torch.int64
|
||||
)
|
||||
req_indices = torch.searchsorted(q_starts[1:], positions, right=True)
|
||||
col_indices = positions - q_starts[req_indices]
|
||||
|
||||
packed_tokens = x_p.new_zeros((num_prefills, max_len, hidden_size))
|
||||
packed_tokens[req_indices, col_indices] = x_p
|
||||
packed_tokens = packed_tokens.transpose(1, 2).contiguous()
|
||||
|
||||
state_indices = state_indices_tensor_p[:num_prefills].to(
|
||||
device=conv_state.device, dtype=torch.int64
|
||||
)
|
||||
valid_state = state_indices != NULL_BLOCK_ID
|
||||
state_indices = torch.where(
|
||||
valid_state, state_indices, torch.zeros_like(state_indices)
|
||||
)
|
||||
has_initial = has_initial_states_p[:num_prefills].to(
|
||||
device=conv_state.device, dtype=torch.bool
|
||||
)
|
||||
if self.conv_state_len > 0:
|
||||
if conv_state.shape[0] == 0:
|
||||
state = conv_state.new_zeros(
|
||||
(num_prefills, hidden_size, self.conv_state_len),
|
||||
dtype=x_p.dtype,
|
||||
)
|
||||
else:
|
||||
state = conv_state.index_select(0, state_indices)[
|
||||
..., : self.conv_state_len
|
||||
].to(x_p.dtype)
|
||||
use_initial_mask = (valid_state & has_initial).view(num_prefills, 1, 1)
|
||||
initial_state = torch.where(
|
||||
use_initial_mask,
|
||||
state,
|
||||
torch.zeros_like(state),
|
||||
)
|
||||
history = torch.cat((initial_state, packed_tokens), dim=-1)
|
||||
else:
|
||||
history = packed_tokens
|
||||
|
||||
conv_output = F.conv1d(
|
||||
history,
|
||||
conv_weights.unsqueeze(1).contiguous(),
|
||||
groups=history.size(1),
|
||||
dilation=self.short_conv_dilation,
|
||||
)
|
||||
conv_output = F.silu(conv_output).transpose(1, 2).contiguous()
|
||||
|
||||
token_positions = torch.arange(max_len, device=x_p.device, dtype=torch.int64)
|
||||
valid_tokens = token_positions.view(1, max_len) < lengths.view(num_prefills, 1)
|
||||
valid_output_mask = valid_tokens & valid_state.to(device=x_p.device).view(
|
||||
num_prefills, 1
|
||||
)
|
||||
conv_output.masked_fill_(~valid_output_mask.unsqueeze(-1), 0)
|
||||
output.copy_(conv_output[req_indices, col_indices])
|
||||
|
||||
if self.conv_state_len > 0 and conv_state.shape[0] > 0:
|
||||
state_starts = lengths.to(device=history.device, dtype=torch.int64).view(
|
||||
num_prefills, 1, 1
|
||||
)
|
||||
state_offsets = torch.arange(
|
||||
self.conv_state_len, device=history.device, dtype=torch.int64
|
||||
).view(1, 1, self.conv_state_len)
|
||||
next_state = history.gather(
|
||||
dim=2,
|
||||
index=(state_starts + state_offsets).expand(-1, history.size(1), -1),
|
||||
)
|
||||
# Write back without a host synchronization. Valid, non-empty rows
|
||||
# receive their new state; padding and zero-length rows keep the
|
||||
# current cache value.
|
||||
existing_state = conv_state.index_select(0, state_indices)
|
||||
existing_base_state = existing_state[..., : self.conv_state_len]
|
||||
update_mask = valid_state & (lengths.to(device=conv_state.device) > 0)
|
||||
safe_next_state = torch.where(
|
||||
update_mask.view(num_prefills, 1, 1),
|
||||
next_state.to(conv_state.dtype),
|
||||
existing_base_state,
|
||||
)
|
||||
existing_state[..., : self.conv_state_len] = safe_next_state
|
||||
conv_state.index_copy_(0, state_indices, existing_state)
|
||||
return output
|
||||
|
||||
def _short_conv_dilated_spec_batched(
|
||||
self,
|
||||
x_spec: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
conv_weights: torch.Tensor,
|
||||
spec_state_indices_tensor: torch.Tensor,
|
||||
spec_query_start_loc: torch.Tensor,
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
spec_query_len: int,
|
||||
) -> torch.Tensor:
|
||||
"""Dilated short-conv for speculative-decode (MTP) requests.
|
||||
|
||||
Each spec request feeds multiple (draft + 1) query tokens. The conv
|
||||
outputs are computed causally after rolling back the previous draft
|
||||
state by ``num_accepted_tokens - 1``. The current candidate inputs stay
|
||||
in the extended cache for the next forward, matching
|
||||
``causal_conv1d_update``.
|
||||
|
||||
``spec_query_len`` (== num_speculative_tokens + 1) is the maximum query
|
||||
length and is a Python int, so no host synchronization is needed; this
|
||||
keeps the path safe for full CUDA-graph capture/replay where the buffers
|
||||
are padded at the request level.
|
||||
"""
|
||||
num_reqs = spec_state_indices_tensor.numel()
|
||||
hidden_size = x_spec.size(-1)
|
||||
# Use a fixed packing width instead of synchronizing on lengths.max().
|
||||
max_len = spec_query_len
|
||||
# Full CUDA graphs can pad these buffers. Only the first num_reqs
|
||||
# accepted-token counts belong to actual speculative requests.
|
||||
num_accepted_tokens = num_accepted_tokens[:num_reqs]
|
||||
q_starts = spec_query_start_loc[: num_reqs + 1].to(torch.int64)
|
||||
# Keep the number of real speculative tokens on the device.
|
||||
total_real_tokens = q_starts[num_reqs]
|
||||
|
||||
state_indices = spec_state_indices_tensor.to(
|
||||
device=conv_state.device, dtype=torch.int64
|
||||
)
|
||||
valid_state = state_indices != NULL_BLOCK_ID
|
||||
state_indices = torch.where(
|
||||
valid_state, state_indices, torch.zeros_like(state_indices)
|
||||
)
|
||||
positions = torch.arange(
|
||||
x_spec.size(0), device=x_spec.device, dtype=torch.int64
|
||||
)
|
||||
# Route graph-padded token rows to the discarded dummy request so that
|
||||
# they cannot overwrite real packed data.
|
||||
req_indices = torch.searchsorted(q_starts[1:], positions, right=True)
|
||||
valid_tokens = (positions < total_real_tokens) & (req_indices < num_reqs)
|
||||
clamped_req_indices = req_indices.clamp_max(max(num_reqs - 1, 0))
|
||||
col_indices = (positions - q_starts[clamped_req_indices]).clamp_(0, max_len - 1)
|
||||
pack_req_indices = torch.where(
|
||||
valid_tokens,
|
||||
clamped_req_indices,
|
||||
torch.full_like(req_indices, num_reqs),
|
||||
)
|
||||
pack_col_indices = torch.where(
|
||||
valid_tokens, col_indices, torch.zeros_like(col_indices)
|
||||
)
|
||||
|
||||
# The last request row is the dummy sink for graph padding.
|
||||
packed = x_spec.new_zeros((num_reqs + 1, max_len, hidden_size))
|
||||
packed[pack_req_indices, pack_col_indices] = x_spec
|
||||
packed = packed.transpose(1, 2).contiguous()
|
||||
|
||||
if self.conv_state_len > 0:
|
||||
cached_state = conv_state.index_select(0, state_indices)
|
||||
rollback_offsets = num_accepted_tokens.to(
|
||||
device=conv_state.device, dtype=torch.int64
|
||||
).sub(1)
|
||||
rollback_offsets = torch.where(
|
||||
valid_state,
|
||||
rollback_offsets.clamp_(0, max_len - 1),
|
||||
torch.zeros_like(rollback_offsets),
|
||||
)
|
||||
state_offsets = torch.arange(
|
||||
self.conv_state_len, device=conv_state.device, dtype=torch.int64
|
||||
).view(1, 1, self.conv_state_len)
|
||||
rollback_indices = rollback_offsets.view(-1, 1, 1) + state_offsets
|
||||
state = cached_state.gather(
|
||||
2, rollback_indices.expand(-1, hidden_size, -1)
|
||||
).to(x_spec.dtype)
|
||||
state = torch.where(
|
||||
valid_state.view(num_reqs, 1, 1),
|
||||
state,
|
||||
torch.zeros_like(state),
|
||||
)
|
||||
# Append a zeroed dummy-row state to match the [num_reqs + 1] pack.
|
||||
dummy_state = state.new_zeros((1, hidden_size, self.conv_state_len))
|
||||
state_full = torch.cat((state, dummy_state), dim=0)
|
||||
history = torch.cat((state_full, packed), dim=-1)
|
||||
else:
|
||||
history = packed
|
||||
|
||||
conv_output = F.conv1d(
|
||||
history,
|
||||
conv_weights.unsqueeze(1).contiguous(),
|
||||
groups=history.size(1),
|
||||
dilation=self.short_conv_dilation,
|
||||
)
|
||||
conv_output = F.silu(conv_output).transpose(1, 2).contiguous()
|
||||
|
||||
output = conv_output[pack_req_indices, pack_col_indices]
|
||||
output = output * valid_tokens.view(-1, 1).to(output.dtype)
|
||||
|
||||
# Keep all current candidate inputs in the extended state. On the next
|
||||
# target forward, ``num_accepted_tokens - 1`` selects the rollback
|
||||
# window before processing the newly scheduled tokens.
|
||||
if self.conv_state_len > 0:
|
||||
state_capacity = self.conv_state_len + max_len - 1
|
||||
if conv_state.size(-1) < state_capacity:
|
||||
raise RuntimeError(
|
||||
"PLE short-conv cache cannot retain speculative tokens: "
|
||||
f"got {conv_state.size(-1)}, need {state_capacity}."
|
||||
)
|
||||
candidate_state = history[:num_reqs, :, 1 : state_capacity + 1]
|
||||
query_lengths = q_starts[1:] - q_starts[:-1]
|
||||
state_positions = torch.arange(
|
||||
state_capacity, device=history.device, dtype=torch.int64
|
||||
).view(1, 1, state_capacity)
|
||||
update_lengths = (self.conv_state_len + query_lengths - 1).view(
|
||||
num_reqs, 1, 1
|
||||
)
|
||||
update_mask = valid_state.view(num_reqs, 1, 1) & (
|
||||
state_positions < update_lengths
|
||||
)
|
||||
existing_state = cached_state[..., :state_capacity]
|
||||
next_state = torch.where(
|
||||
update_mask,
|
||||
candidate_state.to(conv_state.dtype),
|
||||
existing_state,
|
||||
)
|
||||
cached_state[..., :state_capacity] = next_state
|
||||
conv_state.index_copy_(0, state_indices, cached_state)
|
||||
|
||||
return output
|
||||
|
||||
def _short_conv_dilated_dispatch(
|
||||
self,
|
||||
inputs: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
metadata: PleShortConvAttentionMetadata,
|
||||
conv_state: torch.Tensor,
|
||||
conv_weights: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
) -> None:
|
||||
num_prefills = metadata.num_prefills
|
||||
num_decodes = metadata.num_decodes
|
||||
num_decode_tokens = metadata.num_decode_tokens
|
||||
@@ -1015,110 +655,129 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
has_prefill = num_prefills > 0
|
||||
has_decode = num_decodes > 0
|
||||
has_spec = metadata.spec_sequence_masks is not None
|
||||
x = inputs[: metadata.num_actual_tokens]
|
||||
has_non_spec = has_prefill or has_decode
|
||||
inputs = inputs[: metadata.num_actual_tokens]
|
||||
residual = residual[: metadata.num_actual_tokens]
|
||||
|
||||
# Split spec / non-spec tokens.
|
||||
if has_spec:
|
||||
if has_prefill or has_decode:
|
||||
assert metadata.spec_token_indx is not None
|
||||
assert metadata.non_spec_token_indx is not None
|
||||
x_spec = x.index_select(0, metadata.spec_token_indx.long())
|
||||
x_non_spec = x.index_select(0, metadata.non_spec_token_indx.long())
|
||||
else:
|
||||
x_spec = x
|
||||
x_non_spec = None
|
||||
else:
|
||||
x_spec = None
|
||||
x_non_spec = x
|
||||
spec_token_indices = None
|
||||
non_spec_token_indices = None
|
||||
if has_spec and has_non_spec:
|
||||
assert metadata.spec_token_indx is not None
|
||||
assert metadata.non_spec_token_indx is not None
|
||||
spec_token_indices = metadata.spec_token_indx
|
||||
non_spec_token_indices = metadata.non_spec_token_indx
|
||||
|
||||
spec_output = None
|
||||
# 1. Run the multi-query speculative-decode part.
|
||||
if has_spec:
|
||||
assert metadata.spec_state_indices_tensor is not None
|
||||
assert metadata.spec_query_start_loc is not None
|
||||
assert metadata.num_accepted_tokens is not None
|
||||
spec_output = self._short_conv_dilated_spec_batched(
|
||||
x_spec=x_spec,
|
||||
query_start_loc = metadata.spec_query_start_loc
|
||||
num_accepted_tokens = metadata.num_accepted_tokens
|
||||
assert query_start_loc is not None
|
||||
assert num_accepted_tokens is not None
|
||||
spec_state_indices = metadata.spec_state_indices_tensor[
|
||||
: metadata.num_spec_decodes
|
||||
]
|
||||
# Mixed batches stay in their original row order; the kernels map
|
||||
# logical spec/non-spec rows instead of materializing both groups.
|
||||
ple_conv(
|
||||
inputs=inputs,
|
||||
residual=residual,
|
||||
conv_state=conv_state,
|
||||
conv_weights=conv_weights,
|
||||
spec_state_indices_tensor=metadata.spec_state_indices_tensor[
|
||||
: metadata.num_spec_decodes
|
||||
],
|
||||
spec_query_start_loc=metadata.spec_query_start_loc,
|
||||
num_accepted_tokens=metadata.num_accepted_tokens,
|
||||
state_indices=spec_state_indices,
|
||||
mode="spec",
|
||||
dilation=self.short_conv_dilation,
|
||||
query_start_loc=query_start_loc,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
spec_query_len=metadata.spec_query_len,
|
||||
token_indices=spec_token_indices,
|
||||
)
|
||||
|
||||
# 2. Run regular decode and prefill requests.
|
||||
conv_out_non_spec = None
|
||||
state_indices_tensor = metadata.state_indices_tensor
|
||||
if x_non_spec is not None:
|
||||
assert state_indices_tensor is not None
|
||||
if has_prefill:
|
||||
state_indices_tensor_d, state_indices_tensor_p = torch.split(
|
||||
state_indices_tensor,
|
||||
[num_decodes, num_prefills],
|
||||
dim=0,
|
||||
if not has_non_spec:
|
||||
return
|
||||
|
||||
state_indices = metadata.state_indices_tensor
|
||||
assert state_indices is not None
|
||||
if has_prefill:
|
||||
state_indices_d, state_indices_p = torch.split(
|
||||
state_indices, [num_decodes, num_prefills], dim=0
|
||||
)
|
||||
if non_spec_token_indices is None:
|
||||
inputs_d, inputs_p = torch.split(
|
||||
inputs, [num_decode_tokens, num_prefill_tokens], dim=0
|
||||
)
|
||||
x_d, x_p = torch.split(
|
||||
x_non_spec,
|
||||
residual_d, residual_p = torch.split(
|
||||
residual, [num_decode_tokens, num_prefill_tokens], dim=0
|
||||
)
|
||||
token_indices_d = None
|
||||
token_indices_p = None
|
||||
else:
|
||||
inputs_d = inputs_p = inputs
|
||||
residual_d = residual_p = residual
|
||||
token_indices_d, token_indices_p = torch.split(
|
||||
non_spec_token_indices,
|
||||
[num_decode_tokens, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
non_spec_parts: list[torch.Tensor] = []
|
||||
if has_decode:
|
||||
non_spec_parts.append(
|
||||
self._short_conv_dilated_decode_batched(
|
||||
x_d=x_d,
|
||||
conv_state=conv_state,
|
||||
conv_weights=conv_weights,
|
||||
state_indices_tensor_d=state_indices_tensor_d,
|
||||
has_initial_states_d=metadata.has_initial_states_d,
|
||||
)
|
||||
)
|
||||
non_spec_parts.append(
|
||||
self._short_conv_dilated_prefill_batched(
|
||||
x_p=x_p,
|
||||
metadata=metadata,
|
||||
conv_state=conv_state,
|
||||
conv_weights=conv_weights,
|
||||
state_indices_tensor_p=state_indices_tensor_p,
|
||||
num_prefills=num_prefills,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_prefill_tokens=num_prefill_tokens,
|
||||
)
|
||||
)
|
||||
conv_out_non_spec = torch.vstack(non_spec_parts)
|
||||
else:
|
||||
conv_out_non_spec = self._short_conv_dilated_decode_batched(
|
||||
x_d=x_non_spec,
|
||||
|
||||
if has_decode:
|
||||
ple_conv(
|
||||
inputs=inputs_d,
|
||||
residual=residual_d,
|
||||
conv_state=conv_state,
|
||||
conv_weights=conv_weights,
|
||||
state_indices_tensor_d=state_indices_tensor[: x_non_spec.size(0)],
|
||||
has_initial_states_d=metadata.has_initial_states_d,
|
||||
state_indices=state_indices_d,
|
||||
mode="decode",
|
||||
dilation=self.short_conv_dilation,
|
||||
has_initial_states=metadata.has_initial_states_d,
|
||||
token_indices=token_indices_d,
|
||||
)
|
||||
|
||||
# 3. Merge both parts back into the original token order.
|
||||
if has_spec and conv_out_non_spec is not None:
|
||||
assert metadata.spec_token_indx is not None
|
||||
assert metadata.non_spec_token_indx is not None
|
||||
assert spec_output is not None
|
||||
output = x.new_empty((metadata.num_actual_tokens, x.size(-1)))
|
||||
output.index_copy_(0, metadata.spec_token_indx, spec_output)
|
||||
output.index_copy_(0, metadata.non_spec_token_indx, conv_out_non_spec)
|
||||
return output
|
||||
elif has_spec:
|
||||
assert spec_output is not None
|
||||
return spec_output
|
||||
if conv_out_non_spec is None:
|
||||
return x
|
||||
return conv_out_non_spec
|
||||
query_start_loc = metadata.non_spec_query_start_loc
|
||||
if query_start_loc is None:
|
||||
raise ValueError("query_start_loc is required for prefill short-conv")
|
||||
query_start_loc = query_start_loc[-num_prefills - 1 :] - num_decode_tokens
|
||||
has_initial_states = metadata.has_initial_states_p
|
||||
if has_initial_states is None:
|
||||
raise ValueError(
|
||||
"has_initial_states_p is required for prefill short-conv"
|
||||
)
|
||||
ple_conv(
|
||||
inputs=inputs_p,
|
||||
residual=residual_p,
|
||||
conv_state=conv_state,
|
||||
conv_weights=conv_weights,
|
||||
state_indices=state_indices_p,
|
||||
mode="prefill",
|
||||
dilation=self.short_conv_dilation,
|
||||
query_start_loc=query_start_loc,
|
||||
has_initial_states=has_initial_states,
|
||||
token_indices=token_indices_p,
|
||||
)
|
||||
else:
|
||||
num_decode_rows = (
|
||||
non_spec_token_indices.numel()
|
||||
if non_spec_token_indices is not None
|
||||
else inputs.size(0)
|
||||
)
|
||||
ple_conv(
|
||||
inputs=inputs,
|
||||
residual=residual,
|
||||
conv_state=conv_state,
|
||||
conv_weights=conv_weights,
|
||||
state_indices=state_indices[:num_decode_rows],
|
||||
mode="decode",
|
||||
dilation=self.short_conv_dilation,
|
||||
has_initial_states=metadata.has_initial_states_d,
|
||||
token_indices=non_spec_token_indices,
|
||||
)
|
||||
|
||||
def _short_conv(self, inputs: torch.Tensor) -> torch.Tensor:
|
||||
def _short_conv(self, inputs: torch.Tensor, residual: torch.Tensor) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
# Profiling omits all metadata or this Mamba entry. The residual
|
||||
# already contains the gated output, so short convolution is a no-op.
|
||||
if attn_metadata is None:
|
||||
return self._short_conv_fallback(inputs)
|
||||
return
|
||||
|
||||
if not isinstance(attn_metadata, dict):
|
||||
raise RuntimeError(
|
||||
@@ -1128,8 +787,7 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
|
||||
layer_attn_metadata = attn_metadata.get(self.prefix)
|
||||
if layer_attn_metadata is None:
|
||||
# MRV2 omits Mamba-family metadata during profile warmup.
|
||||
return self._short_conv_fallback(inputs)
|
||||
return
|
||||
if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata):
|
||||
raise TypeError(
|
||||
"Expected PleShortConvAttentionMetadata for layer "
|
||||
@@ -1138,24 +796,27 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
)
|
||||
|
||||
conv_state = self.kv_cache[0]
|
||||
# Canonicalize both backend cache layouts to [slot, channel, window].
|
||||
if not is_conv_state_dim_first():
|
||||
conv_state = conv_state.transpose(-1, -2)
|
||||
conv_weights = self.conv1d.weight.squeeze(1)
|
||||
|
||||
state_capacity = self.conv_state_len + self.num_spec_tokens
|
||||
if state_capacity > 0:
|
||||
if conv_state.size(-1) < state_capacity:
|
||||
state_size = conv_state.size(-1)
|
||||
if state_size < state_capacity:
|
||||
raise RuntimeError(
|
||||
"PLE short-conv cache is smaller than expected for "
|
||||
f"dilated convolution: got {conv_state.size(-1)}, "
|
||||
f"dilated convolution: got {state_size}, "
|
||||
f"expect at least {state_capacity}."
|
||||
)
|
||||
conv_state = conv_state[..., -state_capacity:]
|
||||
return self._short_conv_dilated_dispatch(
|
||||
inputs,
|
||||
layer_attn_metadata,
|
||||
conv_state,
|
||||
conv_weights.to(dtype=inputs.dtype),
|
||||
self._short_conv_dilated_dispatch(
|
||||
inputs=inputs,
|
||||
residual=residual,
|
||||
metadata=layer_attn_metadata,
|
||||
conv_state=conv_state,
|
||||
conv_weights=conv_weights.to(dtype=inputs.dtype),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1174,24 +835,21 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
)
|
||||
embeddings = self.ple_embedding(input_ids, query_start_loc, ngram_context)
|
||||
embeddings = self._dequantize_embeddings(embeddings, hidden_states.dtype)
|
||||
key, _ = self.key_proj(embeddings)
|
||||
value, _ = self.value_proj(embeddings)
|
||||
token_count = hidden_states.shape[0]
|
||||
key = key.reshape(token_count, self.hc_count, self.hidden_size)
|
||||
query = hidden_states.reshape(token_count, self.hc_count, self.hidden_size)
|
||||
key = self._apply_norm(self.norm_key, key)
|
||||
query = self._apply_norm(self.norm_query, query)
|
||||
gate = (key * query).sum(dim=-1, keepdim=True) / math.sqrt(self.hidden_size)
|
||||
gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt())
|
||||
gated_value = gate * value.unsqueeze(-2)
|
||||
normalized = self._apply_norm(self.norm_conv, gated_value).flatten(-2)
|
||||
conv_output = torch.zeros_like(normalized)
|
||||
torch.ops.vllm.qwen4_exp_ple_short_conv(
|
||||
normalized,
|
||||
conv_output,
|
||||
self.prefix,
|
||||
kv, _ = self.kv_proj(embeddings)
|
||||
key, value = kv.split(self.kv_proj.output_sizes, dim=-1)
|
||||
gated_output, conv_input = ple_gate(
|
||||
key,
|
||||
value,
|
||||
hidden_states,
|
||||
self.norm_key.weight,
|
||||
self.norm_query.weight,
|
||||
self.norm_conv.weight,
|
||||
self.norm_key.eps,
|
||||
)
|
||||
return gated_value.flatten(-2) + conv_output
|
||||
# State routing depends on runtime request metadata and remains outside
|
||||
# the piecewise graph; short convolution accumulates into gated_output.
|
||||
torch.ops.vllm.qwen4_exp_ple_short_conv(conv_input, gated_output, self.prefix)
|
||||
return gated_output
|
||||
|
||||
|
||||
def qwen4_exp_compute_ple_ngram_ids(
|
||||
@@ -1203,12 +861,12 @@ def qwen4_exp_compute_ple_ngram_ids(
|
||||
) -> None:
|
||||
"""Compute request-dependent PLE n-gram IDs outside piecewise graphs."""
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
ngram_ids = layer.ple_embedding.compute_ngram_ids(
|
||||
layer.ple_embedding.compute_ngram_ids(
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
output,
|
||||
)
|
||||
output.copy_(ngram_ids)
|
||||
|
||||
|
||||
def qwen4_exp_compute_ple_ngram_ids_fake(
|
||||
@@ -1223,17 +881,16 @@ def qwen4_exp_compute_ple_ngram_ids_fake(
|
||||
|
||||
def qwen4_exp_ple_short_conv(
|
||||
inputs: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
residual_output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
result = layer._short_conv(inputs)
|
||||
output[: result.shape[0]].copy_(result)
|
||||
layer._short_conv(inputs, residual_output)
|
||||
|
||||
|
||||
def qwen4_exp_ple_short_conv_fake(
|
||||
inputs: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
residual_output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
return
|
||||
@@ -1250,7 +907,7 @@ direct_register_custom_op(
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_ple_short_conv",
|
||||
op_func=qwen4_exp_ple_short_conv,
|
||||
mutates_args=["output"],
|
||||
mutates_args=["residual_output"],
|
||||
fake_impl=qwen4_exp_ple_short_conv_fake,
|
||||
)
|
||||
|
||||
|
||||
@@ -141,12 +141,6 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder):
|
||||
self.spec_sequence_masks = torch.empty(
|
||||
(self.decode_cudagraph_max_bs,), dtype=torch.bool, device=device
|
||||
)
|
||||
self.spec_token_indx = torch.empty(
|
||||
(self.decode_cudagraph_max_tokens,), dtype=torch.int32, device=device
|
||||
)
|
||||
self.non_spec_token_indx = torch.empty(
|
||||
(self.decode_cudagraph_max_tokens,), dtype=torch.int32, device=device
|
||||
)
|
||||
self.spec_query_start_loc = torch.empty(
|
||||
(self.decode_cudagraph_max_bs + 1,), dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user