[PERF] [Qwen3.5] Split mixed prefill+decode batches: route decodes to the recurrent kernel (#44700)

Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Vadim Gimpelson
2026-06-06 22:13:50 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 67d3792d99
commit fa27d4e9cf
5 changed files with 426 additions and 31 deletions
@@ -66,7 +66,7 @@ from vllm.utils.torch_utils import (
)
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
# Optional ROCm AITER Triton kernels for the GDN decode fast-path.
# Optional ROCm AITER Triton kernels for the GDN decode path.
# Availability is checked centrally via rocm_aiter_ops; the actual function
# references are imported here so that they can be called without per-call
# import overhead.
@@ -897,8 +897,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
projected_states_ba,
z,
core_attn_out,
fast_kernel=True,
layer_name=_encode_layer_name(self.prefix),
use_aiter=True,
)
self._output_projection(core_attn_out, z, output, num_tokens)
@@ -958,7 +958,6 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
b,
a,
core_attn_out,
fast_kernel=False,
layer_name=_encode_layer_name(self.prefix),
)
@@ -1206,7 +1205,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
qkvz/ba layout.
For decode-only (no spec, no prefill) interleaved-GQA layouts,
dispatches directly to ``_forward_core_decode_fast``. Otherwise unpacks
dispatches directly to ``_forward_core_decode_aiter``. Otherwise unpacks
the packed layout and falls through to ``_forward_core``.
Args:
@@ -1237,7 +1236,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
and attn_metadata.num_prefills == 0
and attn_metadata.num_decodes > 0
):
return self._forward_core_decode_fast(
return self._forward_core_decode_aiter(
qkvz=qkvz,
ba=ba,
z_out=z_out,
@@ -1391,6 +1390,15 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
mixed_qkv_non_spec = None
query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec)
# Split mixed non-spec-decode+prefill to process independently
split_non_spec = (
spec_sequence_masks is None
and attn_metadata.num_prefills > 0
and attn_metadata.num_decodes > 0
)
num_decode_tokens = attn_metadata.num_decode_tokens
if attn_metadata.num_prefills > 0:
assert mixed_qkv_non_spec is not None, (
"mixed_qkv_non_spec must be provided for prefill path"
@@ -1402,6 +1410,15 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
a_non_spec = a
b_non_spec = b
if split_non_spec:
conv_output_prefill = mixed_qkv_non_spec[num_decode_tokens:]
a_prefill = a_non_spec[num_decode_tokens:]
b_prefill = b_non_spec[num_decode_tokens:]
else:
conv_output_prefill = mixed_qkv_non_spec
a_prefill = a_non_spec
b_prefill = b_non_spec
(
query_non_spec,
key_non_spec,
@@ -1409,9 +1426,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
g_non_spec,
beta_non_spec,
) = fused_post_conv_prep(
conv_output=mixed_qkv_non_spec,
a=a_non_spec,
b=b_non_spec,
conv_output=conv_output_prefill,
a=a_prefill,
b=b_prefill,
A_log=self.A_log,
dt_bias=self.dt_bias,
num_k_heads=self.num_k_heads // self.tp_size,
@@ -1459,12 +1476,42 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
else:
core_attn_out_spec, last_recurrent_state = None, None
# 2.2: Process the remaining part
# 2.2: Process non-spec-decode part
if split_non_spec:
query_decode, key_decode, value_decode = self.rearrange_mixed_qkv(
mixed_qkv_non_spec[:num_decode_tokens] # type: ignore[index]
)
core_attn_out_decode, _ = fused_sigmoid_gating_delta_rule_update(
A_log=self.A_log,
a=a[:num_decode_tokens],
b=b[:num_decode_tokens],
dt_bias=self.dt_bias,
q=query_decode,
k=key_decode,
v=value_decode,
initial_state=ssm_state,
inplace_final_state=True,
cu_seqlens=non_spec_query_start_loc[ # type: ignore[index]
: attn_metadata.num_decodes + 1
],
ssm_state_indices=non_spec_state_indices_tensor,
use_qk_l2norm_in_kernel=True,
)
else:
core_attn_out_decode = None
# 2.3: Process the remaining part (prefill chunk, or non-spec decode-only)
if attn_metadata.num_prefills > 0:
assert non_spec_state_indices_tensor is not None
initial_state = ssm_state[non_spec_state_indices_tensor].contiguous() # type: ignore[index]
assert has_initial_state is not None
initial_state[~has_initial_state, ...] = 0 # type: ignore[operator]
# State indices, initial-state mask and cu_seqlens for the chunk
# kernel are precomputed by the metadata builder (the prefill tail
# when decodes are peeled off, else the full non-spec batch), so they
# don't need to be re-derived per layer.
prefill_state_indices = attn_metadata.prefill_state_indices
prefill_has_initial_state = attn_metadata.prefill_has_initial_state
assert prefill_state_indices is not None
assert prefill_has_initial_state is not None
initial_state = ssm_state[prefill_state_indices]
initial_state[~prefill_has_initial_state, ...] = 0
(
core_attn_out_non_spec,
last_recurrent_state,
@@ -1476,15 +1523,20 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
beta=beta_non_spec,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=non_spec_query_start_loc,
cu_seqlens=attn_metadata.prefill_query_start_loc,
chunk_indices=attn_metadata.chunk_indices,
chunk_offsets=attn_metadata.chunk_offsets,
use_qk_l2norm_in_kernel=False,
)
# Init cache
ssm_state[non_spec_state_indices_tensor] = last_recurrent_state.to(
ssm_state.dtype
)
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
if split_non_spec:
# Stitch the peeled decode outputs in front of the prefill
# outputs (decode-first order).
core_attn_out_non_spec = torch.cat(
[core_attn_out_decode, core_attn_out_non_spec], dim=1
)
elif attn_metadata.num_decodes > 0:
core_attn_out_non_spec, last_recurrent_state = (
fused_sigmoid_gating_delta_rule_update(
@@ -1523,7 +1575,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
else:
core_attn_out[:num_actual_tokens] = core_attn_out_non_spec.squeeze(0)
def _forward_core_decode_fast(
def _forward_core_decode_aiter(
self,
qkvz: torch.Tensor,
ba: torch.Tensor,
@@ -1649,17 +1701,17 @@ def qwen_gdn_attention_core(
b_or_ba: torch.Tensor,
a_or_z_out: torch.Tensor,
core_attn_out: torch.Tensor,
fast_kernel: bool,
layer_name: LayerNameType,
use_aiter: bool = False,
) -> None:
"""Custom op dispatching to _forward_core or _forward_core_rocm.
Handles conv1d + recurrent attention only; input/output projections
are performed by the caller.
When ``fast_kernel=False`` (standard path):
When ``use_aiter=False`` (standard path):
qkv_or_qkvz is [q, k, v], b_or_ba is b, a_or_z_out is a (read-only).
When ``fast_kernel=True`` (AITER Triton fast path, ROCm only):
When ``use_aiter=True`` (AITER Triton path, ROCm only):
qkv_or_qkvz is [q, k, v, z], b_or_ba is [b, a], a_or_z_out is the
z output buffer (mutated in-place).
@@ -1668,7 +1720,7 @@ def qwen_gdn_attention_core(
layer_name = _resolve_layer_name(layer_name)
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
if fast_kernel:
if use_aiter:
self._forward_core_rocm(
qkvz=qkv_or_qkvz,
ba=b_or_ba,
@@ -1689,8 +1741,8 @@ def gdn_attention_core_fake(
b_or_ba: torch.Tensor,
a_or_z_out: torch.Tensor,
core_attn_out: torch.Tensor,
fast_kernel: bool,
layer_name: LayerNameType,
use_aiter: bool = False,
) -> None:
"""Fake implementation for torch.compile."""
return