From f870b9297685bd3c968063b52b769315cb08fd7f Mon Sep 17 00:00:00 2001 From: Thien Tran Date: Wed, 2 Sep 2026 12:32:16 +0800 Subject: [PATCH] [Qwen3.8-Flash-Next] Fuse Qwen4Exp PLE kernels (#54517) Signed-off-by: Thien Tran Co-authored-by: Codex --- tests/models/qwen4_exp/test_ple.py | 1052 +++++++++++++++-- vllm/models/qwen4_exp/nvidia/model.py | 14 +- vllm/models/qwen4_exp/nvidia/mtp.py | 4 +- vllm/models/qwen4_exp/nvidia/ops/ple.py | 669 +++++++++++ vllm/models/qwen4_exp/nvidia/ple_layer.py | 685 +++-------- vllm/v1/attention/backends/short_conv_attn.py | 6 - 6 files changed, 1836 insertions(+), 594 deletions(-) create mode 100644 vllm/models/qwen4_exp/nvidia/ops/ple.py diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py index 49a64148cd..35c24176f8 100644 --- a/tests/models/qwen4_exp/test_ple.py +++ b/tests/models/qwen4_exp/test_ple.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass from functools import partial +from itertools import accumulate from types import SimpleNamespace import pytest @@ -17,13 +19,16 @@ from vllm.models.qwen4_exp.common.ple import ( compute_ple_shard_overlap, copy_ple_embedding_shard_, ) -from vllm.models.qwen4_exp.nvidia import ple_layer as ple_layer_module from vllm.models.qwen4_exp.nvidia.ple_layer import ( Qwen4ExpNGramEmbedding, Qwen4ExpPLEFp8EmbeddingMethod, Qwen4ExpPLELayer, _get_ple_embedding_quant_method, ) +from vllm.v1.attention.backends.short_conv_attn import ( + PleShortConvAttentionMetadata, +) +from vllm.v1.attention.backends.utils import NULL_BLOCK_ID def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding: @@ -283,54 +288,9 @@ def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None: assert _get_ple_embedding_quant_method(quant_config, prefix) is None -def test_ple_ngram_ids_custom_op_uses_current_request_layout(monkeypatch) -> None: - class RuntimeNGramEmbedding(nn.Module): - def compute_ngram_ids( - self, - input_ids: torch.Tensor, - query_start_loc: torch.Tensor, - ngram_context: torch.Tensor, - ) -> torch.Tensor: - del input_ids, ngram_context - num_reqs = query_start_loc.numel() - 1 - return torch.full((4, 2), num_reqs, dtype=torch.long) - - layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer) - nn.Module.__init__(layer) - layer.ple_embedding = RuntimeNGramEmbedding() - monkeypatch.setattr( - ple_layer_module, - "get_forward_context", - lambda: SimpleNamespace(no_compile_layers={"ple": layer}), - ) - input_ids = torch.arange(4) - ngram_context = torch.zeros(2, 2, dtype=torch.long) - output = torch.empty(4, 2, dtype=torch.long) - - ple_layer_module.qwen4_exp_compute_ple_ngram_ids( - input_ids, - torch.tensor([0, 4]), - ngram_context, - output, - "ple", - ) - assert torch.equal(output, torch.ones_like(output)) - - ple_layer_module.qwen4_exp_compute_ple_ngram_ids( - input_ids, - torch.tensor([0, 2, 4]), - ngram_context, - output, - "ple", - ) - assert torch.equal(output, torch.full_like(output, 2)) - - def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None: - module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer) - nn.Module.__init__(module) - module.conv_state_len = 6 - module.short_conv_dilation = 2 + conv_state_len = 6 + dilation = 2 conv_weights = torch.tensor([[0.25, -0.5, 0.75, 1.0]]) conv_state = torch.zeros(2, 1, 9) @@ -338,12 +298,12 @@ def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None: first_inputs = torch.tensor([[10.0], [20.0], [30.0], [40.0]]) initial_state = conv_state[1:].clone() first_history = torch.cat( - (initial_state[..., : module.conv_state_len], first_inputs.T.unsqueeze(0)), + (initial_state[..., :conv_state_len], first_inputs.T.unsqueeze(0)), dim=-1, ) graph_padded_inputs = F.pad(first_inputs, (0, 0, 0, 4)) - first_output = module._short_conv_dilated_spec_batched( + first_output = _short_conv_dilated_spec_pytorch( graph_padded_inputs, conv_state, conv_weights, @@ -351,6 +311,8 @@ def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None: torch.tensor([0, 4, 4]), torch.tensor([1, 0]), spec_query_len=4, + conv_state_len=conv_state_len, + dilation=dilation, ) expected_first_output = F.silu( @@ -358,7 +320,7 @@ def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None: first_history, conv_weights.unsqueeze(1), groups=1, - dilation=module.short_conv_dilation, + dilation=dilation, ) ).transpose(1, 2)[0] expected_first_state = first_history[..., 1:10] @@ -374,7 +336,7 @@ def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None: expected_second_state = expected_first_state.clone() expected_second_state[..., :7] = second_history[..., 1:8] - second_output = module._short_conv_dilated_spec_batched( + second_output = _short_conv_dilated_spec_pytorch( second_inputs, conv_state, conv_weights, @@ -382,6 +344,8 @@ def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None: torch.tensor([0, 2]), torch.tensor([2]), spec_query_len=4, + conv_state_len=conv_state_len, + dilation=dilation, ) expected_second_output = F.silu( @@ -389,28 +353,982 @@ def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None: second_history, conv_weights.unsqueeze(1), groups=1, - dilation=module.short_conv_dilation, + dilation=dilation, ) ).transpose(1, 2)[0, :2] torch.testing.assert_close(second_output, expected_second_output) torch.testing.assert_close(conv_state[1:], expected_second_state) -def test_ple_short_conv_uses_fallback_when_profile_metadata_is_omitted( - monkeypatch: pytest.MonkeyPatch, +def _reference_ngram_ids( + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + layer_multipliers: torch.Tensor, + ngram_heads_vocab_sizes: torch.Tensor, + ngram_heads_offsets: torch.Tensor, + eos_token_id: int, + heads_per_ngram: int, +) -> torch.Tensor: + tokens = input_ids.tolist() + starts = query_start_loc.tolist() + contexts = ngram_context.tolist() + multipliers = layer_multipliers.cpu() + sizes = ngram_heads_vocab_sizes.cpu() + offsets = ngram_heads_offsets.cpu() + rows = [] + context_len = len(contexts[0]) + for req, (start, end) in enumerate(zip(starts, starts[1:])): + history = contexts[req] + tokens[start:end] + for pos in range(context_len, len(history)): + shifted = [history[pos]] + crossed_eos = False + for shift in range(1, context_len + 1): + token = eos_token_id if crossed_eos else history[pos - shift] + shifted.append(token) + crossed_eos |= token == eos_token_id + + row = [] + mixed = torch.tensor(shifted[0], dtype=torch.int64) * multipliers[0] + for ngram_order in range(2, context_len + 2): + shift = ngram_order - 1 + mixed ^= shifted[shift] * multipliers[shift] + head_start = (ngram_order - 2) * heads_per_ngram + for head in range(head_start, head_start + heads_per_ngram): + row.append(torch.remainder(mixed, sizes[head]) + offsets[head]) + rows.append(torch.stack(row)) + return torch.stack(rows).to(input_ids.device) + + +_NGRAM_MULTIPLIERS = ( + 18_014_398_509_481_983, + 17_114_398_509_481_981, + 16_214_398_509_481_979, + 15_314_398_509_481_977, +) +_NGRAM_HEADS_VOCAB_SIZES = ( + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137, + 139, + 149, + 151, + 157, + 163, + 167, + 173, + 179, + 181, + 191, + 193, + 197, + 199, + 211, + 223, + 227, +) +_NGRAM_EOS_TOKEN_ID = 251 +_NGRAM_HEADS_PER_NGRAM = 8 + + +def _ngram_hash_params(device: torch.device, context_len: int) -> dict: + num_heads = context_len * _NGRAM_HEADS_PER_NGRAM + sizes = torch.tensor( + _NGRAM_HEADS_VOCAB_SIZES[:num_heads], dtype=torch.int64, device=device + ) + offsets = torch.zeros_like(sizes) + offsets[1:] = torch.cumsum(sizes, dim=0)[:-1] + return { + "layer_multipliers": torch.tensor( + _NGRAM_MULTIPLIERS[: context_len + 1], + dtype=torch.int64, + device=device, + ), + "ngram_heads_vocab_sizes": sizes, + "ngram_heads_offsets": offsets, + "eos_token_id": _NGRAM_EOS_TOKEN_ID, + "heads_per_ngram": _NGRAM_HEADS_PER_NGRAM, + } + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused PLE needs CUDA") +@pytest.mark.parametrize( + ("query_lens", "eos_offsets", "contexts", "first_token_id"), + [ + ([1], [], [[11, 12]], 20), + ([3], [1], [[11]], 20), + ([4, 4], [0, 7], [[11, 12], [13, 14]], 20), + ([4, 0, 3], [], [[11, 12], [13, 14], [15, 16]], 20), + ( + [1, 33, 2], + [5, 32], + [[_NGRAM_EOS_TOKEN_ID, 11], [12, 13], [14, _NGRAM_EOS_TOKEN_ID]], + 20, + ), + ( + [5, 12, 16, 1, 16, 17], + [10, 40], + [[11, 12], [13, 14], [15, 16], [17, 18], [19, 20], [21, 22]], + 20, + ), + ([5, 3], [3], [[11, 12, 13], [14, 15, 16]], 20), + ( + [4, 0, 3], + [], + [ + [200_000, 200_001], + [250_000, 250_001], + [300_000, 300_001], + ], + 350_000, + ), + ([3], [], [[1_000_000_000, 1_000_000_001]], 1_000_000_002), + ], + ids=[ + "single-token", + "bigram-only", + "power-of-two", + "empty-request", + "three-requests", + "six-requests", + "four-gram", + "int64-overflow", + "large-int32-ids", + ], +) +def test_fused_ngram_ids_correctness( + query_lens: list[int], + eos_offsets: list[int], + contexts: list[list[int]], + first_token_id: int, ) -> None: - module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer) - nn.Module.__init__(module) - module.prefix = "model.layers.1.ple" - inputs = torch.arange(6, dtype=torch.float32).reshape(3, 2) - expected = inputs + 1 - monkeypatch.setattr(module, "_short_conv_fallback", lambda _: expected) - monkeypatch.setattr( - ple_layer_module, - "get_forward_context", - lambda: SimpleNamespace(attn_metadata={"model.layers.0.self_attn": object()}), + from vllm.models.qwen4_exp.nvidia.ops.ple import ple_ngram_ids + + device = torch.device("cuda") + query_start_loc = torch.tensor( + [0, *accumulate(query_lens)], dtype=torch.int32, device=device + ) + input_ids = torch.arange( + first_token_id, + first_token_id + sum(query_lens), + dtype=torch.int32, + device=device, + ) + if eos_offsets: + input_ids[eos_offsets] = _NGRAM_EOS_TOKEN_ID + ngram_context = torch.tensor(contexts, dtype=torch.int32, device=device) + params = _ngram_hash_params(device, ngram_context.shape[1]) + + expected = _reference_ngram_ids(input_ids, query_start_loc, ngram_context, **params) + actual = ple_ngram_ids(input_ids, query_start_loc, ngram_context, **params) + + assert torch.equal(actual, expected) + offsets = params["ngram_heads_offsets"] + sizes = params["ngram_heads_vocab_sizes"] + assert torch.all((actual >= offsets) & (actual < offsets + sizes)) + + +def _short_conv_dilated_decode_pytorch( + x_d: torch.Tensor, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + state_indices_tensor_d: torch.Tensor, + has_initial_states_d: torch.Tensor | None, + *, + conv_state_len: int, + dilation: int, +) -> torch.Tensor: + state_indices = state_indices_tensor_d.to( + device=conv_state.device, dtype=torch.int64 + ) + # FULL cudagraph padded decode rows use NULL_BLOCK_ID. Remap them to + # slot 0 for a safe gather, then zero output and skip write-back. + valid_state = state_indices != NULL_BLOCK_ID + state_indices = torch.where( + valid_state, state_indices, torch.zeros_like(state_indices) + ) + if has_initial_states_d is None: + has_initial_state = valid_state + else: + if has_initial_states_d.numel() < state_indices_tensor_d.numel(): + raise ValueError( + "has_initial_states_d size mismatch: " + f"got {has_initial_states_d.numel()}, " + f"need >= {state_indices_tensor_d.numel()}." + ) + has_initial_state = has_initial_states_d[: state_indices_tensor_d.numel()].to( + device=conv_state.device, dtype=torch.bool + ) + has_initial_state = has_initial_state & valid_state + + cached_state = conv_state.index_select(0, state_indices) + state = cached_state[..., :conv_state_len].to(x_d.dtype) + if conv_state_len > 0: + initial_state = torch.where( + has_initial_state.view(-1, 1, 1), + state, + torch.zeros_like(state), + ) + history = torch.cat((initial_state, x_d.unsqueeze(-1)), dim=-1) + else: + history = x_d.unsqueeze(-1) + + conv_output = F.conv1d( + history, + conv_weights.unsqueeze(1).contiguous(), + groups=history.size(1), + dilation=dilation, + ).squeeze(-1) + output = F.silu(conv_output) + output = output * valid_state.view(-1, 1).to(output.dtype) + + if conv_state_len > 0: + next_state = history[..., -conv_state_len:] + # Padded rows are remapped to the reserved null slot. Preserve its + # existing value while writing the new states for valid rows. + existing_base_state = cached_state[..., :conv_state_len] + safe_next_state = torch.where( + valid_state.view(-1, 1, 1), + next_state.to(conv_state.dtype), + existing_base_state, + ) + cached_state[..., :conv_state_len] = safe_next_state + conv_state.index_copy_(0, state_indices, cached_state) + + return output + + +def _short_conv_dilated_prefill_pytorch( + x_p: torch.Tensor, + metadata: PleShortConvAttentionMetadata, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + state_indices_tensor_p: torch.Tensor, + num_prefills: int, + num_decode_tokens: int, + num_prefill_tokens: int, + *, + conv_state_len: int, + dilation: int, +) -> torch.Tensor: + # ``non_spec_query_start_loc`` covers the non-spec (decode + prefill) + # requests and equals ``query_start_loc`` when spec-decode is inactive. + non_spec_query_start_loc = metadata.non_spec_query_start_loc + if non_spec_query_start_loc is None: + raise ValueError("query_start_loc is required for prefill short-conv") + query_start_loc_p = ( + non_spec_query_start_loc[-num_prefills - 1 :] - num_decode_tokens + ) + # The metadata builder guarantees that the prefill query offsets start + # at 0 and end at num_prefill_tokens. Avoid reading those values here, + # since doing so would force a device-to-host synchronization. + has_initial_states_p = metadata.has_initial_states_p + if has_initial_states_p is None: + raise ValueError("has_initial_states_p is required for prefill short-conv") + + output = torch.empty_like(x_p) + q_starts = query_start_loc_p.to(torch.int64) + if state_indices_tensor_p.numel() < num_prefills: + raise ValueError( + "state_indices_tensor_p size mismatch: " + f"got {state_indices_tensor_p.numel()}, " + f"need >= {num_prefills}." + ) + if has_initial_states_p.numel() < num_prefills: + raise ValueError( + "has_initial_states_p size mismatch: " + f"got {has_initial_states_p.numel()}, " + f"need >= {num_prefills}." + ) + if num_prefills == 0 or x_p.numel() == 0: + return output + lengths = q_starts[1:] - q_starts[:-1] + # Use the CPU-computed packing width from the metadata builder instead + # of synchronizing on lengths.max(). + max_len = metadata.max_prefill_query_len + if max_len <= 0: + return output + + hidden_size = x_p.shape[1] + positions = torch.arange(num_prefill_tokens, device=x_p.device, dtype=torch.int64) + req_indices = torch.searchsorted(q_starts[1:], positions, right=True) + col_indices = positions - q_starts[req_indices] + + packed_tokens = x_p.new_zeros((num_prefills, max_len, hidden_size)) + packed_tokens[req_indices, col_indices] = x_p + packed_tokens = packed_tokens.transpose(1, 2).contiguous() + + state_indices = state_indices_tensor_p[:num_prefills].to( + device=conv_state.device, dtype=torch.int64 + ) + valid_state = state_indices != NULL_BLOCK_ID + state_indices = torch.where( + valid_state, state_indices, torch.zeros_like(state_indices) + ) + has_initial = has_initial_states_p[:num_prefills].to( + device=conv_state.device, dtype=torch.bool + ) + if conv_state_len > 0: + if conv_state.shape[0] == 0: + state = conv_state.new_zeros( + (num_prefills, hidden_size, conv_state_len), + dtype=x_p.dtype, + ) + else: + state = conv_state.index_select(0, state_indices)[..., :conv_state_len].to( + x_p.dtype + ) + use_initial_mask = (valid_state & has_initial).view(num_prefills, 1, 1) + initial_state = torch.where( + use_initial_mask, + state, + torch.zeros_like(state), + ) + history = torch.cat((initial_state, packed_tokens), dim=-1) + else: + history = packed_tokens + + conv_output = F.conv1d( + history, + conv_weights.unsqueeze(1).contiguous(), + groups=history.size(1), + dilation=dilation, + ) + conv_output = F.silu(conv_output).transpose(1, 2).contiguous() + + token_positions = torch.arange(max_len, device=x_p.device, dtype=torch.int64) + valid_tokens = token_positions.view(1, max_len) < lengths.view(num_prefills, 1) + valid_output_mask = valid_tokens & valid_state.to(device=x_p.device).view( + num_prefills, 1 + ) + conv_output.masked_fill_(~valid_output_mask.unsqueeze(-1), 0) + output.copy_(conv_output[req_indices, col_indices]) + + if conv_state_len > 0 and conv_state.shape[0] > 0: + state_starts = lengths.to(device=history.device, dtype=torch.int64).view( + num_prefills, 1, 1 + ) + state_offsets = torch.arange( + conv_state_len, device=history.device, dtype=torch.int64 + ).view(1, 1, conv_state_len) + next_state = history.gather( + dim=2, + index=(state_starts + state_offsets).expand(-1, history.size(1), -1), + ) + # Write back without a host synchronization. Valid, non-empty rows + # receive their new state; padding and zero-length rows keep the + # current cache value. + existing_state = conv_state.index_select(0, state_indices) + existing_base_state = existing_state[..., :conv_state_len] + update_mask = valid_state & (lengths.to(device=conv_state.device) > 0) + safe_next_state = torch.where( + update_mask.view(num_prefills, 1, 1), + next_state.to(conv_state.dtype), + existing_base_state, + ) + existing_state[..., :conv_state_len] = safe_next_state + conv_state.index_copy_(0, state_indices, existing_state) + return output + + +def _short_conv_dilated_spec_pytorch( + x_spec: torch.Tensor, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + spec_state_indices_tensor: torch.Tensor, + spec_query_start_loc: torch.Tensor, + num_accepted_tokens: torch.Tensor, + spec_query_len: int, + *, + conv_state_len: int, + dilation: int, +) -> torch.Tensor: + """Dilated short-conv for speculative-decode (MTP) requests. + + Each spec request feeds multiple (draft + 1) query tokens. The conv + outputs are computed causally after rolling back the previous draft + state by ``num_accepted_tokens - 1``. The current candidate inputs stay + in the extended cache for the next forward, matching + ``causal_conv1d_update``. + + ``spec_query_len`` (== num_speculative_tokens + 1) is the maximum query + length and is a Python int, so no host synchronization is needed; this + keeps the path safe for full CUDA-graph capture/replay where the buffers + are padded at the request level. + """ + num_reqs = spec_state_indices_tensor.numel() + hidden_size = x_spec.size(-1) + # Use a fixed packing width instead of synchronizing on lengths.max(). + max_len = spec_query_len + # Full CUDA graphs can pad these buffers. Only the first num_reqs + # accepted-token counts belong to actual speculative requests. + num_accepted_tokens = num_accepted_tokens[:num_reqs] + q_starts = spec_query_start_loc[: num_reqs + 1].to(torch.int64) + # Keep the number of real speculative tokens on the device. + total_real_tokens = q_starts[num_reqs] + + state_indices = spec_state_indices_tensor.to( + device=conv_state.device, dtype=torch.int64 + ) + valid_state = state_indices != NULL_BLOCK_ID + state_indices = torch.where( + valid_state, state_indices, torch.zeros_like(state_indices) + ) + positions = torch.arange(x_spec.size(0), device=x_spec.device, dtype=torch.int64) + # Route graph-padded token rows to the discarded dummy request so that + # they cannot overwrite real packed data. + req_indices = torch.searchsorted(q_starts[1:], positions, right=True) + valid_tokens = (positions < total_real_tokens) & (req_indices < num_reqs) + clamped_req_indices = req_indices.clamp_max(max(num_reqs - 1, 0)) + col_indices = (positions - q_starts[clamped_req_indices]).clamp_(0, max_len - 1) + pack_req_indices = torch.where( + valid_tokens, + clamped_req_indices, + torch.full_like(req_indices, num_reqs), + ) + pack_col_indices = torch.where( + valid_tokens, col_indices, torch.zeros_like(col_indices) ) - output = module._short_conv(inputs) + # The last request row is the dummy sink for graph padding. + packed = x_spec.new_zeros((num_reqs + 1, max_len, hidden_size)) + packed[pack_req_indices, pack_col_indices] = x_spec + packed = packed.transpose(1, 2).contiguous() - assert output is expected + if conv_state_len > 0: + cached_state = conv_state.index_select(0, state_indices) + rollback_offsets = num_accepted_tokens.to( + device=conv_state.device, dtype=torch.int64 + ).sub(1) + rollback_offsets = torch.where( + valid_state, + rollback_offsets.clamp_(0, max_len - 1), + torch.zeros_like(rollback_offsets), + ) + state_offsets = torch.arange( + conv_state_len, device=conv_state.device, dtype=torch.int64 + ).view(1, 1, conv_state_len) + rollback_indices = rollback_offsets.view(-1, 1, 1) + state_offsets + state = cached_state.gather(2, rollback_indices.expand(-1, hidden_size, -1)).to( + x_spec.dtype + ) + state = torch.where( + valid_state.view(num_reqs, 1, 1), + state, + torch.zeros_like(state), + ) + # Append a zeroed dummy-row state to match the [num_reqs + 1] pack. + dummy_state = state.new_zeros((1, hidden_size, conv_state_len)) + state_full = torch.cat((state, dummy_state), dim=0) + history = torch.cat((state_full, packed), dim=-1) + else: + history = packed + + conv_output = F.conv1d( + history, + conv_weights.unsqueeze(1).contiguous(), + groups=history.size(1), + dilation=dilation, + ) + conv_output = F.silu(conv_output).transpose(1, 2).contiguous() + + output = conv_output[pack_req_indices, pack_col_indices] + output = output * valid_tokens.view(-1, 1).to(output.dtype) + + # Keep all current candidate inputs in the extended state. On the next + # target forward, ``num_accepted_tokens - 1`` selects the rollback + # window before processing the newly scheduled tokens. + if conv_state_len > 0: + state_capacity = conv_state_len + max_len - 1 + if conv_state.size(-1) < state_capacity: + raise RuntimeError( + "PLE short-conv cache cannot retain speculative tokens: " + f"got {conv_state.size(-1)}, need {state_capacity}." + ) + candidate_state = history[:num_reqs, :, 1 : state_capacity + 1] + query_lengths = q_starts[1:] - q_starts[:-1] + state_positions = torch.arange( + state_capacity, device=history.device, dtype=torch.int64 + ).view(1, 1, state_capacity) + update_lengths = (conv_state_len + query_lengths - 1).view(num_reqs, 1, 1) + update_mask = valid_state.view(num_reqs, 1, 1) & ( + state_positions < update_lengths + ) + existing_state = cached_state[..., :state_capacity] + next_state = torch.where( + update_mask, + candidate_state.to(conv_state.dtype), + existing_state, + ) + cached_state[..., :state_capacity] = next_state + conv_state.index_copy_(0, state_indices, cached_state) + + return output + + +def _short_conv_dilated_dispatch_pytorch( + inputs: torch.Tensor, + residual: torch.Tensor, + metadata: PleShortConvAttentionMetadata, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + *, + conv_state_len: int, + dilation: int, +) -> None: + """PyTorch reference for fused short-convolution dispatch.""" + num_prefills = metadata.num_prefills + num_decodes = metadata.num_decodes + num_decode_tokens = metadata.num_decode_tokens + num_prefill_tokens = metadata.num_prefill_tokens + has_prefill = num_prefills > 0 + has_decode = num_decodes > 0 + has_spec = metadata.spec_sequence_masks is not None + x = inputs[: metadata.num_actual_tokens] + residual = residual[: metadata.num_actual_tokens] + + if has_spec: + if has_prefill or has_decode: + assert metadata.spec_token_indx is not None + assert metadata.non_spec_token_indx is not None + spec_token_indices = metadata.spec_token_indx.long() + non_spec_token_indices = metadata.non_spec_token_indx.long() + x_spec = x.index_select(0, spec_token_indices) + x_non_spec = x.index_select(0, non_spec_token_indices) + residual_spec = residual.index_select(0, spec_token_indices) + residual_non_spec = residual.index_select(0, non_spec_token_indices) + else: + spec_token_indices = None + non_spec_token_indices = None + x_spec = x + x_non_spec = None + residual_spec = residual + residual_non_spec = None + else: + spec_token_indices = None + non_spec_token_indices = None + x_spec = None + x_non_spec = x + residual_spec = None + residual_non_spec = residual + + if has_spec: + assert metadata.spec_state_indices_tensor is not None + assert metadata.spec_query_start_loc is not None + assert metadata.num_accepted_tokens is not None + assert x_spec is not None + assert residual_spec is not None + spec_state_indices = metadata.spec_state_indices_tensor[ + : metadata.num_spec_decodes + ] + conv_output = _short_conv_dilated_spec_pytorch( + x_spec=x_spec, + conv_state=conv_state, + conv_weights=conv_weights, + spec_state_indices_tensor=spec_state_indices, + spec_query_start_loc=metadata.spec_query_start_loc, + num_accepted_tokens=metadata.num_accepted_tokens, + spec_query_len=metadata.spec_query_len, + conv_state_len=conv_state_len, + dilation=dilation, + ) + residual_spec.add_(conv_output) + + state_indices = metadata.state_indices_tensor + if x_non_spec is not None: + assert state_indices is not None + assert residual_non_spec is not None + if has_prefill: + state_indices_d, state_indices_p = torch.split( + state_indices, [num_decodes, num_prefills], dim=0 + ) + x_d, x_p = torch.split( + x_non_spec, [num_decode_tokens, num_prefill_tokens], dim=0 + ) + residual_d, residual_p = torch.split( + residual_non_spec, + [num_decode_tokens, num_prefill_tokens], + dim=0, + ) + if has_decode: + conv_output = _short_conv_dilated_decode_pytorch( + x_d=x_d, + conv_state=conv_state, + conv_weights=conv_weights, + state_indices_tensor_d=state_indices_d, + has_initial_states_d=metadata.has_initial_states_d, + conv_state_len=conv_state_len, + dilation=dilation, + ) + residual_d.add_(conv_output) + conv_output = _short_conv_dilated_prefill_pytorch( + x_p=x_p, + metadata=metadata, + conv_state=conv_state, + conv_weights=conv_weights, + state_indices_tensor_p=state_indices_p, + num_prefills=num_prefills, + num_decode_tokens=num_decode_tokens, + num_prefill_tokens=num_prefill_tokens, + conv_state_len=conv_state_len, + dilation=dilation, + ) + residual_p.add_(conv_output) + else: + conv_output = _short_conv_dilated_decode_pytorch( + x_d=x_non_spec, + conv_state=conv_state, + conv_weights=conv_weights, + state_indices_tensor_d=state_indices[: x_non_spec.size(0)], + has_initial_states_d=metadata.has_initial_states_d, + conv_state_len=conv_state_len, + dilation=dilation, + ) + residual_non_spec.add_(conv_output) + + if has_spec and residual_non_spec is not None: + assert spec_token_indices is not None + assert non_spec_token_indices is not None + assert residual_spec is not None + residual.index_copy_(0, spec_token_indices, residual_spec) + residual.index_copy_(0, non_spec_token_indices, residual_non_spec) + + +_KERNEL_STATE_BLOCKS = 64 + + +def _make_conv_case( + device: torch.device, + seed: int, + channels: int, + kernel_size: int, + dilation: int, + state_layout: str, + spec_query_len: int = 1, +) -> tuple[torch.Generator, torch.Tensor, torch.Tensor, torch.Tensor]: + rng = torch.Generator(device=device).manual_seed(seed) + state_len = (kernel_size - 1) * dilation + state_width = state_len + spec_query_len - 1 + state_shape = ( + (_KERNEL_STATE_BLOCKS, state_width, channels) + if state_layout == "SD" + else (_KERNEL_STATE_BLOCKS, channels, state_width) + ) + state_kernel = torch.randn( + state_shape, device=device, dtype=torch.bfloat16, generator=rng + ) + conv_state = ( + state_kernel.transpose(-1, -2) if state_layout == "SD" else state_kernel + ) + state_reference = conv_state.clone() + weights = torch.randn( + channels, kernel_size, device=device, dtype=torch.bfloat16, generator=rng + ) + return rng, state_reference, conv_state, weights + + +@dataclass(frozen=True) +class _ConvBatchCase: + spec_query_lens: tuple[int, ...] = () + num_accepted: tuple[int, ...] = () + num_decodes: int = 0 + prefill_query_lens: tuple[int, ...] = () + channels: int = 2048 + kernel_size: int = 4 + dilation: int = 3 + spec_query_len: int = 1 + graph_padding: int = 0 + + +def _make_conv_metadata( + case: _ConvBatchCase, + device: torch.device, +) -> tuple[SimpleNamespace, int]: + request_groups: dict[str, list[int]] = { + "spec": [], + "decode": [], + "prefill": [], + } + token_offset = 0 + max_num_reqs = max( + len(case.spec_query_lens), + case.num_decodes, + len(case.prefill_query_lens), + ) + request_kinds = [] + for req_idx in range(max_num_reqs): + if req_idx < len(case.spec_query_lens): + request_kinds.append(("spec", case.spec_query_lens[req_idx])) + if req_idx < case.num_decodes: + request_kinds.append(("decode", 1)) + if req_idx < len(case.prefill_query_lens): + request_kinds.append(("prefill", case.prefill_query_lens[req_idx])) + for kind, query_len in request_kinds: + request_groups[kind].extend(range(token_offset, token_offset + query_len)) + token_offset += query_len + + num_spec_reqs = len(case.spec_query_lens) + num_prefills = len(case.prefill_query_lens) + has_spec = num_spec_reqs > 0 + has_non_spec = case.num_decodes > 0 or num_prefills > 0 + mixed_batch = has_spec and has_non_spec + + spec_state_indices = torch.arange( + 1, num_spec_reqs + 1, dtype=torch.int32, device=device + ) + if num_spec_reqs >= 3: + spec_state_indices[1] = NULL_BLOCK_ID + non_spec_state_indices = torch.arange( + num_spec_reqs + 1, + num_spec_reqs + case.num_decodes + num_prefills + 1, + dtype=torch.int32, + device=device, + ) + if case.num_decodes > 1: + non_spec_state_indices[case.num_decodes - 1] = NULL_BLOCK_ID + if num_prefills > 1: + non_spec_state_indices[case.num_decodes + 1] = NULL_BLOCK_ID + + spec_query_start_loc = torch.tensor( + [0, *accumulate(case.spec_query_lens)], dtype=torch.int32, device=device + ) + non_spec_query_lens = (1,) * case.num_decodes + case.prefill_query_lens + non_spec_query_start_loc = torch.tensor( + [0, *accumulate(non_spec_query_lens)], dtype=torch.int32, device=device + ) + spec_token_indices = torch.tensor( + request_groups["spec"], dtype=torch.int64, device=device + ) + non_spec_token_indices = torch.tensor( + request_groups["decode"] + request_groups["prefill"], + dtype=torch.int64, + device=device, + ) + + metadata = SimpleNamespace( + num_prefills=num_prefills, + num_decodes=case.num_decodes, + num_decode_tokens=case.num_decodes, + num_prefill_tokens=sum(case.prefill_query_lens), + spec_sequence_masks=( + torch.tensor([kind == "spec" for kind, _ in request_kinds], device=device) + if has_spec + else None + ), + spec_token_indx=spec_token_indices if mixed_batch else None, + non_spec_token_indx=non_spec_token_indices if mixed_batch else None, + num_actual_tokens=token_offset + case.graph_padding, + spec_state_indices_tensor=spec_state_indices if has_spec else None, + spec_query_start_loc=spec_query_start_loc if has_spec else None, + num_accepted_tokens=( + torch.tensor(case.num_accepted, dtype=torch.int32, device=device) + if has_spec + else None + ), + spec_query_len=case.spec_query_len, + state_indices_tensor=non_spec_state_indices if has_non_spec else None, + has_initial_states_d=( + torch.arange(case.num_decodes, device=device) % 2 == 0 + if case.num_decodes + else None + ), + non_spec_query_start_loc=(non_spec_query_start_loc if has_non_spec else None), + has_initial_states_p=( + torch.arange(num_prefills, device=device) % 2 == 0 if num_prefills else None + ), + max_prefill_query_len=max(case.prefill_query_lens, default=0), + num_spec_decodes=num_spec_reqs, + ) + return metadata, token_offset + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused conv needs CUDA") +@pytest.mark.parametrize("state_layout", ["SD", "DS"]) +@pytest.mark.parametrize( + "case", + [ + pytest.param( + _ConvBatchCase(num_decodes=1, channels=512), + id="decode-single", + ), + pytest.param( + _ConvBatchCase( + num_decodes=33, + channels=2048, + kernel_size=5, + dilation=1, + ), + id="decode-padded", + ), + pytest.param( + _ConvBatchCase(prefill_query_lens=(37, 0, 5, 128)), + id="prefill", + ), + pytest.param( + _ConvBatchCase( + spec_query_lens=(1, 4, 4, 0), + num_accepted=(1, 2, 4, 1), + spec_query_len=4, + graph_padding=4, + ), + id="spec-graph-padded", + ), + pytest.param( + _ConvBatchCase( + spec_query_lens=(3, 4), + num_accepted=(2, 4), + num_decodes=1, + prefill_query_lens=(5,), + spec_query_len=4, + ), + id="mixed", + ), + pytest.param( + _ConvBatchCase( + spec_query_lens=(1, 2), + num_accepted=(0, 1), + num_decodes=2, + prefill_query_lens=(0, 8, 3), + channels=640, + kernel_size=3, + dilation=2, + spec_query_len=3, + ), + id="mixed-varied", + ), + ], +) +def test_fused_conv_correctness( + case: _ConvBatchCase, + state_layout: str, +) -> None: + device = torch.device("cuda") + metadata, num_real_tokens = _make_conv_metadata(case, device) + module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer) + nn.Module.__init__(module) + module.conv_state_len = (case.kernel_size - 1) * case.dilation + module.short_conv_dilation = case.dilation + + rng, state_reference, conv_state, weights = _make_conv_case( + device, + seed=num_real_tokens + case.channels, + channels=case.channels, + kernel_size=case.kernel_size, + dilation=case.dilation, + state_layout=state_layout, + spec_query_len=case.spec_query_len, + ) + inputs = torch.randn( + metadata.num_actual_tokens, + case.channels, + device=device, + dtype=torch.bfloat16, + generator=rng, + ) + residual = torch.randn( + inputs.shape, + device=device, + dtype=torch.bfloat16, + generator=rng, + ) + null_state = conv_state[NULL_BLOCK_ID].clone() + residual_kernel = residual.clone() + residual_reference = residual.clone() + + module._short_conv_dilated_dispatch( + inputs=inputs, + residual=residual_kernel, + metadata=metadata, + conv_state=conv_state, + conv_weights=weights, + ) + _short_conv_dilated_dispatch_pytorch( + inputs=inputs, + residual=residual_reference, + metadata=metadata, + conv_state=state_reference, + conv_weights=weights, + conv_state_len=module.conv_state_len, + dilation=module.short_conv_dilation, + ) + + torch.testing.assert_close( + residual_kernel.float(), residual_reference.float(), atol=3e-2, rtol=3e-2 + ) + assert torch.equal(conv_state, state_reference) + assert torch.equal(conv_state[NULL_BLOCK_ID], null_state) + if case.graph_padding: + assert torch.equal( + residual_kernel[num_real_tokens:], residual[num_real_tokens:] + ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused gate needs CUDA") +@pytest.mark.parametrize(("num_tokens", "strided_kv"), [(1, False), (64, True)]) +def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None: + import math + + from vllm.models.qwen4_exp.nvidia.ops.ple import ple_gate + + device = torch.device("cuda") + hc, h = 4, 2560 + generator = torch.Generator(device=device).manual_seed(num_tokens) + kv = torch.randn( + num_tokens, + hc * h + h, + device=device, + dtype=torch.bfloat16, + generator=generator, + ) + key = kv[:, : hc * h] + value = kv[:, hc * h :] + if not strided_kv: + key = key.contiguous() + value = value.contiguous() + hidden = torch.randn( + num_tokens, + hc * h, + device=device, + dtype=torch.bfloat16, + generator=generator, + ) + + def make_norm_weight() -> torch.Tensor: + weight = torch.empty(hc * h, device=device, dtype=torch.bfloat16) + return weight.normal_(mean=-0.1, std=0.1, generator=generator) + + norm_key = make_norm_weight() + norm_query = make_norm_weight() + norm_conv = make_norm_weight() + + def grouped_norm(inputs: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + grouped = inputs.float().unflatten(-1, (hc, h)) + var = grouped.square().mean(dim=-1, keepdim=True) + normalized = grouped * torch.rsqrt(var + 1e-6) + return (normalized.flatten(-2) * (1.0 + weight.float())).to(inputs.dtype) + + gated, normed = ple_gate( + key, + value, + hidden, + norm_key, + norm_query, + norm_conv, + 1e-6, + ) + key_normalized = grouped_norm(key, norm_key).reshape(num_tokens, hc, h) + query_normalized = grouped_norm(hidden, norm_query).reshape(num_tokens, hc, h) + dot = (key_normalized * query_normalized).sum(dim=-1, keepdim=True) + dot = (dot / math.sqrt(h)).to(torch.bfloat16) + gate = torch.sigmoid(dot.sign() * dot.abs().clamp_min(1e-6).sqrt()).to( + torch.bfloat16 + ) + expected_gated = (gate * value.unsqueeze(-2)).flatten(-2) + expected_normed = grouped_norm(expected_gated, norm_conv) + assert torch.equal(gated, expected_gated) + torch.testing.assert_close(normed, expected_normed, atol=1e-2, rtol=1e-2) diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index aeb3a14dad..927ad50d5c 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -141,9 +141,9 @@ _QWEN4_EXP_IGNORED_MISSING_SUFFIXES = [ "_input_scale", ] -# The checkpoint keeps down and injection projections separate; runtime packs -# them into adjacent logical shards of one MergedColumnParallelLinear. -_HC_WEIGHTS_MAPPER = WeightsMapper( +# The checkpoint stores these projections separately; runtime packs each group +# into adjacent logical shards of a MergedColumnParallelLinear. +_EXTRA_WEIGHTS_MAPPER = WeightsMapper( orig_to_new_stacked={ "hyper_connection.input_mix_weight_down.weight": ( "hyper_connection.input_mix_weight_down_block_inject.weight", @@ -153,6 +153,8 @@ _HC_WEIGHTS_MAPPER = WeightsMapper( "hyper_connection.input_mix_weight_down_block_inject.weight", 1, ), + "ple.key_proj": ("ple.kv_proj", 0), + "ple.value_proj": ("ple.kv_proj", 1), } ) @@ -389,7 +391,7 @@ class Qwen4ExpMixtureOfExperts(MixtureOfExperts): } ) class Qwen4ExpModel(nn.Module): - hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER + hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: super().__init__() @@ -612,6 +614,7 @@ class Qwen4ExpForCausalLM( packed_modules_mapping = { "qkv_proj": ["q_proj", "k_proj", "v_proj"], "gate_up_proj": ["gate_proj", "up_proj"], + "kv_proj": ["key_proj", "value_proj"], "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"], "in_proj_ba": ["in_proj_b", "in_proj_a"], "input_mix_weight_down_block_inject": [ @@ -848,11 +851,12 @@ class Qwen4ExpForConditionalGeneration( requires_raw_input_tokens = True packed_modules_mapping = Qwen3_5ForConditionalGeneration.packed_modules_mapping | { + "kv_proj": ["key_proj", "value_proj"], "input_mix_weight_down_block_inject": [ "input_mix_weight_down", "block_inject_weight", "_input_mix_padding", - ] + ], } @staticmethod diff --git a/vllm/models/qwen4_exp/nvidia/mtp.py b/vllm/models/qwen4_exp/nvidia/mtp.py index 5a77448eca..5238aab5f9 100644 --- a/vllm/models/qwen4_exp/nvidia/mtp.py +++ b/vllm/models/qwen4_exp/nvidia/mtp.py @@ -52,7 +52,7 @@ from vllm.transformers_utils.configs.qwen4_exp import ( from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm from .model import ( - _HC_WEIGHTS_MAPPER, + _EXTRA_WEIGHTS_MAPPER, _QWEN4_EXP_IGNORED_MISSING_SUFFIXES, Qwen4ExpDecoderLayer, Qwen4ExpMixtureOfExperts, @@ -157,7 +157,7 @@ def _make_draft_vllm_config( } ) class Qwen4ExpMultiTokenPredictor(nn.Module): - hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER + hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: super().__init__() diff --git a/vllm/models/qwen4_exp/nvidia/ops/ple.py b/vllm/models/qwen4_exp/nvidia/ops/ple.py new file mode 100644 index 0000000000..3d90773ded --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/ple.py @@ -0,0 +1,669 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Fused Qwen4Exp PLE kernels. + +N-gram IDs hash each suffix with +``offset[h] + (xor_i(token[t-i] * multiplier[i]) % size[h])``. The gate computes +``d = dot(RMSNorm(key), RMSNorm(query)) / sqrt(H)`` and +``g = sigmoid(sign(d) * sqrt(max(abs(d), 1e-6)))``. Short convolution adds +``silu(sum_k weight[k] * history[t + k * dilation])`` to the gated output and +updates its persistent history. +""" + +from typing import Literal + +import torch + +from vllm.platforms import current_platform +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import direct_register_custom_op +from vllm.v1.attention.backends.utils import NULL_BLOCK_ID + + +# N-gram ID generation +@triton.jit(do_not_specialize=["num_tokens", "num_reqs", "binary_search_iters"]) +def _ple_ngram_ids_kernel( + input_ids_ptr, + qsl_ptr, + ctx_ptr, + multipliers_ptr, + sizes_ptr, + offsets_ptr, + out_ptr, + num_tokens, + num_reqs, + eos_token_id, + binary_search_iters, + NGRAM_CONTEXT_LEN: tl.constexpr, + HEADS_PER_NGRAM: tl.constexpr, + BLOCK_T: tl.constexpr, + launch_pdl: tl.constexpr, +): + NGRAM_HEADS: tl.constexpr = NGRAM_CONTEXT_LEN * HEADS_PER_NGRAM + BLOCK_H: tl.constexpr = triton.next_power_of_2(NGRAM_HEADS) + pid = tl.program_id(0) + token_offsets = pid * BLOCK_T + tl.arange(0, BLOCK_T) + token_mask = token_offsets < num_tokens + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + # A flattened token offset does not identify its request when query chunks + # have different lengths. Binary-search the request boundaries instead. + p_lo = tl.full([BLOCK_T], 1, tl.int32) + p_hi = tl.full([BLOCK_T], num_reqs + 1, tl.int32) + for _ in range(binary_search_iters): + mid = (p_lo + p_hi) // 2 + qmid = tl.load(qsl_ptr + mid, mask=token_mask & (mid <= num_reqs), other=0) + pred = qmid <= token_offsets + p_lo = tl.where(pred, mid + 1, p_lo) + p_hi = tl.where(pred, p_hi, mid) + req = tl.minimum(p_lo - 1, num_reqs - 1).to(tl.int64) + request_start = tl.load(qsl_ptr + req, mask=token_mask, other=0) + chunk_pos = token_offsets - request_start + + # Initialize every head with the current-token term. + current_token = tl.load(input_ids_ptr + token_offsets, mask=token_mask, other=0).to( + tl.int64 + ) + current_multiplier = tl.load(multipliers_ptr) + mixed = current_token[:, None] * current_multiplier + + g = tl.arange(0, BLOCK_H) + head_mask = g < NGRAM_HEADS + ngram_order = g // HEADS_PER_NGRAM + 2 + + # Walk predecessors from newest to oldest. At chunk boundaries they come + # from ngram_context; otherwise they come from this step's input_ids. + crossed = tl.zeros([BLOCK_T], tl.int1) + for shift in tl.static_range(1, NGRAM_CONTEXT_LEN + 1): + in_step = chunk_pos >= shift + ctx_col = NGRAM_CONTEXT_LEN - shift + chunk_pos + step_token = tl.load( + input_ids_ptr + token_offsets - shift, + mask=token_mask & in_step, + other=0, + ) + context_token = tl.load( + ctx_ptr + req * NGRAM_CONTEXT_LEN + ctx_col, + mask=token_mask & (~in_step), + other=0, + ) + candidate = tl.where(in_step, step_token, context_token).to(tl.int64) + # Older positions remain behind the first EOS boundary. + candidate = tl.where(crossed, eos_token_id, candidate) + crossed = crossed | (candidate == eos_token_id) + multiplier = tl.load(multipliers_ptr + shift) + term = candidate[:, None] * multiplier + mixed = mixed ^ tl.where((ngram_order > shift)[None, :], term, 0) + + # Map each hash into its embedding-table partition. + sizes = tl.load(sizes_ptr + g, mask=head_mask, other=1)[None, :] + head_offsets = tl.load(offsets_ptr + g, mask=head_mask, other=0)[None, :] + # Hash products may overflow int64; preserve torch.remainder semantics. + remainders = mixed % sizes + remainders = tl.where(remainders < 0, remainders + sizes, remainders) + ids = remainders + head_offsets + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store( + out_ptr + token_offsets[:, None] * NGRAM_HEADS + g[None, :], + ids, + mask=token_mask[:, None] & head_mask[None, :], + ) + + +def _ple_ngram_ids( + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + layer_multipliers: torch.Tensor, + ngram_heads_vocab_sizes: torch.Tensor, + ngram_heads_offsets: torch.Tensor, + output: torch.Tensor, + eos_token_id: int, + heads_per_ngram: int, +) -> None: + input_ids = input_ids.reshape(-1) + num_tokens = input_ids.shape[0] + num_reqs = query_start_loc.numel() - 1 + ctx_len = ngram_context.shape[1] + BLOCK_T = 8 + launch_pdl = current_platform.is_arch_support_pdl() + _ple_ngram_ids_kernel[(triton.cdiv(num_tokens, BLOCK_T),)]( + input_ids, + query_start_loc, + ngram_context, + layer_multipliers, + ngram_heads_vocab_sizes, + ngram_heads_offsets, + output, + num_tokens, + num_reqs, + eos_token_id, + binary_search_iters=num_reqs.bit_length(), + NGRAM_CONTEXT_LEN=ctx_len, + HEADS_PER_NGRAM=heads_per_ngram, + BLOCK_T=BLOCK_T, + launch_pdl=launch_pdl, + num_warps=4, + ) + + +def ple_ngram_ids( + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + layer_multipliers: torch.Tensor, + ngram_heads_vocab_sizes: torch.Tensor, + ngram_heads_offsets: torch.Tensor, + eos_token_id: int, + heads_per_ngram: int, + output: torch.Tensor | None = None, +) -> torch.Tensor: + if output is None: + output = torch.empty( + (input_ids.numel(), ngram_context.shape[1] * heads_per_ngram), + dtype=torch.int64, + device=input_ids.device, + ) + _ple_ngram_ids( + input_ids, + query_start_loc, + ngram_context, + layer_multipliers, + ngram_heads_vocab_sizes, + ngram_heads_offsets, + output, + eos_token_id, + heads_per_ngram, + ) + return output + + +# Gating and normalization +@triton.jit +def _ple_gate_kernel( + key_ptr, # [T, HC*H] with row stride key_rs + value_ptr, # [T, H] with row stride value_rs + hidden_ptr, # [T, HC*H] + nk_ptr, # [HC*H] norm_key weight + nq_ptr, # [HC*H] norm_query weight + ncw_ptr, # [HC*H] norm_conv weight + gated_ptr, # [T, HC*H] out + normed_ptr, # [T, HC*H] out + key_rs, + value_rs, + eps, + H: tl.constexpr, + HC: tl.constexpr, + BLOCK_H: tl.constexpr, + launch_pdl: tl.constexpr, +): + t = tl.program_id(0) + s = tl.program_id(1) + lanes = tl.arange(0, BLOCK_H) + mask = lanes < H + offs = s * H + lanes + dtype: tl.constexpr = key_ptr.dtype.element_ty + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + k = tl.load(key_ptr + t * key_rs + offs, mask=mask, other=0.0).to(tl.float32) + q = tl.load(hidden_ptr + t * HC * H + offs, mask=mask, other=0.0).to(tl.float32) + nk = tl.load(nk_ptr + offs, mask=mask, other=0.0).to(tl.float32) + nq = tl.load(nq_ptr + offs, mask=mask, other=0.0).to(tl.float32) + + # Match eager materialization at each intermediate tensor boundary. + k_n = (k * tl.rsqrt(tl.sum(k * k) / H + eps) * (1.0 + nk)).to(dtype) + q_n = (q * tl.rsqrt(tl.sum(q * q) / H + eps) * (1.0 + nq)).to(dtype) + products = (k_n.to(tl.float32) * q_n.to(tl.float32)).to(dtype) + dot = tl.sum(products.to(tl.float32)).to(dtype).to(tl.float32) + d = dot / tl.sqrt(float(H)) + d = d.to(dtype).to(tl.float32) + sign = tl.where(d < 0, -1.0, 0.0) + sign = tl.where(d > 0, 1.0, sign) + magnitude = tl.sqrt(tl.maximum(tl.abs(d), 1e-6)).to(dtype) + g = tl.sigmoid(sign * magnitude) + g = g.to(dtype).to(tl.float32) + + v = tl.load(value_ptr + t * value_rs + lanes, mask=mask, other=0.0).to(tl.float32) + gated = (g * v).to(dtype) + gf = gated.to(tl.float32) + ncw = tl.load(ncw_ptr + offs, mask=mask, other=0.0).to(tl.float32) + normed = gf * tl.rsqrt(tl.sum(gf * gf) / H + eps) * (1.0 + ncw) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(gated_ptr + t * HC * H + offs, gated, mask=mask) + tl.store(normed_ptr + t * HC * H + offs, normed, mask=mask) + + +def _ple_gate( + key: torch.Tensor, + value: torch.Tensor, + hidden: torch.Tensor, + norm_key_w: torch.Tensor, + norm_query_w: torch.Tensor, + norm_conv_w: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor]: + num_tokens = hidden.shape[0] + h = value.shape[-1] + hc = hidden.shape[-1] // h + assert key.stride(1) == 1 and value.stride(1) == 1 + assert hidden.is_contiguous() + if key.dtype != value.dtype or key.dtype != hidden.dtype: + raise ValueError("key, value, and hidden must have the same dtype") + if key.dtype not in (torch.bfloat16, torch.float16): + raise ValueError("PLE gate supports BF16 and FP16 inputs") + gated = torch.empty_like(hidden) + normed = torch.empty_like(hidden) + _ple_gate_kernel[(num_tokens, hc)]( + key, + value, + hidden, + norm_key_w, + norm_query_w, + norm_conv_w, + gated, + normed, + key.stride(0), + value.stride(0), + eps, + H=h, + HC=hc, + BLOCK_H=triton.next_power_of_2(h), + num_warps=4, + launch_pdl=current_platform.is_arch_support_pdl(), + ) + return gated, normed + + +def _ple_gate_fake( + key: torch.Tensor, + value: torch.Tensor, + hidden: torch.Tensor, + norm_key_w: torch.Tensor, + norm_query_w: torch.Tensor, + norm_conv_w: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor]: + return torch.empty_like(hidden), torch.empty_like(hidden) + + +direct_register_custom_op( + op_name="qwen4_exp_ple_gate", + op_func=_ple_gate, + mutates_args=[], + fake_impl=_ple_gate_fake, +) + + +def ple_gate( + key: torch.Tensor, + value: torch.Tensor, + hidden: torch.Tensor, + norm_key_w: torch.Tensor, + norm_query_w: torch.Tensor, + norm_conv_w: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor]: + return torch.ops.vllm.qwen4_exp_ple_gate( + key, value, hidden, norm_key_w, norm_query_w, norm_conv_w, eps + ) + + +# Dilated short convolution +@triton.jit(do_not_specialize=["num_reqs", "bs_iters", "has_token_map"]) +def _ple_conv_kernel( + x_ptr, + state_ptr, + w_ptr, + residual_ptr, + state_idx_ptr, + qsl_ptr, + num_acc_ptr, + has_init_ptr, + token_idx_ptr, + has_token_map, + num_reqs, + bs_iters, + state_bs, + state_ws, + state_cs, + C: tl.constexpr, + BLOCK_C: tl.constexpr, + STATE_LEN: tl.constexpr, + DILATION: tl.constexpr, + KERNEL_SIZE: tl.constexpr, + SPEC_QUERY_LEN: tl.constexpr, + MODE: tl.constexpr, + HAS_INIT: tl.constexpr, + NULL_STATE_ID: tl.constexpr, + launch_pdl: tl.constexpr, +): + t = tl.program_id(0) + pid_c = tl.program_id(1) + c_offs = pid_c * BLOCK_C + tl.arange(0, BLOCK_C) + c_mask = c_offs < C + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + # Mode-specific launches use logical row indices. Mixed batches map them + # back to the original token order without gathering inputs or residuals. + output_t = tl.load(token_idx_ptr + t, mask=has_token_map, other=t).to(tl.int64) + + if MODE == "decode": + r = t + q_start = t + j = tl.full([], 0, tl.int32) + slot_off = tl.full([], 0, tl.int32) + in_range = True + else: + # Locate the request containing this token. + lo = tl.full([], 1, tl.int32) + hi = tl.full([], num_reqs + 1, tl.int32) + for _ in range(bs_iters): + mid = (lo + hi) // 2 + qmid = tl.load(qsl_ptr + mid, mask=mid <= num_reqs, other=0) + pred = qmid <= t + lo = tl.where(pred, mid + 1, lo) + hi = tl.where(pred, hi, mid) + r = tl.minimum(lo - 1, num_reqs - 1) + q_start = tl.load(qsl_ptr + r) + j = (t - q_start).to(tl.int32) + total_real = tl.load(qsl_ptr + num_reqs) + in_range = t < total_real + if MODE == "spec": + num_acc = tl.load(num_acc_ptr + r) + # Roll back draft state to the last accepted token. + slot_off = tl.minimum(tl.maximum(num_acc - 1, 0), SPEC_QUERY_LEN - 1).to( + tl.int32 + ) + else: + slot_off = tl.full([], 0, tl.int32) + + sid = tl.load(state_idx_ptr + r).to(tl.int64) + state_ok = sid != NULL_STATE_ID + sid_safe = tl.where(state_ok, sid, 0) + if HAS_INIT: + has_init = tl.load(has_init_ptr + r, mask=state_ok, other=0) != 0 + else: + has_init = state_ok + if MODE == "spec": + # A new spec chunk can produce output from its own tokens without a + # prior state; null slots in other modes denote inactive rows. + read_state = state_ok + out_ok = in_range + else: + read_state = state_ok & has_init + out_ok = in_range & state_ok + + # state[sid, w, c] may be a strided view into a shared pool. + base_state = state_ptr + sid_safe * state_bs + acc = tl.zeros([BLOCK_C], tl.float32) + for k in tl.static_range(0, KERNEL_SIZE): + h = j + DILATION * k + from_state = h <= STATE_LEN - 1 + state_tap = tl.load( + base_state + (h + slot_off) * state_ws + c_offs * state_cs, + mask=c_mask & read_state & from_state, + other=0.0, + ) + input_t = q_start + h - STATE_LEN + input_t = tl.load( + token_idx_ptr + input_t, + mask=has_token_map & out_ok & (~from_state), + other=input_t, + ).to(tl.int64) + input_tap = tl.load( + x_ptr + input_t * C + c_offs, + mask=c_mask & out_ok & (~from_state), + other=0.0, + ) + tap = tl.where(from_state, state_tap, input_tap).to(tl.float32) + weight = tl.load( + w_ptr + c_offs * KERNEL_SIZE + k, + mask=c_mask, + other=0.0, + ).to(tl.float32) + acc += weight * tap + + # F.conv1d materializes its output dtype before SiLU. + conv = acc.to(residual_ptr.dtype.element_ty).to(tl.float32) + y = conv * tl.sigmoid(conv) + conv_output = tl.where(out_ok, y, 0.0).to(residual_ptr.dtype.element_ty) + residual = tl.load( + residual_ptr + output_t * C + c_offs, + mask=c_mask, + other=0.0, + ) + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store( + residual_ptr + output_t * C + c_offs, + residual + conv_output, + mask=c_mask, + ) + + if MODE == "decode": + # Decode has one program per request and channel block, so no peer can + # still be reading this state slice when it is updated. + decode_input = tl.load( + x_ptr + output_t * C + c_offs, + mask=c_mask & state_ok, + other=0.0, + ) + for i in tl.static_range(0, STATE_LEN): + if i < STATE_LEN - 1: + next_state = tl.load( + base_state + (i + 1) * state_ws + c_offs * state_cs, + mask=c_mask & state_ok & has_init, + other=0.0, + ) + else: + next_state = decode_input + tl.store( + base_state + i * state_ws + c_offs * state_cs, + next_state, + mask=c_mask & state_ok, + ) + + +# Prefill and spec write back separately so every token reads the old state. +@triton.jit(do_not_specialize=["has_token_map"]) +def _ple_conv_writeback_kernel( + x_ptr, + state_ptr, + state_idx_ptr, + qsl_ptr, + num_acc_ptr, + has_init_ptr, + token_idx_ptr, + has_token_map, + state_bs, + state_ws, + state_cs, + C: tl.constexpr, + BLOCK_C: tl.constexpr, + STATE_LEN: tl.constexpr, + SPEC_QUERY_LEN: tl.constexpr, + STATE_WIDTH: tl.constexpr, + MODE: tl.constexpr, + HAS_INIT: tl.constexpr, + NULL_STATE_ID: tl.constexpr, +): + r = tl.program_id(0) + pid_c = tl.program_id(1) + c_offs = pid_c * BLOCK_C + tl.arange(0, BLOCK_C) + c_mask = c_offs < C + + sid = tl.load(state_idx_ptr + r).to(tl.int64) + state_ok = sid != NULL_STATE_ID + if not state_ok: + return + + q_start = tl.load(qsl_ptr + r) + q_end = tl.load(qsl_ptr + r + 1) + qlen = (q_end - q_start).to(tl.int32) + if MODE == "spec": + num_acc = tl.load(num_acc_ptr + r) + # Use the same rollback window as the output kernel. + slot_off = tl.minimum(tl.maximum(num_acc - 1, 0), SPEC_QUERY_LEN - 1).to( + tl.int32 + ) + shift = 1 + else: + slot_off = tl.full([], 0, tl.int32) + shift = qlen + if qlen <= 0: + return + + has_init = tl.load(has_init_ptr + r) != 0 if HAS_INIT else True + src_ok = state_ok if MODE == "spec" else state_ok & has_init + + WRITE_W: tl.constexpr = STATE_WIDTH if MODE == "spec" else STATE_LEN + base_state = state_ptr + sid * state_bs + for i in tl.static_range(0, WRITE_W): + m = shift + i + from_state = m <= STATE_LEN - 1 + # Short or graph-padded spec chunks preserve the unused state tail. + do_write = i < STATE_LEN + qlen - 1 if MODE == "spec" else True + state_value = tl.load( + base_state + (slot_off + m) * state_ws + c_offs * state_cs, + mask=c_mask & from_state & src_ok, + other=0.0, + ) + input_t = q_start + m - STATE_LEN + input_t = tl.load( + token_idx_ptr + input_t, + mask=has_token_map & (~from_state) & do_write, + other=input_t, + ).to(tl.int64) + input_value = tl.load( + x_ptr + input_t * C + c_offs, + mask=c_mask & (~from_state) & do_write, + other=0.0, + ) + value = tl.where(from_state, state_value, input_value) + tl.store( + base_state + i * state_ws + c_offs * state_cs, + value, + mask=c_mask & do_write, + ) + + +def ple_conv( + inputs: torch.Tensor, + residual: torch.Tensor, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + state_indices: torch.Tensor, + *, + mode: Literal["decode", "spec", "prefill"], + dilation: int, + query_start_loc: torch.Tensor | None = None, + num_accepted_tokens: torch.Tensor | None = None, + has_initial_states: torch.Tensor | None = None, + spec_query_len: int = 1, + token_indices: torch.Tensor | None = None, +) -> None: + """Add short-convolution output to ``residual`` and update its state.""" + BLOCK_C = 512 + kernel_spec_query_len = spec_query_len if mode == "spec" else 1 + T, C = inputs.shape + K = conv_weights.shape[1] + state_len = (K - 1) * dilation + state_width = state_len + kernel_spec_query_len - 1 + if token_indices is not None: + T = token_indices.numel() + if conv_state.shape[1] != C or conv_state.shape[2] < state_width: + raise ValueError( + "conv_state must have shape [slots, channels, window], with " + f"channels={C} and window >= {state_width}" + ) + state_bs, state_cs, state_ws = conv_state.stride() + + if mode == "decode": + num_reqs = T + binary_search_iters = 1 + has_initial_states_arg = has_initial_states is not None + elif mode == "spec": + if query_start_loc is None or num_accepted_tokens is None: + raise ValueError( + "query_start_loc and num_accepted_tokens are required for spec decode" + ) + num_reqs = state_indices.numel() + binary_search_iters = max(num_reqs, 1).bit_length() + has_initial_states_arg = False + elif mode == "prefill": + if query_start_loc is None or has_initial_states is None: + raise ValueError( + "query_start_loc and has_initial_states are required for prefill" + ) + num_reqs = state_indices.numel() + binary_search_iters = max(num_reqs, 1).bit_length() + has_initial_states_arg = True + else: + raise ValueError(f"Unsupported short-conv mode: {mode}") + + num_warps = 4 if mode == "prefill" else 8 + launch_pdl = current_platform.is_arch_support_pdl() + + # Constexpr flags eliminate accesses to optional None arguments. Without a + # token map, state_indices is an unused but device-resident placeholder. + _ple_conv_kernel[(T, triton.cdiv(C, BLOCK_C))]( + inputs, + conv_state, + conv_weights, + residual, + state_indices, + query_start_loc, + num_accepted_tokens, + has_initial_states, + token_indices if token_indices is not None else state_indices, + token_indices is not None, + num_reqs, + binary_search_iters, + state_bs, + state_ws, + state_cs, + C=C, + BLOCK_C=BLOCK_C, + STATE_LEN=state_len, + DILATION=dilation, + KERNEL_SIZE=K, + SPEC_QUERY_LEN=kernel_spec_query_len, + MODE=mode, + HAS_INIT=has_initial_states_arg, + NULL_STATE_ID=NULL_BLOCK_ID, + launch_pdl=launch_pdl, + num_warps=num_warps, + ) + # conv state update is fused with the kernel above for decode + if mode != "decode": + _ple_conv_writeback_kernel[(num_reqs, triton.cdiv(C, BLOCK_C))]( + inputs, + conv_state, + state_indices, + query_start_loc, + num_accepted_tokens, + has_initial_states, + token_indices if token_indices is not None else state_indices, + token_indices is not None, + state_bs, + state_ws, + state_cs, + C=C, + BLOCK_C=BLOCK_C, + STATE_LEN=state_len, + SPEC_QUERY_LEN=kernel_spec_query_len, + STATE_WIDTH=state_width, + MODE=mode, + HAS_INIT=has_initial_states_arg, + NULL_STATE_ID=NULL_BLOCK_ID, + num_warps=num_warps, + ) diff --git a/vllm/models/qwen4_exp/nvidia/ple_layer.py b/vllm/models/qwen4_exp/nvidia/ple_layer.py index 2f5cecb57e..c0d909e1c3 100644 --- a/vllm/models/qwen4_exp/nvidia/ple_layer.py +++ b/vllm/models/qwen4_exp/nvidia/ple_layer.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """GPU-resident Qwen4Exp position-learning enhancement layers.""" -import math from collections.abc import Iterable, Sequence import torch @@ -11,7 +10,7 @@ from torch import nn from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config from vllm.forward_context import get_forward_context -from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.linear import MergedColumnParallelLinear from vllm.model_executor.layers.mamba.abstract import MambaBase from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateDtypeCalculator, @@ -42,9 +41,9 @@ from vllm.v1.attention.backends.short_conv_attn import ( PleShortConvAttentionBackend, PleShortConvAttentionMetadata, ) -from vllm.v1.attention.backends.utils import NULL_BLOCK_ID from ..common.ple import PLEVocabParallelEmbedding +from .ops.ple import ple_conv, ple_gate, ple_ngram_ids class Qwen4ExpPLEGroupedNorm(nn.Module): @@ -256,8 +255,6 @@ class Qwen4ExpNGramEmbedding(nn.Module): config: Qwen4ExpTextConfig, embedding_dim: int, ple_dense_layer_id: int, - max_total_tokens: int, - max_num_reqs: int, prefix: str, layer_name: str, quant_config: QuantizationConfig | None = None, @@ -324,20 +321,6 @@ class Qwen4ExpNGramEmbedding(nn.Module): quant_config, f"{prefix}.ngram_embedding" ), ) - self.register_buffer( - "positions_buffer", - torch.arange(max_total_tokens, dtype=torch.int64), - persistent=False, - ) - self.register_buffer( - "padded_buffer", - torch.full( - (max_num_reqs, max_total_tokens), - self.eos_token_id, - dtype=torch.int64, - ), - persistent=False, - ) @staticmethod def _shift_precompute( @@ -379,26 +362,34 @@ class Qwen4ExpNGramEmbedding(nn.Module): input_ids: torch.Tensor, query_start_loc: torch.Tensor, ngram_context: torch.Tensor, + output: torch.Tensor | None = None, ) -> torch.Tensor: """Compute n-gram embedding indices for the current request layout.""" - input_ids = input_ids.reshape(-1).long() - query_start_loc = query_start_loc.long() + input_ids = input_ids.reshape(-1) num_reqs = query_start_loc.numel() - 1 num_tokens = input_ids.shape[0] - if num_tokens > self.positions_buffer.numel(): - raise ValueError( - f"PLE received {num_tokens} tokens, but its workspace supports " - f"at most {self.positions_buffer.numel()}" - ) - if num_reqs > self.padded_buffer.shape[0]: - raise ValueError( - f"PLE received {num_reqs} requests, but its workspace supports " - f"at most {self.padded_buffer.shape[0]}" - ) - positions = self.positions_buffer[:num_tokens] - packed = self.padded_buffer[:num_reqs, :num_tokens] - packed.fill_(self.eos_token_id) + if input_ids.is_cuda: + return ple_ngram_ids( + input_ids=input_ids, + query_start_loc=query_start_loc, + ngram_context=ngram_context, + layer_multipliers=self.layer_multipliers, + ngram_heads_vocab_sizes=self.ngram_heads_vocab_sizes, + ngram_heads_offsets=self.ngram_heads_offsets, + eos_token_id=self.eos_token_id, + heads_per_ngram=self.heads_per_ngram, + output=output, + ) + input_ids = input_ids.long() + query_start_loc = query_start_loc.long() + positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64) + packed = torch.full( + (num_reqs, num_tokens), + self.eos_token_id, + device=input_ids.device, + dtype=torch.int64, + ) request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1 request_indices.clamp_(max=num_reqs - 1) columns = (positions - query_start_loc[request_indices]).clamp( @@ -446,12 +437,13 @@ class Qwen4ExpNGramEmbedding(nn.Module): query_start_loc: torch.Tensor, ngram_context: torch.Tensor, ) -> torch.Tensor: - ngram_ids = input_ids.new_empty( - (input_ids.shape[0], self.ngram_heads), - dtype=torch.long, - ) # Keep num_reqs-dependent ID generation outside PIECEWISE CUDA graphs, # which dispatch only on the padded token count. + # torch.compile requires the splitting op to write graph-owned storage. + # Once compilation is removed, the op can return the IDs directly. + ngram_ids = input_ids.new_empty( + (input_ids.numel(), self.ngram_heads), dtype=torch.long + ) torch.ops.vllm.qwen4_exp_compute_ple_ngram_ids( input_ids, query_start_loc, @@ -562,26 +554,20 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): config, int(config.ple_embed_dim), self.ple_dense_layer_id, - vllm_config.scheduler_config.max_num_batched_tokens, - vllm_config.scheduler_config.max_num_seqs, prefix=f"{prefix}.ple_embedding", layer_name=prefix, quant_config=quant_config, params_dtype=model_config.dtype, ) - self.key_proj = ReplicatedLinear( + # The PLE cache is TP-replicated, so this merged projection is too. + self.kv_proj = MergedColumnParallelLinear( int(config.ple_embed_dim), - self.hc_hidden_size, + [self.hc_hidden_size, self.hidden_size], bias=False, + params_dtype=model_config.dtype, quant_config=quant_config, - prefix=f"{prefix}.key_proj", - ) - self.value_proj = ReplicatedLinear( - int(config.ple_embed_dim), - self.hidden_size, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.value_proj", + prefix=f"{prefix}.kv_proj", + disable_tp=True, ) norm_args = ( self.hc_hidden_size, @@ -654,360 +640,14 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): num_spec=self.num_spec_tokens, ) - def _apply_norm( - self, norm: Qwen4ExpPLEGroupedNorm, hidden_states: torch.Tensor - ) -> torch.Tensor: - shape = hidden_states.shape - return norm(hidden_states.flatten(-2)).reshape(shape) - - def _short_conv_fallback(self, inputs: torch.Tensor) -> torch.Tensor: - # Profiling / CUDA graph capture only; conv state is not updated. - inputs_t = inputs.transpose(0, 1).unsqueeze(0) - output = self.conv1d(inputs_t)[..., : inputs_t.size(-1)] - return F.silu(output).squeeze(0).transpose(0, 1) - - def _short_conv_dilated_decode_batched( - self, - x_d: torch.Tensor, - conv_state: torch.Tensor, - conv_weights: torch.Tensor, - state_indices_tensor_d: torch.Tensor, - has_initial_states_d: torch.Tensor | None, - ) -> torch.Tensor: - state_indices = state_indices_tensor_d.to( - device=conv_state.device, dtype=torch.int64 - ) - # FULL cudagraph padded decode rows use NULL_BLOCK_ID. Remap them to - # slot 0 for a safe gather, then zero output and skip write-back. - valid_state = state_indices != NULL_BLOCK_ID - state_indices = torch.where( - valid_state, state_indices, torch.zeros_like(state_indices) - ) - if has_initial_states_d is None: - has_initial_state = valid_state - else: - if has_initial_states_d.numel() < state_indices_tensor_d.numel(): - raise ValueError( - "has_initial_states_d size mismatch: " - f"got {has_initial_states_d.numel()}, " - f"need >= {state_indices_tensor_d.numel()}." - ) - has_initial_state = has_initial_states_d[ - : state_indices_tensor_d.numel() - ].to(device=conv_state.device, dtype=torch.bool) - has_initial_state = has_initial_state & valid_state - - cached_state = conv_state.index_select(0, state_indices) - state = cached_state[..., : self.conv_state_len].to(x_d.dtype) - if self.conv_state_len > 0: - initial_state = torch.where( - has_initial_state.view(-1, 1, 1), - state, - torch.zeros_like(state), - ) - history = torch.cat((initial_state, x_d.unsqueeze(-1)), dim=-1) - else: - history = x_d.unsqueeze(-1) - - conv_output = F.conv1d( - history, - conv_weights.unsqueeze(1).contiguous(), - groups=history.size(1), - dilation=self.short_conv_dilation, - ).squeeze(-1) - output = F.silu(conv_output) - output = output * valid_state.view(-1, 1).to(output.dtype) - - if self.conv_state_len > 0: - next_state = history[..., -self.conv_state_len :] - # Padded rows are remapped to the reserved null slot. Preserve its - # existing value while writing the new states for valid rows. - existing_base_state = cached_state[..., : self.conv_state_len] - safe_next_state = torch.where( - valid_state.view(-1, 1, 1), - next_state.to(conv_state.dtype), - existing_base_state, - ) - cached_state[..., : self.conv_state_len] = safe_next_state - conv_state.index_copy_(0, state_indices, cached_state) - - return output - - def _short_conv_dilated_prefill_batched( - self, - x_p: torch.Tensor, - metadata: PleShortConvAttentionMetadata, - conv_state: torch.Tensor, - conv_weights: torch.Tensor, - state_indices_tensor_p: torch.Tensor, - num_prefills: int, - num_decode_tokens: int, - num_prefill_tokens: int, - ) -> torch.Tensor: - # ``non_spec_query_start_loc`` covers the non-spec (decode + prefill) - # requests and equals ``query_start_loc`` when spec-decode is inactive. - non_spec_query_start_loc = metadata.non_spec_query_start_loc - if non_spec_query_start_loc is None: - raise ValueError("query_start_loc is required for prefill short-conv") - query_start_loc_p = ( - non_spec_query_start_loc[-num_prefills - 1 :] - num_decode_tokens - ) - # The metadata builder guarantees that the prefill query offsets start - # at 0 and end at num_prefill_tokens. Avoid reading those values here, - # since doing so would force a device-to-host synchronization. - has_initial_states_p = metadata.has_initial_states_p - if has_initial_states_p is None: - raise ValueError("has_initial_states_p is required for prefill short-conv") - - output = torch.empty_like(x_p) - q_starts = query_start_loc_p.to(torch.int64) - if state_indices_tensor_p.numel() < num_prefills: - raise ValueError( - "state_indices_tensor_p size mismatch: " - f"got {state_indices_tensor_p.numel()}, " - f"need >= {num_prefills}." - ) - if has_initial_states_p.numel() < num_prefills: - raise ValueError( - "has_initial_states_p size mismatch: " - f"got {has_initial_states_p.numel()}, " - f"need >= {num_prefills}." - ) - if num_prefills == 0 or x_p.numel() == 0: - return output - lengths = q_starts[1:] - q_starts[:-1] - # Use the CPU-computed packing width from the metadata builder instead - # of synchronizing on lengths.max(). - max_len = metadata.max_prefill_query_len - if max_len <= 0: - return output - - hidden_size = x_p.shape[1] - positions = torch.arange( - num_prefill_tokens, device=x_p.device, dtype=torch.int64 - ) - req_indices = torch.searchsorted(q_starts[1:], positions, right=True) - col_indices = positions - q_starts[req_indices] - - packed_tokens = x_p.new_zeros((num_prefills, max_len, hidden_size)) - packed_tokens[req_indices, col_indices] = x_p - packed_tokens = packed_tokens.transpose(1, 2).contiguous() - - state_indices = state_indices_tensor_p[:num_prefills].to( - device=conv_state.device, dtype=torch.int64 - ) - valid_state = state_indices != NULL_BLOCK_ID - state_indices = torch.where( - valid_state, state_indices, torch.zeros_like(state_indices) - ) - has_initial = has_initial_states_p[:num_prefills].to( - device=conv_state.device, dtype=torch.bool - ) - if self.conv_state_len > 0: - if conv_state.shape[0] == 0: - state = conv_state.new_zeros( - (num_prefills, hidden_size, self.conv_state_len), - dtype=x_p.dtype, - ) - else: - state = conv_state.index_select(0, state_indices)[ - ..., : self.conv_state_len - ].to(x_p.dtype) - use_initial_mask = (valid_state & has_initial).view(num_prefills, 1, 1) - initial_state = torch.where( - use_initial_mask, - state, - torch.zeros_like(state), - ) - history = torch.cat((initial_state, packed_tokens), dim=-1) - else: - history = packed_tokens - - conv_output = F.conv1d( - history, - conv_weights.unsqueeze(1).contiguous(), - groups=history.size(1), - dilation=self.short_conv_dilation, - ) - conv_output = F.silu(conv_output).transpose(1, 2).contiguous() - - token_positions = torch.arange(max_len, device=x_p.device, dtype=torch.int64) - valid_tokens = token_positions.view(1, max_len) < lengths.view(num_prefills, 1) - valid_output_mask = valid_tokens & valid_state.to(device=x_p.device).view( - num_prefills, 1 - ) - conv_output.masked_fill_(~valid_output_mask.unsqueeze(-1), 0) - output.copy_(conv_output[req_indices, col_indices]) - - if self.conv_state_len > 0 and conv_state.shape[0] > 0: - state_starts = lengths.to(device=history.device, dtype=torch.int64).view( - num_prefills, 1, 1 - ) - state_offsets = torch.arange( - self.conv_state_len, device=history.device, dtype=torch.int64 - ).view(1, 1, self.conv_state_len) - next_state = history.gather( - dim=2, - index=(state_starts + state_offsets).expand(-1, history.size(1), -1), - ) - # Write back without a host synchronization. Valid, non-empty rows - # receive their new state; padding and zero-length rows keep the - # current cache value. - existing_state = conv_state.index_select(0, state_indices) - existing_base_state = existing_state[..., : self.conv_state_len] - update_mask = valid_state & (lengths.to(device=conv_state.device) > 0) - safe_next_state = torch.where( - update_mask.view(num_prefills, 1, 1), - next_state.to(conv_state.dtype), - existing_base_state, - ) - existing_state[..., : self.conv_state_len] = safe_next_state - conv_state.index_copy_(0, state_indices, existing_state) - return output - - def _short_conv_dilated_spec_batched( - self, - x_spec: torch.Tensor, - conv_state: torch.Tensor, - conv_weights: torch.Tensor, - spec_state_indices_tensor: torch.Tensor, - spec_query_start_loc: torch.Tensor, - num_accepted_tokens: torch.Tensor, - spec_query_len: int, - ) -> torch.Tensor: - """Dilated short-conv for speculative-decode (MTP) requests. - - Each spec request feeds multiple (draft + 1) query tokens. The conv - outputs are computed causally after rolling back the previous draft - state by ``num_accepted_tokens - 1``. The current candidate inputs stay - in the extended cache for the next forward, matching - ``causal_conv1d_update``. - - ``spec_query_len`` (== num_speculative_tokens + 1) is the maximum query - length and is a Python int, so no host synchronization is needed; this - keeps the path safe for full CUDA-graph capture/replay where the buffers - are padded at the request level. - """ - num_reqs = spec_state_indices_tensor.numel() - hidden_size = x_spec.size(-1) - # Use a fixed packing width instead of synchronizing on lengths.max(). - max_len = spec_query_len - # Full CUDA graphs can pad these buffers. Only the first num_reqs - # accepted-token counts belong to actual speculative requests. - num_accepted_tokens = num_accepted_tokens[:num_reqs] - q_starts = spec_query_start_loc[: num_reqs + 1].to(torch.int64) - # Keep the number of real speculative tokens on the device. - total_real_tokens = q_starts[num_reqs] - - state_indices = spec_state_indices_tensor.to( - device=conv_state.device, dtype=torch.int64 - ) - valid_state = state_indices != NULL_BLOCK_ID - state_indices = torch.where( - valid_state, state_indices, torch.zeros_like(state_indices) - ) - positions = torch.arange( - x_spec.size(0), device=x_spec.device, dtype=torch.int64 - ) - # Route graph-padded token rows to the discarded dummy request so that - # they cannot overwrite real packed data. - req_indices = torch.searchsorted(q_starts[1:], positions, right=True) - valid_tokens = (positions < total_real_tokens) & (req_indices < num_reqs) - clamped_req_indices = req_indices.clamp_max(max(num_reqs - 1, 0)) - col_indices = (positions - q_starts[clamped_req_indices]).clamp_(0, max_len - 1) - pack_req_indices = torch.where( - valid_tokens, - clamped_req_indices, - torch.full_like(req_indices, num_reqs), - ) - pack_col_indices = torch.where( - valid_tokens, col_indices, torch.zeros_like(col_indices) - ) - - # The last request row is the dummy sink for graph padding. - packed = x_spec.new_zeros((num_reqs + 1, max_len, hidden_size)) - packed[pack_req_indices, pack_col_indices] = x_spec - packed = packed.transpose(1, 2).contiguous() - - if self.conv_state_len > 0: - cached_state = conv_state.index_select(0, state_indices) - rollback_offsets = num_accepted_tokens.to( - device=conv_state.device, dtype=torch.int64 - ).sub(1) - rollback_offsets = torch.where( - valid_state, - rollback_offsets.clamp_(0, max_len - 1), - torch.zeros_like(rollback_offsets), - ) - state_offsets = torch.arange( - self.conv_state_len, device=conv_state.device, dtype=torch.int64 - ).view(1, 1, self.conv_state_len) - rollback_indices = rollback_offsets.view(-1, 1, 1) + state_offsets - state = cached_state.gather( - 2, rollback_indices.expand(-1, hidden_size, -1) - ).to(x_spec.dtype) - state = torch.where( - valid_state.view(num_reqs, 1, 1), - state, - torch.zeros_like(state), - ) - # Append a zeroed dummy-row state to match the [num_reqs + 1] pack. - dummy_state = state.new_zeros((1, hidden_size, self.conv_state_len)) - state_full = torch.cat((state, dummy_state), dim=0) - history = torch.cat((state_full, packed), dim=-1) - else: - history = packed - - conv_output = F.conv1d( - history, - conv_weights.unsqueeze(1).contiguous(), - groups=history.size(1), - dilation=self.short_conv_dilation, - ) - conv_output = F.silu(conv_output).transpose(1, 2).contiguous() - - output = conv_output[pack_req_indices, pack_col_indices] - output = output * valid_tokens.view(-1, 1).to(output.dtype) - - # Keep all current candidate inputs in the extended state. On the next - # target forward, ``num_accepted_tokens - 1`` selects the rollback - # window before processing the newly scheduled tokens. - if self.conv_state_len > 0: - state_capacity = self.conv_state_len + max_len - 1 - if conv_state.size(-1) < state_capacity: - raise RuntimeError( - "PLE short-conv cache cannot retain speculative tokens: " - f"got {conv_state.size(-1)}, need {state_capacity}." - ) - candidate_state = history[:num_reqs, :, 1 : state_capacity + 1] - query_lengths = q_starts[1:] - q_starts[:-1] - state_positions = torch.arange( - state_capacity, device=history.device, dtype=torch.int64 - ).view(1, 1, state_capacity) - update_lengths = (self.conv_state_len + query_lengths - 1).view( - num_reqs, 1, 1 - ) - update_mask = valid_state.view(num_reqs, 1, 1) & ( - state_positions < update_lengths - ) - existing_state = cached_state[..., :state_capacity] - next_state = torch.where( - update_mask, - candidate_state.to(conv_state.dtype), - existing_state, - ) - cached_state[..., :state_capacity] = next_state - conv_state.index_copy_(0, state_indices, cached_state) - - return output - def _short_conv_dilated_dispatch( self, inputs: torch.Tensor, + residual: torch.Tensor, metadata: PleShortConvAttentionMetadata, conv_state: torch.Tensor, conv_weights: torch.Tensor, - ) -> torch.Tensor: + ) -> None: num_prefills = metadata.num_prefills num_decodes = metadata.num_decodes num_decode_tokens = metadata.num_decode_tokens @@ -1015,110 +655,129 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): has_prefill = num_prefills > 0 has_decode = num_decodes > 0 has_spec = metadata.spec_sequence_masks is not None - x = inputs[: metadata.num_actual_tokens] + has_non_spec = has_prefill or has_decode + inputs = inputs[: metadata.num_actual_tokens] + residual = residual[: metadata.num_actual_tokens] - # Split spec / non-spec tokens. - if has_spec: - if has_prefill or has_decode: - assert metadata.spec_token_indx is not None - assert metadata.non_spec_token_indx is not None - x_spec = x.index_select(0, metadata.spec_token_indx.long()) - x_non_spec = x.index_select(0, metadata.non_spec_token_indx.long()) - else: - x_spec = x - x_non_spec = None - else: - x_spec = None - x_non_spec = x + spec_token_indices = None + non_spec_token_indices = None + if has_spec and has_non_spec: + assert metadata.spec_token_indx is not None + assert metadata.non_spec_token_indx is not None + spec_token_indices = metadata.spec_token_indx + non_spec_token_indices = metadata.non_spec_token_indx - spec_output = None - # 1. Run the multi-query speculative-decode part. if has_spec: assert metadata.spec_state_indices_tensor is not None - assert metadata.spec_query_start_loc is not None - assert metadata.num_accepted_tokens is not None - spec_output = self._short_conv_dilated_spec_batched( - x_spec=x_spec, + query_start_loc = metadata.spec_query_start_loc + num_accepted_tokens = metadata.num_accepted_tokens + assert query_start_loc is not None + assert num_accepted_tokens is not None + spec_state_indices = metadata.spec_state_indices_tensor[ + : metadata.num_spec_decodes + ] + # Mixed batches stay in their original row order; the kernels map + # logical spec/non-spec rows instead of materializing both groups. + ple_conv( + inputs=inputs, + residual=residual, conv_state=conv_state, conv_weights=conv_weights, - spec_state_indices_tensor=metadata.spec_state_indices_tensor[ - : metadata.num_spec_decodes - ], - spec_query_start_loc=metadata.spec_query_start_loc, - num_accepted_tokens=metadata.num_accepted_tokens, + state_indices=spec_state_indices, + mode="spec", + dilation=self.short_conv_dilation, + query_start_loc=query_start_loc, + num_accepted_tokens=num_accepted_tokens, spec_query_len=metadata.spec_query_len, + token_indices=spec_token_indices, ) - # 2. Run regular decode and prefill requests. - conv_out_non_spec = None - state_indices_tensor = metadata.state_indices_tensor - if x_non_spec is not None: - assert state_indices_tensor is not None - if has_prefill: - state_indices_tensor_d, state_indices_tensor_p = torch.split( - state_indices_tensor, - [num_decodes, num_prefills], - dim=0, + if not has_non_spec: + return + + state_indices = metadata.state_indices_tensor + assert state_indices is not None + if has_prefill: + state_indices_d, state_indices_p = torch.split( + state_indices, [num_decodes, num_prefills], dim=0 + ) + if non_spec_token_indices is None: + inputs_d, inputs_p = torch.split( + inputs, [num_decode_tokens, num_prefill_tokens], dim=0 ) - x_d, x_p = torch.split( - x_non_spec, + residual_d, residual_p = torch.split( + residual, [num_decode_tokens, num_prefill_tokens], dim=0 + ) + token_indices_d = None + token_indices_p = None + else: + inputs_d = inputs_p = inputs + residual_d = residual_p = residual + token_indices_d, token_indices_p = torch.split( + non_spec_token_indices, [num_decode_tokens, num_prefill_tokens], dim=0, ) - non_spec_parts: list[torch.Tensor] = [] - if has_decode: - non_spec_parts.append( - self._short_conv_dilated_decode_batched( - x_d=x_d, - conv_state=conv_state, - conv_weights=conv_weights, - state_indices_tensor_d=state_indices_tensor_d, - has_initial_states_d=metadata.has_initial_states_d, - ) - ) - non_spec_parts.append( - self._short_conv_dilated_prefill_batched( - x_p=x_p, - metadata=metadata, - conv_state=conv_state, - conv_weights=conv_weights, - state_indices_tensor_p=state_indices_tensor_p, - num_prefills=num_prefills, - num_decode_tokens=num_decode_tokens, - num_prefill_tokens=num_prefill_tokens, - ) - ) - conv_out_non_spec = torch.vstack(non_spec_parts) - else: - conv_out_non_spec = self._short_conv_dilated_decode_batched( - x_d=x_non_spec, + + if has_decode: + ple_conv( + inputs=inputs_d, + residual=residual_d, conv_state=conv_state, conv_weights=conv_weights, - state_indices_tensor_d=state_indices_tensor[: x_non_spec.size(0)], - has_initial_states_d=metadata.has_initial_states_d, + state_indices=state_indices_d, + mode="decode", + dilation=self.short_conv_dilation, + has_initial_states=metadata.has_initial_states_d, + token_indices=token_indices_d, ) - # 3. Merge both parts back into the original token order. - if has_spec and conv_out_non_spec is not None: - assert metadata.spec_token_indx is not None - assert metadata.non_spec_token_indx is not None - assert spec_output is not None - output = x.new_empty((metadata.num_actual_tokens, x.size(-1))) - output.index_copy_(0, metadata.spec_token_indx, spec_output) - output.index_copy_(0, metadata.non_spec_token_indx, conv_out_non_spec) - return output - elif has_spec: - assert spec_output is not None - return spec_output - if conv_out_non_spec is None: - return x - return conv_out_non_spec + query_start_loc = metadata.non_spec_query_start_loc + if query_start_loc is None: + raise ValueError("query_start_loc is required for prefill short-conv") + query_start_loc = query_start_loc[-num_prefills - 1 :] - num_decode_tokens + has_initial_states = metadata.has_initial_states_p + if has_initial_states is None: + raise ValueError( + "has_initial_states_p is required for prefill short-conv" + ) + ple_conv( + inputs=inputs_p, + residual=residual_p, + conv_state=conv_state, + conv_weights=conv_weights, + state_indices=state_indices_p, + mode="prefill", + dilation=self.short_conv_dilation, + query_start_loc=query_start_loc, + has_initial_states=has_initial_states, + token_indices=token_indices_p, + ) + else: + num_decode_rows = ( + non_spec_token_indices.numel() + if non_spec_token_indices is not None + else inputs.size(0) + ) + ple_conv( + inputs=inputs, + residual=residual, + conv_state=conv_state, + conv_weights=conv_weights, + state_indices=state_indices[:num_decode_rows], + mode="decode", + dilation=self.short_conv_dilation, + has_initial_states=metadata.has_initial_states_d, + token_indices=non_spec_token_indices, + ) - def _short_conv(self, inputs: torch.Tensor) -> torch.Tensor: + def _short_conv(self, inputs: torch.Tensor, residual: torch.Tensor) -> None: forward_context = get_forward_context() attn_metadata = forward_context.attn_metadata + # Profiling omits all metadata or this Mamba entry. The residual + # already contains the gated output, so short convolution is a no-op. if attn_metadata is None: - return self._short_conv_fallback(inputs) + return if not isinstance(attn_metadata, dict): raise RuntimeError( @@ -1128,8 +787,7 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): layer_attn_metadata = attn_metadata.get(self.prefix) if layer_attn_metadata is None: - # MRV2 omits Mamba-family metadata during profile warmup. - return self._short_conv_fallback(inputs) + return if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata): raise TypeError( "Expected PleShortConvAttentionMetadata for layer " @@ -1138,24 +796,27 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): ) conv_state = self.kv_cache[0] + # Canonicalize both backend cache layouts to [slot, channel, window]. if not is_conv_state_dim_first(): conv_state = conv_state.transpose(-1, -2) conv_weights = self.conv1d.weight.squeeze(1) state_capacity = self.conv_state_len + self.num_spec_tokens if state_capacity > 0: - if conv_state.size(-1) < state_capacity: + state_size = conv_state.size(-1) + if state_size < state_capacity: raise RuntimeError( "PLE short-conv cache is smaller than expected for " - f"dilated convolution: got {conv_state.size(-1)}, " + f"dilated convolution: got {state_size}, " f"expect at least {state_capacity}." ) conv_state = conv_state[..., -state_capacity:] - return self._short_conv_dilated_dispatch( - inputs, - layer_attn_metadata, - conv_state, - conv_weights.to(dtype=inputs.dtype), + self._short_conv_dilated_dispatch( + inputs=inputs, + residual=residual, + metadata=layer_attn_metadata, + conv_state=conv_state, + conv_weights=conv_weights.to(dtype=inputs.dtype), ) def forward( @@ -1174,24 +835,21 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): ) embeddings = self.ple_embedding(input_ids, query_start_loc, ngram_context) embeddings = self._dequantize_embeddings(embeddings, hidden_states.dtype) - key, _ = self.key_proj(embeddings) - value, _ = self.value_proj(embeddings) - token_count = hidden_states.shape[0] - key = key.reshape(token_count, self.hc_count, self.hidden_size) - query = hidden_states.reshape(token_count, self.hc_count, self.hidden_size) - key = self._apply_norm(self.norm_key, key) - query = self._apply_norm(self.norm_query, query) - gate = (key * query).sum(dim=-1, keepdim=True) / math.sqrt(self.hidden_size) - gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt()) - gated_value = gate * value.unsqueeze(-2) - normalized = self._apply_norm(self.norm_conv, gated_value).flatten(-2) - conv_output = torch.zeros_like(normalized) - torch.ops.vllm.qwen4_exp_ple_short_conv( - normalized, - conv_output, - self.prefix, + kv, _ = self.kv_proj(embeddings) + key, value = kv.split(self.kv_proj.output_sizes, dim=-1) + gated_output, conv_input = ple_gate( + key, + value, + hidden_states, + self.norm_key.weight, + self.norm_query.weight, + self.norm_conv.weight, + self.norm_key.eps, ) - return gated_value.flatten(-2) + conv_output + # State routing depends on runtime request metadata and remains outside + # the piecewise graph; short convolution accumulates into gated_output. + torch.ops.vllm.qwen4_exp_ple_short_conv(conv_input, gated_output, self.prefix) + return gated_output def qwen4_exp_compute_ple_ngram_ids( @@ -1203,12 +861,12 @@ def qwen4_exp_compute_ple_ngram_ids( ) -> None: """Compute request-dependent PLE n-gram IDs outside piecewise graphs.""" layer = get_forward_context().no_compile_layers[layer_name] - ngram_ids = layer.ple_embedding.compute_ngram_ids( + layer.ple_embedding.compute_ngram_ids( input_ids, query_start_loc, ngram_context, + output, ) - output.copy_(ngram_ids) def qwen4_exp_compute_ple_ngram_ids_fake( @@ -1223,17 +881,16 @@ def qwen4_exp_compute_ple_ngram_ids_fake( def qwen4_exp_ple_short_conv( inputs: torch.Tensor, - output: torch.Tensor, + residual_output: torch.Tensor, layer_name: str, ) -> None: layer = get_forward_context().no_compile_layers[layer_name] - result = layer._short_conv(inputs) - output[: result.shape[0]].copy_(result) + layer._short_conv(inputs, residual_output) def qwen4_exp_ple_short_conv_fake( inputs: torch.Tensor, - output: torch.Tensor, + residual_output: torch.Tensor, layer_name: str, ) -> None: return @@ -1250,7 +907,7 @@ direct_register_custom_op( direct_register_custom_op( op_name="qwen4_exp_ple_short_conv", op_func=qwen4_exp_ple_short_conv, - mutates_args=["output"], + mutates_args=["residual_output"], fake_impl=qwen4_exp_ple_short_conv_fake, ) diff --git a/vllm/v1/attention/backends/short_conv_attn.py b/vllm/v1/attention/backends/short_conv_attn.py index 3e888e69b7..c2365ad61c 100644 --- a/vllm/v1/attention/backends/short_conv_attn.py +++ b/vllm/v1/attention/backends/short_conv_attn.py @@ -141,12 +141,6 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): self.spec_sequence_masks = torch.empty( (self.decode_cudagraph_max_bs,), dtype=torch.bool, device=device ) - self.spec_token_indx = torch.empty( - (self.decode_cudagraph_max_tokens,), dtype=torch.int32, device=device - ) - self.non_spec_token_indx = torch.empty( - (self.decode_cudagraph_max_tokens,), dtype=torch.int32, device=device - ) self.spec_query_start_loc = torch.empty( (self.decode_cudagraph_max_bs + 1,), dtype=torch.int32, device=device )