[Qwen3.8-Flash-Next] Remove torch.compile for NVIDIA implementation (#55272)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Thien Tran
2026-09-06 21:57:56 -07:00
committed by GitHub
parent 9cc7793e32
commit d9105ea800
13 changed files with 165 additions and 185 deletions
+37 -14
View File
@@ -147,9 +147,9 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
model_state.uses_ngram_embedding = True
model_state.ngram_context_len = 3
model_state.ngram_eos_token_id = 99
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
input_batch = SimpleNamespace(
num_reqs=2,
@@ -167,34 +167,57 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
model_inputs = model_state.prepare_inputs(input_batch, req_states)
expected_query_start_loc = torch.full((9,), 3, dtype=torch.int32)
expected_query_start_loc[0] = 0
expected_query_start_loc[1] = 2
torch.testing.assert_close(
model_inputs["query_start_loc"],
torch.tensor([0, 2, 3, 3], dtype=torch.int32),
model_inputs["query_start_loc"], expected_query_start_loc
)
expected_context = torch.full((8, 3), 99, dtype=torch.int32)
expected_context[:2] = torch.tensor([[99, 99, 20], [1, 2, 3]])
torch.testing.assert_close(model_inputs["ngram_context"], expected_context)
# Retain the views to detect reallocations as the request layout changes.
query_start_loc = model_inputs["query_start_loc"]
ngram_context = model_inputs["ngram_context"]
input_batch.num_reqs = 1
input_batch.num_reqs_after_padding = 1
input_batch.idx_mapping = torch.tensor([0])
input_batch.query_start_loc = torch.tensor([0, 3], dtype=torch.int32)
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
model_inputs = model_state.prepare_inputs(input_batch, req_states)
expected_query_start_loc.fill_(3)
expected_query_start_loc[0] = 0
torch.testing.assert_close(
model_inputs["ngram_context"],
torch.tensor([[99, 99, 20], [1, 2, 3], [99, 99, 99]], dtype=torch.int32),
model_inputs["query_start_loc"], expected_query_start_loc
)
expected_context.fill_(99)
expected_context[0] = torch.tensor([1, 2, 3])
torch.testing.assert_close(model_inputs["ngram_context"], expected_context)
assert model_inputs["query_start_loc"].data_ptr() == query_start_loc.data_ptr()
assert model_inputs["ngram_context"].data_ptr() == ngram_context.data_ptr()
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
model_state.ngram_eos_token_id = 99
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}):
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4)
# Dummy runs establish the addresses used during CUDA graph capture.
query_start_loc_ptr = first["query_start_loc"].data_ptr()
ngram_context_ptr = first["ngram_context"].data_ptr()
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4)
expected_query_start_loc = torch.full((9,), 4, dtype=torch.int32)
expected_query_start_loc[:4] = torch.tensor([0, 1, 2, 4], dtype=torch.int32)
torch.testing.assert_close(second["query_start_loc"], expected_query_start_loc)
torch.testing.assert_close(
second["query_start_loc"], torch.tensor([0, 2, 5, 8], dtype=torch.int32)
)
torch.testing.assert_close(
second["ngram_context"], torch.full((3, 3), 99, dtype=torch.int32)
second["ngram_context"], torch.full((8, 3), 99, dtype=torch.int32)
)
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
assert second["ngram_context"].data_ptr() == ngram_context_ptr
+12
View File
@@ -498,6 +498,17 @@ def _ngram_hash_params(device: torch.device, context_len: int) -> dict:
([3], [1], [[11]], 20),
([4, 4], [0, 7], [[11, 12], [13, 14]], 20),
([4, 0, 3], [], [[11, 12], [13, 14], [15, 16]], 20),
(
[3, 2, 0, 0],
[],
[
[11, 12],
[13, 14],
[_NGRAM_EOS_TOKEN_ID, _NGRAM_EOS_TOKEN_ID],
[_NGRAM_EOS_TOKEN_ID, _NGRAM_EOS_TOKEN_ID],
],
20,
),
(
[1, 33, 2],
[5, 32],
@@ -528,6 +539,7 @@ def _ngram_hash_params(device: torch.device, context_len: int) -> dict:
"bigram-only",
"power-of-two",
"empty-request",
"trailing-padded-requests",
"three-requests",
"six-requests",
"four-gram",
+66 -2
View File
@@ -45,7 +45,6 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
indexer = SimpleNamespace(
skip_topk=True,
_metadata=lambda: (raw_metadata, compressed_metadata),
index_qk_proj=lambda hidden: (torch.zeros(2, 2), None),
index_n_heads=1,
index_kv_heads=1,
index_head_dim=1,
@@ -80,7 +79,7 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
actual = indexer_qsa.QSAIndexer.forward(
indexer,
torch.zeros(2, 4),
torch.zeros(2, 2),
torch.tensor([7, 8]),
rows,
)
@@ -446,6 +445,71 @@ def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
assert metadata.k_work_metadata.tolist() == [[0, 0], [2, 0], [2, 1], [-1, -1]]
@requires_qsa_kernels
@pytest.mark.usefixtures("default_vllm_config")
def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
"""Padded projected Q/K rows must not affect either side cache."""
from vllm.model_executor.layers.rotary_embedding import get_rope
device = torch.device("cuda")
# Five tokens complete one compressed group and retain four keys in the ring.
raw_metadata = SimpleNamespace(
num_actual_tokens=5,
slot_mapping=torch.tensor([-1, 1, 2, 3, 0], device=device),
block_table=torch.zeros((1, 1), dtype=torch.int32, device=device),
token_to_req=torch.zeros(5, dtype=torch.int32, device=device),
query_start_loc=torch.tensor([0, 5], dtype=torch.int32, device=device),
logical_positions=torch.arange(5, device=device),
)
compressed_metadata = SimpleNamespace(
slot_mapping=torch.tensor([-1, -1, -1, 0, -1], device=device),
)
with torch.device(device):
rope = get_rope(
head_size=128,
max_position=32,
rope_parameters={"rope_type": "default", "partial_rotary_factor": 0.5},
dtype=torch.bfloat16,
)
raw_cache = torch.zeros((1, 4, 1, 64), dtype=torch.bfloat16, device=device)
compressed_cache = torch.zeros((1, 2, 1, 64), dtype=torch.bfloat16, device=device)
norm = SimpleNamespace(
weight=torch.zeros(64, dtype=torch.bfloat16, device=device),
variance_epsilon=1e-6,
)
indexer = SimpleNamespace(
_metadata=lambda: (raw_metadata, compressed_metadata),
skip_topk=True,
index_kv_heads=1,
use_fused_pre_indexer=False,
index_n_heads=1,
index_head_dim=64,
q_layernorm=norm,
k_layernorm=norm,
rotary_emb=rope,
compress_ratio=4,
raw_key_cache=SimpleNamespace(
kv_cache=raw_cache, key_cache=raw_cache, rope_position_cache=None
),
compressed_key_cache=SimpleNamespace(kv_cache=compressed_cache),
)
keys = torch.arange(1, 6, dtype=torch.bfloat16, device=device)[:, None].expand(
5, 64
)
padded_keys = torch.full((8, 64), torch.nan, dtype=torch.bfloat16, device=device)
padded_keys[:5].copy_(keys)
indexer_qsa.QSAIndexer.forward(
indexer,
torch.cat((torch.ones_like(padded_keys), padded_keys), dim=-1),
torch.zeros(8, dtype=torch.long, device=device),
torch.full((5, 5), -1, dtype=torch.int32, device=device),
)
torch.testing.assert_close(raw_cache[0, :, 0], keys[[4, 1, 2, 3]])
expected_compressed = torch.zeros_like(compressed_cache)
expected_compressed[0, 0] = 1
torch.testing.assert_close(compressed_cache, expected_compressed)
@requires_qsa_kernels
@pytest.mark.parametrize("compress_ratio", [1, 4])
@pytest.mark.parametrize("num_reqs", [2, 3, 4, 7, 8, 9])
+7 -1
View File
@@ -338,9 +338,15 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
("DeepseekV32MTPModel", True, False),
("GlmMoeDsaForCausalLM", False, True),
("GlmMoeDsaForCausalLM", True, False),
("Qwen4ExpForCausalLM", False, True),
("Qwen4ExpForCausalLM", True, False),
("Qwen4ExpForConditionalGeneration", False, True),
("Qwen4ExpForConditionalGeneration", True, False),
("Qwen4ExpMTP", False, True),
("Qwen4ExpMTP", True, False),
],
)
def test_dsa_breakable_cudagraph_platform_default(
def test_breakable_cudagraph_platform_default(
monkeypatch, architecture, is_rocm, expected
):
from vllm.config.vllm import default_breakable_cudagraph_architectures
+1 -1
View File
@@ -768,7 +768,7 @@ class CompilationConfig:
"vllm::mamba_mixer2",
"vllm::mamba_mixer",
"vllm::short_conv",
"vllm::qwen4_exp_compute_ple_ngram_ids",
# Qwen4Exp's AMD backend still uses these splitting ops.
"vllm::qwen4_exp_ple_short_conv",
"vllm::qwen4_exp_qsa_with_output",
"vllm::linear_attention",
+3
View File
@@ -94,6 +94,9 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
"KimiLinearForCausalLM",
"MiniMaxM3SparseForCausalLM",
"MiniMaxM3SparseForConditionalGeneration",
"Qwen4ExpForCausalLM",
"Qwen4ExpForConditionalGeneration",
"Qwen4ExpMTP",
}
)
@@ -12,6 +12,7 @@ from torch import nn
from vllm import _custom_ops as ops
from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import (
VllmConfig,
get_current_vllm_config,
@@ -1909,6 +1910,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
)
@eager_break_during_capture
def qwen_gdn_attention_core(
qkv_or_qkvz: torch.Tensor,
b_or_ba: torch.Tensor,
@@ -1956,6 +1958,7 @@ direct_register_custom_op(
)
@eager_break_during_capture
def qwen_gdn_attention_core_fused_norm_packed(
mixed_qkvz: torch.Tensor,
ba: torch.Tensor,
+6 -8
View File
@@ -88,7 +88,7 @@ def _supports_fused_pre_indexer(
class QSAIndexer(nn.Module):
"""Replicated Q/K projection plus paged, weight-free QSA selection.
"""QSA projection weights, side caches, and paged, weight-free selection.
``prefix`` must be the checkpoint's indexer prefix, normally
``model.layers.N.self_attn.indexer``. Consequently the trainable names are
@@ -218,11 +218,11 @@ class QSAIndexer(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
projected_qk: torch.Tensor,
positions: torch.Tensor,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Select each query row's token indices.
"""Update side caches and select token indices from pre-projected Q/K.
Returns the packed buffer of shape [num_tokens, output_width + 1]:
the leading ``output_width`` columns are ``-1``-padded
@@ -237,10 +237,10 @@ class QSAIndexer(nn.Module):
if self.skip_topk and out is not None:
return out
result = torch.full(
(hidden_states.shape[0], self.packed_output_width),
(projected_qk.shape[0], self.packed_output_width),
-1,
dtype=torch.int32,
device=hidden_states.device,
device=projected_qk.device,
)
# Inert rows carry a zero valid count (empty loop bound), not -1.
result[:, -1] = 0
@@ -258,11 +258,9 @@ class QSAIndexer(nn.Module):
raw_metadata, compressed_metadata = metadata
num_tokens = raw_metadata.num_actual_tokens
hidden_states = hidden_states[:num_tokens]
projected_qk = projected_qk[:num_tokens]
positions = positions[..., :num_tokens]
# Q/K projection
projected_qk, _ = self.index_qk_proj(hidden_states)
projected_q, raw_keys = projected_qk.split(
(
self.index_n_heads * self.index_head_dim,
-12
View File
@@ -8,7 +8,6 @@ from itertools import islice
import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.fused_moe.utils import (
@@ -381,17 +380,6 @@ class Qwen4ExpMixtureOfExperts(MixtureOfExperts):
moe.experts.update_expert_map()
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"query_start_loc": 0,
"ngram_context": 0,
"deepstack_input_embeds": 0,
}
)
class Qwen4ExpModel(nn.Module):
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
+11 -7
View File
@@ -44,6 +44,8 @@ class Qwen4ExpModelState(MambaHybridModelState):
if self.ngram_context_len <= 0:
raise ValueError("N-gram embedding requires context length >= 1.")
self.ngram_eos_token_id = int(config.eos_token_id)
# PLE runs inside captured regions, so these buffers keep a fixed shape
# and address as the active request count changes between replays.
self.ngram_context = torch.full(
(self.max_num_reqs, self.ngram_context_len),
self.ngram_eos_token_id,
@@ -68,8 +70,7 @@ class Qwen4ExpModelState(MambaHybridModelState):
req_states: RequestState,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
num_reqs_padded = input_batch.num_reqs_after_padding
context = self.ngram_context[:num_reqs_padded]
context = self.ngram_context
context.fill_(self.ngram_eos_token_id)
if num_reqs == 0:
return context
@@ -101,8 +102,10 @@ class Qwen4ExpModelState(MambaHybridModelState):
return model_inputs
num_reqs_padded = input_batch.num_reqs_after_padding
query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1]
query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1])
query_start_loc = self.ple_query_start_loc
query_start_loc[: num_reqs_padded + 1].copy_(input_batch.query_start_loc)
# Represent unused capacity as trailing zero-length requests.
query_start_loc[num_reqs_padded + 1 :].copy_(input_batch.query_start_loc[-1])
model_inputs.update(
query_start_loc=query_start_loc,
ngram_context=self._prepare_ngram_context(input_batch, req_states),
@@ -118,7 +121,7 @@ class Qwen4ExpModelState(MambaHybridModelState):
if not self.uses_ngram_embedding:
return model_inputs
query_start_loc = self.ple_query_start_loc[: num_reqs + 1]
query_start_loc = self.ple_query_start_loc
query_start_loc[0] = 0
tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs)
query_lens = torch.full(
@@ -129,9 +132,10 @@ class Qwen4ExpModelState(MambaHybridModelState):
)
if num_extra_tokens > 0:
query_lens[-num_extra_tokens:] += 1
torch.cumsum(query_lens, dim=0, out=query_start_loc[1:])
torch.cumsum(query_lens, dim=0, out=query_start_loc[1 : num_reqs + 1])
query_start_loc[num_reqs + 1 :].fill_(num_tokens)
ngram_context = self.ngram_context[:num_reqs]
ngram_context = self.ngram_context
ngram_context.fill_(self.ngram_eos_token_id)
model_inputs.update(
query_start_loc=query_start_loc,
-19
View File
@@ -19,7 +19,6 @@ import regex as re
import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig, replace, set_current_vllm_config
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.fused_moe.utils import (
@@ -147,15 +146,6 @@ def _make_draft_vllm_config(
return draft_vllm_config
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"hidden_states": 0,
}
)
class Qwen4ExpMultiTokenPredictor(nn.Module):
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
@@ -355,15 +345,6 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
return loader.load_weights(weights, mapper=mapper)
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"hidden_states": 0,
}
)
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
+5 -61
View File
@@ -8,6 +8,7 @@ import torch
import torch.nn.functional as F
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
@@ -38,7 +39,6 @@ from vllm.model_executor.parameter import PerTensorScaleParameter
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionBackend,
@@ -264,12 +264,10 @@ class Qwen4ExpNGramEmbedding(nn.Module):
embedding_dim: int,
ple_dense_layer_id: int,
prefix: str,
layer_name: str,
quant_config: QuantizationConfig | None = None,
params_dtype: torch.dtype | None = None,
) -> None:
super().__init__()
self.layer_name = layer_name
self.embedding_dim = embedding_dim
self.ngram_size = int(config.ngram_size)
self.heads_per_ngram = int(config.heads_per_ngram)
@@ -445,20 +443,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
# Keep num_reqs-dependent ID generation outside PIECEWISE CUDA graphs,
# which dispatch only on the padded token count.
# torch.compile requires the splitting op to write graph-owned storage.
# Once compilation is removed, the op can return the IDs directly.
ngram_ids = input_ids.new_empty(
(input_ids.numel(), self.ngram_heads), dtype=torch.long
)
torch.ops.vllm.qwen4_exp_compute_ple_ngram_ids(
input_ids,
query_start_loc,
ngram_context,
ngram_ids,
self.layer_name,
)
ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context)
return self.ngram_embedding(ngram_ids).flatten(-2)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -563,7 +548,6 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
int(config.ple_embed_dim),
self.ple_dense_layer_id,
prefix=f"{prefix}.ple_embedding",
layer_name=prefix,
quant_config=quant_config,
params_dtype=model_config.dtype,
)
@@ -779,6 +763,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
token_indices=non_spec_token_indices,
)
# State routing consumes the current request metadata on every replay.
@eager_break_during_capture
def _short_conv(self, inputs: torch.Tensor, residual: torch.Tensor) -> None:
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
@@ -854,52 +840,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
self.norm_conv.weight,
self.norm_key.eps,
)
# State routing depends on runtime request metadata and remains outside
# the piecewise graph; short convolution accumulates into gated_output.
torch.ops.vllm.qwen4_exp_ple_short_conv(conv_input, gated_output, self.prefix)
self._short_conv(conv_input, gated_output)
return gated_output
def qwen4_exp_compute_ple_ngram_ids(
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
output: torch.Tensor,
layer_name: str,
) -> None:
"""Compute request-dependent PLE n-gram IDs outside piecewise graphs."""
layer = get_forward_context().no_compile_layers[layer_name]
layer.ple_embedding.compute_ngram_ids(
input_ids,
query_start_loc,
ngram_context,
output,
)
def qwen4_exp_ple_short_conv(
inputs: torch.Tensor,
residual_output: torch.Tensor,
layer_name: str,
) -> None:
layer = get_forward_context().no_compile_layers[layer_name]
layer._short_conv(inputs, residual_output)
direct_register_custom_op(
op_name="qwen4_exp_compute_ple_ngram_ids",
op_func=qwen4_exp_compute_ple_ngram_ids,
mutates_args=["output"],
)
direct_register_custom_op(
op_name="qwen4_exp_ple_short_conv",
op_func=qwen4_exp_ple_short_conv,
mutates_args=["residual_output"],
)
__all__ = [
"Qwen4ExpNGramEmbedding",
"Qwen4ExpPLEGroupedNorm",
+14 -60
View File
@@ -9,6 +9,7 @@ from typing import ClassVar, cast
import torch
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.distributed import get_tensor_model_parallel_world_size
@@ -27,10 +28,6 @@ from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
kv_cache_dtype_str_to_dtype,
)
from vllm.v1.attention.backend import (
@@ -347,9 +344,10 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
)
@eager_break_during_capture
def _run_qsa(
self,
hidden_states: torch.Tensor,
projected_qk: torch.Tensor,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
@@ -374,7 +372,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
if side_metadata.num_actual_tokens != num_tokens:
raise RuntimeError("QSA main and side metadata token counts disagree")
selected = self.indexer(
hidden_states,
projected_qk,
positions,
self.topk_indices_buffer[:num_tokens],
)
@@ -412,27 +410,16 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
attn_output = torch.empty_like(query)
encoded_layer_name = _encode_layer_name(self.layer_name)
if current_platform.opaque_attention_op():
torch.ops.vllm.qwen4_exp_qsa_with_output(
hidden_states,
positions,
query,
key,
value,
attn_output,
encoded_layer_name,
)
else:
qwen4_exp_qsa_with_output(
hidden_states,
positions,
query,
key,
value,
attn_output,
encoded_layer_name,
)
# Keep the index projection outside the eager break.
projected_qk, _ = self.indexer.index_qk_proj(hidden_states)
self._run_qsa(
projected_qk,
positions,
query,
key,
value,
attn_output,
)
flat_output = attn_output.view(num_tokens, -1)
if gate is not None:
flat_output = flat_output * torch.sigmoid(gate)
@@ -440,42 +427,9 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
return output
def qwen4_exp_qsa_with_output(
hidden_states: torch.Tensor,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
output: torch.Tensor,
layer_name: LayerNameType,
) -> None:
"""Run the complete QSA state/update/attend transaction."""
layer_name = _resolve_layer_name(layer_name)
layer = get_forward_context().no_compile_layers[layer_name]
if not isinstance(layer, Qwen4ExpQSAAttention):
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
layer._run_qsa(
hidden_states,
positions,
query,
key,
value,
output,
)
direct_register_custom_op(
op_name="qwen4_exp_qsa_with_output",
op_func=qwen4_exp_qsa_with_output,
mutates_args=["output"],
)
__all__ = [
"QSAIndexer",
"Qwen4ExpQSAAttention",
"Qwen4ExpQSAFlashAttentionBackend",
"Qwen4ExpQSAFlashAttentionImpl",
"qwen4_exp_qsa_with_output",
]