From d9105ea8001e0a6d77a96327d17515bb5791fb36 Mon Sep 17 00:00:00 2001 From: Thien Tran Date: Mon, 7 Sep 2026 12:57:56 +0800 Subject: [PATCH] [Qwen3.8-Flash-Next] Remove torch.compile for NVIDIA implementation (#55272) Signed-off-by: Thien Tran --- tests/models/qwen4_exp/test_config.py | 51 +++++++++---- tests/models/qwen4_exp/test_ple.py | 12 +++ tests/models/qwen4_exp/test_qsa_reference.py | 68 ++++++++++++++++- tests/test_config.py | 8 +- vllm/config/compilation.py | 2 +- vllm/config/vllm.py | 3 + .../layers/mamba/gdn/qwen_gdn_linear_attn.py | 3 + vllm/models/qwen4_exp/nvidia/indexer_qsa.py | 14 ++-- vllm/models/qwen4_exp/nvidia/model.py | 12 --- vllm/models/qwen4_exp/nvidia/model_state.py | 18 +++-- vllm/models/qwen4_exp/nvidia/mtp.py | 19 ----- vllm/models/qwen4_exp/nvidia/ple_layer.py | 66 ++--------------- vllm/models/qwen4_exp/nvidia/qsa.py | 74 ++++--------------- 13 files changed, 165 insertions(+), 185 deletions(-) diff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.py index b883aa24b8..45af884a86 100644 --- a/tests/models/qwen4_exp/test_config.py +++ b/tests/models/qwen4_exp/test_config.py @@ -147,9 +147,9 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None: model_state.uses_ngram_embedding = True model_state.ngram_context_len = 3 model_state.ngram_eos_token_id = 99 - model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32) + model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32) model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64) - model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32) + model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32) input_batch = SimpleNamespace( num_reqs=2, @@ -167,34 +167,57 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None: with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}): model_inputs = model_state.prepare_inputs(input_batch, req_states) + expected_query_start_loc = torch.full((9,), 3, dtype=torch.int32) + expected_query_start_loc[0] = 0 + expected_query_start_loc[1] = 2 torch.testing.assert_close( - model_inputs["query_start_loc"], - torch.tensor([0, 2, 3, 3], dtype=torch.int32), + model_inputs["query_start_loc"], expected_query_start_loc ) + expected_context = torch.full((8, 3), 99, dtype=torch.int32) + expected_context[:2] = torch.tensor([[99, 99, 20], [1, 2, 3]]) + torch.testing.assert_close(model_inputs["ngram_context"], expected_context) + + # Retain the views to detect reallocations as the request layout changes. + query_start_loc = model_inputs["query_start_loc"] + ngram_context = model_inputs["ngram_context"] + input_batch.num_reqs = 1 + input_batch.num_reqs_after_padding = 1 + input_batch.idx_mapping = torch.tensor([0]) + input_batch.query_start_loc = torch.tensor([0, 3], dtype=torch.int32) + with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}): + model_inputs = model_state.prepare_inputs(input_batch, req_states) + + expected_query_start_loc.fill_(3) + expected_query_start_loc[0] = 0 torch.testing.assert_close( - model_inputs["ngram_context"], - torch.tensor([[99, 99, 20], [1, 2, 3], [99, 99, 99]], dtype=torch.int32), + model_inputs["query_start_loc"], expected_query_start_loc ) + expected_context.fill_(99) + expected_context[0] = torch.tensor([1, 2, 3]) + torch.testing.assert_close(model_inputs["ngram_context"], expected_context) + assert model_inputs["query_start_loc"].data_ptr() == query_start_loc.data_ptr() + assert model_inputs["ngram_context"].data_ptr() == ngram_context.data_ptr() def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None: model_state = object.__new__(Qwen4ExpModelState) model_state.uses_ngram_embedding = True model_state.ngram_eos_token_id = 99 - model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32) - model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32) + model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32) + model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32) with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}): - first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8) + first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4) + # Dummy runs establish the addresses used during CUDA graph capture. query_start_loc_ptr = first["query_start_loc"].data_ptr() ngram_context_ptr = first["ngram_context"].data_ptr() - second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8) + second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4) + expected_query_start_loc = torch.full((9,), 4, dtype=torch.int32) + expected_query_start_loc[:4] = torch.tensor([0, 1, 2, 4], dtype=torch.int32) + torch.testing.assert_close(second["query_start_loc"], expected_query_start_loc) torch.testing.assert_close( - second["query_start_loc"], torch.tensor([0, 2, 5, 8], dtype=torch.int32) - ) - torch.testing.assert_close( - second["ngram_context"], torch.full((3, 3), 99, dtype=torch.int32) + second["ngram_context"], torch.full((8, 3), 99, dtype=torch.int32) ) assert second["query_start_loc"].data_ptr() == query_start_loc_ptr assert second["ngram_context"].data_ptr() == ngram_context_ptr diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py index 767f38898b..9d188af81f 100644 --- a/tests/models/qwen4_exp/test_ple.py +++ b/tests/models/qwen4_exp/test_ple.py @@ -498,6 +498,17 @@ def _ngram_hash_params(device: torch.device, context_len: int) -> dict: ([3], [1], [[11]], 20), ([4, 4], [0, 7], [[11, 12], [13, 14]], 20), ([4, 0, 3], [], [[11, 12], [13, 14], [15, 16]], 20), + ( + [3, 2, 0, 0], + [], + [ + [11, 12], + [13, 14], + [_NGRAM_EOS_TOKEN_ID, _NGRAM_EOS_TOKEN_ID], + [_NGRAM_EOS_TOKEN_ID, _NGRAM_EOS_TOKEN_ID], + ], + 20, + ), ( [1, 33, 2], [5, 32], @@ -528,6 +539,7 @@ def _ngram_hash_params(device: torch.device, context_len: int) -> dict: "bigram-only", "power-of-two", "empty-request", + "trailing-padded-requests", "three-requests", "six-requests", "four-gram", diff --git a/tests/models/qwen4_exp/test_qsa_reference.py b/tests/models/qwen4_exp/test_qsa_reference.py index 75ade4b95f..eb026e2205 100644 --- a/tests/models/qwen4_exp/test_qsa_reference.py +++ b/tests/models/qwen4_exp/test_qsa_reference.py @@ -45,7 +45,6 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection( indexer = SimpleNamespace( skip_topk=True, _metadata=lambda: (raw_metadata, compressed_metadata), - index_qk_proj=lambda hidden: (torch.zeros(2, 2), None), index_n_heads=1, index_kv_heads=1, index_head_dim=1, @@ -80,7 +79,7 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection( actual = indexer_qsa.QSAIndexer.forward( indexer, - torch.zeros(2, 4), + torch.zeros(2, 2), torch.tensor([7, 8]), rows, ) @@ -446,6 +445,71 @@ def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None: assert metadata.k_work_metadata.tolist() == [[0, 0], [2, 0], [2, 1], [-1, -1]] +@requires_qsa_kernels +@pytest.mark.usefixtures("default_vllm_config") +def test_qsa_unfused_cache_update_ignores_padded_qk() -> None: + """Padded projected Q/K rows must not affect either side cache.""" + from vllm.model_executor.layers.rotary_embedding import get_rope + + device = torch.device("cuda") + # Five tokens complete one compressed group and retain four keys in the ring. + raw_metadata = SimpleNamespace( + num_actual_tokens=5, + slot_mapping=torch.tensor([-1, 1, 2, 3, 0], device=device), + block_table=torch.zeros((1, 1), dtype=torch.int32, device=device), + token_to_req=torch.zeros(5, dtype=torch.int32, device=device), + query_start_loc=torch.tensor([0, 5], dtype=torch.int32, device=device), + logical_positions=torch.arange(5, device=device), + ) + compressed_metadata = SimpleNamespace( + slot_mapping=torch.tensor([-1, -1, -1, 0, -1], device=device), + ) + with torch.device(device): + rope = get_rope( + head_size=128, + max_position=32, + rope_parameters={"rope_type": "default", "partial_rotary_factor": 0.5}, + dtype=torch.bfloat16, + ) + raw_cache = torch.zeros((1, 4, 1, 64), dtype=torch.bfloat16, device=device) + compressed_cache = torch.zeros((1, 2, 1, 64), dtype=torch.bfloat16, device=device) + norm = SimpleNamespace( + weight=torch.zeros(64, dtype=torch.bfloat16, device=device), + variance_epsilon=1e-6, + ) + indexer = SimpleNamespace( + _metadata=lambda: (raw_metadata, compressed_metadata), + skip_topk=True, + index_kv_heads=1, + use_fused_pre_indexer=False, + index_n_heads=1, + index_head_dim=64, + q_layernorm=norm, + k_layernorm=norm, + rotary_emb=rope, + compress_ratio=4, + raw_key_cache=SimpleNamespace( + kv_cache=raw_cache, key_cache=raw_cache, rope_position_cache=None + ), + compressed_key_cache=SimpleNamespace(kv_cache=compressed_cache), + ) + keys = torch.arange(1, 6, dtype=torch.bfloat16, device=device)[:, None].expand( + 5, 64 + ) + padded_keys = torch.full((8, 64), torch.nan, dtype=torch.bfloat16, device=device) + padded_keys[:5].copy_(keys) + indexer_qsa.QSAIndexer.forward( + indexer, + torch.cat((torch.ones_like(padded_keys), padded_keys), dim=-1), + torch.zeros(8, dtype=torch.long, device=device), + torch.full((5, 5), -1, dtype=torch.int32, device=device), + ) + torch.testing.assert_close(raw_cache[0, :, 0], keys[[4, 1, 2, 3]]) + expected_compressed = torch.zeros_like(compressed_cache) + expected_compressed[0, 0] = 1 + torch.testing.assert_close(compressed_cache, expected_compressed) + + @requires_qsa_kernels @pytest.mark.parametrize("compress_ratio", [1, 4]) @pytest.mark.parametrize("num_reqs", [2, 3, 4, 7, 8, 9]) diff --git a/tests/test_config.py b/tests/test_config.py index a888f97a1f..28fd185060 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -338,9 +338,15 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph( ("DeepseekV32MTPModel", True, False), ("GlmMoeDsaForCausalLM", False, True), ("GlmMoeDsaForCausalLM", True, False), + ("Qwen4ExpForCausalLM", False, True), + ("Qwen4ExpForCausalLM", True, False), + ("Qwen4ExpForConditionalGeneration", False, True), + ("Qwen4ExpForConditionalGeneration", True, False), + ("Qwen4ExpMTP", False, True), + ("Qwen4ExpMTP", True, False), ], ) -def test_dsa_breakable_cudagraph_platform_default( +def test_breakable_cudagraph_platform_default( monkeypatch, architecture, is_rocm, expected ): from vllm.config.vllm import default_breakable_cudagraph_architectures diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 9989c4d2a0..1219607d3a 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -768,7 +768,7 @@ class CompilationConfig: "vllm::mamba_mixer2", "vllm::mamba_mixer", "vllm::short_conv", - "vllm::qwen4_exp_compute_ple_ngram_ids", + # Qwen4Exp's AMD backend still uses these splitting ops. "vllm::qwen4_exp_ple_short_conv", "vllm::qwen4_exp_qsa_with_output", "vllm::linear_attention", diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index a59e5bfc38..2af8400c26 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -94,6 +94,9 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset( "KimiLinearForCausalLM", "MiniMaxM3SparseForCausalLM", "MiniMaxM3SparseForConditionalGeneration", + "Qwen4ExpForCausalLM", + "Qwen4ExpForConditionalGeneration", + "Qwen4ExpMTP", } ) diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 718f8de05c..b7bd5b7b0b 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -12,6 +12,7 @@ from torch import nn from vllm import _custom_ops as ops from vllm import envs from vllm._aiter_ops import rocm_aiter_ops +from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import ( VllmConfig, get_current_vllm_config, @@ -1909,6 +1910,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): ) +@eager_break_during_capture def qwen_gdn_attention_core( qkv_or_qkvz: torch.Tensor, b_or_ba: torch.Tensor, @@ -1956,6 +1958,7 @@ direct_register_custom_op( ) +@eager_break_during_capture def qwen_gdn_attention_core_fused_norm_packed( mixed_qkvz: torch.Tensor, ba: torch.Tensor, diff --git a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py index 982e15a34a..b14174e295 100644 --- a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py +++ b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py @@ -88,7 +88,7 @@ def _supports_fused_pre_indexer( class QSAIndexer(nn.Module): - """Replicated Q/K projection plus paged, weight-free QSA selection. + """QSA projection weights, side caches, and paged, weight-free selection. ``prefix`` must be the checkpoint's indexer prefix, normally ``model.layers.N.self_attn.indexer``. Consequently the trainable names are @@ -218,11 +218,11 @@ class QSAIndexer(nn.Module): def forward( self, - hidden_states: torch.Tensor, + projected_qk: torch.Tensor, positions: torch.Tensor, out: torch.Tensor | None = None, ) -> torch.Tensor: - """Select each query row's token indices. + """Update side caches and select token indices from pre-projected Q/K. Returns the packed buffer of shape [num_tokens, output_width + 1]: the leading ``output_width`` columns are ``-1``-padded @@ -237,10 +237,10 @@ class QSAIndexer(nn.Module): if self.skip_topk and out is not None: return out result = torch.full( - (hidden_states.shape[0], self.packed_output_width), + (projected_qk.shape[0], self.packed_output_width), -1, dtype=torch.int32, - device=hidden_states.device, + device=projected_qk.device, ) # Inert rows carry a zero valid count (empty loop bound), not -1. result[:, -1] = 0 @@ -258,11 +258,9 @@ class QSAIndexer(nn.Module): raw_metadata, compressed_metadata = metadata num_tokens = raw_metadata.num_actual_tokens - hidden_states = hidden_states[:num_tokens] + projected_qk = projected_qk[:num_tokens] positions = positions[..., :num_tokens] - # Q/K projection - projected_qk, _ = self.index_qk_proj(hidden_states) projected_q, raw_keys = projected_qk.split( ( self.index_n_heads * self.index_head_dim, diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index 4f03011820..da74356a62 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -8,7 +8,6 @@ from itertools import islice import torch from torch import nn -from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed import get_pp_group from vllm.model_executor.layers.fused_moe.utils import ( @@ -381,17 +380,6 @@ class Qwen4ExpMixtureOfExperts(MixtureOfExperts): moe.experts.update_expert_map() -@support_torch_compile( - dynamic_arg_dims={ - "input_ids": 0, - "positions": -1, - "intermediate_tensors": 0, - "inputs_embeds": 0, - "query_start_loc": 0, - "ngram_context": 0, - "deepstack_input_embeds": 0, - } -) class Qwen4ExpModel(nn.Module): hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER diff --git a/vllm/models/qwen4_exp/nvidia/model_state.py b/vllm/models/qwen4_exp/nvidia/model_state.py index ec2a0fef8b..246168509b 100644 --- a/vllm/models/qwen4_exp/nvidia/model_state.py +++ b/vllm/models/qwen4_exp/nvidia/model_state.py @@ -44,6 +44,8 @@ class Qwen4ExpModelState(MambaHybridModelState): if self.ngram_context_len <= 0: raise ValueError("N-gram embedding requires context length >= 1.") self.ngram_eos_token_id = int(config.eos_token_id) + # PLE runs inside captured regions, so these buffers keep a fixed shape + # and address as the active request count changes between replays. self.ngram_context = torch.full( (self.max_num_reqs, self.ngram_context_len), self.ngram_eos_token_id, @@ -68,8 +70,7 @@ class Qwen4ExpModelState(MambaHybridModelState): req_states: RequestState, ) -> torch.Tensor: num_reqs = input_batch.num_reqs - num_reqs_padded = input_batch.num_reqs_after_padding - context = self.ngram_context[:num_reqs_padded] + context = self.ngram_context context.fill_(self.ngram_eos_token_id) if num_reqs == 0: return context @@ -101,8 +102,10 @@ class Qwen4ExpModelState(MambaHybridModelState): return model_inputs num_reqs_padded = input_batch.num_reqs_after_padding - query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1] - query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1]) + query_start_loc = self.ple_query_start_loc + query_start_loc[: num_reqs_padded + 1].copy_(input_batch.query_start_loc) + # Represent unused capacity as trailing zero-length requests. + query_start_loc[num_reqs_padded + 1 :].copy_(input_batch.query_start_loc[-1]) model_inputs.update( query_start_loc=query_start_loc, ngram_context=self._prepare_ngram_context(input_batch, req_states), @@ -118,7 +121,7 @@ class Qwen4ExpModelState(MambaHybridModelState): if not self.uses_ngram_embedding: return model_inputs - query_start_loc = self.ple_query_start_loc[: num_reqs + 1] + query_start_loc = self.ple_query_start_loc query_start_loc[0] = 0 tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs) query_lens = torch.full( @@ -129,9 +132,10 @@ class Qwen4ExpModelState(MambaHybridModelState): ) if num_extra_tokens > 0: query_lens[-num_extra_tokens:] += 1 - torch.cumsum(query_lens, dim=0, out=query_start_loc[1:]) + torch.cumsum(query_lens, dim=0, out=query_start_loc[1 : num_reqs + 1]) + query_start_loc[num_reqs + 1 :].fill_(num_tokens) - ngram_context = self.ngram_context[:num_reqs] + ngram_context = self.ngram_context ngram_context.fill_(self.ngram_eos_token_id) model_inputs.update( query_start_loc=query_start_loc, diff --git a/vllm/models/qwen4_exp/nvidia/mtp.py b/vllm/models/qwen4_exp/nvidia/mtp.py index 932e16338f..1650f84a30 100644 --- a/vllm/models/qwen4_exp/nvidia/mtp.py +++ b/vllm/models/qwen4_exp/nvidia/mtp.py @@ -19,7 +19,6 @@ import regex as re import torch from torch import nn -from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig, replace, set_current_vllm_config from vllm.distributed import get_pp_group from vllm.model_executor.layers.fused_moe.utils import ( @@ -147,15 +146,6 @@ def _make_draft_vllm_config( return draft_vllm_config -@support_torch_compile( - dynamic_arg_dims={ - "input_ids": 0, - "positions": -1, - "intermediate_tensors": 0, - "inputs_embeds": 0, - "hidden_states": 0, - } -) class Qwen4ExpMultiTokenPredictor(nn.Module): hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER @@ -355,15 +345,6 @@ class Qwen4ExpMultiTokenPredictor(nn.Module): return loader.load_weights(weights, mapper=mapper) -@support_torch_compile( - dynamic_arg_dims={ - "input_ids": 0, - "positions": -1, - "intermediate_tensors": 0, - "inputs_embeds": 0, - "hidden_states": 0, - } -) class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts): packed_modules_mapping = { "qkv_proj": ["q_proj", "k_proj", "v_proj"], diff --git a/vllm/models/qwen4_exp/nvidia/ple_layer.py b/vllm/models/qwen4_exp/nvidia/ple_layer.py index 7c2ab8a8d5..7650e2e3a9 100644 --- a/vllm/models/qwen4_exp/nvidia/ple_layer.py +++ b/vllm/models/qwen4_exp/nvidia/ple_layer.py @@ -8,6 +8,7 @@ import torch import torch.nn.functional as F from torch import nn +from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config from vllm.forward_context import get_forward_context from vllm.model_executor.layers.linear import MergedColumnParallelLinear @@ -38,7 +39,6 @@ from vllm.model_executor.parameter import PerTensorScaleParameter from vllm.transformers_utils.configs.qwen4_exp import ( Qwen4ExpTextConfig, ) -from vllm.utils.torch_utils import direct_register_custom_op from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.attention.backends.short_conv_attn import ( PleShortConvAttentionBackend, @@ -264,12 +264,10 @@ class Qwen4ExpNGramEmbedding(nn.Module): embedding_dim: int, ple_dense_layer_id: int, prefix: str, - layer_name: str, quant_config: QuantizationConfig | None = None, params_dtype: torch.dtype | None = None, ) -> None: super().__init__() - self.layer_name = layer_name self.embedding_dim = embedding_dim self.ngram_size = int(config.ngram_size) self.heads_per_ngram = int(config.heads_per_ngram) @@ -445,20 +443,7 @@ class Qwen4ExpNGramEmbedding(nn.Module): query_start_loc: torch.Tensor, ngram_context: torch.Tensor, ) -> torch.Tensor: - # Keep num_reqs-dependent ID generation outside PIECEWISE CUDA graphs, - # which dispatch only on the padded token count. - # torch.compile requires the splitting op to write graph-owned storage. - # Once compilation is removed, the op can return the IDs directly. - ngram_ids = input_ids.new_empty( - (input_ids.numel(), self.ngram_heads), dtype=torch.long - ) - torch.ops.vllm.qwen4_exp_compute_ple_ngram_ids( - input_ids, - query_start_loc, - ngram_context, - ngram_ids, - self.layer_name, - ) + ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context) return self.ngram_embedding(ngram_ids).flatten(-2) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: @@ -563,7 +548,6 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): int(config.ple_embed_dim), self.ple_dense_layer_id, prefix=f"{prefix}.ple_embedding", - layer_name=prefix, quant_config=quant_config, params_dtype=model_config.dtype, ) @@ -779,6 +763,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): token_indices=non_spec_token_indices, ) + # State routing consumes the current request metadata on every replay. + @eager_break_during_capture def _short_conv(self, inputs: torch.Tensor, residual: torch.Tensor) -> None: forward_context = get_forward_context() attn_metadata = forward_context.attn_metadata @@ -854,52 +840,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): self.norm_conv.weight, self.norm_key.eps, ) - # State routing depends on runtime request metadata and remains outside - # the piecewise graph; short convolution accumulates into gated_output. - torch.ops.vllm.qwen4_exp_ple_short_conv(conv_input, gated_output, self.prefix) + self._short_conv(conv_input, gated_output) return gated_output -def qwen4_exp_compute_ple_ngram_ids( - input_ids: torch.Tensor, - query_start_loc: torch.Tensor, - ngram_context: torch.Tensor, - output: torch.Tensor, - layer_name: str, -) -> None: - """Compute request-dependent PLE n-gram IDs outside piecewise graphs.""" - layer = get_forward_context().no_compile_layers[layer_name] - layer.ple_embedding.compute_ngram_ids( - input_ids, - query_start_loc, - ngram_context, - output, - ) - - -def qwen4_exp_ple_short_conv( - inputs: torch.Tensor, - residual_output: torch.Tensor, - layer_name: str, -) -> None: - layer = get_forward_context().no_compile_layers[layer_name] - layer._short_conv(inputs, residual_output) - - -direct_register_custom_op( - op_name="qwen4_exp_compute_ple_ngram_ids", - op_func=qwen4_exp_compute_ple_ngram_ids, - mutates_args=["output"], -) - - -direct_register_custom_op( - op_name="qwen4_exp_ple_short_conv", - op_func=qwen4_exp_ple_short_conv, - mutates_args=["residual_output"], -) - - __all__ = [ "Qwen4ExpNGramEmbedding", "Qwen4ExpPLEGroupedNorm", diff --git a/vllm/models/qwen4_exp/nvidia/qsa.py b/vllm/models/qwen4_exp/nvidia/qsa.py index 53ad18362e..90449d5706 100644 --- a/vllm/models/qwen4_exp/nvidia/qsa.py +++ b/vllm/models/qwen4_exp/nvidia/qsa.py @@ -9,6 +9,7 @@ from typing import ClassVar, cast import torch from torch import nn +from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import VllmConfig from vllm.config.cache import CacheDType from vllm.distributed import get_tensor_model_parallel_world_size @@ -27,10 +28,6 @@ from vllm.transformers_utils.configs.qwen4_exp import ( Qwen4ExpTextConfig, ) from vllm.utils.torch_utils import ( - LayerNameType, - _encode_layer_name, - _resolve_layer_name, - direct_register_custom_op, kv_cache_dtype_str_to_dtype, ) from vllm.v1.attention.backend import ( @@ -347,9 +344,10 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), ) + @eager_break_during_capture def _run_qsa( self, - hidden_states: torch.Tensor, + projected_qk: torch.Tensor, positions: torch.Tensor, query: torch.Tensor, key: torch.Tensor, @@ -374,7 +372,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): if side_metadata.num_actual_tokens != num_tokens: raise RuntimeError("QSA main and side metadata token counts disagree") selected = self.indexer( - hidden_states, + projected_qk, positions, self.topk_indices_buffer[:num_tokens], ) @@ -412,27 +410,16 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): key = k.view(num_tokens, self.num_kv_heads, self.head_dim) value = v.view(num_tokens, self.num_kv_heads, self.head_dim) attn_output = torch.empty_like(query) - encoded_layer_name = _encode_layer_name(self.layer_name) - if current_platform.opaque_attention_op(): - torch.ops.vllm.qwen4_exp_qsa_with_output( - hidden_states, - positions, - query, - key, - value, - attn_output, - encoded_layer_name, - ) - else: - qwen4_exp_qsa_with_output( - hidden_states, - positions, - query, - key, - value, - attn_output, - encoded_layer_name, - ) + # Keep the index projection outside the eager break. + projected_qk, _ = self.indexer.index_qk_proj(hidden_states) + self._run_qsa( + projected_qk, + positions, + query, + key, + value, + attn_output, + ) flat_output = attn_output.view(num_tokens, -1) if gate is not None: flat_output = flat_output * torch.sigmoid(gate) @@ -440,42 +427,9 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): return output -def qwen4_exp_qsa_with_output( - hidden_states: torch.Tensor, - positions: torch.Tensor, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - output: torch.Tensor, - layer_name: LayerNameType, -) -> None: - """Run the complete QSA state/update/attend transaction.""" - - layer_name = _resolve_layer_name(layer_name) - layer = get_forward_context().no_compile_layers[layer_name] - if not isinstance(layer, Qwen4ExpQSAAttention): - raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner") - layer._run_qsa( - hidden_states, - positions, - query, - key, - value, - output, - ) - - -direct_register_custom_op( - op_name="qwen4_exp_qsa_with_output", - op_func=qwen4_exp_qsa_with_output, - mutates_args=["output"], -) - - __all__ = [ "QSAIndexer", "Qwen4ExpQSAAttention", "Qwen4ExpQSAFlashAttentionBackend", "Qwen4ExpQSAFlashAttentionImpl", - "qwen4_exp_qsa_with_output", ]