[Qwen3.8-Flash-Next] Remove torch.compile for NVIDIA implementation (#55272)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
@@ -147,9 +147,9 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
|
||||
model_state.uses_ngram_embedding = True
|
||||
model_state.ngram_context_len = 3
|
||||
model_state.ngram_eos_token_id = 99
|
||||
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
|
||||
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
|
||||
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
|
||||
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
||||
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
|
||||
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=2,
|
||||
@@ -167,34 +167,57 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
|
||||
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
|
||||
model_inputs = model_state.prepare_inputs(input_batch, req_states)
|
||||
|
||||
expected_query_start_loc = torch.full((9,), 3, dtype=torch.int32)
|
||||
expected_query_start_loc[0] = 0
|
||||
expected_query_start_loc[1] = 2
|
||||
torch.testing.assert_close(
|
||||
model_inputs["query_start_loc"],
|
||||
torch.tensor([0, 2, 3, 3], dtype=torch.int32),
|
||||
model_inputs["query_start_loc"], expected_query_start_loc
|
||||
)
|
||||
expected_context = torch.full((8, 3), 99, dtype=torch.int32)
|
||||
expected_context[:2] = torch.tensor([[99, 99, 20], [1, 2, 3]])
|
||||
torch.testing.assert_close(model_inputs["ngram_context"], expected_context)
|
||||
|
||||
# Retain the views to detect reallocations as the request layout changes.
|
||||
query_start_loc = model_inputs["query_start_loc"]
|
||||
ngram_context = model_inputs["ngram_context"]
|
||||
input_batch.num_reqs = 1
|
||||
input_batch.num_reqs_after_padding = 1
|
||||
input_batch.idx_mapping = torch.tensor([0])
|
||||
input_batch.query_start_loc = torch.tensor([0, 3], dtype=torch.int32)
|
||||
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
|
||||
model_inputs = model_state.prepare_inputs(input_batch, req_states)
|
||||
|
||||
expected_query_start_loc.fill_(3)
|
||||
expected_query_start_loc[0] = 0
|
||||
torch.testing.assert_close(
|
||||
model_inputs["ngram_context"],
|
||||
torch.tensor([[99, 99, 20], [1, 2, 3], [99, 99, 99]], dtype=torch.int32),
|
||||
model_inputs["query_start_loc"], expected_query_start_loc
|
||||
)
|
||||
expected_context.fill_(99)
|
||||
expected_context[0] = torch.tensor([1, 2, 3])
|
||||
torch.testing.assert_close(model_inputs["ngram_context"], expected_context)
|
||||
assert model_inputs["query_start_loc"].data_ptr() == query_start_loc.data_ptr()
|
||||
assert model_inputs["ngram_context"].data_ptr() == ngram_context.data_ptr()
|
||||
|
||||
|
||||
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
|
||||
model_state = object.__new__(Qwen4ExpModelState)
|
||||
model_state.uses_ngram_embedding = True
|
||||
model_state.ngram_eos_token_id = 99
|
||||
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
|
||||
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
||||
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
|
||||
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
|
||||
|
||||
with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}):
|
||||
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
|
||||
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4)
|
||||
# Dummy runs establish the addresses used during CUDA graph capture.
|
||||
query_start_loc_ptr = first["query_start_loc"].data_ptr()
|
||||
ngram_context_ptr = first["ngram_context"].data_ptr()
|
||||
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
|
||||
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4)
|
||||
|
||||
expected_query_start_loc = torch.full((9,), 4, dtype=torch.int32)
|
||||
expected_query_start_loc[:4] = torch.tensor([0, 1, 2, 4], dtype=torch.int32)
|
||||
torch.testing.assert_close(second["query_start_loc"], expected_query_start_loc)
|
||||
torch.testing.assert_close(
|
||||
second["query_start_loc"], torch.tensor([0, 2, 5, 8], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
second["ngram_context"], torch.full((3, 3), 99, dtype=torch.int32)
|
||||
second["ngram_context"], torch.full((8, 3), 99, dtype=torch.int32)
|
||||
)
|
||||
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
|
||||
assert second["ngram_context"].data_ptr() == ngram_context_ptr
|
||||
|
||||
@@ -498,6 +498,17 @@ def _ngram_hash_params(device: torch.device, context_len: int) -> dict:
|
||||
([3], [1], [[11]], 20),
|
||||
([4, 4], [0, 7], [[11, 12], [13, 14]], 20),
|
||||
([4, 0, 3], [], [[11, 12], [13, 14], [15, 16]], 20),
|
||||
(
|
||||
[3, 2, 0, 0],
|
||||
[],
|
||||
[
|
||||
[11, 12],
|
||||
[13, 14],
|
||||
[_NGRAM_EOS_TOKEN_ID, _NGRAM_EOS_TOKEN_ID],
|
||||
[_NGRAM_EOS_TOKEN_ID, _NGRAM_EOS_TOKEN_ID],
|
||||
],
|
||||
20,
|
||||
),
|
||||
(
|
||||
[1, 33, 2],
|
||||
[5, 32],
|
||||
@@ -528,6 +539,7 @@ def _ngram_hash_params(device: torch.device, context_len: int) -> dict:
|
||||
"bigram-only",
|
||||
"power-of-two",
|
||||
"empty-request",
|
||||
"trailing-padded-requests",
|
||||
"three-requests",
|
||||
"six-requests",
|
||||
"four-gram",
|
||||
|
||||
@@ -45,7 +45,6 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
|
||||
indexer = SimpleNamespace(
|
||||
skip_topk=True,
|
||||
_metadata=lambda: (raw_metadata, compressed_metadata),
|
||||
index_qk_proj=lambda hidden: (torch.zeros(2, 2), None),
|
||||
index_n_heads=1,
|
||||
index_kv_heads=1,
|
||||
index_head_dim=1,
|
||||
@@ -80,7 +79,7 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
|
||||
|
||||
actual = indexer_qsa.QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.zeros(2, 4),
|
||||
torch.zeros(2, 2),
|
||||
torch.tensor([7, 8]),
|
||||
rows,
|
||||
)
|
||||
@@ -446,6 +445,71 @@ def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
|
||||
assert metadata.k_work_metadata.tolist() == [[0, 0], [2, 0], [2, 1], [-1, -1]]
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
|
||||
"""Padded projected Q/K rows must not affect either side cache."""
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
|
||||
device = torch.device("cuda")
|
||||
# Five tokens complete one compressed group and retain four keys in the ring.
|
||||
raw_metadata = SimpleNamespace(
|
||||
num_actual_tokens=5,
|
||||
slot_mapping=torch.tensor([-1, 1, 2, 3, 0], device=device),
|
||||
block_table=torch.zeros((1, 1), dtype=torch.int32, device=device),
|
||||
token_to_req=torch.zeros(5, dtype=torch.int32, device=device),
|
||||
query_start_loc=torch.tensor([0, 5], dtype=torch.int32, device=device),
|
||||
logical_positions=torch.arange(5, device=device),
|
||||
)
|
||||
compressed_metadata = SimpleNamespace(
|
||||
slot_mapping=torch.tensor([-1, -1, -1, 0, -1], device=device),
|
||||
)
|
||||
with torch.device(device):
|
||||
rope = get_rope(
|
||||
head_size=128,
|
||||
max_position=32,
|
||||
rope_parameters={"rope_type": "default", "partial_rotary_factor": 0.5},
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
raw_cache = torch.zeros((1, 4, 1, 64), dtype=torch.bfloat16, device=device)
|
||||
compressed_cache = torch.zeros((1, 2, 1, 64), dtype=torch.bfloat16, device=device)
|
||||
norm = SimpleNamespace(
|
||||
weight=torch.zeros(64, dtype=torch.bfloat16, device=device),
|
||||
variance_epsilon=1e-6,
|
||||
)
|
||||
indexer = SimpleNamespace(
|
||||
_metadata=lambda: (raw_metadata, compressed_metadata),
|
||||
skip_topk=True,
|
||||
index_kv_heads=1,
|
||||
use_fused_pre_indexer=False,
|
||||
index_n_heads=1,
|
||||
index_head_dim=64,
|
||||
q_layernorm=norm,
|
||||
k_layernorm=norm,
|
||||
rotary_emb=rope,
|
||||
compress_ratio=4,
|
||||
raw_key_cache=SimpleNamespace(
|
||||
kv_cache=raw_cache, key_cache=raw_cache, rope_position_cache=None
|
||||
),
|
||||
compressed_key_cache=SimpleNamespace(kv_cache=compressed_cache),
|
||||
)
|
||||
keys = torch.arange(1, 6, dtype=torch.bfloat16, device=device)[:, None].expand(
|
||||
5, 64
|
||||
)
|
||||
padded_keys = torch.full((8, 64), torch.nan, dtype=torch.bfloat16, device=device)
|
||||
padded_keys[:5].copy_(keys)
|
||||
indexer_qsa.QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.cat((torch.ones_like(padded_keys), padded_keys), dim=-1),
|
||||
torch.zeros(8, dtype=torch.long, device=device),
|
||||
torch.full((5, 5), -1, dtype=torch.int32, device=device),
|
||||
)
|
||||
torch.testing.assert_close(raw_cache[0, :, 0], keys[[4, 1, 2, 3]])
|
||||
expected_compressed = torch.zeros_like(compressed_cache)
|
||||
expected_compressed[0, 0] = 1
|
||||
torch.testing.assert_close(compressed_cache, expected_compressed)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 4])
|
||||
@pytest.mark.parametrize("num_reqs", [2, 3, 4, 7, 8, 9])
|
||||
|
||||
@@ -338,9 +338,15 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
|
||||
("DeepseekV32MTPModel", True, False),
|
||||
("GlmMoeDsaForCausalLM", False, True),
|
||||
("GlmMoeDsaForCausalLM", True, False),
|
||||
("Qwen4ExpForCausalLM", False, True),
|
||||
("Qwen4ExpForCausalLM", True, False),
|
||||
("Qwen4ExpForConditionalGeneration", False, True),
|
||||
("Qwen4ExpForConditionalGeneration", True, False),
|
||||
("Qwen4ExpMTP", False, True),
|
||||
("Qwen4ExpMTP", True, False),
|
||||
],
|
||||
)
|
||||
def test_dsa_breakable_cudagraph_platform_default(
|
||||
def test_breakable_cudagraph_platform_default(
|
||||
monkeypatch, architecture, is_rocm, expected
|
||||
):
|
||||
from vllm.config.vllm import default_breakable_cudagraph_architectures
|
||||
|
||||
@@ -768,7 +768,7 @@ class CompilationConfig:
|
||||
"vllm::mamba_mixer2",
|
||||
"vllm::mamba_mixer",
|
||||
"vllm::short_conv",
|
||||
"vllm::qwen4_exp_compute_ple_ngram_ids",
|
||||
# Qwen4Exp's AMD backend still uses these splitting ops.
|
||||
"vllm::qwen4_exp_ple_short_conv",
|
||||
"vllm::qwen4_exp_qsa_with_output",
|
||||
"vllm::linear_attention",
|
||||
|
||||
@@ -94,6 +94,9 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
|
||||
"KimiLinearForCausalLM",
|
||||
"MiniMaxM3SparseForCausalLM",
|
||||
"MiniMaxM3SparseForConditionalGeneration",
|
||||
"Qwen4ExpForCausalLM",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
"Qwen4ExpMTP",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ from torch import nn
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
@@ -1909,6 +1910,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def qwen_gdn_attention_core(
|
||||
qkv_or_qkvz: torch.Tensor,
|
||||
b_or_ba: torch.Tensor,
|
||||
@@ -1956,6 +1958,7 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def qwen_gdn_attention_core_fused_norm_packed(
|
||||
mixed_qkvz: torch.Tensor,
|
||||
ba: torch.Tensor,
|
||||
|
||||
@@ -88,7 +88,7 @@ def _supports_fused_pre_indexer(
|
||||
|
||||
|
||||
class QSAIndexer(nn.Module):
|
||||
"""Replicated Q/K projection plus paged, weight-free QSA selection.
|
||||
"""QSA projection weights, side caches, and paged, weight-free selection.
|
||||
|
||||
``prefix`` must be the checkpoint's indexer prefix, normally
|
||||
``model.layers.N.self_attn.indexer``. Consequently the trainable names are
|
||||
@@ -218,11 +218,11 @@ class QSAIndexer(nn.Module):
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
projected_qk: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Select each query row's token indices.
|
||||
"""Update side caches and select token indices from pre-projected Q/K.
|
||||
|
||||
Returns the packed buffer of shape [num_tokens, output_width + 1]:
|
||||
the leading ``output_width`` columns are ``-1``-padded
|
||||
@@ -237,10 +237,10 @@ class QSAIndexer(nn.Module):
|
||||
if self.skip_topk and out is not None:
|
||||
return out
|
||||
result = torch.full(
|
||||
(hidden_states.shape[0], self.packed_output_width),
|
||||
(projected_qk.shape[0], self.packed_output_width),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=hidden_states.device,
|
||||
device=projected_qk.device,
|
||||
)
|
||||
# Inert rows carry a zero valid count (empty loop bound), not -1.
|
||||
result[:, -1] = 0
|
||||
@@ -258,11 +258,9 @@ class QSAIndexer(nn.Module):
|
||||
|
||||
raw_metadata, compressed_metadata = metadata
|
||||
num_tokens = raw_metadata.num_actual_tokens
|
||||
hidden_states = hidden_states[:num_tokens]
|
||||
projected_qk = projected_qk[:num_tokens]
|
||||
positions = positions[..., :num_tokens]
|
||||
|
||||
# Q/K projection
|
||||
projected_qk, _ = self.index_qk_proj(hidden_states)
|
||||
projected_q, raw_keys = projected_qk.split(
|
||||
(
|
||||
self.index_n_heads * self.index_head_dim,
|
||||
|
||||
@@ -8,7 +8,6 @@ from itertools import islice
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
@@ -381,17 +380,6 @@ class Qwen4ExpMixtureOfExperts(MixtureOfExperts):
|
||||
moe.experts.update_expert_map()
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
"query_start_loc": 0,
|
||||
"ngram_context": 0,
|
||||
"deepstack_input_embeds": 0,
|
||||
}
|
||||
)
|
||||
class Qwen4ExpModel(nn.Module):
|
||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
|
||||
|
||||
|
||||
@@ -44,6 +44,8 @@ class Qwen4ExpModelState(MambaHybridModelState):
|
||||
if self.ngram_context_len <= 0:
|
||||
raise ValueError("N-gram embedding requires context length >= 1.")
|
||||
self.ngram_eos_token_id = int(config.eos_token_id)
|
||||
# PLE runs inside captured regions, so these buffers keep a fixed shape
|
||||
# and address as the active request count changes between replays.
|
||||
self.ngram_context = torch.full(
|
||||
(self.max_num_reqs, self.ngram_context_len),
|
||||
self.ngram_eos_token_id,
|
||||
@@ -68,8 +70,7 @@ class Qwen4ExpModelState(MambaHybridModelState):
|
||||
req_states: RequestState,
|
||||
) -> torch.Tensor:
|
||||
num_reqs = input_batch.num_reqs
|
||||
num_reqs_padded = input_batch.num_reqs_after_padding
|
||||
context = self.ngram_context[:num_reqs_padded]
|
||||
context = self.ngram_context
|
||||
context.fill_(self.ngram_eos_token_id)
|
||||
if num_reqs == 0:
|
||||
return context
|
||||
@@ -101,8 +102,10 @@ class Qwen4ExpModelState(MambaHybridModelState):
|
||||
return model_inputs
|
||||
|
||||
num_reqs_padded = input_batch.num_reqs_after_padding
|
||||
query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1]
|
||||
query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1])
|
||||
query_start_loc = self.ple_query_start_loc
|
||||
query_start_loc[: num_reqs_padded + 1].copy_(input_batch.query_start_loc)
|
||||
# Represent unused capacity as trailing zero-length requests.
|
||||
query_start_loc[num_reqs_padded + 1 :].copy_(input_batch.query_start_loc[-1])
|
||||
model_inputs.update(
|
||||
query_start_loc=query_start_loc,
|
||||
ngram_context=self._prepare_ngram_context(input_batch, req_states),
|
||||
@@ -118,7 +121,7 @@ class Qwen4ExpModelState(MambaHybridModelState):
|
||||
if not self.uses_ngram_embedding:
|
||||
return model_inputs
|
||||
|
||||
query_start_loc = self.ple_query_start_loc[: num_reqs + 1]
|
||||
query_start_loc = self.ple_query_start_loc
|
||||
query_start_loc[0] = 0
|
||||
tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs)
|
||||
query_lens = torch.full(
|
||||
@@ -129,9 +132,10 @@ class Qwen4ExpModelState(MambaHybridModelState):
|
||||
)
|
||||
if num_extra_tokens > 0:
|
||||
query_lens[-num_extra_tokens:] += 1
|
||||
torch.cumsum(query_lens, dim=0, out=query_start_loc[1:])
|
||||
torch.cumsum(query_lens, dim=0, out=query_start_loc[1 : num_reqs + 1])
|
||||
query_start_loc[num_reqs + 1 :].fill_(num_tokens)
|
||||
|
||||
ngram_context = self.ngram_context[:num_reqs]
|
||||
ngram_context = self.ngram_context
|
||||
ngram_context.fill_(self.ngram_eos_token_id)
|
||||
model_inputs.update(
|
||||
query_start_loc=query_start_loc,
|
||||
|
||||
@@ -19,7 +19,6 @@ import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig, replace, set_current_vllm_config
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
@@ -147,15 +146,6 @@ def _make_draft_vllm_config(
|
||||
return draft_vllm_config
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
"hidden_states": 0,
|
||||
}
|
||||
)
|
||||
class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
|
||||
|
||||
@@ -355,15 +345,6 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
"hidden_states": 0,
|
||||
}
|
||||
)
|
||||
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
@@ -38,7 +39,6 @@ from vllm.model_executor.parameter import PerTensorScaleParameter
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionBackend,
|
||||
@@ -264,12 +264,10 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
embedding_dim: int,
|
||||
ple_dense_layer_id: int,
|
||||
prefix: str,
|
||||
layer_name: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.layer_name = layer_name
|
||||
self.embedding_dim = embedding_dim
|
||||
self.ngram_size = int(config.ngram_size)
|
||||
self.heads_per_ngram = int(config.heads_per_ngram)
|
||||
@@ -445,20 +443,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# Keep num_reqs-dependent ID generation outside PIECEWISE CUDA graphs,
|
||||
# which dispatch only on the padded token count.
|
||||
# torch.compile requires the splitting op to write graph-owned storage.
|
||||
# Once compilation is removed, the op can return the IDs directly.
|
||||
ngram_ids = input_ids.new_empty(
|
||||
(input_ids.numel(), self.ngram_heads), dtype=torch.long
|
||||
)
|
||||
torch.ops.vllm.qwen4_exp_compute_ple_ngram_ids(
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
ngram_ids,
|
||||
self.layer_name,
|
||||
)
|
||||
ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context)
|
||||
return self.ngram_embedding(ngram_ids).flatten(-2)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
@@ -563,7 +548,6 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
int(config.ple_embed_dim),
|
||||
self.ple_dense_layer_id,
|
||||
prefix=f"{prefix}.ple_embedding",
|
||||
layer_name=prefix,
|
||||
quant_config=quant_config,
|
||||
params_dtype=model_config.dtype,
|
||||
)
|
||||
@@ -779,6 +763,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
token_indices=non_spec_token_indices,
|
||||
)
|
||||
|
||||
# State routing consumes the current request metadata on every replay.
|
||||
@eager_break_during_capture
|
||||
def _short_conv(self, inputs: torch.Tensor, residual: torch.Tensor) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
@@ -854,52 +840,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
self.norm_conv.weight,
|
||||
self.norm_key.eps,
|
||||
)
|
||||
# State routing depends on runtime request metadata and remains outside
|
||||
# the piecewise graph; short convolution accumulates into gated_output.
|
||||
torch.ops.vllm.qwen4_exp_ple_short_conv(conv_input, gated_output, self.prefix)
|
||||
self._short_conv(conv_input, gated_output)
|
||||
return gated_output
|
||||
|
||||
|
||||
def qwen4_exp_compute_ple_ngram_ids(
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Compute request-dependent PLE n-gram IDs outside piecewise graphs."""
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
layer.ple_embedding.compute_ngram_ids(
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
output,
|
||||
)
|
||||
|
||||
|
||||
def qwen4_exp_ple_short_conv(
|
||||
inputs: torch.Tensor,
|
||||
residual_output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
layer._short_conv(inputs, residual_output)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_compute_ple_ngram_ids",
|
||||
op_func=qwen4_exp_compute_ple_ngram_ids,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_ple_short_conv",
|
||||
op_func=qwen4_exp_ple_short_conv,
|
||||
mutates_args=["residual_output"],
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Qwen4ExpNGramEmbedding",
|
||||
"Qwen4ExpPLEGroupedNorm",
|
||||
|
||||
@@ -9,6 +9,7 @@ from typing import ClassVar, cast
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
@@ -27,10 +28,6 @@ from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
@@ -347,9 +344,10 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
)
|
||||
|
||||
@eager_break_during_capture
|
||||
def _run_qsa(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
projected_qk: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
@@ -374,7 +372,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
if side_metadata.num_actual_tokens != num_tokens:
|
||||
raise RuntimeError("QSA main and side metadata token counts disagree")
|
||||
selected = self.indexer(
|
||||
hidden_states,
|
||||
projected_qk,
|
||||
positions,
|
||||
self.topk_indices_buffer[:num_tokens],
|
||||
)
|
||||
@@ -412,27 +410,16 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||
attn_output = torch.empty_like(query)
|
||||
encoded_layer_name = _encode_layer_name(self.layer_name)
|
||||
if current_platform.opaque_attention_op():
|
||||
torch.ops.vllm.qwen4_exp_qsa_with_output(
|
||||
hidden_states,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_output,
|
||||
encoded_layer_name,
|
||||
)
|
||||
else:
|
||||
qwen4_exp_qsa_with_output(
|
||||
hidden_states,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_output,
|
||||
encoded_layer_name,
|
||||
)
|
||||
# Keep the index projection outside the eager break.
|
||||
projected_qk, _ = self.indexer.index_qk_proj(hidden_states)
|
||||
self._run_qsa(
|
||||
projected_qk,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_output,
|
||||
)
|
||||
flat_output = attn_output.view(num_tokens, -1)
|
||||
if gate is not None:
|
||||
flat_output = flat_output * torch.sigmoid(gate)
|
||||
@@ -440,42 +427,9 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
return output
|
||||
|
||||
|
||||
def qwen4_exp_qsa_with_output(
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
"""Run the complete QSA state/update/attend transaction."""
|
||||
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
if not isinstance(layer, Qwen4ExpQSAAttention):
|
||||
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
|
||||
layer._run_qsa(
|
||||
hidden_states,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
output,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_qsa_with_output",
|
||||
op_func=qwen4_exp_qsa_with_output,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"QSAIndexer",
|
||||
"Qwen4ExpQSAAttention",
|
||||
"Qwen4ExpQSAFlashAttentionBackend",
|
||||
"Qwen4ExpQSAFlashAttentionImpl",
|
||||
"qwen4_exp_qsa_with_output",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user