[Qwen3.8-Flash-Next] Remove torch.compile for NVIDIA implementation (#55272)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Thien Tran
2026-09-06 21:57:56 -07:00
committed by GitHub
parent 9cc7793e32
commit d9105ea800
13 changed files with 165 additions and 185 deletions
+6 -8
View File
@@ -88,7 +88,7 @@ def _supports_fused_pre_indexer(
class QSAIndexer(nn.Module):
"""Replicated Q/K projection plus paged, weight-free QSA selection.
"""QSA projection weights, side caches, and paged, weight-free selection.
``prefix`` must be the checkpoint's indexer prefix, normally
``model.layers.N.self_attn.indexer``. Consequently the trainable names are
@@ -218,11 +218,11 @@ class QSAIndexer(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
projected_qk: torch.Tensor,
positions: torch.Tensor,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Select each query row's token indices.
"""Update side caches and select token indices from pre-projected Q/K.
Returns the packed buffer of shape [num_tokens, output_width + 1]:
the leading ``output_width`` columns are ``-1``-padded
@@ -237,10 +237,10 @@ class QSAIndexer(nn.Module):
if self.skip_topk and out is not None:
return out
result = torch.full(
(hidden_states.shape[0], self.packed_output_width),
(projected_qk.shape[0], self.packed_output_width),
-1,
dtype=torch.int32,
device=hidden_states.device,
device=projected_qk.device,
)
# Inert rows carry a zero valid count (empty loop bound), not -1.
result[:, -1] = 0
@@ -258,11 +258,9 @@ class QSAIndexer(nn.Module):
raw_metadata, compressed_metadata = metadata
num_tokens = raw_metadata.num_actual_tokens
hidden_states = hidden_states[:num_tokens]
projected_qk = projected_qk[:num_tokens]
positions = positions[..., :num_tokens]
# Q/K projection
projected_qk, _ = self.index_qk_proj(hidden_states)
projected_q, raw_keys = projected_qk.split(
(
self.index_n_heads * self.index_head_dim,
-12
View File
@@ -8,7 +8,6 @@ from itertools import islice
import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.fused_moe.utils import (
@@ -381,17 +380,6 @@ class Qwen4ExpMixtureOfExperts(MixtureOfExperts):
moe.experts.update_expert_map()
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"query_start_loc": 0,
"ngram_context": 0,
"deepstack_input_embeds": 0,
}
)
class Qwen4ExpModel(nn.Module):
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
+11 -7
View File
@@ -44,6 +44,8 @@ class Qwen4ExpModelState(MambaHybridModelState):
if self.ngram_context_len <= 0:
raise ValueError("N-gram embedding requires context length >= 1.")
self.ngram_eos_token_id = int(config.eos_token_id)
# PLE runs inside captured regions, so these buffers keep a fixed shape
# and address as the active request count changes between replays.
self.ngram_context = torch.full(
(self.max_num_reqs, self.ngram_context_len),
self.ngram_eos_token_id,
@@ -68,8 +70,7 @@ class Qwen4ExpModelState(MambaHybridModelState):
req_states: RequestState,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
num_reqs_padded = input_batch.num_reqs_after_padding
context = self.ngram_context[:num_reqs_padded]
context = self.ngram_context
context.fill_(self.ngram_eos_token_id)
if num_reqs == 0:
return context
@@ -101,8 +102,10 @@ class Qwen4ExpModelState(MambaHybridModelState):
return model_inputs
num_reqs_padded = input_batch.num_reqs_after_padding
query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1]
query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1])
query_start_loc = self.ple_query_start_loc
query_start_loc[: num_reqs_padded + 1].copy_(input_batch.query_start_loc)
# Represent unused capacity as trailing zero-length requests.
query_start_loc[num_reqs_padded + 1 :].copy_(input_batch.query_start_loc[-1])
model_inputs.update(
query_start_loc=query_start_loc,
ngram_context=self._prepare_ngram_context(input_batch, req_states),
@@ -118,7 +121,7 @@ class Qwen4ExpModelState(MambaHybridModelState):
if not self.uses_ngram_embedding:
return model_inputs
query_start_loc = self.ple_query_start_loc[: num_reqs + 1]
query_start_loc = self.ple_query_start_loc
query_start_loc[0] = 0
tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs)
query_lens = torch.full(
@@ -129,9 +132,10 @@ class Qwen4ExpModelState(MambaHybridModelState):
)
if num_extra_tokens > 0:
query_lens[-num_extra_tokens:] += 1
torch.cumsum(query_lens, dim=0, out=query_start_loc[1:])
torch.cumsum(query_lens, dim=0, out=query_start_loc[1 : num_reqs + 1])
query_start_loc[num_reqs + 1 :].fill_(num_tokens)
ngram_context = self.ngram_context[:num_reqs]
ngram_context = self.ngram_context
ngram_context.fill_(self.ngram_eos_token_id)
model_inputs.update(
query_start_loc=query_start_loc,
-19
View File
@@ -19,7 +19,6 @@ import regex as re
import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig, replace, set_current_vllm_config
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.fused_moe.utils import (
@@ -147,15 +146,6 @@ def _make_draft_vllm_config(
return draft_vllm_config
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"hidden_states": 0,
}
)
class Qwen4ExpMultiTokenPredictor(nn.Module):
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
@@ -355,15 +345,6 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
return loader.load_weights(weights, mapper=mapper)
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"hidden_states": 0,
}
)
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
+5 -61
View File
@@ -8,6 +8,7 @@ import torch
import torch.nn.functional as F
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
@@ -38,7 +39,6 @@ from vllm.model_executor.parameter import PerTensorScaleParameter
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionBackend,
@@ -264,12 +264,10 @@ class Qwen4ExpNGramEmbedding(nn.Module):
embedding_dim: int,
ple_dense_layer_id: int,
prefix: str,
layer_name: str,
quant_config: QuantizationConfig | None = None,
params_dtype: torch.dtype | None = None,
) -> None:
super().__init__()
self.layer_name = layer_name
self.embedding_dim = embedding_dim
self.ngram_size = int(config.ngram_size)
self.heads_per_ngram = int(config.heads_per_ngram)
@@ -445,20 +443,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
# Keep num_reqs-dependent ID generation outside PIECEWISE CUDA graphs,
# which dispatch only on the padded token count.
# torch.compile requires the splitting op to write graph-owned storage.
# Once compilation is removed, the op can return the IDs directly.
ngram_ids = input_ids.new_empty(
(input_ids.numel(), self.ngram_heads), dtype=torch.long
)
torch.ops.vllm.qwen4_exp_compute_ple_ngram_ids(
input_ids,
query_start_loc,
ngram_context,
ngram_ids,
self.layer_name,
)
ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context)
return self.ngram_embedding(ngram_ids).flatten(-2)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -563,7 +548,6 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
int(config.ple_embed_dim),
self.ple_dense_layer_id,
prefix=f"{prefix}.ple_embedding",
layer_name=prefix,
quant_config=quant_config,
params_dtype=model_config.dtype,
)
@@ -779,6 +763,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
token_indices=non_spec_token_indices,
)
# State routing consumes the current request metadata on every replay.
@eager_break_during_capture
def _short_conv(self, inputs: torch.Tensor, residual: torch.Tensor) -> None:
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
@@ -854,52 +840,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
self.norm_conv.weight,
self.norm_key.eps,
)
# State routing depends on runtime request metadata and remains outside
# the piecewise graph; short convolution accumulates into gated_output.
torch.ops.vllm.qwen4_exp_ple_short_conv(conv_input, gated_output, self.prefix)
self._short_conv(conv_input, gated_output)
return gated_output
def qwen4_exp_compute_ple_ngram_ids(
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
output: torch.Tensor,
layer_name: str,
) -> None:
"""Compute request-dependent PLE n-gram IDs outside piecewise graphs."""
layer = get_forward_context().no_compile_layers[layer_name]
layer.ple_embedding.compute_ngram_ids(
input_ids,
query_start_loc,
ngram_context,
output,
)
def qwen4_exp_ple_short_conv(
inputs: torch.Tensor,
residual_output: torch.Tensor,
layer_name: str,
) -> None:
layer = get_forward_context().no_compile_layers[layer_name]
layer._short_conv(inputs, residual_output)
direct_register_custom_op(
op_name="qwen4_exp_compute_ple_ngram_ids",
op_func=qwen4_exp_compute_ple_ngram_ids,
mutates_args=["output"],
)
direct_register_custom_op(
op_name="qwen4_exp_ple_short_conv",
op_func=qwen4_exp_ple_short_conv,
mutates_args=["residual_output"],
)
__all__ = [
"Qwen4ExpNGramEmbedding",
"Qwen4ExpPLEGroupedNorm",
+14 -60
View File
@@ -9,6 +9,7 @@ from typing import ClassVar, cast
import torch
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.distributed import get_tensor_model_parallel_world_size
@@ -27,10 +28,6 @@ from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
kv_cache_dtype_str_to_dtype,
)
from vllm.v1.attention.backend import (
@@ -347,9 +344,10 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
)
@eager_break_during_capture
def _run_qsa(
self,
hidden_states: torch.Tensor,
projected_qk: torch.Tensor,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
@@ -374,7 +372,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
if side_metadata.num_actual_tokens != num_tokens:
raise RuntimeError("QSA main and side metadata token counts disagree")
selected = self.indexer(
hidden_states,
projected_qk,
positions,
self.topk_indices_buffer[:num_tokens],
)
@@ -412,27 +410,16 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
attn_output = torch.empty_like(query)
encoded_layer_name = _encode_layer_name(self.layer_name)
if current_platform.opaque_attention_op():
torch.ops.vllm.qwen4_exp_qsa_with_output(
hidden_states,
positions,
query,
key,
value,
attn_output,
encoded_layer_name,
)
else:
qwen4_exp_qsa_with_output(
hidden_states,
positions,
query,
key,
value,
attn_output,
encoded_layer_name,
)
# Keep the index projection outside the eager break.
projected_qk, _ = self.indexer.index_qk_proj(hidden_states)
self._run_qsa(
projected_qk,
positions,
query,
key,
value,
attn_output,
)
flat_output = attn_output.view(num_tokens, -1)
if gate is not None:
flat_output = flat_output * torch.sigmoid(gate)
@@ -440,42 +427,9 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
return output
def qwen4_exp_qsa_with_output(
hidden_states: torch.Tensor,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
output: torch.Tensor,
layer_name: LayerNameType,
) -> None:
"""Run the complete QSA state/update/attend transaction."""
layer_name = _resolve_layer_name(layer_name)
layer = get_forward_context().no_compile_layers[layer_name]
if not isinstance(layer, Qwen4ExpQSAAttention):
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
layer._run_qsa(
hidden_states,
positions,
query,
key,
value,
output,
)
direct_register_custom_op(
op_name="qwen4_exp_qsa_with_output",
op_func=qwen4_exp_qsa_with_output,
mutates_args=["output"],
)
__all__ = [
"QSAIndexer",
"Qwen4ExpQSAAttention",
"Qwen4ExpQSAFlashAttentionBackend",
"Qwen4ExpQSAFlashAttentionImpl",
"qwen4_exp_qsa_with_output",
]