[Qwen3.8-Flash-Next] Improve QSA sparse GQA for prefill and short-ctx decode (#54873)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Kimi <noreply@moonshot.cn>
This commit is contained in:
Thien Tran
2026-09-04 06:08:02 -07:00
committed by GitHub
co-authored by Kimi
parent 8340fe1bb9
commit 31a8a26662
6 changed files with 365 additions and 106 deletions
@@ -5,6 +5,8 @@
import sys
from typing import TYPE_CHECKING, cast
import torch
from vllm.logger import init_logger
if TYPE_CHECKING:
@@ -15,37 +17,41 @@ logger = init_logger(__name__)
def qwen4_exp_qsa_triton_warmup(worker: "Worker") -> None:
"""Warm every reachable QSA decode-query-length specialization."""
"""Warm every reachable QSA specialization: indexer decode-query-length
profiles plus the sparse attention split-K/merge configs."""
qsa_module = sys.modules.get("vllm.models.qwen4_exp.nvidia.indexer_qsa")
if qsa_module is None:
attn_module = sys.modules.get("vllm.models.qwen4_exp.nvidia.qsa")
if qsa_module is None or attn_module is None:
return
indexer = next(
(
layer
for layer in worker.get_model().modules()
if isinstance(layer, qsa_module.QSAIndexer)
),
None,
)
if indexer is None:
indexer = None
owner = None
for layer in worker.get_model().modules():
if indexer is None and isinstance(layer, qsa_module.QSAIndexer):
indexer = layer
elif owner is None and isinstance(layer, attn_module.Qwen4ExpQSAAttention):
owner = layer
if indexer is None or owner is None:
return
runner = worker.model_runner
prefix = indexer.compressed_key_cache.prefix
group_id = next(
i
for i, group in enumerate(runner.kv_cache_config.kv_cache_groups)
if prefix in group.layer_names
)
if worker.use_v2_model_runner:
runner_v2 = cast("GPUModelRunnerV2", runner)
block_table = runner_v2.block_tables.input_block_tables[group_id]
max_decode_query_len = runner_v2.decode_query_len
else:
block_table = runner.input_batch.block_table[group_id].get_device_tensor(
def block_table_for(prefix: str) -> torch.Tensor:
group_id = next(
i
for i, group in enumerate(runner.kv_cache_config.kv_cache_groups)
if prefix in group.layer_names
)
if worker.use_v2_model_runner:
runner_v2 = cast("GPUModelRunnerV2", runner)
return runner_v2.block_tables.input_block_tables[group_id]
return runner.input_batch.block_table[group_id].get_device_tensor(
runner.max_num_reqs
)
if worker.use_v2_model_runner:
max_decode_query_len = cast("GPUModelRunnerV2", runner).decode_query_len
else:
max_decode_query_len = runner.uniform_decode_query_len
from vllm.models.qwen4_exp.nvidia.ops.qsa_indexer import (
@@ -56,7 +62,7 @@ def qwen4_exp_qsa_triton_warmup(worker: "Worker") -> None:
assert k_cache.numel()
profiles = warmup_qsa_mqa_paged_decode(
k_cache,
block_table,
block_table_for(indexer.compressed_key_cache.prefix),
num_heads=indexer.index_n_heads,
head_dim=indexer.index_head_dim,
max_decode_query_len=max_decode_query_len,
@@ -64,3 +70,20 @@ def qwen4_exp_qsa_triton_warmup(worker: "Worker") -> None:
max_num_batched_tokens=runner.max_num_tokens,
)
logger.info("Warmed up Qwen4Exp QSA decode kernels: %s.", profiles)
from vllm.models.qwen4_exp.nvidia.ops.qsa import (
warmup_qsa_sparse_paged_attention,
)
kv_cache = owner.kv_cache
assert kv_cache.numel()
attention_profiles = warmup_qsa_sparse_paged_attention(
kv_cache,
block_table_for(owner.layer_name),
num_query_heads=owner.num_heads,
selection_width=indexer.output_width,
)
logger.info(
"Warmed up Qwen4Exp QSA sparse attention kernels: %s.",
attention_profiles,
)