[Qwen3.8-Flash-Next] Improve QSA sparse GQA for prefill and short-ctx decode (#54873)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Kimi <noreply@moonshot.cn>
This commit is contained in:
@@ -5,6 +5,8 @@
|
||||
import sys
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -15,37 +17,41 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
def qwen4_exp_qsa_triton_warmup(worker: "Worker") -> None:
|
||||
"""Warm every reachable QSA decode-query-length specialization."""
|
||||
"""Warm every reachable QSA specialization: indexer decode-query-length
|
||||
profiles plus the sparse attention split-K/merge configs."""
|
||||
|
||||
qsa_module = sys.modules.get("vllm.models.qwen4_exp.nvidia.indexer_qsa")
|
||||
if qsa_module is None:
|
||||
attn_module = sys.modules.get("vllm.models.qwen4_exp.nvidia.qsa")
|
||||
if qsa_module is None or attn_module is None:
|
||||
return
|
||||
indexer = next(
|
||||
(
|
||||
layer
|
||||
for layer in worker.get_model().modules()
|
||||
if isinstance(layer, qsa_module.QSAIndexer)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if indexer is None:
|
||||
indexer = None
|
||||
owner = None
|
||||
for layer in worker.get_model().modules():
|
||||
if indexer is None and isinstance(layer, qsa_module.QSAIndexer):
|
||||
indexer = layer
|
||||
elif owner is None and isinstance(layer, attn_module.Qwen4ExpQSAAttention):
|
||||
owner = layer
|
||||
if indexer is None or owner is None:
|
||||
return
|
||||
|
||||
runner = worker.model_runner
|
||||
prefix = indexer.compressed_key_cache.prefix
|
||||
group_id = next(
|
||||
i
|
||||
for i, group in enumerate(runner.kv_cache_config.kv_cache_groups)
|
||||
if prefix in group.layer_names
|
||||
)
|
||||
if worker.use_v2_model_runner:
|
||||
runner_v2 = cast("GPUModelRunnerV2", runner)
|
||||
block_table = runner_v2.block_tables.input_block_tables[group_id]
|
||||
max_decode_query_len = runner_v2.decode_query_len
|
||||
else:
|
||||
block_table = runner.input_batch.block_table[group_id].get_device_tensor(
|
||||
|
||||
def block_table_for(prefix: str) -> torch.Tensor:
|
||||
group_id = next(
|
||||
i
|
||||
for i, group in enumerate(runner.kv_cache_config.kv_cache_groups)
|
||||
if prefix in group.layer_names
|
||||
)
|
||||
if worker.use_v2_model_runner:
|
||||
runner_v2 = cast("GPUModelRunnerV2", runner)
|
||||
return runner_v2.block_tables.input_block_tables[group_id]
|
||||
return runner.input_batch.block_table[group_id].get_device_tensor(
|
||||
runner.max_num_reqs
|
||||
)
|
||||
|
||||
if worker.use_v2_model_runner:
|
||||
max_decode_query_len = cast("GPUModelRunnerV2", runner).decode_query_len
|
||||
else:
|
||||
max_decode_query_len = runner.uniform_decode_query_len
|
||||
|
||||
from vllm.models.qwen4_exp.nvidia.ops.qsa_indexer import (
|
||||
@@ -56,7 +62,7 @@ def qwen4_exp_qsa_triton_warmup(worker: "Worker") -> None:
|
||||
assert k_cache.numel()
|
||||
profiles = warmup_qsa_mqa_paged_decode(
|
||||
k_cache,
|
||||
block_table,
|
||||
block_table_for(indexer.compressed_key_cache.prefix),
|
||||
num_heads=indexer.index_n_heads,
|
||||
head_dim=indexer.index_head_dim,
|
||||
max_decode_query_len=max_decode_query_len,
|
||||
@@ -64,3 +70,20 @@ def qwen4_exp_qsa_triton_warmup(worker: "Worker") -> None:
|
||||
max_num_batched_tokens=runner.max_num_tokens,
|
||||
)
|
||||
logger.info("Warmed up Qwen4Exp QSA decode kernels: %s.", profiles)
|
||||
|
||||
from vllm.models.qwen4_exp.nvidia.ops.qsa import (
|
||||
warmup_qsa_sparse_paged_attention,
|
||||
)
|
||||
|
||||
kv_cache = owner.kv_cache
|
||||
assert kv_cache.numel()
|
||||
attention_profiles = warmup_qsa_sparse_paged_attention(
|
||||
kv_cache,
|
||||
block_table_for(owner.layer_name),
|
||||
num_query_heads=owner.num_heads,
|
||||
selection_width=indexer.output_width,
|
||||
)
|
||||
logger.info(
|
||||
"Warmed up Qwen4Exp QSA sparse attention kernels: %s.",
|
||||
attention_profiles,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user