[Perf][Kernel] Integrate DeepSelect TopK for the DSA sparse indexer (#56464)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Kimi Code <noreply@moonshot.ai> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Kimi Code
Roger Wang
parent
1cfd972816
commit
fa008bdccf
@@ -20,6 +20,10 @@ from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
|
||||
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
|
||||
select_candidate_blocks as _select_candidate_blocks,
|
||||
)
|
||||
from vllm.model_executor.layers.indexer_topk import (
|
||||
RADIX_TOPK_WORKSPACE_SIZE,
|
||||
get_indexer_topk,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
@@ -46,8 +50,6 @@ from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
|
||||
|
||||
# MXFP4 layout: 2 values packed per byte, ue8m0 (1-byte) scale per block of 32.
|
||||
MXFP4_BLOCK_SIZE = 32
|
||||
|
||||
@@ -343,6 +345,7 @@ def sparse_attn_indexer(
|
||||
candidate_blocks: torch.Tensor | None = None,
|
||||
candidate_block_size: int = 0,
|
||||
candidate_write: bool = False,
|
||||
topk_backend: str = "auto",
|
||||
) -> torch.Tensor:
|
||||
# careful! this will be None in dummy run
|
||||
forward_context = get_forward_context()
|
||||
@@ -750,56 +753,16 @@ def sparse_attn_indexer(
|
||||
)
|
||||
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
|
||||
|
||||
use_cooperative_topk = (
|
||||
current_platform.is_cuda()
|
||||
and topk_tokens in (512, 1024, 2048)
|
||||
and num_rows <= 64
|
||||
and logits.stride(0) % 4 == 0 # TMA 16-byte alignment
|
||||
and current_platform.has_device_capability(90)
|
||||
and not current_platform.is_device_capability_family(120)
|
||||
# The backend comes from the layer (config is only readable at model
|
||||
# construction); dispatchers are cached per backend.
|
||||
get_indexer_topk(topk_backend)(
|
||||
logits,
|
||||
seq_lens,
|
||||
next_n,
|
||||
topk_indices,
|
||||
topk_tokens,
|
||||
attn_metadata_narrowed.max_seq_len,
|
||||
)
|
||||
use_persistent_topk = current_platform.is_cuda() and topk_tokens in (
|
||||
512,
|
||||
1024,
|
||||
2048,
|
||||
)
|
||||
if use_cooperative_topk:
|
||||
workspace_manager = current_workspace_manager()
|
||||
(topk_workspace,) = workspace_manager.get_simultaneous(
|
||||
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
|
||||
)
|
||||
torch.ops._C.cooperative_topk(
|
||||
logits,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
topk_workspace,
|
||||
topk_tokens,
|
||||
attn_metadata_narrowed.max_seq_len,
|
||||
)
|
||||
elif use_persistent_topk:
|
||||
workspace_manager = current_workspace_manager()
|
||||
(topk_workspace,) = workspace_manager.get_simultaneous(
|
||||
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
|
||||
)
|
||||
torch.ops._C.persistent_topk(
|
||||
logits,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
topk_workspace,
|
||||
topk_tokens,
|
||||
logits.shape[1],
|
||||
)
|
||||
else:
|
||||
ops.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
)
|
||||
|
||||
if decode_metadata.global_seq_lens is not None:
|
||||
_merge_dcp_topk_global(
|
||||
@@ -851,6 +814,7 @@ def sparse_attn_indexer_fake(
|
||||
candidate_blocks: torch.Tensor | None = None,
|
||||
candidate_block_size: int = 0,
|
||||
candidate_write: bool = False,
|
||||
topk_backend: str = "auto",
|
||||
) -> torch.Tensor:
|
||||
return topk_indices_buffer
|
||||
|
||||
@@ -917,6 +881,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
# than threading them through per-step metadata.
|
||||
vllm_config = get_current_vllm_config()
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.topk_backend = vllm_config.kernel_config.sparse_indexer_topk_backend
|
||||
self._parallel_config = parallel_config
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
||||
@@ -1026,6 +991,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
candidate_blocks=self.candidate_blocks,
|
||||
candidate_block_size=self.candidate_block_size,
|
||||
candidate_write=self.candidate_write,
|
||||
topk_backend=self.topk_backend,
|
||||
)
|
||||
|
||||
def forward_xpu(
|
||||
|
||||
Reference in New Issue
Block a user