[Attention][DSA] support dcp for FLASHINFER_MLA_SPARSE (#46076)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Jingyi Yang <girasoleyang@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: GirasoleY <girasoleyang@gmail.com> Co-authored-by: Jingyi Yang <girasoleyang@gmail.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jingyi Yang
Lucas Wilkinson
Claude Opus 4.8
parent
3c1396bab6
commit
c5200d3565
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,420 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import Float32, Int32, Uint32, Uint64
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import recast_val
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
def stable_topk_from_gathered_candidates_cutedsl(
|
||||
gathered: torch.Tensor,
|
||||
topk: int,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if out is None:
|
||||
out = torch.empty(
|
||||
(gathered.shape[0], topk),
|
||||
dtype=torch.int32,
|
||||
device=gathered.device,
|
||||
)
|
||||
StableTopKFromGatheredCandidatesKernel.compile(topk, gathered.shape[1])(
|
||||
gathered, out
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def pack_dcp_topk_candidates_cutedsl(
|
||||
logits: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
packed: torch.Tensor,
|
||||
dcp_rank: int,
|
||||
dcp_world_size: int,
|
||||
cp_interleave: int,
|
||||
row_starts: torch.Tensor | None,
|
||||
) -> None:
|
||||
topk = topk_indices.shape[1]
|
||||
grid = (topk_indices.shape[0], triton.cdiv(topk, 512))
|
||||
row_starts_arg = row_starts if row_starts is not None else topk_indices
|
||||
_pack_dcp_topk_candidates_triton_kernel[grid](
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
row_starts_arg,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_indices.stride(0),
|
||||
topk_indices.stride(1),
|
||||
packed.stride(0),
|
||||
packed.stride(1),
|
||||
packed.stride(2),
|
||||
logits.shape[1],
|
||||
DCP_RANK=dcp_rank,
|
||||
DCP_WORLD_SIZE=dcp_world_size,
|
||||
CP_INTERLEAVE=cp_interleave,
|
||||
HAS_ROW_STARTS=row_starts is not None,
|
||||
TOPK=topk,
|
||||
BLOCK_SIZE=512,
|
||||
num_warps=8,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_dcp_topk_candidates_triton_kernel(
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
row_starts,
|
||||
logits_stride0: tl.constexpr,
|
||||
logits_stride1: tl.constexpr,
|
||||
topk_stride0: tl.constexpr,
|
||||
topk_stride1: tl.constexpr,
|
||||
packed_stride0: tl.constexpr,
|
||||
packed_stride1: tl.constexpr,
|
||||
packed_stride2: tl.constexpr,
|
||||
num_cols,
|
||||
DCP_RANK: tl.constexpr,
|
||||
DCP_WORLD_SIZE: tl.constexpr,
|
||||
CP_INTERLEAVE: tl.constexpr,
|
||||
HAS_ROW_STARTS: tl.constexpr,
|
||||
TOPK: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
tile = tl.program_id(1)
|
||||
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = cols < TOPK
|
||||
|
||||
local_idx = tl.load(
|
||||
topk_indices + row * topk_stride0 + cols * topk_stride1,
|
||||
mask=mask,
|
||||
other=-1,
|
||||
)
|
||||
valid = local_idx >= 0
|
||||
safe_local_idx = tl.maximum(local_idx, 0)
|
||||
|
||||
row_start = 0
|
||||
if HAS_ROW_STARTS:
|
||||
row_start = tl.load(row_starts + row)
|
||||
|
||||
score_col = safe_local_idx + row_start
|
||||
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
|
||||
score = tl.load(
|
||||
logits + row * logits_stride0 + score_col * logits_stride1,
|
||||
mask=mask & valid,
|
||||
other=-float("inf"),
|
||||
)
|
||||
|
||||
global_id = (
|
||||
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
|
||||
+ DCP_RANK * CP_INTERLEAVE
|
||||
+ safe_local_idx % CP_INTERLEAVE
|
||||
)
|
||||
global_id = tl.where(valid, global_id, -1)
|
||||
|
||||
packed_base = packed + row * packed_stride0 + cols * packed_stride1
|
||||
tl.store(packed_base, score, mask=mask)
|
||||
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
|
||||
|
||||
|
||||
@cute.jit
|
||||
def _warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
|
||||
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
|
||||
offset = 1 << i
|
||||
partial = cute.arch.shuffle_sync_up(val, offset=offset, mask_and_clamp=0)
|
||||
if lane >= offset:
|
||||
val += partial
|
||||
return val
|
||||
|
||||
|
||||
@cute.jit
|
||||
def _block_scan_inclusive_i32(
|
||||
val: Int32,
|
||||
lane: Int32,
|
||||
warp_id: Int32,
|
||||
warp_scratch: cute.Tensor,
|
||||
warps_per_block: int,
|
||||
) -> Int32:
|
||||
prefix = _warp_scan_inclusive_i32(val, lane)
|
||||
if lane == Int32(cute.arch.WARP_SIZE - 1):
|
||||
warp_scratch[0, warp_id] = prefix
|
||||
cute.arch.sync_threads()
|
||||
|
||||
if warp_id == Int32(0):
|
||||
warp_total = Int32(0)
|
||||
if lane < Int32(warps_per_block):
|
||||
warp_total = warp_scratch[0, lane]
|
||||
warp_prefix = _warp_scan_inclusive_i32(warp_total, lane)
|
||||
if lane < Int32(warps_per_block):
|
||||
warp_scratch[0, lane] = warp_prefix - warp_total
|
||||
cute.arch.sync_threads()
|
||||
|
||||
return prefix + warp_scratch[0, warp_id]
|
||||
|
||||
|
||||
class StableTopKFromGatheredCandidatesKernel:
|
||||
tb_size = 512
|
||||
hist_bins = 2048
|
||||
radix_bits = (hist_bins - 1).bit_length()
|
||||
assert hist_bins == 1 << radix_bits
|
||||
key_bits = Uint64.width
|
||||
radix_passes = (key_bits + radix_bits - 1) // radix_bits
|
||||
final_radix_bits = key_bits - radix_bits * (radix_passes - 1)
|
||||
hist_chunks = (hist_bins + tb_size - 1) // tb_size
|
||||
warps_per_block = tb_size // cute.arch.WARP_SIZE
|
||||
|
||||
def __init__(self, topk: int, num_candidates: int):
|
||||
assert num_candidates % self.tb_size == 0, (
|
||||
"StableTopKFromGatheredCandidatesKernel requires candidate count "
|
||||
f"to be a multiple of {self.tb_size}, got {num_candidates}"
|
||||
)
|
||||
self.topk = topk
|
||||
self.keys_per_thread = num_candidates // self.tb_size
|
||||
|
||||
@cute.struct
|
||||
class SharedStorage:
|
||||
hist: cute.struct.MemRange[Int32, self.hist_bins]
|
||||
committed_count: cute.struct.MemRange[Int32, 1]
|
||||
running_count: cute.struct.MemRange[Int32, 1]
|
||||
threshold_bin: cute.struct.MemRange[Int32, 1]
|
||||
threshold_found: cute.struct.MemRange[Int32, 1]
|
||||
include_threshold_bin: cute.struct.MemRange[Int32, 1]
|
||||
prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8]
|
||||
warp_totals: cute.struct.MemRange[Int32, self.warps_per_block]
|
||||
|
||||
self.shared_storage = SharedStorage
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
gathered: cute.Tensor,
|
||||
out: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
grid = (gathered.shape[0], 1, 1)
|
||||
self.kernel(gathered, out).launch(
|
||||
grid=grid,
|
||||
block=(self.tb_size, 1, 1),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def _stable_key(self, score: Float32, token_id: Int32) -> Uint64:
|
||||
bits = recast_val(score, Uint32)
|
||||
mask = Uint32(0x80000000)
|
||||
if (bits & Uint32(0x80000000)) != Uint32(0):
|
||||
mask = Uint32(0xFFFFFFFF)
|
||||
score_key = Uint64(bits ^ mask) << Uint64(32)
|
||||
id_key = Uint64(~Uint32(token_id))
|
||||
key = score_key | id_key
|
||||
if token_id < Int32(0):
|
||||
key = Uint64(0)
|
||||
return key
|
||||
|
||||
@cute.jit
|
||||
def _prefix_matches(
|
||||
self,
|
||||
key: Uint64,
|
||||
prefix: Uint64,
|
||||
prefix_bits: Int32,
|
||||
):
|
||||
matches = prefix_bits == Int32(0)
|
||||
if prefix_bits != Int32(0):
|
||||
shift = Int32(self.key_bits) - prefix_bits
|
||||
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
|
||||
return matches
|
||||
|
||||
@cute.jit
|
||||
def _radix_pass(
|
||||
self,
|
||||
keys: cute.Tensor,
|
||||
output: cute.Tensor,
|
||||
storage,
|
||||
tid: Int32,
|
||||
step: Int32,
|
||||
bits: int,
|
||||
is_final_pass: bool,
|
||||
):
|
||||
hist_smem = storage.hist.get_tensor(cute.make_layout((self.hist_bins,)))
|
||||
committed_count_smem = storage.committed_count.data_ptr()
|
||||
running_count_smem = storage.running_count.data_ptr()
|
||||
threshold_bin_smem = storage.threshold_bin.data_ptr()
|
||||
threshold_found_smem = storage.threshold_found.data_ptr()
|
||||
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
|
||||
prefix_smem = storage.prefix_s.data_ptr()
|
||||
warp_totals_smem = storage.warp_totals.get_tensor(
|
||||
cute.make_layout((1, self.warps_per_block))
|
||||
)
|
||||
|
||||
prefix_bits = step * Int32(self.radix_bits)
|
||||
num_bins = 1 << bits
|
||||
block_scan_iterations = (num_bins + self.tb_size - 1) // self.tb_size
|
||||
shift = Int32(self.key_bits) - prefix_bits - Int32(bits)
|
||||
bin_mask = Uint64(num_bins - 1)
|
||||
prefix = prefix_smem.load()
|
||||
|
||||
for chunk in cutlass.range_constexpr(self.hist_chunks):
|
||||
hist_smem[tid + Int32(chunk * self.tb_size)] = Int32(0)
|
||||
if tid == Int32(0):
|
||||
running_count_smem.store(committed_count_smem.load())
|
||||
include_threshold_bin_smem.store(Int32(0))
|
||||
threshold_found_smem.store(Int32(0))
|
||||
cute.arch.sync_threads()
|
||||
|
||||
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
|
||||
key = keys[key_idx]
|
||||
if self._prefix_matches(key, prefix, prefix_bits):
|
||||
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
|
||||
cute.arch.atomic_add(
|
||||
hist_smem.iterator + bin_idx,
|
||||
Int32(1),
|
||||
sem="relaxed",
|
||||
scope="cta",
|
||||
)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
lane = cute.arch.lane_idx()
|
||||
warp_id = cute.arch.warp_idx()
|
||||
# Each iteration scans one tb_size-wide slice of bins, high to low.
|
||||
iter = Int32(0)
|
||||
threshold_found = threshold_found_smem.load()
|
||||
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
|
||||
bin_idx = Int32(num_bins - 1) - (iter * Int32(self.tb_size) + tid)
|
||||
count = hist_smem[bin_idx]
|
||||
chunk_inclusive = _block_scan_inclusive_i32(
|
||||
count,
|
||||
lane,
|
||||
warp_id,
|
||||
warp_totals_smem,
|
||||
self.warps_per_block,
|
||||
)
|
||||
running_count = running_count_smem.load()
|
||||
prior_in_scan_slice = chunk_inclusive - count
|
||||
remaining = Int32(self.topk) - running_count - prior_in_scan_slice
|
||||
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
|
||||
threshold_bin_smem.store(bin_idx)
|
||||
if count <= remaining or cutlass.const_expr(is_final_pass):
|
||||
include_threshold_bin_smem.store(Int32(1))
|
||||
threshold_found_smem.store(Int32(1))
|
||||
# Barrier: every thread must finish reading running_count for this
|
||||
# slice before tb_size-1 advances it, else a warp racing ahead to
|
||||
# the store makes a lagging thread double-count the slice total
|
||||
# (-> remaining too small -> threshold too high -> under-fill).
|
||||
cute.arch.sync_threads()
|
||||
if tid == Int32(self.tb_size - 1):
|
||||
running_count_smem.store(running_count + chunk_inclusive)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
threshold_found = threshold_found_smem.load()
|
||||
iter += Int32(1)
|
||||
|
||||
threshold = threshold_bin_smem.load()
|
||||
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
|
||||
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
|
||||
key = keys[key_idx]
|
||||
if self._prefix_matches(key, prefix, prefix_bits):
|
||||
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
|
||||
selected = bin_idx > threshold
|
||||
if should_include_threshold:
|
||||
selected = selected or bin_idx == threshold
|
||||
if selected:
|
||||
dst = cute.arch.atomic_add(
|
||||
committed_count_smem,
|
||||
Int32(1),
|
||||
sem="relaxed",
|
||||
scope="cta",
|
||||
)
|
||||
if dst < Int32(self.topk):
|
||||
output[dst] = recast_val(~Uint32(key), Int32)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
pass_finished = include_threshold_bin_smem.load()
|
||||
if tid == Int32(0) and pass_finished == Int32(0):
|
||||
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
|
||||
cute.arch.sync_threads()
|
||||
return pass_finished
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
input: cute.Tensor,
|
||||
out: cute.Tensor,
|
||||
):
|
||||
row, _, _ = cute.arch.block_idx()
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
input_row = input[row, None, None]
|
||||
output_row = out[row, None]
|
||||
keys = cute.make_rmem_tensor((self.keys_per_thread,), Uint64)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
storage = smem.allocate(self.shared_storage, 8)
|
||||
committed_count_smem = storage.committed_count.data_ptr()
|
||||
prefix_smem = storage.prefix_s.data_ptr()
|
||||
for i in range(tid, self.topk, self.tb_size):
|
||||
output_row[i] = Int32(-1)
|
||||
|
||||
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
|
||||
col = tid + Int32(key_idx * self.tb_size)
|
||||
score = Float32(input_row[col, 0])
|
||||
token_id = Int32(input_row[col, 1])
|
||||
keys[key_idx] = self._stable_key(score, token_id)
|
||||
|
||||
if tid == Int32(0):
|
||||
committed_count_smem.store(Int32(0))
|
||||
prefix_smem.store(Uint64(0))
|
||||
cute.arch.sync_threads()
|
||||
|
||||
step = Int32(0)
|
||||
finished = Int32(0)
|
||||
while finished == Int32(0) and step < Int32(self.radix_passes - 1):
|
||||
finished = self._radix_pass(
|
||||
keys,
|
||||
output_row,
|
||||
storage,
|
||||
tid,
|
||||
step,
|
||||
self.radix_bits,
|
||||
False,
|
||||
)
|
||||
step += Int32(1)
|
||||
|
||||
if finished == Int32(0):
|
||||
self._radix_pass(
|
||||
keys,
|
||||
output_row,
|
||||
storage,
|
||||
tid,
|
||||
Int32(self.radix_passes - 1),
|
||||
self.final_radix_bits,
|
||||
True,
|
||||
)
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(topk: int, num_candidates: int):
|
||||
num_rows = cute.sym_int()
|
||||
|
||||
gathered = cute.runtime.make_fake_tensor(
|
||||
Float32,
|
||||
(num_rows, num_candidates, 2),
|
||||
stride=(cute.sym_int64(divisibility=2), 2, 1),
|
||||
assumed_align=8,
|
||||
)
|
||||
out = make_fake_tensor(Int32, (num_rows, topk), divisibility=1)
|
||||
|
||||
kernel = StableTopKFromGatheredCandidatesKernel(topk, num_candidates)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
gathered,
|
||||
out,
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
@@ -8,6 +8,8 @@ import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_dcp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
@@ -21,6 +23,7 @@ from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
has_deep_gemm,
|
||||
)
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
@@ -41,6 +44,85 @@ RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
|
||||
MXFP4_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
def _assert_cutedsl_dcp_merge_supported(
|
||||
logits: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
k: int,
|
||||
) -> None:
|
||||
# The DCP merge only supports the CuteDSL path (Triton pack kernel + CuteDSL
|
||||
# stable-topk selector); there is no PyTorch fallback. The first cut targets
|
||||
# Blackwell/Hopper with index_topk in (512, 1024, 2048) (the selector's radix
|
||||
# sizing); the Triton pack itself has no shape/topk constraints.
|
||||
if not has_cutedsl():
|
||||
raise RuntimeError(
|
||||
"DCP sparse-indexer merge requires CuteDSL; install it or disable DCP."
|
||||
)
|
||||
if logits.device.type != "cuda":
|
||||
raise RuntimeError("DCP sparse-indexer merge requires CUDA tensors.")
|
||||
if logits.dtype != torch.float32 or topk_indices.dtype != torch.int32:
|
||||
raise RuntimeError(
|
||||
"DCP sparse-indexer merge requires fp32 logits and int32 indices."
|
||||
)
|
||||
if k not in (512, 1024, 2048):
|
||||
raise RuntimeError(
|
||||
f"DCP sparse-indexer merge requires index_topk in (512, 1024, 2048); "
|
||||
f"got {k}."
|
||||
)
|
||||
|
||||
|
||||
def _merge_dcp_topk_global(
|
||||
logits: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
topk_tokens: int,
|
||||
dcp_rank: int,
|
||||
dcp_world_size: int,
|
||||
cp_interleave: int,
|
||||
row_starts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Merge each DCP rank's local top-K into the global top-K.
|
||||
|
||||
``topk_indices`` are this rank's local top-K positions into its 1/N KV
|
||||
shard. A token in the global top-K must also be in its owning rank's local
|
||||
top-K (at most ``topk_tokens - 1`` tokens rank globally above it, hence at
|
||||
most that many on its own rank), so exchanging only the per-rank local
|
||||
candidates is exact -- equivalent to all-gathering the full logit matrix,
|
||||
but it ships ``dcp_world_size * topk_tokens`` candidates instead of the whole
|
||||
score row. Overwrites ``topk_indices`` with global token ids (``-1`` for
|
||||
padding); the attention backend localizes them back to physical slots per
|
||||
rank.
|
||||
"""
|
||||
if dcp_world_size <= 1:
|
||||
return
|
||||
|
||||
# CuteDSL-only path (no PyTorch fallback): Triton-pack each rank's
|
||||
# (score, global_id) candidates on-device, all-gather, then the CuteDSL
|
||||
# stable-topk selector.
|
||||
_assert_cutedsl_dcp_merge_supported(logits, topk_indices, topk_tokens)
|
||||
from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import (
|
||||
pack_dcp_topk_candidates_cutedsl,
|
||||
stable_topk_from_gathered_candidates_cutedsl,
|
||||
)
|
||||
|
||||
packed = torch.empty(
|
||||
(*topk_indices.shape, 2),
|
||||
dtype=torch.float32,
|
||||
device=topk_indices.device,
|
||||
)
|
||||
pack_dcp_topk_candidates_cutedsl(
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_interleave,
|
||||
row_starts,
|
||||
)
|
||||
gathered = get_dcp_group().all_gather(packed, dim=1)
|
||||
stable_topk_from_gathered_candidates_cutedsl(
|
||||
gathered, topk_tokens, out=topk_indices
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_indexer_q_rope_quant_kernel(
|
||||
positions,
|
||||
@@ -227,6 +309,9 @@ def sparse_attn_indexer(
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
skip_topk_buffer_clear: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# careful! this will be None in dummy run
|
||||
@@ -328,16 +413,18 @@ def sparse_attn_indexer(
|
||||
scales_spec,
|
||||
)
|
||||
for chunk in prefill_metadata.chunks:
|
||||
k_quant = k_quant_full[: chunk.total_seq_lens]
|
||||
k_scale = k_scale_full[: chunk.total_seq_lens]
|
||||
|
||||
if not chunk.skip_kv_gather:
|
||||
cu_seqlen_ks = chunk.cu_seqlen_ks
|
||||
cu_seqlen_ke = chunk.cu_seqlen_ke
|
||||
assert chunk.local_cu_seq_lens is not None
|
||||
k_quant = k_quant_full[: chunk.max_local_total_seq_lens]
|
||||
k_scale = k_scale_full[: chunk.max_local_total_seq_lens]
|
||||
if not chunk.skip_kv_gather and chunk.local_total_seq_lens > 0:
|
||||
ops.cp_gather_indexer_k_quant_cache(
|
||||
kv_cache,
|
||||
k_quant,
|
||||
k_scale,
|
||||
chunk.block_table,
|
||||
chunk.cu_seq_lens,
|
||||
chunk.local_cu_seq_lens,
|
||||
)
|
||||
|
||||
q_slice = q_quant[chunk.token_start : chunk.token_end]
|
||||
@@ -346,51 +433,64 @@ def sparse_attn_indexer(
|
||||
if q_scale is not None
|
||||
else None
|
||||
)
|
||||
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
|
||||
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
|
||||
if use_fp4_cache:
|
||||
q_slice_cast = q_slice.view(torch.int8)
|
||||
k_quant_cast = k_quant.view(torch.int8)
|
||||
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
|
||||
else:
|
||||
q_slice_cast = q_slice
|
||||
k_quant_cast = k_quant
|
||||
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
|
||||
if current_platform.is_xpu():
|
||||
if q_scale_slice is not None:
|
||||
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
|
||||
logits = torch.ops.vllm.xpu_fp8_mqa_logits(
|
||||
q_slice_cast,
|
||||
k_quant_cast,
|
||||
k_scale_cast,
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
)
|
||||
else:
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
|
||||
topk_indices = topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
|
||||
ops.top_k_per_row_prefill(
|
||||
if chunk.local_total_seq_lens == 0:
|
||||
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
|
||||
topk_indices.fill_(-1)
|
||||
else:
|
||||
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
|
||||
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
|
||||
if use_fp4_cache:
|
||||
q_slice_cast = q_slice.view(torch.int8)
|
||||
k_quant_cast = k_quant.view(torch.int8)
|
||||
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
|
||||
else:
|
||||
q_slice_cast = q_slice
|
||||
k_quant_cast = k_quant
|
||||
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
|
||||
if current_platform.is_xpu():
|
||||
if q_scale_slice is not None:
|
||||
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
|
||||
logits = torch.ops.vllm.xpu_fp8_mqa_logits(
|
||||
q_slice_cast,
|
||||
k_quant_cast,
|
||||
k_scale_cast,
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
)
|
||||
else:
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
ops.top_k_per_row_prefill(
|
||||
logits,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
)
|
||||
|
||||
_merge_dcp_topk_global(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
row_starts=chunk.cu_seqlen_ks,
|
||||
)
|
||||
|
||||
if has_decode:
|
||||
@@ -508,7 +608,7 @@ def sparse_attn_indexer(
|
||||
topk_indices,
|
||||
topk_workspace,
|
||||
topk_tokens,
|
||||
attn_metadata_narrowed.max_seq_len,
|
||||
logits.shape[1],
|
||||
)
|
||||
else:
|
||||
ops.top_k_per_row_decode(
|
||||
@@ -522,6 +622,16 @@ def sparse_attn_indexer(
|
||||
topk_tokens,
|
||||
)
|
||||
|
||||
if decode_metadata.global_seq_lens is not None:
|
||||
_merge_dcp_topk_global(
|
||||
logits,
|
||||
topk_indices,
|
||||
topk_tokens,
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
)
|
||||
|
||||
if decode_metadata.requires_padding:
|
||||
# if padded, we need to unpack
|
||||
# the topk indices removing padded tokens
|
||||
@@ -553,6 +663,9 @@ def sparse_attn_indexer_fake(
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
skip_topk_buffer_clear: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return topk_indices_buffer
|
||||
@@ -604,6 +717,13 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
self.skip_k_cache_insert = skip_k_cache_insert
|
||||
self.use_fp4_cache = use_fp4_cache
|
||||
# DCP scalars are constant for the run; resolve them here (config is set
|
||||
# during model construction) and pass them into the custom op, rather
|
||||
# than threading them through per-step metadata.
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
||||
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||
if current_platform.is_cuda() and not has_deep_gemm():
|
||||
raise RuntimeError(
|
||||
"Sparse Attention Indexer CUDA op requires DeepGEMM support in "
|
||||
@@ -657,6 +777,9 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.topk_indices_buffer,
|
||||
self.skip_k_cache_insert,
|
||||
self.use_fp4_cache,
|
||||
self.dcp_rank,
|
||||
self.dcp_world_size,
|
||||
self.cp_kv_cache_interleave_size,
|
||||
)
|
||||
|
||||
def forward_xpu(
|
||||
|
||||
Reference in New Issue
Block a user