[Perf] Fuse DSV4.1 input metadata preparation with Triton (#56562)
This commit is contained in:
@@ -484,19 +484,21 @@ class CommonAttentionMetadata:
|
||||
# Built from the device query_start_loc: adaptive verification decides the
|
||||
# per-request draft split on device, so the CPU copy carries the right total
|
||||
# but not the right per-request boundaries. Padding requests have a query
|
||||
# length of zero and drop out of the repeat.
|
||||
# length of zero and are skipped by the device boundary search.
|
||||
num_mapped_tokens = int(self.query_start_loc_cpu[-1])
|
||||
query_lens = self.query_start_loc[1:] - self.query_start_loc[:-1]
|
||||
assert buffer.shape[0] >= max(num_mapped_tokens, num_tokens)
|
||||
token_to_req_indices = torch.repeat_interleave(
|
||||
torch.arange(query_lens.shape[0], dtype=torch.int32, device=buffer.device),
|
||||
query_lens,
|
||||
output_size=num_mapped_tokens,
|
||||
from vllm.v1.attention.ops.metadata import _token_request_mapping_kernel
|
||||
|
||||
num_output_tokens = max(num_mapped_tokens, num_tokens)
|
||||
assert buffer.shape[0] >= num_output_tokens
|
||||
_token_request_mapping_kernel[((num_output_tokens + 255) // 256,)](
|
||||
self.query_start_loc,
|
||||
buffer,
|
||||
self.query_start_loc.shape[0] - 1,
|
||||
num_mapped_tokens,
|
||||
num_output_tokens,
|
||||
num_warps=4,
|
||||
)
|
||||
buffer[:num_mapped_tokens].copy_(token_to_req_indices)
|
||||
if num_mapped_tokens < num_tokens:
|
||||
buffer[num_mapped_tokens:num_tokens].zero_()
|
||||
self._token_to_req_indices_cache = buffer[: max(num_mapped_tokens, num_tokens)]
|
||||
self._token_to_req_indices_cache = buffer[:num_output_tokens]
|
||||
return self._token_to_req_indices_cache[:num_tokens]
|
||||
|
||||
# TODO(lucas): remove once we have FULL-CG spec-decode support
|
||||
|
||||
@@ -960,28 +960,6 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
self.global_decode_seq_lens_buffer[actual_expanded:num_decode_tokens] = 0
|
||||
return self.global_decode_seq_lens_buffer[:num_decode_tokens]
|
||||
|
||||
def _build_varlen_decode_indices(
|
||||
self,
|
||||
decode_lens: torch.Tensor,
|
||||
decode_lens_cpu: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
) -> torch.Tensor:
|
||||
"""Build request ids for flattened SM100 varlen rows."""
|
||||
indices = self.decode_indices_buffer[:num_decode_tokens]
|
||||
actual_expanded = int(decode_lens_cpu.sum().item())
|
||||
num_decodes = decode_lens.shape[0]
|
||||
indices[:actual_expanded] = torch.repeat_interleave(
|
||||
self.arange_buffer[:num_decodes],
|
||||
decode_lens,
|
||||
output_size=actual_expanded,
|
||||
)
|
||||
if actual_expanded < num_decode_tokens:
|
||||
pad = num_decode_tokens - actual_expanded
|
||||
indices[actual_expanded:num_decode_tokens] = (
|
||||
num_decodes + self.arange_buffer[:pad]
|
||||
)
|
||||
return indices
|
||||
|
||||
@staticmethod
|
||||
def _split_indexer_prefill_chunks(
|
||||
compressed_seq_lens_cpu: torch.Tensor,
|
||||
@@ -1156,12 +1134,15 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
|
||||
decode_metadata = None
|
||||
if num_decodes > 0:
|
||||
torch.diff(
|
||||
common_attn_metadata.query_start_loc[: num_decodes + 1],
|
||||
out=self.decode_lens_buffer[:num_decodes],
|
||||
)
|
||||
if not self.supports_varlen:
|
||||
torch.diff(
|
||||
common_attn_metadata.query_start_loc[: num_decodes + 1],
|
||||
out=self.decode_lens_buffer[:num_decodes],
|
||||
)
|
||||
self.per_req_decode_lens_buffer[:num_decodes].copy_(
|
||||
self.decode_lens_buffer[:num_decodes]
|
||||
)
|
||||
decode_lens = self.decode_lens_buffer[:num_decodes]
|
||||
self.per_req_decode_lens_buffer[:num_decodes].copy_(decode_lens)
|
||||
decode_lens_cpu = torch.diff(
|
||||
common_attn_metadata.query_start_loc_cpu[: num_decodes + 1]
|
||||
)
|
||||
@@ -1196,38 +1177,74 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
and step_next_n_ok
|
||||
)
|
||||
|
||||
global_seq_lens_for_decode = self._prepare_global_decode_seq_lens(
|
||||
global_seq_lens=global_seq_lens_for_decode,
|
||||
decode_lens=decode_lens,
|
||||
decode_lens_cpu=decode_lens_cpu,
|
||||
query_start_loc=common_attn_metadata.query_start_loc[:num_decodes],
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
use_native=use_native,
|
||||
max_decode_len=max_decode_len,
|
||||
)
|
||||
|
||||
decode_indices = None
|
||||
if self.supports_varlen:
|
||||
decode_indices = self._build_varlen_decode_indices(
|
||||
decode_lens=decode_lens,
|
||||
decode_lens_cpu=decode_lens_cpu,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
)
|
||||
|
||||
seq_lens, block_table, decode_lens, batch_size, requires_padding = (
|
||||
self._prepare_decode_tensors(
|
||||
seq_lens=seq_lens,
|
||||
block_table=block_table,
|
||||
if not self.supports_varlen:
|
||||
global_seq_lens_for_decode = self._prepare_global_decode_seq_lens(
|
||||
global_seq_lens=global_seq_lens_for_decode,
|
||||
decode_lens=decode_lens,
|
||||
decode_lens_cpu=decode_lens_cpu,
|
||||
query_start_loc=common_attn_metadata.query_start_loc[:num_decodes],
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
use_native=use_native,
|
||||
next_n=next_n,
|
||||
max_decode_len=max_decode_len,
|
||||
)
|
||||
)
|
||||
|
||||
decode_indices = None
|
||||
if self.supports_varlen:
|
||||
from vllm.v1.attention.ops.metadata import (
|
||||
_indexer_decode_metadata_kernel,
|
||||
)
|
||||
|
||||
capacity = self.decode_seq_lens_buffer.numel()
|
||||
grid = max(
|
||||
num_decodes,
|
||||
num_decode_tokens + triton.cdiv(capacity - num_decode_tokens, 256),
|
||||
)
|
||||
_indexer_decode_metadata_kernel[(grid,)](
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
block_table,
|
||||
self.decode_seq_lens_buffer,
|
||||
self.expanded_block_table_buffer,
|
||||
self.decode_lens_buffer,
|
||||
self.decode_indices_buffer,
|
||||
self.per_req_decode_lens_buffer,
|
||||
num_decodes,
|
||||
num_decode_tokens,
|
||||
capacity,
|
||||
block_table.stride(0),
|
||||
self.expanded_block_table_buffer.stride(0),
|
||||
BLOCK_COLS=block_table.shape[1],
|
||||
num_warps=4,
|
||||
)
|
||||
seq_lens = self.decode_seq_lens_buffer[:num_decode_tokens]
|
||||
block_table = self.expanded_block_table_buffer[:num_decode_tokens]
|
||||
decode_lens = self.decode_lens_buffer[:num_decode_tokens]
|
||||
decode_indices = self.decode_indices_buffer[:num_decode_tokens]
|
||||
requires_padding = False
|
||||
if global_seq_lens_for_decode is not None and max_decode_len > 1:
|
||||
self.global_decode_seq_lens_buffer[:num_decode_tokens].copy_(
|
||||
seq_lens
|
||||
)
|
||||
global_seq_lens_for_decode = self.global_decode_seq_lens_buffer[
|
||||
:num_decode_tokens
|
||||
]
|
||||
else:
|
||||
seq_lens, block_table, decode_lens, batch_size, requires_padding = (
|
||||
self._prepare_decode_tensors(
|
||||
seq_lens=seq_lens,
|
||||
block_table=block_table,
|
||||
decode_lens=decode_lens,
|
||||
decode_lens_cpu=decode_lens_cpu,
|
||||
query_start_loc=common_attn_metadata.query_start_loc[
|
||||
:num_decodes
|
||||
],
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
use_native=use_native,
|
||||
next_n=next_n,
|
||||
max_decode_len=max_decode_len,
|
||||
)
|
||||
)
|
||||
|
||||
if self.compress_ratio > 1:
|
||||
kernel_block_size = self.kernel_block_size
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Device-side request mapping and sparse indexer metadata."""
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _token_request(query_start_loc, token, num_reqs):
|
||||
lo = tl.full(token.shape, 0, tl.int32)
|
||||
hi = tl.full(token.shape, num_reqs, tl.int32)
|
||||
while tl.sum((lo < hi).to(tl.int32), 0) > 0:
|
||||
mid = (lo + hi) // 2
|
||||
end = tl.load(query_start_loc + mid + 1, mid < num_reqs, other=0x7FFFFFFF)
|
||||
right = (lo < hi) & (end <= token)
|
||||
lo = tl.where(right, mid + 1, lo)
|
||||
hi = tl.where((lo < hi) & ~right, mid, hi)
|
||||
return tl.minimum(lo, num_reqs - 1)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_reqs", "num_mapped", "num_tokens"])
|
||||
def _token_request_mapping_kernel(
|
||||
query_start_loc, output, num_reqs, num_mapped, num_tokens
|
||||
):
|
||||
token = tl.program_id(0) * 256 + tl.arange(0, 256)
|
||||
req = _token_request(query_start_loc, token, num_reqs)
|
||||
tl.store(output + token, tl.where(token < num_mapped, req, 0), token < num_tokens)
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"num_reqs",
|
||||
"num_tokens",
|
||||
"capacity",
|
||||
"input_stride",
|
||||
"output_stride",
|
||||
]
|
||||
)
|
||||
def _indexer_decode_metadata_kernel(
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
block_table,
|
||||
output_seq_lens,
|
||||
output_block_table,
|
||||
decode_lens,
|
||||
indices,
|
||||
per_req_lens,
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
capacity,
|
||||
input_stride,
|
||||
output_stride,
|
||||
BLOCK_COLS: tl.constexpr,
|
||||
):
|
||||
token = tl.program_id(0)
|
||||
cols = tl.arange(0, triton.next_power_of_2(BLOCK_COLS))
|
||||
actual_tokens = tl.load(query_start_loc + num_reqs)
|
||||
if token < num_tokens:
|
||||
req = _token_request(query_start_loc, tl.full((1,), token, tl.int32), num_reqs)
|
||||
req = tl.sum(req, 0)
|
||||
valid = token < actual_tokens
|
||||
start = tl.load(query_start_loc + req)
|
||||
end = tl.load(query_start_loc + req + 1)
|
||||
seq = tl.load(seq_lens + req)
|
||||
length = tl.where(valid, seq - (end - start) + token - start + 1, 0)
|
||||
tl.store(output_seq_lens + token, length)
|
||||
tl.store(decode_lens + token, 1)
|
||||
tl.store(
|
||||
indices + token, tl.where(valid, req, num_reqs + token - actual_tokens)
|
||||
)
|
||||
blocks = tl.load(
|
||||
block_table + req.to(tl.int64) * input_stride + cols,
|
||||
valid & (cols < BLOCK_COLS),
|
||||
other=0,
|
||||
)
|
||||
tl.store(
|
||||
output_block_table + token.to(tl.int64) * output_stride + cols,
|
||||
blocks,
|
||||
cols < BLOCK_COLS,
|
||||
)
|
||||
else:
|
||||
tail = num_tokens + (token - num_tokens) * 256 + tl.arange(0, 256)
|
||||
tl.store(output_seq_lens + tail, 0, tail < capacity)
|
||||
if token < num_reqs:
|
||||
start = tl.load(query_start_loc + token)
|
||||
end = tl.load(query_start_loc + token + 1)
|
||||
tl.store(per_req_lens + token, end - start)
|
||||
Reference in New Issue
Block a user