Files
2026-10-16 13:53:49 -05:00

195 lines
6.5 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm Kimi-K3 specialization of GDN attention metadata.
The request classification and cudagraph staging intentionally mirror
``GDNAttentionMetadataBuilder``. Only the FLA chunk metadata is built
differently on device rather than on the host.
"""
from dataclasses import replace
import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
from vllm.v1.attention.backend import max_decode_query_len
from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionBackend,
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
from vllm.v1.kv_cache_interface import KVCacheSpec
logger = init_logger(__name__)
_BLOCK_T = 256
_MIN_BLOCK_N = 128
@triton.jit(do_not_specialize=["N"])
def _chunk_metadata_kernel(
cu_seqlens,
chunk_indices,
chunk_offsets,
N,
BT: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_T: tl.constexpr,
):
i_n = tl.program_id(0)
offs_n = tl.arange(0, BLOCK_N)
is_seq = offs_n < N
bos = tl.load(cu_seqlens + offs_n, mask=is_seq, other=0).to(tl.int32)
eos = tl.load(cu_seqlens + offs_n + 1, mask=is_seq, other=0).to(tl.int32)
nt = tl.where(is_seq, tl.cdiv(eos - bos, BT), 0)
base = tl.sum(tl.where(offs_n < i_n, nt, 0))
num_chunks = tl.sum(tl.where(offs_n == i_n, nt, 0))
tl.store(chunk_offsets + i_n, base)
if i_n == 0:
tl.store(chunk_offsets + N, tl.sum(nt))
for t0 in range(0, num_chunks, BLOCK_T):
offs_t = t0 + tl.arange(0, BLOCK_T)
mask_t = offs_t < num_chunks
row = (base + offs_t) * 2
tl.store(chunk_indices + row, tl.full([BLOCK_T], i_n, tl.int32), mask=mask_t)
tl.store(chunk_indices + row + 1, offs_t.to(tl.int32), mask=mask_t)
def prepare_chunk_metadata_device(
cu_seqlens: torch.Tensor,
cu_seqlens_cpu: torch.Tensor,
chunk_size: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Build FLA chunk metadata on device, with no host<->device transfer."""
num_seqs = cu_seqlens_cpu.numel() - 1
seq_lens = cu_seqlens_cpu[1:] - cu_seqlens_cpu[:-1]
num_chunks = int(((seq_lens + chunk_size - 1) // chunk_size).sum())
chunk_indices = torch.empty(
num_chunks, 2, dtype=cu_seqlens.dtype, device=cu_seqlens.device
)
chunk_offsets = torch.empty(
num_seqs + 1, dtype=torch.int64, device=cu_seqlens.device
)
_chunk_metadata_kernel[(num_seqs,)](
cu_seqlens,
chunk_indices,
chunk_offsets,
num_seqs,
BT=chunk_size,
BLOCK_N=max(_MIN_BLOCK_N, next_power_of_2(num_seqs + 1)),
BLOCK_T=_BLOCK_T,
num_warps=4,
)
return chunk_indices, chunk_offsets
def _adaptive_verification_enabled(vllm_config: VllmConfig) -> bool:
speculative_config = vllm_config.speculative_config
return speculative_config is not None and bool(
getattr(speculative_config, "enable_adaptive_verification", False)
)
class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
@classmethod
def get_varlen_cudagraph_max_query_len(
cls: type["KimiK3ROCmKDAMetadataBuilder"],
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> int | None:
if not _adaptive_verification_enabled(vllm_config):
return None
return max_decode_query_len(vllm_config)
def build( # type: ignore[override]
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
num_accepted_tokens: torch.Tensor | None = None,
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
fast_build: bool = False,
) -> GDNAttentionMetadata:
attn_metadata = super().build(
common_prefix_len,
common_attn_metadata,
num_accepted_tokens,
num_decode_draft_tokens_cpu,
fast_build,
)
if (
attn_metadata.uniform_spec_sequence_length is not None
and _adaptive_verification_enabled(self.vllm_config)
):
attn_metadata = replace(attn_metadata, uniform_spec_sequence_length=None)
if attn_metadata.checkpoint is None:
return attn_metadata
# The shared builder keys checkpoint rows on every request; the ROCm
# prefill path indexes them by prefill_query_start_loc, so rebuild with
# those rows.
return replace(
attn_metadata,
checkpoint=self._build_checkpoint_metadata(
common_attn_metadata, attn_metadata, num_decode_draft_tokens_cpu
),
)
def _build_checkpoint_metadata(
self,
m: CommonAttentionMetadata,
attn_metadata: GDNAttentionMetadata,
num_decode_draft_tokens_cpu: torch.Tensor | None,
) -> MambaPrefillCheckpointMetadata | None:
if attn_metadata.num_prefills == 0:
return None
# request_rows must line up with prefill_query_start_loc, either the
# prefill tail of a decode-first batch, or every non-spec row
# including padding
num_decodes = attn_metadata.num_decodes
request_rows = list(
range(num_decodes, num_decodes + attn_metadata.num_prefills)
)
if attn_metadata.spec_sequence_masks is not None:
assert num_decode_draft_tokens_cpu is not None
request_rows = (
(num_decode_draft_tokens_cpu < 0).nonzero().flatten().tolist()
)
return self.checkpoint_builder.build(m, request_rows)
def _build_chunk_metadata(
self,
prefill_query_start_loc: torch.Tensor,
prefill_query_start_loc_cpu: torch.Tensor,
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor]:
return prepare_chunk_metadata_device(
prefill_query_start_loc,
prefill_query_start_loc_cpu,
FLA_CHUNK_SIZE,
)
class KimiK3ROCmKDABackend(GDNAttentionBackend):
@staticmethod
def get_name() -> str:
return "KIMI_K3_KDA_ROCM"
@staticmethod
def get_builder_cls() -> type[KimiK3ROCmKDAMetadataBuilder]:
return KimiK3ROCmKDAMetadataBuilder
@classmethod
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
return True