Signed-off-by: Oxana Korzh <okorzh@amd.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
195 lines
6.5 KiB
Python
195 lines
6.5 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""ROCm Kimi-K3 specialization of GDN attention metadata.
|
|
|
|
The request classification and cudagraph staging intentionally mirror
|
|
``GDNAttentionMetadataBuilder``. Only the FLA chunk metadata is built
|
|
differently on device rather than on the host.
|
|
"""
|
|
|
|
from dataclasses import replace
|
|
|
|
import torch
|
|
|
|
from vllm.config import VllmConfig
|
|
from vllm.logger import init_logger
|
|
from vllm.model_executor.layers.mamba.checkpoint import (
|
|
MambaPrefillCheckpointMetadata,
|
|
)
|
|
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
|
from vllm.triton_utils import tl, triton
|
|
from vllm.utils.math_utils import next_power_of_2
|
|
from vllm.v1.attention.backend import max_decode_query_len
|
|
from vllm.v1.attention.backends.gdn_attn import (
|
|
GDNAttentionBackend,
|
|
GDNAttentionMetadata,
|
|
GDNAttentionMetadataBuilder,
|
|
)
|
|
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
|
|
from vllm.v1.kv_cache_interface import KVCacheSpec
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
_BLOCK_T = 256
|
|
_MIN_BLOCK_N = 128
|
|
|
|
|
|
@triton.jit(do_not_specialize=["N"])
|
|
def _chunk_metadata_kernel(
|
|
cu_seqlens,
|
|
chunk_indices,
|
|
chunk_offsets,
|
|
N,
|
|
BT: tl.constexpr,
|
|
BLOCK_N: tl.constexpr,
|
|
BLOCK_T: tl.constexpr,
|
|
):
|
|
i_n = tl.program_id(0)
|
|
offs_n = tl.arange(0, BLOCK_N)
|
|
is_seq = offs_n < N
|
|
bos = tl.load(cu_seqlens + offs_n, mask=is_seq, other=0).to(tl.int32)
|
|
eos = tl.load(cu_seqlens + offs_n + 1, mask=is_seq, other=0).to(tl.int32)
|
|
nt = tl.where(is_seq, tl.cdiv(eos - bos, BT), 0)
|
|
|
|
base = tl.sum(tl.where(offs_n < i_n, nt, 0))
|
|
num_chunks = tl.sum(tl.where(offs_n == i_n, nt, 0))
|
|
|
|
tl.store(chunk_offsets + i_n, base)
|
|
if i_n == 0:
|
|
tl.store(chunk_offsets + N, tl.sum(nt))
|
|
|
|
for t0 in range(0, num_chunks, BLOCK_T):
|
|
offs_t = t0 + tl.arange(0, BLOCK_T)
|
|
mask_t = offs_t < num_chunks
|
|
row = (base + offs_t) * 2
|
|
tl.store(chunk_indices + row, tl.full([BLOCK_T], i_n, tl.int32), mask=mask_t)
|
|
tl.store(chunk_indices + row + 1, offs_t.to(tl.int32), mask=mask_t)
|
|
|
|
|
|
def prepare_chunk_metadata_device(
|
|
cu_seqlens: torch.Tensor,
|
|
cu_seqlens_cpu: torch.Tensor,
|
|
chunk_size: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Build FLA chunk metadata on device, with no host<->device transfer."""
|
|
num_seqs = cu_seqlens_cpu.numel() - 1
|
|
seq_lens = cu_seqlens_cpu[1:] - cu_seqlens_cpu[:-1]
|
|
num_chunks = int(((seq_lens + chunk_size - 1) // chunk_size).sum())
|
|
|
|
chunk_indices = torch.empty(
|
|
num_chunks, 2, dtype=cu_seqlens.dtype, device=cu_seqlens.device
|
|
)
|
|
chunk_offsets = torch.empty(
|
|
num_seqs + 1, dtype=torch.int64, device=cu_seqlens.device
|
|
)
|
|
_chunk_metadata_kernel[(num_seqs,)](
|
|
cu_seqlens,
|
|
chunk_indices,
|
|
chunk_offsets,
|
|
num_seqs,
|
|
BT=chunk_size,
|
|
BLOCK_N=max(_MIN_BLOCK_N, next_power_of_2(num_seqs + 1)),
|
|
BLOCK_T=_BLOCK_T,
|
|
num_warps=4,
|
|
)
|
|
return chunk_indices, chunk_offsets
|
|
|
|
|
|
def _adaptive_verification_enabled(vllm_config: VllmConfig) -> bool:
|
|
speculative_config = vllm_config.speculative_config
|
|
return speculative_config is not None and bool(
|
|
getattr(speculative_config, "enable_adaptive_verification", False)
|
|
)
|
|
|
|
|
|
class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
|
@classmethod
|
|
def get_varlen_cudagraph_max_query_len(
|
|
cls: type["KimiK3ROCmKDAMetadataBuilder"],
|
|
vllm_config: VllmConfig,
|
|
kv_cache_spec: KVCacheSpec,
|
|
) -> int | None:
|
|
if not _adaptive_verification_enabled(vllm_config):
|
|
return None
|
|
return max_decode_query_len(vllm_config)
|
|
|
|
def build( # type: ignore[override]
|
|
self,
|
|
common_prefix_len: int,
|
|
common_attn_metadata: CommonAttentionMetadata,
|
|
num_accepted_tokens: torch.Tensor | None = None,
|
|
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
|
|
fast_build: bool = False,
|
|
) -> GDNAttentionMetadata:
|
|
attn_metadata = super().build(
|
|
common_prefix_len,
|
|
common_attn_metadata,
|
|
num_accepted_tokens,
|
|
num_decode_draft_tokens_cpu,
|
|
fast_build,
|
|
)
|
|
if (
|
|
attn_metadata.uniform_spec_sequence_length is not None
|
|
and _adaptive_verification_enabled(self.vllm_config)
|
|
):
|
|
attn_metadata = replace(attn_metadata, uniform_spec_sequence_length=None)
|
|
if attn_metadata.checkpoint is None:
|
|
return attn_metadata
|
|
# The shared builder keys checkpoint rows on every request; the ROCm
|
|
# prefill path indexes them by prefill_query_start_loc, so rebuild with
|
|
# those rows.
|
|
return replace(
|
|
attn_metadata,
|
|
checkpoint=self._build_checkpoint_metadata(
|
|
common_attn_metadata, attn_metadata, num_decode_draft_tokens_cpu
|
|
),
|
|
)
|
|
|
|
def _build_checkpoint_metadata(
|
|
self,
|
|
m: CommonAttentionMetadata,
|
|
attn_metadata: GDNAttentionMetadata,
|
|
num_decode_draft_tokens_cpu: torch.Tensor | None,
|
|
) -> MambaPrefillCheckpointMetadata | None:
|
|
if attn_metadata.num_prefills == 0:
|
|
return None
|
|
# request_rows must line up with prefill_query_start_loc, either the
|
|
# prefill tail of a decode-first batch, or every non-spec row
|
|
# including padding
|
|
num_decodes = attn_metadata.num_decodes
|
|
request_rows = list(
|
|
range(num_decodes, num_decodes + attn_metadata.num_prefills)
|
|
)
|
|
if attn_metadata.spec_sequence_masks is not None:
|
|
assert num_decode_draft_tokens_cpu is not None
|
|
request_rows = (
|
|
(num_decode_draft_tokens_cpu < 0).nonzero().flatten().tolist()
|
|
)
|
|
return self.checkpoint_builder.build(m, request_rows)
|
|
|
|
def _build_chunk_metadata(
|
|
self,
|
|
prefill_query_start_loc: torch.Tensor,
|
|
prefill_query_start_loc_cpu: torch.Tensor,
|
|
device: torch.device,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
return prepare_chunk_metadata_device(
|
|
prefill_query_start_loc,
|
|
prefill_query_start_loc_cpu,
|
|
FLA_CHUNK_SIZE,
|
|
)
|
|
|
|
|
|
class KimiK3ROCmKDABackend(GDNAttentionBackend):
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "KIMI_K3_KDA_ROCM"
|
|
|
|
@staticmethod
|
|
def get_builder_cls() -> type[KimiK3ROCmKDAMetadataBuilder]:
|
|
return KimiK3ROCmKDAMetadataBuilder
|
|
|
|
@classmethod
|
|
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
|
|
return True
|