[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,267 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Attention layer with FlashAttention."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
canonicalize_singleton_dim_strides,
|
||||
is_quantized_kv_cache,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
get_flash_attn_version,
|
||||
is_flash_attn_varlen_func_available,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
||||
triton_reshape_and_cache_flash_diffkv,
|
||||
)
|
||||
|
||||
if is_flash_attn_varlen_func_available():
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func
|
||||
|
||||
from .flash_attn import (
|
||||
FlashAttentionBackend,
|
||||
FlashAttentionImpl,
|
||||
FlashAttentionMetadata,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class FlashAttentionDiffKVBackend(FlashAttentionBackend):
|
||||
# Default to 128 for this backend
|
||||
head_size_v: int = 128
|
||||
|
||||
@classmethod
|
||||
def set_head_size_v(cls, head_size_v: int) -> None:
|
||||
cls.head_size_v = head_size_v
|
||||
|
||||
@classmethod
|
||||
def is_supported_on_current_device(
|
||||
cls,
|
||||
head_size: int,
|
||||
head_size_v: int,
|
||||
has_sinks: bool,
|
||||
) -> bool:
|
||||
"""Check whether FA3/4 with this DiffKV config is usable here.
|
||||
|
||||
DiffKV (hdim_qk != hdim_v) requires FA3 or FA4
|
||||
"""
|
||||
if not is_flash_attn_varlen_func_available():
|
||||
return False
|
||||
try:
|
||||
version = get_flash_attn_version(
|
||||
requires_alibi=False,
|
||||
head_size=head_size,
|
||||
head_size_v=head_size_v,
|
||||
has_sinks=has_sinks,
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
return version in (3, 4)
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASH_ATTN_DIFFKV"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["FlashAttentionImpl"]:
|
||||
return FlashAttentionDiffKVImpl
|
||||
|
||||
|
||||
class FlashAttentionDiffKVImpl(FlashAttentionImpl):
|
||||
vllm_flash_attn_version: int | None
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
# Re-derive the FA version with diff-kv context so that
|
||||
# get_flash_attn_version can apply the FA3 -> FA4 upgrade rule
|
||||
# for sinks + hdim != hdim_v.
|
||||
self.vllm_flash_attn_version = get_flash_attn_version(
|
||||
requires_alibi=self.alibi_slopes is not None,
|
||||
head_size=self.head_size,
|
||||
head_size_v=FlashAttentionDiffKVBackend.head_size_v,
|
||||
has_sinks=self.sinks is not None,
|
||||
)
|
||||
self.fa4_hd256 = False
|
||||
|
||||
def do_kv_cache_update(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
if self.attn_type in (AttentionType.ENCODER_ONLY, AttentionType.ENCODER):
|
||||
# For encoder attention,
|
||||
# we use direct Q, K, V tensors without caching
|
||||
return
|
||||
|
||||
# DiffKV packs K and V into a single tensor along the last dim:
|
||||
# kv_cache shape: [num_blocks, block_size, num_kv_heads,
|
||||
# head_size_k + head_size_v]
|
||||
# (B, H, N, C) -> (B, N, H, C) for kernel compatibility.
|
||||
triton_reshape_and_cache_flash_diffkv(
|
||||
key,
|
||||
value,
|
||||
kv_cache.transpose(1, 2),
|
||||
slot_mapping,
|
||||
self.kv_cache_dtype,
|
||||
layer._k_scale,
|
||||
layer._v_scale,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
attn_metadata: FlashAttentionMetadata,
|
||||
output: torch.Tensor,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
output_block_scale: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Forward pass with FlashAttention.
|
||||
|
||||
Args:
|
||||
layer: The attention layer, providing the q/k/v quantization scales.
|
||||
query: shape = [num_tokens, num_heads, head_size]
|
||||
key: shape = [num_tokens, num_kv_heads, head_size]
|
||||
value: shape = [num_tokens, num_kv_heads, head_size_v]
|
||||
kv_cache: shape =
|
||||
[num_blocks, block_size, num_kv_heads, head_size + head_size_v]
|
||||
attn_metadata: Metadata for attention.
|
||||
output: Tensor that the attention result is written into.
|
||||
output_scale: Scale for fused output quantization; not supported
|
||||
by this backend.
|
||||
output_block_scale: Block scale for fused output quantization;
|
||||
not supported by this backend.
|
||||
|
||||
Returns:
|
||||
shape = [num_tokens, num_heads * head_size_v]
|
||||
NOTE: FP8 quantization, flash-attn expect the size of
|
||||
{q,k,v}_descale to be (num_sequences, num_kv_heads).
|
||||
We use torch's .expand() to avoid duplicating values
|
||||
|
||||
"""
|
||||
assert self.vllm_flash_attn_version is not None, (
|
||||
"FlashAttention version not detected."
|
||||
)
|
||||
|
||||
if output_scale is not None or output_block_scale is not None:
|
||||
raise NotImplementedError(
|
||||
"fused output quantization is not yet supported for FlashAttentionImpl"
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
# Profiling run.
|
||||
return output.fill_(0)
|
||||
|
||||
attn_type = self.attn_type
|
||||
|
||||
# IMPORTANT!
|
||||
# NOTE(woosuk): With piece-wise CUDA graphs, this method is executed in
|
||||
# eager-mode PyTorch. Thus, we need to be careful about any CPU overhead
|
||||
# in this method. For example, `view` and `slice` (or `[:n]`) operations
|
||||
# are surprisingly slow even in the case they do not invoke any GPU ops.
|
||||
# Minimize the PyTorch ops in this method as much as possible.
|
||||
# Whenever making a change in this method, please benchmark the
|
||||
# performance to make sure it does not introduce any overhead.
|
||||
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
|
||||
# Handle encoder attention differently - no KV cache needed
|
||||
if attn_type in (AttentionType.ENCODER_ONLY, AttentionType.ENCODER):
|
||||
# For encoder attention,
|
||||
# we use direct Q, K, V tensors without caching
|
||||
return self._forward_encoder_attention(
|
||||
query[:num_actual_tokens],
|
||||
key[:num_actual_tokens],
|
||||
value[:num_actual_tokens],
|
||||
output[:num_actual_tokens],
|
||||
attn_metadata,
|
||||
layer,
|
||||
)
|
||||
|
||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
# Fix degenerate strides on size-1 dims (e.g. num_kv_heads=1 with TP).
|
||||
# FA3/4 on H100+ uses TMA, which requires ≥16-byte stride alignment.
|
||||
# See vllm.utils.torch_utils.canonicalize_singleton_dim_strides.
|
||||
fixed_k = canonicalize_singleton_dim_strides(key_cache)
|
||||
fixed_v = canonicalize_singleton_dim_strides(value_cache)
|
||||
if fixed_k is not key_cache or fixed_v is not value_cache:
|
||||
logger.debug(
|
||||
"Canonicalized degenerate KV cache strides (FlashAttentionDiffKV): "
|
||||
"shape=%s, key strides before=%s after=%s, "
|
||||
"value strides before=%s after=%s",
|
||||
key_cache.shape,
|
||||
key_cache.stride(),
|
||||
fixed_k.stride(),
|
||||
value_cache.stride(),
|
||||
fixed_v.stride(),
|
||||
)
|
||||
key_cache, value_cache = fixed_k, fixed_v
|
||||
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
# queries are quantized in the attention layer
|
||||
key_cache = key_cache.view(current_platform.fp8_dtype())
|
||||
value_cache = value_cache.view(current_platform.fp8_dtype())
|
||||
|
||||
cu_seqlens_q = attn_metadata.query_start_loc
|
||||
seqused_k = attn_metadata.seq_lens
|
||||
max_seqlen_q = attn_metadata.max_query_len
|
||||
max_seqlen_k = attn_metadata.max_seq_len
|
||||
block_table = attn_metadata.block_table
|
||||
scheduler_metadata = attn_metadata.scheduler_metadata
|
||||
|
||||
descale_shape = (cu_seqlens_q.shape[0] - 1, self.num_kv_heads)
|
||||
|
||||
if self.dcp_world_size > 1:
|
||||
self._forward_with_dcp(
|
||||
query[:num_actual_tokens],
|
||||
key[:num_actual_tokens],
|
||||
value[:num_actual_tokens],
|
||||
key_cache,
|
||||
value_cache,
|
||||
output[:num_actual_tokens],
|
||||
attn_metadata,
|
||||
q_descale=layer._q_scale.expand(descale_shape),
|
||||
k_descale=layer._k_scale.expand(descale_shape),
|
||||
v_descale=layer._v_scale.expand(descale_shape),
|
||||
)
|
||||
return output
|
||||
else:
|
||||
sliding_window_size = (
|
||||
list(self.sliding_window) if self.sliding_window is not None else None
|
||||
)
|
||||
flash_attn_varlen_func(
|
||||
q=query[:num_actual_tokens],
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
out=output[:num_actual_tokens],
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
seqused_k=seqused_k,
|
||||
max_seqlen_k=max_seqlen_k,
|
||||
softmax_scale=self.scale,
|
||||
causal=attn_metadata.causal,
|
||||
alibi_slopes=self.alibi_slopes,
|
||||
window_size=sliding_window_size,
|
||||
block_table=block_table,
|
||||
softcap=self.logits_soft_cap,
|
||||
scheduler_metadata=scheduler_metadata,
|
||||
fa_version=self.vllm_flash_attn_version,
|
||||
q_descale=layer._q_scale.expand(descale_shape),
|
||||
k_descale=layer._k_scale.expand(descale_shape),
|
||||
v_descale=layer._v_scale.expand(descale_shape),
|
||||
num_splits=attn_metadata.max_num_splits,
|
||||
s_aux=self.sinks,
|
||||
)
|
||||
return output
|
||||
@@ -1,223 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton attention backend with different K/V head dimensions (DiffKV).
|
||||
|
||||
The KV cache layout is identical to ``FlashAttentionDiffKVBackend``: K and V
|
||||
are packed along the last dim in the logical shape
|
||||
``[num_blocks, num_kv_heads, block_size, head_size_qk + head_size_v]``.
|
||||
"""
|
||||
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.backend import AttentionLayer, AttentionType
|
||||
from vllm.v1.attention.backends.triton_attn import (
|
||||
TritonAttentionBackend,
|
||||
TritonAttentionImpl,
|
||||
TritonAttentionMetadata,
|
||||
TritonAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
||||
triton_reshape_and_cache_flash_diffkv,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_unified_attention_diffkv import (
|
||||
unified_attention_diffkv,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TritonAttentionDiffKVMetadataBuilder(TritonAttentionMetadataBuilder):
|
||||
"""Override the parent's softmax buffer last-dim to head_size_v.
|
||||
|
||||
The parent allocates ``softmax_segm_output`` with last-dim sized to
|
||||
``next_power_of_2(head_size)`` (== Q/K head size). For DiffKV the
|
||||
accumulator and per-segment partial outputs are V-shaped, so we
|
||||
re-allocate with ``next_power_of_2(head_size_v)`` instead.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
layer_names: list[str],
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
):
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
|
||||
head_size_v = TritonAttentionDiffKVBackend.head_size_v
|
||||
head_size_v_padded = next_power_of_2(head_size_v)
|
||||
self.softmax_segm_output = torch.empty(
|
||||
(
|
||||
self.softmax_segm_max.shape[0],
|
||||
self.num_heads_q,
|
||||
self.num_par_softmax_segments,
|
||||
head_size_v_padded,
|
||||
),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
class TritonAttentionDiffKVBackend(TritonAttentionBackend):
|
||||
# V head dim — set per layer via ``set_head_size_v`` before instantiation.
|
||||
head_size_v: int = 128
|
||||
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"bfloat16",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def set_head_size_v(cls, head_size_v: int) -> None:
|
||||
cls.head_size_v = head_size_v
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "TRITON_ATTN_DIFFKV"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["TritonAttentionDiffKVImpl"]:
|
||||
return TritonAttentionDiffKVImpl
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["TritonAttentionDiffKVMetadataBuilder"]:
|
||||
return TritonAttentionDiffKVMetadataBuilder
|
||||
|
||||
@classmethod
|
||||
def supports_head_size(cls, head_size: int) -> bool:
|
||||
# DiffKV K head sizes (e.g. 192 for MiMo-V2.5) need to be allowed.
|
||||
return head_size >= 32
|
||||
|
||||
@classmethod
|
||||
def supports_attn_type(cls, attn_type: str) -> bool:
|
||||
# DiffKV only implements decoder self-attention. Unlike the parent
|
||||
# TritonAttentionBackend (which advertises all types), encoder
|
||||
# attention is not supported, so gate it here at backend selection.
|
||||
return attn_type == AttentionType.DECODER
|
||||
|
||||
|
||||
class TritonAttentionDiffKVImpl(TritonAttentionImpl):
|
||||
"""Triton attention impl for the DiffKV packed KV cache layout."""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
if (
|
||||
is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
and self.kv_cache_dtype
|
||||
not in TritonAttentionDiffKVBackend.supported_kv_cache_dtypes
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"TritonAttentionDiffKVBackend supports only fp8/fp8_e4m3 quantized "
|
||||
f"KV cache (got kv_cache_dtype={self.kv_cache_dtype!r})."
|
||||
)
|
||||
if self.chunk_lookback > -1:
|
||||
raise NotImplementedError(
|
||||
"TritonAttentionDiffKVBackend does not support chunked "
|
||||
"attention with lookback."
|
||||
)
|
||||
|
||||
def do_kv_cache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
# Cache is logical (B, H, N, C); the diffkv reshape kernel expects
|
||||
# (B, N, H, C).
|
||||
triton_reshape_and_cache_flash_diffkv(
|
||||
key,
|
||||
value,
|
||||
kv_cache.transpose(1, 2),
|
||||
slot_mapping,
|
||||
self.kv_cache_dtype,
|
||||
layer._k_scale,
|
||||
layer._v_scale,
|
||||
)
|
||||
|
||||
def fused_rope_kvcache_supported(self):
|
||||
# The fused rope+cache path assumes the standard 2-tensor layout.
|
||||
return False
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
attn_metadata: TritonAttentionMetadata,
|
||||
output: torch.Tensor,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
output_block_scale: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Forward pass.
|
||||
|
||||
Shapes:
|
||||
query: [num_tokens, num_heads, head_size_qk]
|
||||
key: [num_tokens, num_kv_heads, head_size_qk]
|
||||
value: [num_tokens, num_kv_heads, head_size_v]
|
||||
kv_cache: [num_blocks, num_kv_heads, block_size,
|
||||
head_size_qk + head_size_v]
|
||||
output: [num_tokens, num_heads, head_size_v]
|
||||
"""
|
||||
if output_scale is not None or output_block_scale is not None:
|
||||
raise NotImplementedError(
|
||||
"fused output quantization is not supported for "
|
||||
"TritonAttentionDiffKVImpl"
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
return output.fill_(0)
|
||||
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
head_size_qk = self.head_size
|
||||
head_size_v = TritonAttentionDiffKVBackend.head_size_v
|
||||
|
||||
# Triton DiffKV kernels consume (B, N, H, D) cache views.
|
||||
kv_cache = kv_cache.transpose(1, 2)
|
||||
quantized = is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
if quantized:
|
||||
kv_cache = kv_cache.view(self.fp8_dtype)
|
||||
key_cache = kv_cache[..., :head_size_qk]
|
||||
value_cache = kv_cache[..., head_size_qk : head_size_qk + head_size_v]
|
||||
|
||||
unified_attention_diffkv(
|
||||
q=query[:num_actual_tokens],
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
out=output[:num_actual_tokens],
|
||||
cu_seqlens_q=attn_metadata.query_start_loc,
|
||||
seqused_k=attn_metadata.seq_lens,
|
||||
softmax_scale=self.scale,
|
||||
q_descale=(
|
||||
layer._q_scale if quantized and query.dtype == self.fp8_dtype else None
|
||||
),
|
||||
k_descale=layer._k_scale if quantized else None,
|
||||
v_descale=layer._v_scale if quantized else None,
|
||||
causal=True,
|
||||
alibi_slopes=self.alibi_slopes,
|
||||
use_alibi_sqrt=self.use_alibi_sqrt,
|
||||
window_size=self.sliding_window,
|
||||
block_table=attn_metadata.block_table,
|
||||
softcap=self.logits_soft_cap,
|
||||
sinks=self.sinks,
|
||||
max_seqlen_q=attn_metadata.max_query_len,
|
||||
seq_threshold_3D=attn_metadata.seq_threshold_3D,
|
||||
num_par_softmax_segments=attn_metadata.num_par_softmax_segments,
|
||||
softmax_segm_output=attn_metadata.softmax_segm_output,
|
||||
softmax_segm_max=attn_metadata.softmax_segm_max,
|
||||
softmax_segm_expsum=attn_metadata.softmax_segm_expsum,
|
||||
)
|
||||
return output
|
||||
@@ -1,65 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton image-mask and FlashAttention causal attention composite."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.composite import (
|
||||
MMPrefixAttentionRouting,
|
||||
create_composite_attention_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
|
||||
|
||||
|
||||
class _FABackend(FlashAttentionBackend):
|
||||
@classmethod
|
||||
def supports_combination(
|
||||
cls,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType | None,
|
||||
block_size: int | None,
|
||||
use_mla: bool,
|
||||
has_sink: bool,
|
||||
use_sparse: bool,
|
||||
use_mm_prefix: bool,
|
||||
device_capability: DeviceCapability,
|
||||
) -> str | None:
|
||||
reason = super().supports_combination(
|
||||
head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
block_size,
|
||||
use_mla,
|
||||
has_sink,
|
||||
use_sparse,
|
||||
use_mm_prefix,
|
||||
device_capability,
|
||||
)
|
||||
if reason is not None:
|
||||
return reason
|
||||
fa_version = get_flash_attn_version(
|
||||
head_size=head_size,
|
||||
has_sinks=has_sink,
|
||||
kv_cache_block_size=block_size,
|
||||
supports_fa4_hd256=True,
|
||||
)
|
||||
if fa_version not in (3, 4):
|
||||
return "causal route requires FlashAttention v3 or v4"
|
||||
return None
|
||||
|
||||
|
||||
TritonFlashAttentionBackend = create_composite_attention_backend(
|
||||
TritonAttentionBackend,
|
||||
_FABackend,
|
||||
name="TritonFlashAttentionBackend",
|
||||
backend_name="TRITON_FLASH_ATTN",
|
||||
module=__name__,
|
||||
routing_policy=MMPrefixAttentionRouting,
|
||||
head_sizes=(256, 512),
|
||||
device_major=9,
|
||||
)
|
||||
@@ -1,527 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Authors:
|
||||
# - Burkhard Ringlein <ngl@zurich.ibm.com>
|
||||
# - Jan van Lunteren <jvl@zurich.ibm.com>
|
||||
# - Chih-Chieh Yang <chih.chieh.yang@ibm.com>
|
||||
# - Thomas Parnell <tpa@zurich.ibm.com>
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from .prefix_prefill import context_attention_fwd
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
float8_info = torch.finfo(current_platform.fp8_dtype())
|
||||
|
||||
|
||||
def has_native_kv_cache_layout(
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
) -> bool:
|
||||
"""Return whether KV cache blocks can use the native ROCm pairing.
|
||||
|
||||
The native reshape_and_cache writer assumes packed blocks. If cache update
|
||||
needs reshape_and_cache_flash for a stride-padded hybrid layout, decode
|
||||
should use the matching Triton path too.
|
||||
"""
|
||||
return (
|
||||
key_cache.stride(0) == key_cache.shape[1:].numel()
|
||||
and value_cache.stride(0) == value_cache.shape[1:].numel()
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def cdiv_fn(x, y):
|
||||
return (x + y - 1) // y
|
||||
|
||||
|
||||
@triton.jit
|
||||
def kernel_paged_attention_2d(
|
||||
output_ptr, # [num_tokens, num_query_heads, head_size]
|
||||
query_ptr, # [num_tokens, num_query_heads, head_size]
|
||||
key_cache_ptr, # [num_blks, num_kv_heads, head_size // x, blk_size, x]
|
||||
value_cache_ptr, # [num_blks, num_kv_heads, head_size, blk_size]
|
||||
sink_ptr, # [num_query_heads]
|
||||
block_tables_ptr, # [num_seqs, max_num_blocks_per_seq]
|
||||
seq_lens_ptr, # [num_seqs]
|
||||
alibi_slopes_ptr, # [num_query_heads]
|
||||
scale, # float32
|
||||
k_scale, # float32
|
||||
v_scale, # float32
|
||||
out_scale_inv,
|
||||
num_query_heads: tl.constexpr, # int
|
||||
num_queries_per_kv: tl.constexpr, # int
|
||||
num_queries_per_kv_padded: tl.constexpr, # int
|
||||
block_table_stride: tl.int64, # int
|
||||
query_stride_0: tl.int64, # int
|
||||
query_stride_1: tl.int64, # int, should be equal to head_size
|
||||
output_stride_0: tl.int64, # int
|
||||
output_stride_1: tl.int64, # int, should be equal to head_size
|
||||
BLOCK_SIZE: tl.constexpr, # int
|
||||
PHYSICAL_BLOCK_SIZE: tl.constexpr, # int
|
||||
HEAD_SIZE: tl.constexpr, # int
|
||||
HEAD_SIZE_PADDED: tl.constexpr, # int, must be power of 2
|
||||
USE_ALIBI_SLOPES: tl.constexpr, # bool
|
||||
SLIDING_WINDOW: tl.constexpr, # int
|
||||
x: tl.constexpr, # int
|
||||
stride_k_cache_0: tl.int64, # int
|
||||
stride_k_cache_1: tl.int64, # int
|
||||
stride_k_cache_2: tl.int64, # int
|
||||
stride_k_cache_3: tl.int64, # int
|
||||
stride_k_cache_4: tl.int64, # int
|
||||
stride_v_cache_0: tl.int64, # int
|
||||
stride_v_cache_1: tl.int64, # int
|
||||
stride_v_cache_2: tl.int64, # int
|
||||
stride_v_cache_3: tl.int64, # int
|
||||
filter_by_query_len: tl.constexpr, # bool
|
||||
query_start_len_ptr, # [num_seqs+1]
|
||||
USE_SINKS: tl.constexpr, # bool
|
||||
USE_FP8: tl.constexpr,
|
||||
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
|
||||
FP8_MIN: tl.constexpr = float8_info.min,
|
||||
FP8_MAX: tl.constexpr = float8_info.max,
|
||||
):
|
||||
seq_idx = tl.program_id(0)
|
||||
kv_head_idx = tl.program_id(1)
|
||||
|
||||
if filter_by_query_len:
|
||||
cur_batch_in_all_start_index = tl.load(query_start_len_ptr + seq_idx)
|
||||
cur_batch_in_all_stop_index = tl.load(query_start_len_ptr + seq_idx + 1)
|
||||
cur_batch_query_len = cur_batch_in_all_stop_index - cur_batch_in_all_start_index
|
||||
if cur_batch_query_len > 1:
|
||||
return
|
||||
else:
|
||||
cur_batch_in_all_start_index = seq_idx
|
||||
|
||||
query_head_idx = kv_head_idx * num_queries_per_kv + tl.arange(
|
||||
0, num_queries_per_kv_padded
|
||||
)
|
||||
|
||||
query_offset = (
|
||||
cur_batch_in_all_start_index * query_stride_0
|
||||
+ query_head_idx[:, None] * query_stride_1
|
||||
)
|
||||
|
||||
head_mask = query_head_idx < (kv_head_idx + 1) * num_queries_per_kv
|
||||
head_mask = head_mask & (query_head_idx < num_query_heads)
|
||||
|
||||
dim_mask = tl.where(tl.arange(0, HEAD_SIZE_PADDED) < HEAD_SIZE, 1, 0).to(tl.int1)
|
||||
|
||||
# Q : (num_queries_per_kv, HEAD_SIZE,)
|
||||
Q = tl.load(
|
||||
query_ptr + query_offset + tl.arange(0, HEAD_SIZE_PADDED)[None, :],
|
||||
mask=dim_mask[None, :] & head_mask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
block_table_offset = seq_idx * block_table_stride
|
||||
|
||||
if not USE_SINKS:
|
||||
M = tl.full([num_queries_per_kv_padded], float("-inf"), dtype=tl.float32)
|
||||
L = tl.zeros([num_queries_per_kv_padded], dtype=tl.float32)
|
||||
else:
|
||||
M = tl.load(
|
||||
sink_ptr + query_head_idx,
|
||||
mask=head_mask,
|
||||
other=float("-inf"),
|
||||
).to(dtype=tl.float32)
|
||||
L = tl.where(float("-inf") < M, 1.0, 0.0)
|
||||
|
||||
acc = tl.zeros([num_queries_per_kv_padded, HEAD_SIZE_PADDED], dtype=tl.float32)
|
||||
|
||||
# sequence len for this particular sequence
|
||||
seq_len = tl.load(seq_lens_ptr + seq_idx)
|
||||
|
||||
# alibi slope for this head
|
||||
if USE_ALIBI_SLOPES:
|
||||
alibi_slope = tl.load(
|
||||
alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||
)
|
||||
|
||||
num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||
|
||||
offs_n = tl.arange(0, BLOCK_SIZE)
|
||||
offs_d = tl.arange(0, HEAD_SIZE_PADDED)
|
||||
# iterate through tiles
|
||||
for j in range(0, num_blocks):
|
||||
start_n = j * BLOCK_SIZE
|
||||
# Calculate the logical location within a non-standard physical block,
|
||||
# such as 544 in Qwen/Qwen3-Next-80B-A3B-Thinking.
|
||||
# Supports non-contiguous mapping
|
||||
# from logical blocks to physical blocks
|
||||
abs_token_idx = start_n + offs_n
|
||||
l_block_idx = abs_token_idx // PHYSICAL_BLOCK_SIZE
|
||||
# Vectorized loading of physical block IDs
|
||||
p_block_idx = tl.load(block_tables_ptr + block_table_offset + l_block_idx)
|
||||
internal_offsets = abs_token_idx % PHYSICAL_BLOCK_SIZE
|
||||
|
||||
# 5D addressing logic of K
|
||||
k_offset = (
|
||||
p_block_idx[None, :] * stride_k_cache_0
|
||||
+ kv_head_idx * stride_k_cache_1
|
||||
+ (offs_d[:, None] // x) * stride_k_cache_2
|
||||
+ internal_offsets[None, :] * stride_k_cache_3
|
||||
+ (offs_d[:, None] % x) * stride_k_cache_4
|
||||
)
|
||||
|
||||
# V addressing: standard 4D or interleaved 5D
|
||||
if INTERLEAVED_V_PACK_FACTOR > 0:
|
||||
v_offset = (
|
||||
p_block_idx[:, None] * stride_v_cache_0
|
||||
+ kv_head_idx * stride_v_cache_1
|
||||
+ (internal_offsets[:, None] // INTERLEAVED_V_PACK_FACTOR)
|
||||
* HEAD_SIZE
|
||||
* INTERLEAVED_V_PACK_FACTOR
|
||||
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
|
||||
+ (internal_offsets[:, None] % INTERLEAVED_V_PACK_FACTOR)
|
||||
)
|
||||
else:
|
||||
v_offset = (
|
||||
p_block_idx[:, None] * stride_v_cache_0
|
||||
+ kv_head_idx * stride_v_cache_1
|
||||
+ offs_d[None, :] * stride_v_cache_2
|
||||
+ internal_offsets[:, None] * stride_v_cache_3
|
||||
)
|
||||
|
||||
# Only the final tile can straddle seq_len. Slots >= seq_len are
|
||||
# unwritten KV cache that may hold NaN/garbage; they are score-masked
|
||||
# below, but 0 * NaN = NaN would still poison the output, so mask them
|
||||
# out of the K/V loads too. Earlier tiles are fully written, so they
|
||||
# use the cheaper token-uniform dim_mask (matching the pre-0.25.0 fast
|
||||
# path) and skip the per-token predicate entirely.
|
||||
# K : (HEAD_SIZE, BLOCK_SIZE), V : (BLOCK_SIZE, HEAD_SIZE)
|
||||
if j == num_blocks - 1:
|
||||
kv_load_mask = abs_token_idx < seq_len
|
||||
K_load = tl.load(
|
||||
key_cache_ptr + k_offset,
|
||||
mask=dim_mask[:, None] & kv_load_mask[None, :],
|
||||
other=0.0,
|
||||
eviction_policy="evict_last",
|
||||
)
|
||||
V_load = tl.load(
|
||||
value_cache_ptr + v_offset,
|
||||
mask=dim_mask[None, :] & kv_load_mask[:, None],
|
||||
other=0.0,
|
||||
eviction_policy="evict_last",
|
||||
)
|
||||
else:
|
||||
K_load = tl.load(
|
||||
key_cache_ptr + k_offset,
|
||||
mask=dim_mask[:, None],
|
||||
other=0.0,
|
||||
eviction_policy="evict_last",
|
||||
)
|
||||
V_load = tl.load(
|
||||
value_cache_ptr + v_offset,
|
||||
mask=dim_mask[None, :],
|
||||
other=0.0,
|
||||
eviction_policy="evict_last",
|
||||
)
|
||||
|
||||
if K_load.dtype.is_fp8():
|
||||
K = (K_load.to(tl.float32) * tl.load(k_scale)).to(Q.dtype)
|
||||
else:
|
||||
K = K_load
|
||||
|
||||
if V_load.dtype.is_fp8():
|
||||
V = (V_load.to(tl.float32) * tl.load(v_scale)).to(Q.dtype)
|
||||
else:
|
||||
V = V_load
|
||||
|
||||
seq_offset = j * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
boundary = tl.full([BLOCK_SIZE], seq_len, dtype=tl.int32)
|
||||
seq_mask = seq_offset[None, :] < boundary
|
||||
|
||||
# First calculate the dot, then apply the mask.
|
||||
qk = scale * tl.dot(Q, K)
|
||||
S = tl.where(head_mask[:, None] & seq_mask, qk, float("-inf"))
|
||||
|
||||
context_len = seq_len - 1
|
||||
|
||||
if SLIDING_WINDOW > 0:
|
||||
S = tl.where((context_len - seq_offset) < SLIDING_WINDOW, S, -10000)
|
||||
|
||||
if USE_ALIBI_SLOPES:
|
||||
S += alibi_slope[:, None] * (seq_offset - context_len)
|
||||
|
||||
# compute running maximum
|
||||
# m_j : (num_queries_per_kv,)
|
||||
m_j = tl.maximum(M, tl.max(S, axis=1))
|
||||
|
||||
# P : (num_queries_per_kv, BLOCK_SIZE,)
|
||||
p = tl.exp(S - m_j[:, None])
|
||||
p = tl.where(m_j[:, None] == float("-inf"), 0.0, p)
|
||||
|
||||
# l_j : (num_queries_per_kv,)
|
||||
l_j = tl.sum(p, axis=1)
|
||||
|
||||
# alpha : (num_queries_per_kv, )
|
||||
alpha = tl.exp(M - m_j)
|
||||
alpha = tl.where(float("-inf") == M, 0.0, alpha)
|
||||
|
||||
# acc : (num_queries_per_kv, BLOCK_SIZE,)
|
||||
acc = acc * alpha[:, None]
|
||||
|
||||
# update constants
|
||||
L = L * alpha + l_j
|
||||
M = m_j
|
||||
|
||||
# acc : (num_queries_per_kv, BLOCK_SIZE,)
|
||||
acc += tl.dot(p.to(V.dtype), V)
|
||||
|
||||
# epilogue
|
||||
acc = acc / (L[:, None] + 1e-10)
|
||||
if USE_FP8:
|
||||
acc = acc * tl.load(out_scale_inv)
|
||||
acc = tl.clamp(acc, FP8_MIN, FP8_MAX)
|
||||
|
||||
output_offset = (
|
||||
cur_batch_in_all_start_index * output_stride_0
|
||||
+ query_head_idx * output_stride_1
|
||||
)
|
||||
|
||||
tl.store(
|
||||
output_ptr + output_offset[:, None] + tl.arange(0, HEAD_SIZE_PADDED)[None, :],
|
||||
acc,
|
||||
mask=dim_mask[None, :] & head_mask[:, None],
|
||||
)
|
||||
|
||||
|
||||
def chunked_prefill_paged_decode(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
output,
|
||||
kv_cache_dtype,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
max_seq_len,
|
||||
max_query_len,
|
||||
k_scale,
|
||||
v_scale,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
sm_scale=None,
|
||||
output_scale=None,
|
||||
# Optional tensor for sinks
|
||||
sinks=None,
|
||||
is_block_table_ptr: bool = False,
|
||||
use_interleaved_v_cache: bool = False,
|
||||
causal: bool = True,
|
||||
):
|
||||
if sm_scale is None:
|
||||
sm_scale = 1.0 / (query.shape[2] ** 0.5)
|
||||
|
||||
use_alibi_slopes = alibi_slopes is not None
|
||||
|
||||
if sliding_window is None or sliding_window <= 0:
|
||||
sliding_window = 0
|
||||
|
||||
interleaved_v_pack_factor = (
|
||||
(16 // value_cache.element_size()) if use_interleaved_v_cache else 0
|
||||
)
|
||||
|
||||
if max_query_len > 1:
|
||||
context_attention_fwd(
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
o=output,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
k_cache=key_cache,
|
||||
v_cache=value_cache,
|
||||
b_loc=block_table,
|
||||
b_start_loc=query_start_loc,
|
||||
b_seq_len=seq_lens,
|
||||
max_seq_len=max_seq_len,
|
||||
max_input_len=max_query_len,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
alibi_slopes=alibi_slopes,
|
||||
sliding_window=sliding_window,
|
||||
sm_scale=sm_scale,
|
||||
skip_decode=True,
|
||||
fp8_out_scale=output_scale,
|
||||
sinks=sinks,
|
||||
interleaved_v_pack_factor=interleaved_v_pack_factor,
|
||||
causal=causal,
|
||||
)
|
||||
|
||||
block_size = value_cache.shape[3]
|
||||
num_seqs = len(seq_lens)
|
||||
num_query_heads = query.shape[1]
|
||||
# key may be None in cross-attention decode (already cached from encoder)
|
||||
num_kv_heads = key.shape[1] if key is not None else key_cache.shape[1]
|
||||
num_queries_per_kv = num_query_heads // num_kv_heads
|
||||
head_size = query.shape[2]
|
||||
|
||||
# Conversion of FP8 Tensor from uint8 storage to
|
||||
# appropriate torch.dtype for interpretation by Triton
|
||||
if "fp8" in kv_cache_dtype:
|
||||
assert key_cache.dtype in [torch.uint8, current_platform.fp8_dtype()]
|
||||
assert value_cache.dtype in [torch.uint8, current_platform.fp8_dtype()]
|
||||
|
||||
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
||||
target_dtype = current_platform.fp8_dtype()
|
||||
elif kv_cache_dtype == "fp8_e5m2":
|
||||
target_dtype = torch.float8_e5m2
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported FP8 kv_cache_dtype {kv_cache_dtype}: "
|
||||
f"should be one of 'fp8', 'fp8_e4m3', 'fp8_e5m2'."
|
||||
)
|
||||
|
||||
key_cache = key_cache.view(target_dtype)
|
||||
value_cache = value_cache.view(target_dtype)
|
||||
|
||||
num_queries_per_kv_padded = max(triton.next_power_of_2(num_queries_per_kv), 16)
|
||||
|
||||
from vllm.platforms.rocm import use_rocm_custom_paged_attention
|
||||
|
||||
use_custom = use_rocm_custom_paged_attention(
|
||||
query.dtype,
|
||||
head_size,
|
||||
block_size,
|
||||
num_queries_per_kv,
|
||||
max_seq_len,
|
||||
sliding_window,
|
||||
kv_cache_dtype,
|
||||
alibi_slopes,
|
||||
sinks,
|
||||
)
|
||||
has_native_layout = has_native_kv_cache_layout(key_cache, value_cache)
|
||||
# Force Triton for non-standard blocks like Qwen3's 544 and for
|
||||
# stride-padded hybrid layouts. The latter use reshape_and_cache_flash
|
||||
# during cache update, so keep decode on the matching stride-aware path.
|
||||
is_pow2 = block_size > 0 and (block_size & (block_size - 1) == 0)
|
||||
if not is_pow2 or not has_native_layout:
|
||||
use_custom = False
|
||||
|
||||
if use_custom:
|
||||
_PARTITION_SIZE_ROCM = 256
|
||||
max_num_partitions = (
|
||||
max_seq_len + _PARTITION_SIZE_ROCM - 1
|
||||
) // _PARTITION_SIZE_ROCM
|
||||
assert _PARTITION_SIZE_ROCM % block_size == 0
|
||||
total_num_seq = block_table.shape[0]
|
||||
tmp_output = torch.empty(
|
||||
size=(total_num_seq, num_query_heads, max_num_partitions, head_size),
|
||||
dtype=query.dtype,
|
||||
device=output.device,
|
||||
)
|
||||
exp_sums = torch.empty(
|
||||
size=(total_num_seq, num_query_heads, max_num_partitions),
|
||||
dtype=torch.float32,
|
||||
device=output.device,
|
||||
)
|
||||
max_logits = torch.empty_like(exp_sums)
|
||||
|
||||
ops.paged_attention_rocm(
|
||||
output,
|
||||
exp_sums,
|
||||
max_logits,
|
||||
tmp_output,
|
||||
query,
|
||||
key_cache,
|
||||
value_cache,
|
||||
num_kv_heads,
|
||||
scale=sm_scale,
|
||||
block_tables=block_table,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
block_size=block_size,
|
||||
max_seq_len=max_seq_len,
|
||||
alibi_slopes=alibi_slopes,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
fp8_out_scale=output_scale,
|
||||
use_interleaved_v_cache=use_interleaved_v_cache,
|
||||
)
|
||||
else:
|
||||
logger.warning_once(
|
||||
"Cannot use ROCm custom paged attention kernel,"
|
||||
" falling back to Triton implementation."
|
||||
)
|
||||
real_block_size = value_cache.shape[3]
|
||||
# The standard model directly uses the original block_size.
|
||||
# Non-standard 544 uses 32 to accommodate integer division logic.
|
||||
# Cap at 128 to avoid exceeding GPU shared memory limits
|
||||
# (e.g. hybrid Mamba models inflate block_size to 2048).
|
||||
# The kernel handles TRITON_BLOCK_SIZE != PHYSICAL_BLOCK_SIZE
|
||||
# via the l_block_idx/internal_offsets addressing logic.
|
||||
MAX_TRITON_BLOCK_SIZE = 128
|
||||
TRITON_BLOCK_SIZE = min(block_size, MAX_TRITON_BLOCK_SIZE) if is_pow2 else 32
|
||||
if is_block_table_ptr:
|
||||
# Using the physical base address of tensors
|
||||
kv_element_size = key_cache.element_size()
|
||||
block_byte_stride = key_cache.stride(0) * kv_element_size
|
||||
# Get the starting physical address of the KV Cache
|
||||
base_addr = key_cache.data_ptr()
|
||||
|
||||
# Normalization: Directly calculate the block offset
|
||||
# of the pointer relative to the base address
|
||||
processed_block_table = ((block_table - base_addr) // block_byte_stride).to(
|
||||
torch.int32
|
||||
)
|
||||
else:
|
||||
processed_block_table = block_table.to(torch.int32)
|
||||
|
||||
kernel_paged_attention_2d[
|
||||
(
|
||||
num_seqs,
|
||||
num_kv_heads,
|
||||
)
|
||||
](
|
||||
output_ptr=output,
|
||||
query_ptr=query,
|
||||
key_cache_ptr=key_cache,
|
||||
value_cache_ptr=value_cache,
|
||||
sink_ptr=sinks,
|
||||
block_tables_ptr=processed_block_table,
|
||||
seq_lens_ptr=seq_lens,
|
||||
alibi_slopes_ptr=alibi_slopes,
|
||||
scale=sm_scale,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
out_scale_inv=1.0 / output_scale if output_scale is not None else 1.0,
|
||||
num_query_heads=num_query_heads,
|
||||
num_queries_per_kv=num_queries_per_kv,
|
||||
num_queries_per_kv_padded=num_queries_per_kv_padded,
|
||||
block_table_stride=processed_block_table.stride(0),
|
||||
query_stride_0=query.stride(0),
|
||||
query_stride_1=query.stride(1),
|
||||
output_stride_0=output.stride(0),
|
||||
output_stride_1=output.stride(1),
|
||||
BLOCK_SIZE=TRITON_BLOCK_SIZE,
|
||||
PHYSICAL_BLOCK_SIZE=real_block_size,
|
||||
HEAD_SIZE=head_size,
|
||||
HEAD_SIZE_PADDED=triton.next_power_of_2(head_size),
|
||||
USE_ALIBI_SLOPES=use_alibi_slopes,
|
||||
SLIDING_WINDOW=sliding_window,
|
||||
x=key_cache.shape[4],
|
||||
stride_k_cache_0=key_cache.stride(0),
|
||||
stride_k_cache_1=key_cache.stride(1),
|
||||
stride_k_cache_2=key_cache.stride(2),
|
||||
stride_k_cache_3=key_cache.stride(3),
|
||||
stride_k_cache_4=key_cache.stride(4),
|
||||
stride_v_cache_0=value_cache.stride(0),
|
||||
stride_v_cache_1=value_cache.stride(1),
|
||||
stride_v_cache_2=value_cache.stride(2),
|
||||
stride_v_cache_3=value_cache.stride(3),
|
||||
filter_by_query_len=True,
|
||||
query_start_len_ptr=query_start_loc,
|
||||
USE_SINKS=sinks is not None,
|
||||
USE_FP8=output_scale is not None,
|
||||
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
|
||||
)
|
||||
@@ -1,149 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# adapted from: https://github.com/deepseek-ai/FlashMLA/blob/main/flash_mla/flash_mla_interface.py
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if current_platform.is_cuda():
|
||||
try:
|
||||
import vllm._flashmla_C # noqa: F401
|
||||
|
||||
_flashmla_C_AVAILABLE = True
|
||||
except ImportError:
|
||||
_flashmla_C_AVAILABLE = False
|
||||
else:
|
||||
_flashmla_C_AVAILABLE = False
|
||||
|
||||
if current_platform.is_cuda():
|
||||
try:
|
||||
import vllm._flashmla_extension_C # noqa: F401
|
||||
|
||||
_flashmla_extension_C_AVAILABLE = True
|
||||
except ImportError:
|
||||
_flashmla_extension_C_AVAILABLE = False
|
||||
else:
|
||||
_flashmla_extension_C_AVAILABLE = False
|
||||
|
||||
|
||||
def _is_flashmla_available() -> tuple[bool, str | None]:
|
||||
if not _flashmla_C_AVAILABLE:
|
||||
return (
|
||||
False,
|
||||
"vllm._flashmla_C is not available, likely was not "
|
||||
"compiled due to insufficient nvcc version or a supported arch "
|
||||
"was not in the list of target arches to compile for.",
|
||||
)
|
||||
if not _flashmla_extension_C_AVAILABLE:
|
||||
return (
|
||||
False,
|
||||
"vllm._flashmla_extension_C is not available, likely "
|
||||
"was not compiled due to a build error.",
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
|
||||
def is_flashmla_dense_supported() -> tuple[bool, str | None]:
|
||||
"""Return: is_supported_flag, unsupported_reason (optional)."""
|
||||
is_available, maybe_reason = _is_flashmla_available()
|
||||
if not is_available:
|
||||
return False, maybe_reason
|
||||
if not current_platform.is_device_capability_family(90):
|
||||
return False, "FlashMLA Dense is only supported on Hopper devices."
|
||||
return True, None
|
||||
|
||||
|
||||
def is_flashmla_sparse_supported() -> tuple[bool, str | None]:
|
||||
"""Return: is_supported_flag, unsupported_reason (optional)."""
|
||||
is_available, maybe_reason = _is_flashmla_available()
|
||||
if not is_available:
|
||||
return False, maybe_reason
|
||||
if not (
|
||||
current_platform.is_device_capability_family(90)
|
||||
or current_platform.is_device_capability_family(100)
|
||||
):
|
||||
return (
|
||||
False,
|
||||
"FlashMLA Sparse is only supported on Hopper and Blackwell DC devices.",
|
||||
)
|
||||
return True, None
|
||||
|
||||
|
||||
def _raise_flashmla_unavailable(*_args, **_kwargs):
|
||||
_, reason = _is_flashmla_available()
|
||||
raise RuntimeError(reason or "FlashMLA is not available")
|
||||
|
||||
|
||||
if _is_flashmla_available()[0]:
|
||||
from vllm.third_party.flashmla.flash_mla_interface import ( # noqa: F401
|
||||
FlashMLASchedMeta,
|
||||
flash_attn_varlen_func,
|
||||
flash_attn_varlen_kvpacked_func,
|
||||
flash_attn_varlen_qkvpacked_func,
|
||||
flash_mla_sparse_fwd,
|
||||
flash_mla_with_kvcache,
|
||||
get_mla_metadata,
|
||||
)
|
||||
else:
|
||||
|
||||
class FlashMLASchedMeta: # type: ignore[no-redef]
|
||||
pass
|
||||
|
||||
flash_attn_varlen_func = _raise_flashmla_unavailable # type: ignore[assignment]
|
||||
flash_attn_varlen_kvpacked_func = _raise_flashmla_unavailable # type: ignore[assignment]
|
||||
flash_attn_varlen_qkvpacked_func = _raise_flashmla_unavailable # type: ignore[assignment]
|
||||
flash_mla_sparse_fwd = _raise_flashmla_unavailable # type: ignore[assignment]
|
||||
flash_mla_with_kvcache = _raise_flashmla_unavailable # type: ignore[assignment]
|
||||
get_mla_metadata = _raise_flashmla_unavailable # type: ignore[assignment]
|
||||
|
||||
|
||||
def get_mla_metadata_dense_fp8(
|
||||
cache_seqlens: torch.Tensor,
|
||||
num_q_tokens_per_head_k: int,
|
||||
num_heads_k: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if not _is_flashmla_available()[0]:
|
||||
_raise_flashmla_unavailable()
|
||||
return torch.ops._flashmla_extension_C.get_mla_decoding_metadata_dense_fp8(
|
||||
cache_seqlens,
|
||||
num_q_tokens_per_head_k,
|
||||
num_heads_k,
|
||||
)
|
||||
|
||||
|
||||
def flash_mla_with_kvcache_fp8(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
cache_seqlens: torch.Tensor,
|
||||
head_dim_v: int,
|
||||
tile_scheduler_metadata: torch.Tensor,
|
||||
num_splits: torch.Tensor,
|
||||
softmax_scale: float | None = None,
|
||||
causal: bool = False,
|
||||
descale_q: torch.Tensor | None = None,
|
||||
descale_k: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if not _is_flashmla_available()[0]:
|
||||
_raise_flashmla_unavailable()
|
||||
if softmax_scale is None:
|
||||
softmax_scale = q.shape[-1] ** (-0.5)
|
||||
out, softmax_lse = torch.ops._flashmla_extension_C.fwd_kvcache_mla_fp8(
|
||||
q,
|
||||
k_cache,
|
||||
head_dim_v,
|
||||
cache_seqlens,
|
||||
block_table,
|
||||
softmax_scale,
|
||||
causal,
|
||||
tile_scheduler_metadata,
|
||||
num_splits,
|
||||
descale_q,
|
||||
descale_k,
|
||||
)
|
||||
return out, softmax_lse
|
||||
@@ -1,823 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# TurboQuant decode (FlyDSL) — MI355X / gfx950 / CDNA4
|
||||
#
|
||||
# Decode-only paged attention over a TurboQuant SoA KV cache.
|
||||
# Targets Qwen-class models: HEAD_SIZE=128, GQA group=8, MSE_BITS=4 K, VQB=4 V,
|
||||
# N_CENTROIDS=16, BLOCK_SIZE=16, partitioned mode.
|
||||
#
|
||||
# === Design ===
|
||||
# * 1 wave (64 lanes) per CTA — simpler than 4-warp; trades parallelism for
|
||||
# correctness/debug-ability. A later revision will fan out to 4 warps.
|
||||
# * Flash-Attention-2 online softmax: rescale running PV/sum per K-tile,
|
||||
# no QK score storage across tiles, no P-LDS round-trip (qk_acc layout
|
||||
# matches PV B-operand directly).
|
||||
# * Centroids loaded once into LDS at kernel start (LDS-resident LUT).
|
||||
# * K dequant → LDS [token, head_dim] (natural layout)
|
||||
# * V dequant → LDS [head_dim, token] (TRANSPOSED — required so PV MFMA's
|
||||
# A=V operand can be loaded as 4-bf16 contiguous via 1 ds_read_b64).
|
||||
#
|
||||
# MFMA layouts used:
|
||||
# QK: mfma(A=K, B=Q, C=qk_acc) → C[m=token, n=query], 4 fp32/lane
|
||||
# = 4 tokens for SAME query → softmax via
|
||||
# xor_shuffle(16),(32) reduces per-row
|
||||
# PV: mfma(A=V_T, B=P, C=acc_pv) → C[m=head_dim, n=query], 4 fp32/lane
|
||||
# = 4 contiguous head_dim cols for SAME
|
||||
# query → 1 buffer_store of 8 bytes
|
||||
# per chunk per lane
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import flydsl.compiler as flyc
|
||||
import flydsl.expr as fx
|
||||
from flydsl._mlir import ir
|
||||
from flydsl._mlir.dialects import scf as _scf
|
||||
from flydsl.expr import (
|
||||
arith,
|
||||
buffer_ops,
|
||||
const_expr, # noqa: F401 kept for kernel-DSL API parity
|
||||
gpu,
|
||||
range_constexpr,
|
||||
rocdl,
|
||||
vector,
|
||||
)
|
||||
from flydsl.expr.typing import Int32, T # noqa: F401 Int32 via fx.Int32
|
||||
from flydsl.runtime.device import get_rocm_arch as get_hip_arch
|
||||
from flydsl.utils.smem_allocator import SmemAllocator, SmemPtr
|
||||
|
||||
# === Constants (Qwen-class TQ decode profile) ===============================
|
||||
HEAD_SIZE = 128
|
||||
# default; overridable via build_tq_decode_module(kv_block_size=...)
|
||||
KV_BLOCK_SIZE = 16
|
||||
# MFMA tile = 16 tokens (do not change without re-deriving MFMA shapes)
|
||||
TILE_SIZE = 16
|
||||
N_CENTROIDS = 16
|
||||
# default for tests; overridable via build_tq_decode_module(query_group_size=...)
|
||||
QUERY_GROUP_SIZE = 16
|
||||
WARP_SIZE = 64
|
||||
NUM_WARPS = 1
|
||||
BLOCK_THREADS = NUM_WARPS * WARP_SIZE
|
||||
KV_COMPUTE_BLOCK = 256 # 16 K-tiles × 16 tokens
|
||||
|
||||
# TQ SoA layout
|
||||
NUM_SOA_FIELDS = 3
|
||||
SOA_K_NORM = 0
|
||||
SOA_V_SCALE = 1
|
||||
SOA_V_ZERO = 2
|
||||
KEY_DATA_BYTES = HEAD_SIZE // 2
|
||||
VAL_DATA_BYTES = HEAD_SIZE // 2
|
||||
DATA_BYTES_PER_SLOT = KEY_DATA_BYTES + VAL_DATA_BYTES
|
||||
|
||||
# MFMA
|
||||
MFMA_M = MFMA_N = 16
|
||||
MFMA_K_BF16_QK = 32 # CDNA4 wide-K (mfma_f32_16x16x32_bf16)
|
||||
MFMA_K_BF16_PV = 16 # PV's K = token tile = 16
|
||||
QK_K_CHUNKS = HEAD_SIZE // MFMA_K_BF16_QK # 4
|
||||
PV_N_CHUNKS = HEAD_SIZE // MFMA_N # 8
|
||||
|
||||
# LDS regions
|
||||
CENTROID_LDS_BYTES = N_CENTROIDS * 4 # 64
|
||||
Q_LDS_BYTES = QUERY_GROUP_SIZE * HEAD_SIZE * 2 # 4096
|
||||
KV_TILE_LDS_BYTES = TILE_SIZE * HEAD_SIZE * 2 # 4096
|
||||
|
||||
LOG2E = 1.4426950408889634
|
||||
NEG_INF_VAL = float("-inf")
|
||||
|
||||
|
||||
def _vsplat_mul(vec, scalar):
|
||||
s = scalar.ir_value() if hasattr(scalar, "ir_value") else scalar
|
||||
return vec * vector.broadcast(T.f32x4, s)
|
||||
|
||||
|
||||
allocator = None
|
||||
|
||||
|
||||
def build_tq_decode_module(
|
||||
num_seqs: int,
|
||||
num_kv_heads: int,
|
||||
num_partitions: int,
|
||||
max_blocks_per_seq: int = 512,
|
||||
softmax_scale: float | None = None,
|
||||
query_group_size: int = QUERY_GROUP_SIZE,
|
||||
kv_block_size: int = KV_BLOCK_SIZE,
|
||||
use_hw_v_transpose: bool = False,
|
||||
tile_groups_per_partition: int = 1,
|
||||
):
|
||||
"""Build a TQ decode kernel module.
|
||||
|
||||
``query_group_size`` (= num_query_heads // num_kv_heads) selects the
|
||||
GQA factor. Supported values: 8 and 16. For 8 the MFMA's 16-row
|
||||
capacity is half-used; lanes 8..15 compute garbage and are gated out
|
||||
of all global-memory writes via an OOB-offset trick on the buffer
|
||||
resource. Q load iterates ``QG // 4`` chunks (= 2 for QG=8, 4 for
|
||||
QG=16) so we never read past the real Q rows in HBM.
|
||||
|
||||
``kv_block_size`` is the number of tokens per vLLM cache block. Must
|
||||
be a multiple of TILE_SIZE (=16). Supported values: 16 and 32. With
|
||||
block_size=32, two MFMA K-tiles fit in each block; the kernel walks
|
||||
block-table entries every ``kv_block_size // TILE_SIZE`` tiles and
|
||||
re-uses the entry for sub-tiles within the same block. SoA metadata
|
||||
region grows linearly with block_size.
|
||||
|
||||
``use_hw_v_transpose`` (CDNA4 / gfx950 only) replaces the 32-element
|
||||
scattered ``ds_write_b16`` V-dequant pattern with a row-major
|
||||
``V[token][head_dim]`` LDS layout written as 4 wide ``ds_write_b128``
|
||||
per lane, then read back into the MFMA A operand via the hardware
|
||||
``ds_read_tr16_b64`` transpose. This eliminates ~28 ds_write
|
||||
instructions per lane per K-tile (32 -> 4) and 8 strided ds_read_b64
|
||||
per PV chunk (replaces them with 1 hw-transpose read each).
|
||||
|
||||
``tile_groups_per_partition`` (default 1) controls the FA-2 split-K
|
||||
granularity. With value G each partition processes ``G * 16`` K-tiles
|
||||
= ``G * KV_COMPUTE_BLOCK`` tokens, with the existing 16-tile loop
|
||||
body iterated G times. The launcher uses this to bound
|
||||
``num_partitions`` (e.g. cap at 32) for long context: at 32K /
|
||||
block_size=32 / num_partitions=32, G=5 covers 32*5*256 = 40 960
|
||||
tokens of worst-case context with grid.z=32 instead of 256.
|
||||
Behavior at G=1 is bit-identical to the single-partition kernel.
|
||||
"""
|
||||
assert query_group_size in (8, 16), (
|
||||
f"query_group_size must be 8 or 16; got {query_group_size}"
|
||||
)
|
||||
assert kv_block_size in (16, 32), (
|
||||
f"kv_block_size must be 16 or 32; got {kv_block_size}"
|
||||
)
|
||||
assert kv_block_size % TILE_SIZE == 0
|
||||
assert int(tile_groups_per_partition) >= 1, (
|
||||
f"tile_groups_per_partition must be >= 1; got {tile_groups_per_partition}"
|
||||
)
|
||||
USE_HW_TR = bool(use_hw_v_transpose)
|
||||
TGPP = int(tile_groups_per_partition)
|
||||
PARTITION_EXTENT_TOKENS = TGPP * KV_COMPUTE_BLOCK
|
||||
|
||||
QG = int(query_group_size)
|
||||
QG_LOAD_ITERS = QG // 4 # 2 for QG=8, 4 for QG=16
|
||||
OOB_OFFSET = 0x7FFFFFF0 # noqa: F841 ~2GB byte offset; > any plausible buffer
|
||||
|
||||
_BS = int(kv_block_size)
|
||||
_TILES_PER_BLOCK = _BS // TILE_SIZE # 1 for BS=16, 2 for BS=32
|
||||
|
||||
global allocator
|
||||
arch = get_hip_arch()
|
||||
|
||||
if softmax_scale is None:
|
||||
softmax_scale = 1.0 / (HEAD_SIZE**0.5)
|
||||
_qk_scale = float(softmax_scale)
|
||||
|
||||
# --- Strides ---
|
||||
_Hq = num_kv_heads * QG
|
||||
_stride_q_seq = _Hq * HEAD_SIZE
|
||||
_stride_q_head = HEAD_SIZE
|
||||
_stride_bt_seq = max_blocks_per_seq
|
||||
|
||||
_data_region_bytes = _BS * num_kv_heads * DATA_BYTES_PER_SLOT
|
||||
_meta_region_bytes = num_kv_heads * NUM_SOA_FIELDS * _BS * 2
|
||||
_stride_cache_block = _data_region_bytes + _meta_region_bytes
|
||||
_meta_region_offset = _data_region_bytes
|
||||
|
||||
_stride_out_part = QG * HEAD_SIZE
|
||||
_stride_out_head = num_partitions * QG * HEAD_SIZE
|
||||
_stride_out_seq = num_kv_heads * num_partitions * QG * HEAD_SIZE
|
||||
_stride_es_seq = num_kv_heads * num_partitions * QG
|
||||
_stride_ml_seq = _stride_es_seq
|
||||
|
||||
# --- LDS layout ---
|
||||
allocator = SmemAllocator(None, arch=arch, global_sym_name="tq_smem")
|
||||
centroid_off = 0
|
||||
allocator.ptr = CENTROID_LDS_BYTES
|
||||
q_off = allocator.ptr
|
||||
allocator.ptr += Q_LDS_BYTES
|
||||
kv_off = allocator.ptr
|
||||
allocator.ptr += KV_TILE_LDS_BYTES
|
||||
|
||||
@flyc.kernel
|
||||
def tq_decode_kernel(
|
||||
out_ptr: fx.Tensor,
|
||||
exp_sums_ptr: fx.Tensor,
|
||||
max_logits_ptr: fx.Tensor,
|
||||
query_ptr: fx.Tensor,
|
||||
kv_cache_ptr: fx.Tensor,
|
||||
centroids_ptr: fx.Tensor,
|
||||
block_tables_ptr: fx.Tensor,
|
||||
seq_lens_ptr: fx.Tensor,
|
||||
):
|
||||
# ---- IDs ---------------------------------------------------------
|
||||
tid = gpu.thread_idx.x
|
||||
seq = gpu.block_idx.x
|
||||
kv_h = gpu.block_idx.y
|
||||
part = gpu.block_idx.z
|
||||
lane = tid # 0..63
|
||||
mfma_row = lane & fx.Int32(15) # = query_row (B's N) /
|
||||
# token (A's M for QK)
|
||||
mfma_col_grp = lane >> fx.Int32(4) # 0..3, K-group dim
|
||||
|
||||
# ---- Buffer resources -------------------------------------------
|
||||
q_rsrc = buffer_ops.create_buffer_resource(query_ptr, max_size=True)
|
||||
kv_rsrc = buffer_ops.create_buffer_resource(kv_cache_ptr, max_size=True)
|
||||
bt_rsrc = buffer_ops.create_buffer_resource(block_tables_ptr, max_size=True)
|
||||
sl_rsrc = buffer_ops.create_buffer_resource(seq_lens_ptr, max_size=True)
|
||||
cent_rsrc = buffer_ops.create_buffer_resource(centroids_ptr, max_size=True)
|
||||
out_rsrc = buffer_ops.create_buffer_resource(out_ptr, max_size=True)
|
||||
es_rsrc = buffer_ops.create_buffer_resource(exp_sums_ptr, max_size=True)
|
||||
ml_rsrc = buffer_ops.create_buffer_resource(max_logits_ptr, max_size=True)
|
||||
|
||||
# ---- LDS pointers -----------------------------------------------
|
||||
base = allocator.get_base()
|
||||
cent_lds = SmemPtr(base, centroid_off, T.f32, shape=(N_CENTROIDS,))
|
||||
q_lds_i32 = SmemPtr(base, q_off, T.i32, shape=(Q_LDS_BYTES // 4,)).get()
|
||||
q_lds_i64 = SmemPtr(base, q_off, T.i64, shape=(Q_LDS_BYTES // 8,)).get()
|
||||
kv_lds_i32 = SmemPtr(base, kv_off, T.i32, shape=(KV_TILE_LDS_BYTES // 4,)).get() # noqa: F841
|
||||
kv_lds_i64 = SmemPtr(base, kv_off, T.i64, shape=(KV_TILE_LDS_BYTES // 8,)).get()
|
||||
kv_lds_i16 = SmemPtr(base, kv_off, T.i16, shape=(KV_TILE_LDS_BYTES // 2,)).get()
|
||||
|
||||
# ---- Constants ---------------------------------------------------
|
||||
c_sq = fx.Int32(_stride_q_seq)
|
||||
c_qh = fx.Int32(_stride_q_head)
|
||||
c_qg = fx.Int32(QG)
|
||||
c_bt = fx.Int32(_stride_bt_seq)
|
||||
c_block = fx.Int32(_stride_cache_block)
|
||||
c_meta_off = fx.Int32(_meta_region_offset)
|
||||
c_data_per_slot = fx.Int32(DATA_BYTES_PER_SLOT)
|
||||
c_kv_heads = fx.Int32(num_kv_heads)
|
||||
c_keydata = fx.Int32(KEY_DATA_BYTES)
|
||||
c_w = fx.Int32(WARP_SIZE)
|
||||
|
||||
NEG_INF = arith.constant(NEG_INF_VAL, type=T.f32)
|
||||
ZERO_F = fx.Float32(0.0)
|
||||
ONE_F = fx.Float32(1.0)
|
||||
LOG2E_C = arith.constant(LOG2E, type=T.f32)
|
||||
QK_SCALE = arith.constant(_qk_scale, type=T.f32)
|
||||
|
||||
# Helper: unwrap fx wrapper → raw ir.Value (used in STEP F)
|
||||
def _ival(v):
|
||||
return v.ir_value() if hasattr(v, "ir_value") else v
|
||||
|
||||
# ===== STEP A: Load centroids → LDS (cooperative, race-safe) =====
|
||||
c_idx_safe = lane & fx.Int32(N_CENTROIDS - 1)
|
||||
c_val = buffer_ops.buffer_load(cent_rsrc, c_idx_safe, vec_width=1, dtype=T.f32)
|
||||
cent_lds.store(c_val, [arith.index_cast(T.index, c_idx_safe)])
|
||||
gpu.barrier()
|
||||
|
||||
# ===== STEP B: Load pre-rotated q_rot → row-major LDS ============
|
||||
# q_rsrc points to the externally-rotated q_rot tensor. Load 8 bf16
|
||||
# per lane per iter (vec_width=4 i32) in QG_LOAD_ITERS iterations.
|
||||
for c in range_constexpr(QG_LOAD_ITERS):
|
||||
row_chunk = lane + fx.Int32(c * WARP_SIZE) # 0..QG*16-1
|
||||
row = row_chunk >> fx.Int32(4) # 0..QG-1
|
||||
col_b = row_chunk & fx.Int32(15) # 0..15
|
||||
col_elem = col_b * fx.Int32(8) # 0..120 in bf16
|
||||
q_off_elem = seq * c_sq + (kv_h * c_qg + row) * c_qh + col_elem
|
||||
q_v = buffer_ops.buffer_load(
|
||||
q_rsrc,
|
||||
q_off_elem // fx.Int32(2),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
q_lds_byte = row * fx.Int32(HEAD_SIZE * 2) + col_elem * fx.Int32(2)
|
||||
vector.store(
|
||||
q_v,
|
||||
q_lds_i32,
|
||||
[arith.index_cast(T.index, q_lds_byte // fx.Int32(4))],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# ===== STEP C: Pre-load Q operands for QK_K_CHUNKS = 4 K-chunks ===
|
||||
# Wide-K MFMA: K_per_chunk = 32, K_per_lane = 32/4 = 8 bf16 per chunk.
|
||||
# Lane t holds 8 bf16 at row=mfma_row, K-cols
|
||||
# chunk*32 + mfma_col_grp*8 + 0..7
|
||||
# Byte addr = mfma_row * HEAD_SIZE*2 + (chunk*32 + col_grp*8)*2
|
||||
# = mfma_row * 256 + chunk*64 + col_grp*16
|
||||
# i64 idx = mfma_row * 32 + chunk*8 + col_grp*2 (each i64 = 4 bf16)
|
||||
# Load 16 bytes (= 8 bf16 = 2 i64) per chunk via vec(2, i64).
|
||||
q_chunks = []
|
||||
for chk in range_constexpr(QK_K_CHUNKS):
|
||||
q_idx_i64 = (
|
||||
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
+ fx.Int32(chk * 8)
|
||||
+ mfma_col_grp * fx.Int32(2)
|
||||
)
|
||||
qv = vector.load_op(
|
||||
T.vec(2, T.i64),
|
||||
q_lds_i64,
|
||||
[arith.index_cast(T.index, q_idx_i64)],
|
||||
)
|
||||
q_chunks.append(vector.bitcast(T.vec(8, T.bf16), qv))
|
||||
|
||||
# ===== STEP D: Online softmax + PV state =========================
|
||||
running_max = NEG_INF
|
||||
running_sum = ZERO_F
|
||||
zero_v4 = arith.constant_vector(0.0, T.f32x4)
|
||||
acc_pv = [zero_v4 for _ in range(PV_N_CHUNKS)]
|
||||
|
||||
# ===== STEP E: Sequence-len + partition base ====================
|
||||
seq_len = buffer_ops.buffer_load(sl_rsrc, seq, vec_width=1, dtype=T.i32)
|
||||
# ── Bounded num_partitions + internal looping ───────────────────
|
||||
# With TGPP > 1 this CTA owns a contiguous slice of length
|
||||
# ``PARTITION_EXTENT_TOKENS`` that is iterated as ``TGPP`` groups
|
||||
# of 16 K-tiles each. ``partition_start`` is recomputed per
|
||||
# outer ``tg`` iteration below; the FA-2 online-softmax state
|
||||
# (running_max, running_sum, acc_pv) accumulates across all
|
||||
# ``TGPP * 16`` tiles for this CTA — semantically identical to
|
||||
# processing one large 16*TGPP-tile partition.
|
||||
partition_base = part * fx.Int32(PARTITION_EXTENT_TOKENS)
|
||||
|
||||
# Per-K-tile dequant lane assignment:
|
||||
# 16 tokens × 64 packed bytes per token = 1024 bytes
|
||||
# 64 lanes × 16 bytes/lane = 1024 ✓
|
||||
# Lane t: token = t / 4, sub-chunk = t % 4 (0..3) → 16 bytes contiguous
|
||||
tok_in_tile = lane >> fx.Int32(2)
|
||||
chunk_in_tok = lane & fx.Int32(3)
|
||||
# ``slot`` (= absolute slot index within the cache block, range
|
||||
# 0.._BS-1) is recomputed per tile inside the K-tile loop below
|
||||
# so it reflects ``tile_in_block * TILE_SIZE + tok_in_tile``.
|
||||
|
||||
c_meta_u16_per_kvh = fx.Int32(NUM_SOA_FIELDS * _BS)
|
||||
c_knorm_off_u16 = fx.Int32(SOA_K_NORM * _BS)
|
||||
c_vscale_off_u16 = fx.Int32(SOA_V_SCALE * _BS)
|
||||
c_vzero_off_u16 = fx.Int32(SOA_V_ZERO * _BS)
|
||||
|
||||
# ===== STEP F: K-tile loop =======================================
|
||||
# With kv_block_size > TILE_SIZE there are _TILES_PER_BLOCK tiles
|
||||
# per cache block. Tiles within the same block share a block-table
|
||||
# entry but address different slot ranges within the block.
|
||||
#
|
||||
# ── Per-tile block-table OOB redirect ────────────────────────────
|
||||
# The K-tile loop is unrolled 16 times so every CTA issues 16
|
||||
# block-table reads regardless of (partition, seq_len). When a
|
||||
# partition extends past ``seq_len`` (e.g. num_partitions=2 for
|
||||
# a 256-token sequence — partition 1 covers tokens 256..511, all
|
||||
# masked out) the per-tile bt offset can land beyond the bt
|
||||
# allocation. With ``max_size=True`` the descriptor advertises
|
||||
# 4 GB so the HW returns whatever pre-existing HBM bytes live at
|
||||
# that offset; the resulting garbage ``phys_block`` is multiplied
|
||||
# by ``stride_cache_block`` and the subsequent kv_cache read
|
||||
# jumps into pages that may be unmapped (→ "Memory access fault
|
||||
# by GPU node") or may decode as NaN bytes (→ NaN segm_out).
|
||||
# Allocator-pattern dependent, but reproduces deterministically
|
||||
# at large B (e.g. B=64 seq=256 in the test sweep produces 8 192
|
||||
# NaN entries even on canonical QG=8).
|
||||
#
|
||||
# Fix: when the tile starts at or past ``seq_len`` (so its
|
||||
# qk_acc is going to be killed by the per-token
|
||||
# ``kv_tok < seq_len`` mask anyway), redirect the bt read to
|
||||
# ``bt[seq, 0]`` — always in bounds. The redundant phys_block
|
||||
# decode + kv_cache read is wasted work, but correctness is
|
||||
# preserved without changing the kernel's iteration count.
|
||||
# ===== Runtime-adaptive scf.ForOp w/ iter_args =========
|
||||
# Runtime-adaptive outer loop: trip count derives from
|
||||
# actual seq_len, NOT TGPP_max. At cudagraph-capture warmup
|
||||
# (seq_len=1) only 1 iteration runs; at 32K production
|
||||
# decode all TGPP iterations run. Kernel binary stays small
|
||||
# (single body, looped at runtime) — capture time matches
|
||||
# the legacy 16-tile-only kernel rather than scaling 4x.
|
||||
#
|
||||
# FA-2 state (running_max, running_sum, acc_pv[8]) threads
|
||||
# through scf iter_args; the body reads them at top, runs
|
||||
# the unchanged 16-tile inner body, and yields the new
|
||||
# state at bottom. After the loop, results are pulled out
|
||||
# of for_op.results back into the local Python names so the
|
||||
# downstream STEP G (output) is unmodified.
|
||||
c_kcb = fx.Int32(KV_COMPUTE_BLOCK)
|
||||
c_tgpp = fx.Int32(TGPP)
|
||||
c_zero_i32 = fx.Int32(0)
|
||||
c_one_i32 = fx.Int32(1)
|
||||
# remaining = seq_len - partition_base (signed, may be <=0)
|
||||
remaining = seq_len - partition_base
|
||||
in_range = remaining > c_zero_i32
|
||||
# trip_raw = ceil(remaining / KV_COMPUTE_BLOCK)
|
||||
# (when in_range is false, the divisor branch produces
|
||||
# garbage that the select below overrides with zero, so we
|
||||
# don't pre-clamp)
|
||||
trip_raw = (remaining + c_kcb - c_one_i32) // c_kcb
|
||||
trip_clamped = (trip_raw > c_tgpp).select(c_tgpp, trip_raw)
|
||||
trip_or_zero = in_range.select(trip_clamped, c_zero_i32)
|
||||
c_zero_idx = arith.constant(0, index=True)
|
||||
c_one_idx = arith.constant(1, index=True)
|
||||
trip_idx = arith.index_cast(
|
||||
T.index,
|
||||
trip_or_zero.ir_value()
|
||||
if hasattr(trip_or_zero, "ir_value")
|
||||
else trip_or_zero,
|
||||
)
|
||||
# Initial iter_args list: FA-2 accumulator state.
|
||||
# Order: running_max, running_sum, acc_pv[0..PV_N_CHUNKS-1].
|
||||
# NOTE: scf.ForOp requires ir.Value (with .type). The DSL
|
||||
# constants ZERO_F / ONE_F are fx.Float32 (Numeric) wrappers,
|
||||
# not raw ir.Value, so we unwrap via .ir_value() before
|
||||
# passing. _ival() is defined once near the top of the kernel body.
|
||||
_init_iter = [
|
||||
_ival(running_max),
|
||||
_ival(running_sum),
|
||||
*[_ival(p) for p in acc_pv],
|
||||
]
|
||||
_for_op = _scf.ForOp(
|
||||
c_zero_idx,
|
||||
trip_idx,
|
||||
c_one_idx,
|
||||
_init_iter,
|
||||
)
|
||||
_for_ip = ir.InsertionPoint(_for_op.body)
|
||||
_for_ip.__enter__()
|
||||
try:
|
||||
tg_idx = _for_op.induction_variable
|
||||
tg_i32 = fx.Int32(arith.index_cast(T.i32, tg_idx))
|
||||
partition_start = partition_base + tg_i32 * c_kcb
|
||||
bt_seq_base = seq * c_bt + (partition_start // fx.Int32(_BS))
|
||||
running_max = _for_op.inner_iter_args[0]
|
||||
running_sum = _for_op.inner_iter_args[1]
|
||||
acc_pv = list(_for_op.inner_iter_args[2:])
|
||||
for n_tile in range_constexpr(16):
|
||||
block_in_part = n_tile // _TILES_PER_BLOCK
|
||||
tile_in_block = n_tile % _TILES_PER_BLOCK
|
||||
tile_start_tok = partition_start + fx.Int32(n_tile * TILE_SIZE)
|
||||
tile_in_seq = tile_start_tok < seq_len
|
||||
bt_off = bt_seq_base + fx.Int32(block_in_part)
|
||||
bt_off_safe = tile_in_seq.select(bt_off, seq * c_bt)
|
||||
phys_block = buffer_ops.buffer_load(
|
||||
bt_rsrc,
|
||||
bt_off_safe,
|
||||
vec_width=1,
|
||||
dtype=T.i32,
|
||||
)
|
||||
block_base = phys_block * c_block
|
||||
data_region = block_base
|
||||
meta_region = block_base + c_meta_off
|
||||
|
||||
# ``slot`` is the absolute slot index within the cache block
|
||||
# (range 0.._BS-1). For BS=16 it equals tok_in_tile; for BS=32
|
||||
# it equals tile_in_block * 16 + tok_in_tile. Used both for
|
||||
# the K/V data region and SoA metadata region addressing.
|
||||
slot = fx.Int32(tile_in_block * TILE_SIZE) + tok_in_tile
|
||||
data_bases_byte = (
|
||||
data_region
|
||||
+ slot * (c_kv_heads * c_data_per_slot)
|
||||
+ kv_h * c_data_per_slot
|
||||
)
|
||||
k_byte = data_bases_byte + chunk_in_tok * fx.Int32(16)
|
||||
k_packed = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
k_byte // fx.Int32(4),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
|
||||
# ---- HOISTED: issue V data + meta HBM loads early ---------
|
||||
# These are async; their s_waitcnt is pushed by the compiler
|
||||
# past the K dequant + QK MFMA + softmax block, hiding most
|
||||
# of the V HBM latency behind compute.
|
||||
v_byte = data_bases_byte + c_keydata + chunk_in_tok * fx.Int32(16)
|
||||
v_packed = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
v_byte // fx.Int32(4),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
vscale_u16 = (
|
||||
meta_region // fx.Int32(2)
|
||||
+ kv_h * c_meta_u16_per_kvh
|
||||
+ c_vscale_off_u16
|
||||
+ slot
|
||||
)
|
||||
vzero_u16 = (
|
||||
meta_region // fx.Int32(2)
|
||||
+ kv_h * c_meta_u16_per_kvh
|
||||
+ c_vzero_off_u16
|
||||
+ slot
|
||||
)
|
||||
vscale_raw = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
vscale_u16,
|
||||
vec_width=1,
|
||||
dtype=T.i16,
|
||||
)
|
||||
vzero_raw = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
vzero_u16,
|
||||
vec_width=1,
|
||||
dtype=T.i16,
|
||||
)
|
||||
|
||||
knorm_u16 = (
|
||||
meta_region // fx.Int32(2)
|
||||
+ kv_h * c_meta_u16_per_kvh
|
||||
+ c_knorm_off_u16
|
||||
+ slot
|
||||
)
|
||||
knorm_raw = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
knorm_u16,
|
||||
vec_width=1,
|
||||
dtype=T.i16,
|
||||
)
|
||||
knorm_f16 = arith.bitcast(T.f16, knorm_raw)
|
||||
knorm_f32 = arith.extf(T.f32, knorm_f16)
|
||||
|
||||
# K dequant → LDS [token, head_dim] (natural)
|
||||
# Lane writes 32 bf16 (= 4×8) for token=tok_in_tile,
|
||||
# head_dims chunk_in_tok*32..+31 (4 sub-chunks of 8).
|
||||
tok_kreg = tok_in_tile * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
chunk_kreg = tok_kreg + chunk_in_tok * fx.Int32(8)
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(k_packed, static_position=[w])
|
||||
bf16_elems = []
|
||||
for n in range_constexpr(8):
|
||||
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
|
||||
nibble_idx = arith.index_cast(T.index, nibble)
|
||||
cent_f32 = cent_lds.load([nibble_idx])
|
||||
elem_bf16 = arith.trunc_f(T.bf16, cent_f32 * knorm_f32)
|
||||
bf16_elems.append(elem_bf16)
|
||||
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
|
||||
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
|
||||
vector.store(
|
||||
v_i64,
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, chunk_kreg + fx.Int32(w * 2))],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# QK MFMA (CDNA4 wide-K): A=K[token, head_dim], B=Q (= Q^T).
|
||||
# K read: lane t = K[token=mfma_row, head_dim=chunk*32+col_grp*8..+7]
|
||||
# Same i64×2 indexing as Q.
|
||||
qk_acc = zero_v4
|
||||
for chk in range_constexpr(QK_K_CHUNKS):
|
||||
k_idx_i64 = (
|
||||
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
+ fx.Int32(chk * 8)
|
||||
+ mfma_col_grp * fx.Int32(2)
|
||||
)
|
||||
kv_load = vector.load_op(
|
||||
T.vec(2, T.i64),
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, k_idx_i64)],
|
||||
)
|
||||
k_op = vector.bitcast(T.vec(8, T.bf16), kv_load)
|
||||
qk_acc = rocdl.mfma_f32_16x16x32_bf16(
|
||||
T.f32x4, [k_op, q_chunks[chk], qk_acc, 0, 0, 0]
|
||||
)
|
||||
|
||||
# qk_acc layout: lane t holds C[token=(t/16)*4..+3, query=t%16]
|
||||
# 4 fp32/lane = 4 different tokens at SAME query_row.
|
||||
|
||||
# Scale + mask out-of-context tokens.
|
||||
qk_acc = _vsplat_mul(qk_acc, QK_SCALE)
|
||||
for elem in range_constexpr(4):
|
||||
kv_tok = (
|
||||
partition_start
|
||||
+ fx.Int32(n_tile * TILE_SIZE)
|
||||
+ mfma_col_grp * fx.Int32(4)
|
||||
+ fx.Int32(elem)
|
||||
)
|
||||
in_b = kv_tok < seq_len
|
||||
v = vector.extract(qk_acc, static_position=[elem])
|
||||
qk_acc = vector.insert(
|
||||
in_b.select(v, NEG_INF),
|
||||
qk_acc,
|
||||
static_position=[elem],
|
||||
dynamic_position=[],
|
||||
)
|
||||
|
||||
# FA2 online softmax: per-query-row reduce.
|
||||
# Per-row max: max over 4 fp32 in lane, then xor-shuffle 16, 32
|
||||
# (across the 4 col_grps that share same mfma_row).
|
||||
local_max = vector.reduction(T.f32, "maxnumf", qk_acc)
|
||||
r1 = local_max.shuffle_xor(fx.Int32(16), c_w)
|
||||
local_max = local_max.maximumf(r1)
|
||||
r2 = local_max.shuffle_xor(fx.Int32(32), c_w)
|
||||
tile_max = local_max.maximumf(r2)
|
||||
|
||||
new_max = running_max.maximumf(tile_max)
|
||||
max_diff = running_max - new_max
|
||||
safe_diff = (running_max > NEG_INF).select(max_diff, ZERO_F)
|
||||
scale = (safe_diff * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
|
||||
running_sum = running_sum * scale
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
acc_pv[h] = _vsplat_mul(acc_pv[h], scale)
|
||||
running_max = new_max
|
||||
|
||||
# Compute probs: p = exp((qk - new_max) * LOG2E)
|
||||
tile_sum = ZERO_F
|
||||
for elem in range_constexpr(4):
|
||||
s = vector.extract(qk_acc, static_position=[elem])
|
||||
d = s - new_max
|
||||
d = (new_max > NEG_INF).select(d, NEG_INF)
|
||||
p = (d * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
|
||||
tile_sum = tile_sum + p
|
||||
qk_acc = vector.insert(
|
||||
p, qk_acc, static_position=[elem], dynamic_position=[]
|
||||
)
|
||||
|
||||
ts1 = tile_sum.shuffle_xor(fx.Int32(16), c_w)
|
||||
tile_sum = tile_sum + ts1
|
||||
ts2 = tile_sum.shuffle_xor(fx.Int32(32), c_w)
|
||||
tile_sum = tile_sum + ts2
|
||||
running_sum = running_sum + tile_sum
|
||||
|
||||
# ---- V dequant → LDS (V/scale/zero raw loads were hoisted) -
|
||||
# v_packed, vscale_raw, vzero_raw are already in flight from
|
||||
# the top of this iteration; only the f32 conversion stays
|
||||
# here so the actual consume site is still close to the use.
|
||||
vscale_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vscale_raw))
|
||||
vzero_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vzero_raw))
|
||||
|
||||
if USE_HW_TR:
|
||||
# V dequant → LDS [token][head_dim] (ROW-MAJOR, no transpose).
|
||||
# Each lane writes 32 contiguous bf16 (one token, head_dims
|
||||
# chunk_in_tok*32..+31) as 4× ds_write_b128 = 4× vec(2,i64).
|
||||
# Replaces 32× ds_write_b16 of the legacy transposed path.
|
||||
v_lds_elem_base = tok_in_tile * fx.Int32(
|
||||
HEAD_SIZE
|
||||
) + chunk_in_tok * fx.Int32(32)
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(v_packed, static_position=[w])
|
||||
bf16_elems = []
|
||||
for n in range_constexpr(8):
|
||||
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
|
||||
nibble_f32 = arith.sitofp(T.f32, nibble)
|
||||
elem_f32 = nibble_f32 * vscale_f32 + vzero_f32
|
||||
elem_bf16 = arith.trunc_f(T.bf16, elem_f32)
|
||||
bf16_elems.append(elem_bf16)
|
||||
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
|
||||
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
|
||||
v_lds_i64_idx = (v_lds_elem_base + fx.Int32(w * 8)) // fx.Int32(
|
||||
4
|
||||
)
|
||||
vector.store(
|
||||
v_i64,
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, v_lds_i64_idx)],
|
||||
)
|
||||
# ── HW V transpose: cross-lane LDS sync ─────────────────────
|
||||
# ds_read_tr16_b64 below introduces a cross-lane LDS read:
|
||||
# consumer lane t reads bytes written by source lanes
|
||||
# 0,4,8,12 (etc.). The compiler's automatic waitcnt insertion
|
||||
# is per-lane and has no awareness of the HW transpose's
|
||||
# cross-lane forwarding, so it can sink the next iteration's
|
||||
# ds_read ahead of these ds_write_b128 ops in the schedule.
|
||||
#
|
||||
# Two-part fence:
|
||||
# 1. sched_barrier(0): MachineScheduler reorder barrier
|
||||
# (mask=0 → no instruction class may cross). Pure hint,
|
||||
# generates no runtime instruction. Use the simple
|
||||
# sched_barrier (NOT sched_group_barrier, which is an
|
||||
# IGLP marker that requires partner barriers and
|
||||
# corrupts the schedule when used in isolation).
|
||||
# 2. s_waitcnt lgkmcnt=0: runtime drain of the LDS write
|
||||
# queue. vmcnt/expcnt left at no-wait so we don't
|
||||
# stall on speculatively-issued HBM loads (which can
|
||||
# include OOB-but-masked buffer_load reads from the
|
||||
# next K-tile's hoisted V prefetch).
|
||||
#
|
||||
# The SW path below does NOT need this because each lane
|
||||
# only reads cells it itself wrote (no cross-lane dep), and
|
||||
# the per-lane same-address waitcnt the compiler emits is
|
||||
# correct.
|
||||
#
|
||||
# Without this fence the reordered ds_read produces
|
||||
# wrong results on high-partition-count configs (large
|
||||
# padded num_partitions); low-partition configs are
|
||||
# unaffected.
|
||||
rocdl.sched_barrier(0)
|
||||
# encode_waitcnt(vmcnt=63, expcnt=7, lgkmcnt=0)
|
||||
# = 0xF | (7<<4) | (0<<8) | (3<<14) = 0xC07F
|
||||
rocdl.s_waitcnt(0xC07F)
|
||||
else:
|
||||
# Legacy transposed V_LDS path: 32 ds_write_b16 per lane.
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(v_packed, static_position=[w])
|
||||
for n in range_constexpr(8):
|
||||
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
|
||||
nibble_f32 = arith.sitofp(T.f32, nibble)
|
||||
elem_f32 = nibble_f32 * vscale_f32 + vzero_f32
|
||||
elem_bf16 = arith.trunc_f(T.bf16, elem_f32)
|
||||
elem_i16 = arith.bitcast(T.i16, elem_bf16)
|
||||
head_dim = chunk_in_tok * fx.Int32(32) + fx.Int32(w * 8 + n)
|
||||
v_idx_i16 = head_dim * fx.Int32(TILE_SIZE) + tok_in_tile
|
||||
v_vec = vector.from_elements(T.vec(1, T.i16), [elem_i16])
|
||||
vector.store(
|
||||
v_vec,
|
||||
kv_lds_i16,
|
||||
[arith.index_cast(T.index, v_idx_i16)],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# ---- PV MFMA: A=V[head_dim, token], B=P (=qk_acc bf16) -----
|
||||
# P operand B layout matches qk_acc: lane t holds 4 bf16 at
|
||||
# K=token=(t/16)*4..+3, N=query=t%16. Just trunc_f to bf16.
|
||||
p_bf16 = arith.trunc_f(T.vec(4, T.bf16), qk_acc)
|
||||
p_op = vector.bitcast(T.vec(4, T.i16), p_bf16)
|
||||
|
||||
if USE_HW_TR:
|
||||
# HW-transpose PV path: V_lds is row-major V[token][head_dim].
|
||||
# MFMA A operand layout: lane t holds 4 bf16 at
|
||||
# M=head_dim=mfma_row + h*16, K=token=(t/16)*4..+3
|
||||
# ds_read_tr16_b64 (4-element 16-bit transpose, per-16-lane block):
|
||||
# result[lane=t, elem=e] = Input[source_lane=e*4 + (t%16)//4,
|
||||
# col=t%4]
|
||||
# Per-lane address: token_idx = lane // 4 (covers 0..15 across
|
||||
# all four 16-lane MFMA blocks), hd_sub = (lane % 4)*4 selects
|
||||
# the 4-element column window inside the h*16 chunk.
|
||||
# Total LDS byte offset = kv_off + token_idx*HEAD_SIZE*2
|
||||
# + (h*16 + hd_sub)*2
|
||||
token_idx = lane >> fx.Int32(2)
|
||||
hd_sub = (lane & fx.Int32(3)) * fx.Int32(4)
|
||||
v_lane_byte = (
|
||||
fx.Int32(kv_off)
|
||||
+ token_idx * fx.Int32(HEAD_SIZE * 2)
|
||||
+ hd_sub * fx.Int32(2)
|
||||
)
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
v_byte_off = v_lane_byte + fx.Int32(h * 32)
|
||||
v_byte_i64 = fx.Int64(v_byte_off)
|
||||
v_ptr = buffer_ops.create_llvm_ptr(
|
||||
v_byte_i64,
|
||||
address_space=3,
|
||||
)
|
||||
v_op_raw = rocdl.ds_read_tr16_b64(
|
||||
T.vec(4, T.i16),
|
||||
v_ptr,
|
||||
).result
|
||||
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
|
||||
T.f32x4, [v_op_raw, p_op, acc_pv[h], 0, 0, 0]
|
||||
)
|
||||
else:
|
||||
# Legacy transposed-LDS PV: 1 ds_read_b64 per chunk.
|
||||
# Byte addr = (mfma_row + h*16) * TILE_SIZE * 2 + (col_grp*4) * 2
|
||||
# = (mfma_row + h*16) * 32 + col_grp*8
|
||||
# i64 idx = (mfma_row + h*16) * 4 + col_grp
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
v_idx_i64 = (mfma_row + fx.Int32(h * 16)) * fx.Int32(
|
||||
4
|
||||
) + mfma_col_grp
|
||||
kv_load = vector.load_op(
|
||||
T.vec(1, T.i64),
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, v_idx_i64)],
|
||||
)
|
||||
v_op = vector.bitcast(T.vec(4, T.i16), kv_load)
|
||||
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
|
||||
T.f32x4, [v_op, p_op, acc_pv[h], 0, 0, 0]
|
||||
)
|
||||
|
||||
_scf.YieldOp(
|
||||
[
|
||||
_ival(running_max),
|
||||
_ival(running_sum),
|
||||
*[_ival(p) for p in acc_pv],
|
||||
]
|
||||
)
|
||||
finally:
|
||||
_for_ip.__exit__(None, None, None)
|
||||
# Pull final accumulator values out of the for_op results.
|
||||
running_max = _for_op.results[0]
|
||||
running_sum = _for_op.results[1]
|
||||
acc_pv = list(_for_op.results[2:])
|
||||
|
||||
# ===== STEP G: Output ===========================================
|
||||
# acc_pv[h] layout: lane t holds 4 fp32 at M=head_dim=(t/16)*4..+3 + h*16,
|
||||
# N=query_row=t%16.
|
||||
# 4 fp32/lane = 4 contiguous head_dim values for ONE query_row.
|
||||
safe_sum = (running_sum > ZERO_F).select(running_sum, ONE_F)
|
||||
rcp = ONE_F / safe_sum
|
||||
|
||||
c_os = fx.Int32(_stride_out_seq)
|
||||
c_oh = fx.Int32(_stride_out_head)
|
||||
c_op_ = fx.Int32(_stride_out_part)
|
||||
out_base = seq * c_os + kv_h * c_oh + part * c_op_
|
||||
|
||||
# Gate all global stores by mfma_row < QG. For QG=16 the predicate is
|
||||
# tautological (mfma_row ∈ [0,15]); for QG=8 lanes 8..15 (whose MFMA
|
||||
# outputs are computational waste) skip the stores entirely.
|
||||
valid_row_pred = arith.cmpi(
|
||||
arith.CmpIPredicate.ult,
|
||||
mfma_row.ir_value() if hasattr(mfma_row, "ir_value") else mfma_row,
|
||||
arith.constant(QG, type=T.i32),
|
||||
)
|
||||
_if = _scf.IfOp(valid_row_pred)
|
||||
with ir.InsertionPoint(_if.then_block):
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
pv_norm = _vsplat_mul(acc_pv[h], rcp)
|
||||
pv_bf16 = arith.trunc_f(T.vec(4, T.bf16), pv_norm)
|
||||
pv_i32x2 = vector.bitcast(T.vec(2, T.i32), pv_bf16)
|
||||
head_dim_start = fx.Int32(h * 16) + mfma_col_grp * fx.Int32(4)
|
||||
out_off_elem = (
|
||||
out_base + mfma_row * fx.Int32(HEAD_SIZE) + head_dim_start
|
||||
)
|
||||
buffer_ops.buffer_store(
|
||||
pv_i32x2,
|
||||
out_rsrc,
|
||||
out_off_elem * fx.Int32(2),
|
||||
offset_is_bytes=True,
|
||||
)
|
||||
|
||||
c_npq = fx.Int32(num_partitions * QG)
|
||||
ml_off = (
|
||||
seq * fx.Int32(_stride_ml_seq) + kv_h * c_npq + part * c_qg + mfma_row
|
||||
)
|
||||
es_off = (
|
||||
seq * fx.Int32(_stride_es_seq) + kv_h * c_npq + part * c_qg + mfma_row
|
||||
)
|
||||
buffer_ops.buffer_store(running_max, ml_rsrc, ml_off)
|
||||
buffer_ops.buffer_store(running_sum, es_rsrc, es_off)
|
||||
_scf.YieldOp([])
|
||||
|
||||
return tq_decode_kernel
|
||||
@@ -1,875 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# TurboQuant decode GQA-6 sibling (FlyDSL) — MI355X / gfx950 / CDNA4
|
||||
#
|
||||
# Decode-only paged attention over a TurboQuant SoA KV cache.
|
||||
# Targets MiniMax-M2.5 class models: HEAD_SIZE=128, GQA group=6, MSE_BITS=4 K,
|
||||
# VQB=4 V, N_CENTROIDS=16, BLOCK_SIZE=16, partitioned mode.
|
||||
#
|
||||
# === Sibling-file rationale ===
|
||||
# Structurally identical to tq_decode.py except:
|
||||
# * QUERY_GROUP_SIZE default = 6 (was 8)
|
||||
# * Build-time assert allows query_group_size = 6 only (Qwen sizes go to
|
||||
# the canonical tq_decode kernel — NOT this file)
|
||||
# * QG_LOAD_ITERS = (QG + 3) // 4 (round-up; 2 iters for QG=6 loads
|
||||
# 8 LDS rows of which only 6 carry real Q; rows 6..7 are zero-padded
|
||||
# by buffer_load OOB protection — gated out by the same mfma_row < QG
|
||||
# predicate the canonical kernel already uses for QG=8)
|
||||
#
|
||||
# All cross-lane operations (xor_shuffle softmax reductions, ds_read_tr16_b64
|
||||
# HW V transpose) are mfma_row-isolated and thus already GQA-agnostic. The
|
||||
# kernel is functionally a direct copy of tq_decode.py with the QG knob
|
||||
# turned to 6 — kept as a sibling per FlyDSL convention to preserve the
|
||||
# canonical kernel's invariants for its production callers (Qwen2.5-72B,
|
||||
# Qwen3-32B).
|
||||
#
|
||||
# === Design ===
|
||||
# * 1 wave (64 lanes) per CTA — simpler than 4-warp; trades parallelism for
|
||||
# correctness/debug-ability. A later revision will fan out to 4 warps.
|
||||
# * Flash-Attention-2 online softmax: rescale running PV/sum per K-tile,
|
||||
# no QK score storage across tiles, no P-LDS round-trip (qk_acc layout
|
||||
# matches PV B-operand directly).
|
||||
# * Centroids loaded once into LDS at kernel start (LDS-resident LUT).
|
||||
# * K dequant → LDS [token, head_dim] (natural layout)
|
||||
# * V dequant → LDS [head_dim, token] (TRANSPOSED — required so PV MFMA's
|
||||
# A=V operand can be loaded as 4-bf16 contiguous via 1 ds_read_b64).
|
||||
#
|
||||
# MFMA layouts used:
|
||||
# QK: mfma(A=K, B=Q, C=qk_acc) → C[m=token, n=query], 4 fp32/lane
|
||||
# = 4 tokens for SAME query → softmax via
|
||||
# xor_shuffle(16),(32) reduces per-row
|
||||
# PV: mfma(A=V_T, B=P, C=acc_pv) → C[m=head_dim, n=query], 4 fp32/lane
|
||||
# = 4 contiguous head_dim cols for SAME
|
||||
# query → 1 buffer_store of 8 bytes
|
||||
# per chunk per lane
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import flydsl.compiler as flyc
|
||||
import flydsl.expr as fx
|
||||
from flydsl._mlir import ir
|
||||
from flydsl._mlir.dialects import scf as _scf
|
||||
from flydsl.expr import (
|
||||
arith,
|
||||
buffer_ops,
|
||||
const_expr, # noqa: F401 kept for kernel-DSL API parity
|
||||
gpu,
|
||||
range_constexpr,
|
||||
rocdl,
|
||||
vector,
|
||||
)
|
||||
from flydsl.expr.typing import Int32, T # noqa: F401 Int32 via fx.Int32
|
||||
from flydsl.runtime.device import get_rocm_arch as get_hip_arch
|
||||
from flydsl.utils.smem_allocator import SmemAllocator, SmemPtr
|
||||
|
||||
# === Constants (MiniMax-M2.5-class TQ decode profile) =======================
|
||||
HEAD_SIZE = 128
|
||||
# default; overridable via build_tq_decode_gqa6_module(kv_block_size=...)
|
||||
KV_BLOCK_SIZE = 16
|
||||
# MFMA tile = 16 tokens (do not change without re-deriving MFMA shapes)
|
||||
TILE_SIZE = 16
|
||||
N_CENTROIDS = 16
|
||||
QUERY_GROUP_SIZE = 6 # GQA-6 default for MiniMax-M2.5
|
||||
WARP_SIZE = 64
|
||||
NUM_WARPS = 1
|
||||
BLOCK_THREADS = NUM_WARPS * WARP_SIZE
|
||||
KV_COMPUTE_BLOCK = 256 # 16 K-tiles × 16 tokens
|
||||
|
||||
# TQ SoA layout
|
||||
NUM_SOA_FIELDS = 3
|
||||
SOA_K_NORM = 0
|
||||
SOA_V_SCALE = 1
|
||||
SOA_V_ZERO = 2
|
||||
KEY_DATA_BYTES = HEAD_SIZE // 2
|
||||
VAL_DATA_BYTES = HEAD_SIZE // 2
|
||||
DATA_BYTES_PER_SLOT = KEY_DATA_BYTES + VAL_DATA_BYTES
|
||||
|
||||
# MFMA
|
||||
MFMA_M = MFMA_N = 16
|
||||
MFMA_K_BF16_QK = 32 # CDNA4 wide-K (mfma_f32_16x16x32_bf16)
|
||||
MFMA_K_BF16_PV = 16 # PV's K = token tile = 16
|
||||
QK_K_CHUNKS = HEAD_SIZE // MFMA_K_BF16_QK # 4
|
||||
PV_N_CHUNKS = HEAD_SIZE // MFMA_N # 8
|
||||
|
||||
# LDS regions
|
||||
# Q region uses the full 16-row footprint (4096 B) rather than a tight
|
||||
# QG=6 sizing, keeping the LDS layout identical to the canonical kernel.
|
||||
CENTROID_LDS_BYTES = N_CENTROIDS * 4 # 64
|
||||
Q_LDS_BYTES = 16 * HEAD_SIZE * 2 # 4096
|
||||
KV_TILE_LDS_BYTES = TILE_SIZE * HEAD_SIZE * 2 # 4096
|
||||
|
||||
LOG2E = 1.4426950408889634
|
||||
NEG_INF_VAL = float("-inf")
|
||||
|
||||
|
||||
def _vsplat_mul(vec, scalar):
|
||||
s = scalar.ir_value() if hasattr(scalar, "ir_value") else scalar
|
||||
return vec * vector.broadcast(T.f32x4, s)
|
||||
|
||||
|
||||
allocator = None
|
||||
|
||||
|
||||
def build_tq_decode_gqa6_module(
|
||||
num_seqs: int,
|
||||
num_kv_heads: int,
|
||||
num_partitions: int,
|
||||
max_blocks_per_seq: int = 512,
|
||||
softmax_scale: float | None = None,
|
||||
query_group_size: int = QUERY_GROUP_SIZE,
|
||||
kv_block_size: int = KV_BLOCK_SIZE,
|
||||
use_hw_v_transpose: bool = False,
|
||||
tile_groups_per_partition: int = 1,
|
||||
):
|
||||
"""Build a TQ decode GQA-6 kernel module (MiniMax-M2.5).
|
||||
|
||||
``query_group_size`` (= num_query_heads // num_kv_heads) is fixed at 6
|
||||
for this sibling. The MFMA's 16-row capacity is 6/16 = 37.5% used;
|
||||
lanes 6..15 compute garbage and are gated out of all global-memory
|
||||
writes via the same ``mfma_row < QG`` predicate the canonical kernel
|
||||
uses for QG=8. Q load iterates ``(QG + 3) // 4`` = 2 chunks (loads
|
||||
8 LDS rows of which rows 6..7 are zero-padded by the buffer-resource
|
||||
OOB protection — they're never consumed because the gating lanes
|
||||
don't write).
|
||||
|
||||
For GQA-8 or GQA-16 (Qwen models), use the canonical
|
||||
``build_tq_decode_module`` from ``tq_decode.py`` instead.
|
||||
|
||||
``kv_block_size`` is the number of tokens per vLLM cache block. Must
|
||||
be a multiple of TILE_SIZE (=16). Supported values: 16 and 32.
|
||||
|
||||
``use_hw_v_transpose`` (CDNA4 / gfx950 only) — see canonical kernel
|
||||
docstring for details. Same fence pattern applies here.
|
||||
"""
|
||||
assert query_group_size == 6, (
|
||||
f"build_tq_decode_gqa6_module is GQA-6 only; got "
|
||||
f"query_group_size={query_group_size}. Use build_tq_decode_module "
|
||||
f"from tq_decode.py for GQA-8 or GQA-16."
|
||||
)
|
||||
assert kv_block_size in (16, 32), (
|
||||
f"kv_block_size must be 16 or 32; got {kv_block_size}"
|
||||
)
|
||||
assert kv_block_size % TILE_SIZE == 0
|
||||
assert int(tile_groups_per_partition) >= 1, (
|
||||
f"tile_groups_per_partition must be >= 1; got {tile_groups_per_partition}"
|
||||
)
|
||||
USE_HW_TR = bool(use_hw_v_transpose)
|
||||
TGPP = int(tile_groups_per_partition)
|
||||
PARTITION_EXTENT_TOKENS = TGPP * KV_COMPUTE_BLOCK
|
||||
|
||||
QG = int(query_group_size)
|
||||
# Round-up: 2 iters for QG=6 → loads 8 LDS rows (rows 6..7 are
|
||||
# zero-padded by buffer_load OOB protection; never consumed because
|
||||
# mfma_row >= QG lanes are gated out of all global stores below).
|
||||
QG_LOAD_ITERS = (QG + 3) // 4
|
||||
OOB_OFFSET = 0x7FFFFFF0 # noqa: F841 ~2GB byte offset; > any plausible buffer
|
||||
|
||||
_BS = int(kv_block_size)
|
||||
_TILES_PER_BLOCK = _BS // TILE_SIZE # 1 for BS=16, 2 for BS=32
|
||||
|
||||
global allocator
|
||||
arch = get_hip_arch()
|
||||
|
||||
if softmax_scale is None:
|
||||
softmax_scale = 1.0 / (HEAD_SIZE**0.5)
|
||||
_qk_scale = float(softmax_scale)
|
||||
|
||||
# --- Strides ---
|
||||
_Hq = num_kv_heads * QG
|
||||
_stride_q_seq = _Hq * HEAD_SIZE
|
||||
_stride_q_head = HEAD_SIZE
|
||||
_stride_bt_seq = max_blocks_per_seq
|
||||
|
||||
_data_region_bytes = _BS * num_kv_heads * DATA_BYTES_PER_SLOT
|
||||
_meta_region_bytes = num_kv_heads * NUM_SOA_FIELDS * _BS * 2
|
||||
_stride_cache_block = _data_region_bytes + _meta_region_bytes
|
||||
_meta_region_offset = _data_region_bytes
|
||||
|
||||
_stride_out_part = QG * HEAD_SIZE
|
||||
_stride_out_head = num_partitions * QG * HEAD_SIZE
|
||||
_stride_out_seq = num_kv_heads * num_partitions * QG * HEAD_SIZE
|
||||
_stride_es_seq = num_kv_heads * num_partitions * QG
|
||||
_stride_ml_seq = _stride_es_seq
|
||||
|
||||
# --- LDS layout ---
|
||||
allocator = SmemAllocator(None, arch=arch, global_sym_name="tq_gqa6_smem")
|
||||
centroid_off = 0
|
||||
allocator.ptr = CENTROID_LDS_BYTES
|
||||
q_off = allocator.ptr
|
||||
allocator.ptr += Q_LDS_BYTES
|
||||
kv_off = allocator.ptr
|
||||
allocator.ptr += KV_TILE_LDS_BYTES
|
||||
|
||||
@flyc.kernel
|
||||
def tq_decode_gqa6_kernel(
|
||||
out_ptr: fx.Tensor,
|
||||
exp_sums_ptr: fx.Tensor,
|
||||
max_logits_ptr: fx.Tensor,
|
||||
query_ptr: fx.Tensor,
|
||||
kv_cache_ptr: fx.Tensor,
|
||||
centroids_ptr: fx.Tensor,
|
||||
block_tables_ptr: fx.Tensor,
|
||||
seq_lens_ptr: fx.Tensor,
|
||||
):
|
||||
# ---- IDs ---------------------------------------------------------
|
||||
tid = gpu.thread_idx.x
|
||||
seq = gpu.block_idx.x
|
||||
kv_h = gpu.block_idx.y
|
||||
part = gpu.block_idx.z
|
||||
lane = tid # 0..63
|
||||
mfma_row = lane & fx.Int32(15) # = query_row (B's N) /
|
||||
# token (A's M for QK)
|
||||
mfma_col_grp = lane >> fx.Int32(4) # 0..3, K-group dim
|
||||
|
||||
# ---- Buffer resources -------------------------------------------
|
||||
q_rsrc = buffer_ops.create_buffer_resource(query_ptr, max_size=True)
|
||||
kv_rsrc = buffer_ops.create_buffer_resource(kv_cache_ptr, max_size=True)
|
||||
bt_rsrc = buffer_ops.create_buffer_resource(block_tables_ptr, max_size=True)
|
||||
sl_rsrc = buffer_ops.create_buffer_resource(seq_lens_ptr, max_size=True)
|
||||
cent_rsrc = buffer_ops.create_buffer_resource(centroids_ptr, max_size=True)
|
||||
out_rsrc = buffer_ops.create_buffer_resource(out_ptr, max_size=True)
|
||||
es_rsrc = buffer_ops.create_buffer_resource(exp_sums_ptr, max_size=True)
|
||||
ml_rsrc = buffer_ops.create_buffer_resource(max_logits_ptr, max_size=True)
|
||||
|
||||
# ---- LDS pointers -----------------------------------------------
|
||||
base = allocator.get_base()
|
||||
cent_lds = SmemPtr(base, centroid_off, T.f32, shape=(N_CENTROIDS,))
|
||||
q_lds_i32 = SmemPtr(base, q_off, T.i32, shape=(Q_LDS_BYTES // 4,)).get()
|
||||
q_lds_i64 = SmemPtr(base, q_off, T.i64, shape=(Q_LDS_BYTES // 8,)).get()
|
||||
kv_lds_i32 = SmemPtr(base, kv_off, T.i32, shape=(KV_TILE_LDS_BYTES // 4,)).get() # noqa: F841
|
||||
kv_lds_i64 = SmemPtr(base, kv_off, T.i64, shape=(KV_TILE_LDS_BYTES // 8,)).get()
|
||||
kv_lds_i16 = SmemPtr(base, kv_off, T.i16, shape=(KV_TILE_LDS_BYTES // 2,)).get()
|
||||
|
||||
# ---- Constants ---------------------------------------------------
|
||||
c_sq = fx.Int32(_stride_q_seq)
|
||||
c_qh = fx.Int32(_stride_q_head)
|
||||
c_qg = fx.Int32(QG)
|
||||
c_bt = fx.Int32(_stride_bt_seq)
|
||||
c_block = fx.Int32(_stride_cache_block)
|
||||
c_meta_off = fx.Int32(_meta_region_offset)
|
||||
c_data_per_slot = fx.Int32(DATA_BYTES_PER_SLOT)
|
||||
c_kv_heads = fx.Int32(num_kv_heads)
|
||||
c_keydata = fx.Int32(KEY_DATA_BYTES)
|
||||
c_w = fx.Int32(WARP_SIZE)
|
||||
|
||||
NEG_INF = arith.constant(NEG_INF_VAL, type=T.f32)
|
||||
ZERO_F = fx.Float32(0.0)
|
||||
ONE_F = fx.Float32(1.0)
|
||||
LOG2E_C = arith.constant(LOG2E, type=T.f32)
|
||||
QK_SCALE = arith.constant(_qk_scale, type=T.f32)
|
||||
|
||||
# ===== STEP A: Load centroids → LDS (cooperative, race-safe) =====
|
||||
c_idx_safe = lane & fx.Int32(N_CENTROIDS - 1)
|
||||
c_val = buffer_ops.buffer_load(cent_rsrc, c_idx_safe, vec_width=1, dtype=T.f32)
|
||||
cent_lds.store(c_val, [arith.index_cast(T.index, c_idx_safe)])
|
||||
gpu.barrier()
|
||||
|
||||
# ===== STEP B: Load Q → row-major LDS [query_row, head_dim] ======
|
||||
# Q layout in HBM: [N, Hq, D] bf16, slice = QG rows × D=128 cols × 2B.
|
||||
# QG=6 → 1536 B real, 96 chunks; QG_LOAD_ITERS=2 issues 128 chunks
|
||||
# (8 rows × 16 col_b) so lanes whose row ∈ {6, 7} address heads that
|
||||
# do NOT exist in the Q tensor (Hq = num_kv_heads * 6).
|
||||
#
|
||||
# ── Per-lane Q-load OOB redirect ─────────────────────────────────
|
||||
# Hq = num_kv_heads * QG, so head index = kv_h * QG + row. For QG=6
|
||||
# the upper rows alias to (kv_h+1)'s real heads (still in-bounds
|
||||
# for most kv_h), but for the *last* (seq, kv_h) the read tips
|
||||
# past the Q tensor entirely. With ``max_size=True`` the buffer
|
||||
# descriptor advertises 4 GB so the HW does no OOB clamping; the
|
||||
# padding read then touches unmapped pages and the kernel dies
|
||||
# with "Memory access fault by GPU node" the moment the
|
||||
# surrounding allocator state leaves the next page unmapped
|
||||
# (allocator-pattern-dependent → nondeterministic fault).
|
||||
#
|
||||
# Fix: redirect OOB lanes (``row >= QG``) to load Q[0,0,col_elem]
|
||||
# — a byte offset that is always inside the actual Q tensor.
|
||||
# The redundant load is wasted but harmless; the resulting LDS
|
||||
# write to row ∈ {6, 7} is overwritten/ignored downstream because
|
||||
# every global store is gated by the output-time
|
||||
# ``mfma_row < QG`` predicate (the same gate canonical QG=8 uses
|
||||
# for its garbage rows 8..15).
|
||||
#
|
||||
# NB: A ``mask=`` parameter on ``buffer_load`` substitutes the
|
||||
# masked-out offset with 0x7FFFFFFF *bytes*; with ``max_size=True``
|
||||
# this is still inside the 4 GB descriptor and so the HW happily
|
||||
# dereferences ``q_base + 2 GB`` → another unmapped page → fault.
|
||||
# The address-redirect below is the only safe option short of
|
||||
# threading the true Q size through as a runtime kernel arg.
|
||||
#
|
||||
# The canonical Qwen kernel does not need this redirect because
|
||||
# for QG ∈ {8, 16} the (kv_h, row) iteration covers exactly the
|
||||
# real heads; QG=6 is the only case where
|
||||
# ``QG_LOAD_ITERS * 4 != QG``.
|
||||
# STEP B: load pre-rotated q_rot → Q_LDS. QG=6 over-reads
|
||||
# (QG_LOAD_ITERS * 4 != QG), so rows >= QG redirect to a safe
|
||||
# in-bounds offset to avoid faulting past the real Q rows.
|
||||
for c in range_constexpr(QG_LOAD_ITERS):
|
||||
row_chunk = lane + fx.Int32(c * WARP_SIZE) # 0..127
|
||||
row = row_chunk >> fx.Int32(4) # 0..7
|
||||
col_b = row_chunk & fx.Int32(15) # 0..15
|
||||
col_elem = col_b * fx.Int32(8) # 0..120 in bf16
|
||||
q_off_real = seq * c_sq + (kv_h * c_qg + row) * c_qh + col_elem
|
||||
row_in_qg = row < fx.Int32(QG)
|
||||
q_off_elem = row_in_qg.select(q_off_real, col_elem)
|
||||
q_v = buffer_ops.buffer_load(
|
||||
q_rsrc,
|
||||
q_off_elem // fx.Int32(2),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
q_lds_byte = row * fx.Int32(HEAD_SIZE * 2) + col_elem * fx.Int32(2)
|
||||
vector.store(
|
||||
q_v,
|
||||
q_lds_i32,
|
||||
[arith.index_cast(T.index, q_lds_byte // fx.Int32(4))],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# ===== STEP C: Pre-load Q operands for QK_K_CHUNKS = 4 K-chunks ===
|
||||
# Wide-K MFMA: K_per_chunk = 32, K_per_lane = 32/4 = 8 bf16 per chunk.
|
||||
# Lane t holds 8 bf16 at row=mfma_row, K-cols
|
||||
# chunk*32 + mfma_col_grp*8 + 0..7
|
||||
# Byte addr = mfma_row * HEAD_SIZE*2 + (chunk*32 + col_grp*8)*2
|
||||
# = mfma_row * 256 + chunk*64 + col_grp*16
|
||||
# i64 idx = mfma_row * 32 + chunk*8 + col_grp*2 (each i64 = 4 bf16)
|
||||
# Load 16 bytes (= 8 bf16 = 2 i64) per chunk via vec(2, i64).
|
||||
q_chunks = []
|
||||
for chk in range_constexpr(QK_K_CHUNKS):
|
||||
q_idx_i64 = (
|
||||
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
+ fx.Int32(chk * 8)
|
||||
+ mfma_col_grp * fx.Int32(2)
|
||||
)
|
||||
qv = vector.load_op(
|
||||
T.vec(2, T.i64),
|
||||
q_lds_i64,
|
||||
[arith.index_cast(T.index, q_idx_i64)],
|
||||
)
|
||||
q_chunks.append(vector.bitcast(T.vec(8, T.bf16), qv))
|
||||
|
||||
# ===== STEP D: Online softmax + PV state =========================
|
||||
running_max = NEG_INF
|
||||
running_sum = ZERO_F
|
||||
zero_v4 = arith.constant_vector(0.0, T.f32x4)
|
||||
acc_pv = [zero_v4 for _ in range(PV_N_CHUNKS)]
|
||||
|
||||
# ===== STEP E: Sequence-len + partition base ====================
|
||||
seq_len = buffer_ops.buffer_load(sl_rsrc, seq, vec_width=1, dtype=T.i32)
|
||||
# ── Bounded num_partitions + internal looping ───────────────────
|
||||
# Mirrors the canonical tq_decode kernel: each CTA owns
|
||||
# ``PARTITION_EXTENT_TOKENS = TGPP * KV_COMPUTE_BLOCK`` contiguous
|
||||
# tokens of the seq, processed as TGPP groups of 16 K-tiles each.
|
||||
# FA-2 online-softmax state accumulates across all TGPP * 16 tiles.
|
||||
# With TGPP=1 this is bit-identical to the legacy 16-tile body.
|
||||
partition_base = part * fx.Int32(PARTITION_EXTENT_TOKENS)
|
||||
|
||||
# Per-K-tile dequant lane assignment:
|
||||
# 16 tokens × 64 packed bytes per token = 1024 bytes
|
||||
# 64 lanes × 16 bytes/lane = 1024 ✓
|
||||
# Lane t: token = t / 4, sub-chunk = t % 4 (0..3) → 16 bytes contiguous
|
||||
tok_in_tile = lane >> fx.Int32(2)
|
||||
chunk_in_tok = lane & fx.Int32(3)
|
||||
# ``slot`` (= absolute slot index within the cache block, range
|
||||
# 0.._BS-1) is recomputed per tile inside the K-tile loop below
|
||||
# so it reflects ``tile_in_block * TILE_SIZE + tok_in_tile``.
|
||||
|
||||
c_meta_u16_per_kvh = fx.Int32(NUM_SOA_FIELDS * _BS)
|
||||
c_knorm_off_u16 = fx.Int32(SOA_K_NORM * _BS)
|
||||
c_vscale_off_u16 = fx.Int32(SOA_V_SCALE * _BS)
|
||||
c_vzero_off_u16 = fx.Int32(SOA_V_ZERO * _BS)
|
||||
|
||||
# ===== STEP F: K-tile loop =======================================
|
||||
# With kv_block_size > TILE_SIZE there are _TILES_PER_BLOCK tiles
|
||||
# per cache block. Tiles within the same block share a block-table
|
||||
# entry but address different slot ranges within the block.
|
||||
#
|
||||
# ── Per-tile block-table OOB redirect ────────────────────────────
|
||||
# The K-tile loop is unrolled 16 times so every CTA issues 16
|
||||
# block-table reads regardless of (partition, seq_len). When a
|
||||
# partition extends past ``seq_len`` (e.g. num_partitions=2 for a
|
||||
# 256-token sequence — partition 1 covers tokens 256..511, all
|
||||
# masked out) the per-tile bt offset can land beyond the bt
|
||||
# allocation. With ``max_size=True`` the descriptor is 4 GB so
|
||||
# the HW returns whatever pre-existing HBM bytes live at that
|
||||
# offset; the resulting garbage ``phys_block`` is multiplied by
|
||||
# ``stride_cache_block`` and the subsequent kv_cache read jumps
|
||||
# into unmapped pages, killing the whole launch with a
|
||||
# "Memory access fault by GPU node" exactly as the OOB Q load
|
||||
# used to (same allocator-pattern-dependent fault; B=4 makes
|
||||
# it deterministic by adding more colliding CTAs).
|
||||
#
|
||||
# Fix: when the tile starts at or past ``seq_len`` (so its
|
||||
# qk_acc is going to be killed by the per-token
|
||||
# ``kv_tok < seq_len`` mask anyway), redirect the bt read to
|
||||
# ``bt[seq, 0]`` — always in bounds. The redundant phys_block
|
||||
# decode + kv_cache read is wasted work, but correctness is
|
||||
# preserved without changing the kernel's iteration count.
|
||||
#
|
||||
# The canonical Qwen kernel reads the same tiles and is also
|
||||
# exposed to this issue in principle; in practice it surfaces on
|
||||
# GQA-6 first because the surrounding allocator state differs
|
||||
# (extra Q-tail allocation) and shifts the page maps so the bt
|
||||
# overflow lands on unmapped pages. Applying the same redirect to
|
||||
# the canonical kernel is a possible follow-up.
|
||||
# ===== Runtime-adaptive scf.ForOp w/ iter_args =========
|
||||
# Runtime-adaptive outer loop: trip count derives from
|
||||
# actual seq_len, NOT TGPP_max. At cudagraph-capture warmup
|
||||
# (seq_len=1) only 1 iteration runs; at 32K production
|
||||
# decode all TGPP iterations run. Kernel binary stays small
|
||||
# (single body, looped at runtime) — capture time matches
|
||||
# the legacy 16-tile-only kernel rather than scaling 4x.
|
||||
#
|
||||
# FA-2 state (running_max, running_sum, acc_pv[8]) threads
|
||||
# through scf iter_args; the body reads them at top, runs
|
||||
# the unchanged 16-tile inner body, and yields the new
|
||||
# state at bottom. After the loop, results are pulled out
|
||||
# of for_op.results back into the local Python names so the
|
||||
# downstream STEP G (output) is unmodified.
|
||||
c_kcb = fx.Int32(KV_COMPUTE_BLOCK)
|
||||
c_tgpp = fx.Int32(TGPP)
|
||||
c_zero_i32 = fx.Int32(0)
|
||||
c_one_i32 = fx.Int32(1)
|
||||
# remaining = seq_len - partition_base (signed, may be <=0)
|
||||
remaining = seq_len - partition_base
|
||||
in_range = remaining > c_zero_i32
|
||||
# trip_raw = ceil(remaining / KV_COMPUTE_BLOCK)
|
||||
# (when in_range is false, the divisor branch produces
|
||||
# garbage that the select below overrides with zero, so we
|
||||
# don't pre-clamp)
|
||||
trip_raw = (remaining + c_kcb - c_one_i32) // c_kcb
|
||||
trip_clamped = (trip_raw > c_tgpp).select(c_tgpp, trip_raw)
|
||||
trip_or_zero = in_range.select(trip_clamped, c_zero_i32)
|
||||
c_zero_idx = arith.constant(0, index=True)
|
||||
c_one_idx = arith.constant(1, index=True)
|
||||
trip_idx = arith.index_cast(
|
||||
T.index,
|
||||
trip_or_zero.ir_value()
|
||||
if hasattr(trip_or_zero, "ir_value")
|
||||
else trip_or_zero,
|
||||
)
|
||||
|
||||
# Initial iter_args list: FA-2 accumulator state.
|
||||
# Order: running_max, running_sum, acc_pv[0..PV_N_CHUNKS-1].
|
||||
# NOTE: scf.ForOp requires ir.Value (with .type). The DSL
|
||||
# constants ZERO_F / ONE_F are fx.Float32 (Numeric) wrappers,
|
||||
# not raw ir.Value, so we unwrap via .ir_value() before
|
||||
# passing.
|
||||
def _ival(v):
|
||||
return v.ir_value() if hasattr(v, "ir_value") else v
|
||||
|
||||
_init_iter = [
|
||||
_ival(running_max),
|
||||
_ival(running_sum),
|
||||
*[_ival(p) for p in acc_pv],
|
||||
]
|
||||
_for_op = _scf.ForOp(
|
||||
c_zero_idx,
|
||||
trip_idx,
|
||||
c_one_idx,
|
||||
_init_iter,
|
||||
)
|
||||
_for_ip = ir.InsertionPoint(_for_op.body)
|
||||
_for_ip.__enter__()
|
||||
try:
|
||||
tg_idx = _for_op.induction_variable
|
||||
tg_i32 = fx.Int32(arith.index_cast(T.i32, tg_idx))
|
||||
partition_start = partition_base + tg_i32 * c_kcb
|
||||
bt_seq_base = seq * c_bt + (partition_start // fx.Int32(_BS))
|
||||
running_max = _for_op.inner_iter_args[0]
|
||||
running_sum = _for_op.inner_iter_args[1]
|
||||
acc_pv = list(_for_op.inner_iter_args[2:])
|
||||
for n_tile in range_constexpr(16):
|
||||
block_in_part = n_tile // _TILES_PER_BLOCK
|
||||
tile_in_block = n_tile % _TILES_PER_BLOCK
|
||||
tile_start_tok = partition_start + fx.Int32(n_tile * TILE_SIZE)
|
||||
tile_in_seq = tile_start_tok < seq_len
|
||||
bt_off = bt_seq_base + fx.Int32(block_in_part)
|
||||
bt_off_safe = tile_in_seq.select(bt_off, seq * c_bt)
|
||||
phys_block = buffer_ops.buffer_load(
|
||||
bt_rsrc,
|
||||
bt_off_safe,
|
||||
vec_width=1,
|
||||
dtype=T.i32,
|
||||
)
|
||||
block_base = phys_block * c_block
|
||||
data_region = block_base
|
||||
meta_region = block_base + c_meta_off
|
||||
|
||||
# ``slot`` is the absolute slot index within the cache block
|
||||
# (range 0.._BS-1). For BS=16 it equals tok_in_tile; for BS=32
|
||||
# it equals tile_in_block * 16 + tok_in_tile. Used both for
|
||||
# the K/V data region and SoA metadata region addressing.
|
||||
slot = fx.Int32(tile_in_block * TILE_SIZE) + tok_in_tile
|
||||
data_bases_byte = (
|
||||
data_region
|
||||
+ slot * (c_kv_heads * c_data_per_slot)
|
||||
+ kv_h * c_data_per_slot
|
||||
)
|
||||
k_byte = data_bases_byte + chunk_in_tok * fx.Int32(16)
|
||||
k_packed = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
k_byte // fx.Int32(4),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
|
||||
# ---- HOISTED: issue V data + meta HBM loads early ---------
|
||||
# These are async; their s_waitcnt is pushed by the compiler
|
||||
# past the K dequant + QK MFMA + softmax block, hiding most
|
||||
# of the V HBM latency behind compute.
|
||||
v_byte = data_bases_byte + c_keydata + chunk_in_tok * fx.Int32(16)
|
||||
v_packed = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
v_byte // fx.Int32(4),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
vscale_u16 = (
|
||||
meta_region // fx.Int32(2)
|
||||
+ kv_h * c_meta_u16_per_kvh
|
||||
+ c_vscale_off_u16
|
||||
+ slot
|
||||
)
|
||||
vzero_u16 = (
|
||||
meta_region // fx.Int32(2)
|
||||
+ kv_h * c_meta_u16_per_kvh
|
||||
+ c_vzero_off_u16
|
||||
+ slot
|
||||
)
|
||||
vscale_raw = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
vscale_u16,
|
||||
vec_width=1,
|
||||
dtype=T.i16,
|
||||
)
|
||||
vzero_raw = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
vzero_u16,
|
||||
vec_width=1,
|
||||
dtype=T.i16,
|
||||
)
|
||||
|
||||
knorm_u16 = (
|
||||
meta_region // fx.Int32(2)
|
||||
+ kv_h * c_meta_u16_per_kvh
|
||||
+ c_knorm_off_u16
|
||||
+ slot
|
||||
)
|
||||
knorm_raw = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
knorm_u16,
|
||||
vec_width=1,
|
||||
dtype=T.i16,
|
||||
)
|
||||
knorm_f16 = arith.bitcast(T.f16, knorm_raw)
|
||||
knorm_f32 = arith.extf(T.f32, knorm_f16)
|
||||
|
||||
# K dequant → LDS [token, head_dim] (natural)
|
||||
# Lane writes 32 bf16 (= 4×8) for token=tok_in_tile,
|
||||
# head_dims chunk_in_tok*32..+31 (4 sub-chunks of 8).
|
||||
tok_kreg = tok_in_tile * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
chunk_kreg = tok_kreg + chunk_in_tok * fx.Int32(8)
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(k_packed, static_position=[w])
|
||||
bf16_elems = []
|
||||
for n in range_constexpr(8):
|
||||
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
|
||||
nibble_idx = arith.index_cast(T.index, nibble)
|
||||
cent_f32 = cent_lds.load([nibble_idx])
|
||||
elem_bf16 = arith.trunc_f(T.bf16, cent_f32 * knorm_f32)
|
||||
bf16_elems.append(elem_bf16)
|
||||
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
|
||||
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
|
||||
vector.store(
|
||||
v_i64,
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, chunk_kreg + fx.Int32(w * 2))],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# QK MFMA (CDNA4 wide-K): A=K[token, head_dim], B=Q (= Q^T).
|
||||
# K read: lane t = K[token=mfma_row, head_dim=chunk*32+col_grp*8..+7]
|
||||
# Same i64×2 indexing as Q.
|
||||
qk_acc = zero_v4
|
||||
for chk in range_constexpr(QK_K_CHUNKS):
|
||||
k_idx_i64 = (
|
||||
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
+ fx.Int32(chk * 8)
|
||||
+ mfma_col_grp * fx.Int32(2)
|
||||
)
|
||||
kv_load = vector.load_op(
|
||||
T.vec(2, T.i64),
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, k_idx_i64)],
|
||||
)
|
||||
k_op = vector.bitcast(T.vec(8, T.bf16), kv_load)
|
||||
qk_acc = rocdl.mfma_f32_16x16x32_bf16(
|
||||
T.f32x4, [k_op, q_chunks[chk], qk_acc, 0, 0, 0]
|
||||
)
|
||||
|
||||
# qk_acc layout: lane t holds C[token=(t/16)*4..+3, query=t%16]
|
||||
# 4 fp32/lane = 4 different tokens at SAME query_row.
|
||||
|
||||
# Scale + mask out-of-context tokens.
|
||||
qk_acc = _vsplat_mul(qk_acc, QK_SCALE)
|
||||
for elem in range_constexpr(4):
|
||||
kv_tok = (
|
||||
partition_start
|
||||
+ fx.Int32(n_tile * TILE_SIZE)
|
||||
+ mfma_col_grp * fx.Int32(4)
|
||||
+ fx.Int32(elem)
|
||||
)
|
||||
in_b = kv_tok < seq_len
|
||||
v = vector.extract(qk_acc, static_position=[elem])
|
||||
qk_acc = vector.insert(
|
||||
in_b.select(v, NEG_INF),
|
||||
qk_acc,
|
||||
static_position=[elem],
|
||||
dynamic_position=[],
|
||||
)
|
||||
|
||||
# FA2 online softmax: per-query-row reduce.
|
||||
# Per-row max: max over 4 fp32 in lane, then xor-shuffle 16, 32
|
||||
# (across the 4 col_grps that share same mfma_row).
|
||||
local_max = vector.reduction(T.f32, "maxnumf", qk_acc)
|
||||
r1 = local_max.shuffle_xor(fx.Int32(16), c_w)
|
||||
local_max = local_max.maximumf(r1)
|
||||
r2 = local_max.shuffle_xor(fx.Int32(32), c_w)
|
||||
tile_max = local_max.maximumf(r2)
|
||||
|
||||
new_max = running_max.maximumf(tile_max)
|
||||
max_diff = running_max - new_max
|
||||
safe_diff = (running_max > NEG_INF).select(max_diff, ZERO_F)
|
||||
scale = (safe_diff * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
|
||||
running_sum = running_sum * scale
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
acc_pv[h] = _vsplat_mul(acc_pv[h], scale)
|
||||
running_max = new_max
|
||||
|
||||
# Compute probs: p = exp((qk - new_max) * LOG2E)
|
||||
tile_sum = ZERO_F
|
||||
for elem in range_constexpr(4):
|
||||
s = vector.extract(qk_acc, static_position=[elem])
|
||||
d = s - new_max
|
||||
d = (new_max > NEG_INF).select(d, NEG_INF)
|
||||
p = (d * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
|
||||
tile_sum = tile_sum + p
|
||||
qk_acc = vector.insert(
|
||||
p, qk_acc, static_position=[elem], dynamic_position=[]
|
||||
)
|
||||
|
||||
ts1 = tile_sum.shuffle_xor(fx.Int32(16), c_w)
|
||||
tile_sum = tile_sum + ts1
|
||||
ts2 = tile_sum.shuffle_xor(fx.Int32(32), c_w)
|
||||
tile_sum = tile_sum + ts2
|
||||
running_sum = running_sum + tile_sum
|
||||
|
||||
# ---- V dequant → LDS (V/scale/zero raw loads were hoisted) -
|
||||
# v_packed, vscale_raw, vzero_raw are already in flight from
|
||||
# the top of this iteration; only the f32 conversion stays
|
||||
# here so the actual consume site is still close to the use.
|
||||
vscale_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vscale_raw))
|
||||
vzero_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vzero_raw))
|
||||
|
||||
if USE_HW_TR:
|
||||
# V dequant → LDS [token][head_dim] (ROW-MAJOR, no transpose).
|
||||
# Each lane writes 32 contiguous bf16 (one token, head_dims
|
||||
# chunk_in_tok*32..+31) as 4× ds_write_b128 = 4× vec(2,i64).
|
||||
# Replaces 32× ds_write_b16 of the legacy transposed path.
|
||||
v_lds_elem_base = tok_in_tile * fx.Int32(
|
||||
HEAD_SIZE
|
||||
) + chunk_in_tok * fx.Int32(32)
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(v_packed, static_position=[w])
|
||||
bf16_elems = []
|
||||
for n in range_constexpr(8):
|
||||
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
|
||||
nibble_f32 = arith.sitofp(T.f32, nibble)
|
||||
elem_f32 = nibble_f32 * vscale_f32 + vzero_f32
|
||||
elem_bf16 = arith.trunc_f(T.bf16, elem_f32)
|
||||
bf16_elems.append(elem_bf16)
|
||||
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
|
||||
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
|
||||
v_lds_i64_idx = (v_lds_elem_base + fx.Int32(w * 8)) // fx.Int32(
|
||||
4
|
||||
)
|
||||
vector.store(
|
||||
v_i64,
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, v_lds_i64_idx)],
|
||||
)
|
||||
# ── HW V transpose: cross-lane LDS sync ─────────────────────
|
||||
# ds_read_tr16_b64 below introduces a cross-lane LDS read:
|
||||
# consumer lane t reads bytes written by source lanes
|
||||
# 0,4,8,12 (etc.). The compiler's automatic waitcnt insertion
|
||||
# is per-lane and has no awareness of the HW transpose's
|
||||
# cross-lane forwarding, so it can sink the next iteration's
|
||||
# ds_read ahead of these ds_write_b128 ops in the schedule.
|
||||
#
|
||||
# Two-part fence:
|
||||
# 1. sched_barrier(0): MachineScheduler reorder barrier
|
||||
# (mask=0 → no instruction class may cross). Pure hint,
|
||||
# generates no runtime instruction. Use the simple
|
||||
# sched_barrier (NOT sched_group_barrier, which is an
|
||||
# IGLP marker that requires partner barriers and
|
||||
# corrupts the schedule when used in isolation).
|
||||
# 2. s_waitcnt lgkmcnt=0: runtime drain of the LDS write
|
||||
# queue. vmcnt/expcnt left at no-wait so we don't
|
||||
# stall on speculatively-issued HBM loads (which can
|
||||
# include OOB-but-masked buffer_load reads from the
|
||||
# next K-tile's hoisted V prefetch).
|
||||
#
|
||||
# The SW path below does NOT need this because each lane
|
||||
# only reads cells it itself wrote (no cross-lane dep), and
|
||||
# the per-lane same-address waitcnt the compiler emits is
|
||||
# correct.
|
||||
#
|
||||
# Without this fence the reordered ds_read produces
|
||||
# wrong results on high-partition-count configs (large
|
||||
# padded num_partitions); low-partition configs are
|
||||
# unaffected.
|
||||
rocdl.sched_barrier(0)
|
||||
# encode_waitcnt(vmcnt=63, expcnt=7, lgkmcnt=0)
|
||||
# = 0xF | (7<<4) | (0<<8) | (3<<14) = 0xC07F
|
||||
rocdl.s_waitcnt(0xC07F)
|
||||
else:
|
||||
# Legacy transposed V_LDS path: 32 ds_write_b16 per lane.
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(v_packed, static_position=[w])
|
||||
for n in range_constexpr(8):
|
||||
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
|
||||
nibble_f32 = arith.sitofp(T.f32, nibble)
|
||||
elem_f32 = nibble_f32 * vscale_f32 + vzero_f32
|
||||
elem_bf16 = arith.trunc_f(T.bf16, elem_f32)
|
||||
elem_i16 = arith.bitcast(T.i16, elem_bf16)
|
||||
head_dim = chunk_in_tok * fx.Int32(32) + fx.Int32(w * 8 + n)
|
||||
v_idx_i16 = head_dim * fx.Int32(TILE_SIZE) + tok_in_tile
|
||||
v_vec = vector.from_elements(T.vec(1, T.i16), [elem_i16])
|
||||
vector.store(
|
||||
v_vec,
|
||||
kv_lds_i16,
|
||||
[arith.index_cast(T.index, v_idx_i16)],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# ---- PV MFMA: A=V[head_dim, token], B=P (=qk_acc bf16) -----
|
||||
# P operand B layout matches qk_acc: lane t holds 4 bf16 at
|
||||
# K=token=(t/16)*4..+3, N=query=t%16. Just trunc_f to bf16.
|
||||
p_bf16 = arith.trunc_f(T.vec(4, T.bf16), qk_acc)
|
||||
p_op = vector.bitcast(T.vec(4, T.i16), p_bf16)
|
||||
|
||||
if USE_HW_TR:
|
||||
# HW-transpose PV path: V_lds is row-major V[token][head_dim].
|
||||
# MFMA A operand layout: lane t holds 4 bf16 at
|
||||
# M=head_dim=mfma_row + h*16, K=token=(t/16)*4..+3
|
||||
# ds_read_tr16_b64 (4-element 16-bit transpose, per-16-lane block):
|
||||
# result[lane=t, elem=e] = Input[source_lane=e*4 + (t%16)//4,
|
||||
# col=t%4]
|
||||
# Per-lane address: token_idx = lane // 4 (covers 0..15 across
|
||||
# all four 16-lane MFMA blocks), hd_sub = (lane % 4)*4 selects
|
||||
# the 4-element column window inside the h*16 chunk.
|
||||
# Total LDS byte offset = kv_off + token_idx*HEAD_SIZE*2
|
||||
# + (h*16 + hd_sub)*2
|
||||
token_idx = lane >> fx.Int32(2)
|
||||
hd_sub = (lane & fx.Int32(3)) * fx.Int32(4)
|
||||
v_lane_byte = (
|
||||
fx.Int32(kv_off)
|
||||
+ token_idx * fx.Int32(HEAD_SIZE * 2)
|
||||
+ hd_sub * fx.Int32(2)
|
||||
)
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
v_byte_off = v_lane_byte + fx.Int32(h * 32)
|
||||
v_byte_i64 = fx.Int64(v_byte_off)
|
||||
v_ptr = buffer_ops.create_llvm_ptr(
|
||||
v_byte_i64,
|
||||
address_space=3,
|
||||
)
|
||||
v_op_raw = rocdl.ds_read_tr16_b64(
|
||||
T.vec(4, T.i16),
|
||||
v_ptr,
|
||||
).result
|
||||
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
|
||||
T.f32x4, [v_op_raw, p_op, acc_pv[h], 0, 0, 0]
|
||||
)
|
||||
else:
|
||||
# Legacy transposed-LDS PV: 1 ds_read_b64 per chunk.
|
||||
# Byte addr = (mfma_row + h*16) * TILE_SIZE * 2 + (col_grp*4) * 2
|
||||
# = (mfma_row + h*16) * 32 + col_grp*8
|
||||
# i64 idx = (mfma_row + h*16) * 4 + col_grp
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
v_idx_i64 = (mfma_row + fx.Int32(h * 16)) * fx.Int32(
|
||||
4
|
||||
) + mfma_col_grp
|
||||
kv_load = vector.load_op(
|
||||
T.vec(1, T.i64),
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, v_idx_i64)],
|
||||
)
|
||||
v_op = vector.bitcast(T.vec(4, T.i16), kv_load)
|
||||
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
|
||||
T.f32x4, [v_op, p_op, acc_pv[h], 0, 0, 0]
|
||||
)
|
||||
|
||||
_scf.YieldOp(
|
||||
[
|
||||
_ival(running_max),
|
||||
_ival(running_sum),
|
||||
*[_ival(p) for p in acc_pv],
|
||||
]
|
||||
)
|
||||
finally:
|
||||
_for_ip.__exit__(None, None, None)
|
||||
# Pull final accumulator values out of the for_op results.
|
||||
running_max = _for_op.results[0]
|
||||
running_sum = _for_op.results[1]
|
||||
acc_pv = list(_for_op.results[2:])
|
||||
|
||||
# ===== STEP G: Output ===========================================
|
||||
# acc_pv[h] layout: lane t holds 4 fp32 at M=head_dim=(t/16)*4..+3 + h*16,
|
||||
# N=query_row=t%16.
|
||||
# 4 fp32/lane = 4 contiguous head_dim values for ONE query_row.
|
||||
safe_sum = (running_sum > ZERO_F).select(running_sum, ONE_F)
|
||||
rcp = ONE_F / safe_sum
|
||||
|
||||
c_os = fx.Int32(_stride_out_seq)
|
||||
c_oh = fx.Int32(_stride_out_head)
|
||||
c_op_ = fx.Int32(_stride_out_part)
|
||||
out_base = seq * c_os + kv_h * c_oh + part * c_op_
|
||||
|
||||
# Gate all global stores by mfma_row < QG. For QG=16 the predicate is
|
||||
# tautological (mfma_row ∈ [0,15]); for QG=8 lanes 8..15 (whose MFMA
|
||||
# outputs are computational waste) skip the stores entirely.
|
||||
valid_row_pred = arith.cmpi(
|
||||
arith.CmpIPredicate.ult,
|
||||
mfma_row.ir_value() if hasattr(mfma_row, "ir_value") else mfma_row,
|
||||
arith.constant(QG, type=T.i32),
|
||||
)
|
||||
_if = _scf.IfOp(valid_row_pred)
|
||||
with ir.InsertionPoint(_if.then_block):
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
pv_norm = _vsplat_mul(acc_pv[h], rcp)
|
||||
pv_bf16 = arith.trunc_f(T.vec(4, T.bf16), pv_norm)
|
||||
pv_i32x2 = vector.bitcast(T.vec(2, T.i32), pv_bf16)
|
||||
head_dim_start = fx.Int32(h * 16) + mfma_col_grp * fx.Int32(4)
|
||||
out_off_elem = (
|
||||
out_base + mfma_row * fx.Int32(HEAD_SIZE) + head_dim_start
|
||||
)
|
||||
buffer_ops.buffer_store(
|
||||
pv_i32x2,
|
||||
out_rsrc,
|
||||
out_off_elem * fx.Int32(2),
|
||||
offset_is_bytes=True,
|
||||
)
|
||||
|
||||
c_npq = fx.Int32(num_partitions * QG)
|
||||
ml_off = (
|
||||
seq * fx.Int32(_stride_ml_seq) + kv_h * c_npq + part * c_qg + mfma_row
|
||||
)
|
||||
es_off = (
|
||||
seq * fx.Int32(_stride_es_seq) + kv_h * c_npq + part * c_qg + mfma_row
|
||||
)
|
||||
buffer_ops.buffer_store(running_max, ml_rsrc, ml_off)
|
||||
buffer_ops.buffer_store(running_sum, es_rsrc, es_off)
|
||||
_scf.YieldOp([])
|
||||
|
||||
return tq_decode_gqa6_kernel
|
||||
@@ -1,983 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# UltraQuant D=256 decode (FlyDSL) — gfx950 / CDNA4
|
||||
#
|
||||
# Production path only: scaled FP4×E4M3 QK MFMA, native V CVT, HW V-transpose,
|
||||
# Q hoist, strided tile-groups, and in-kernel Walsh-Hadamard for GQA 6/8/16.
|
||||
#
|
||||
# AoS slot (D=256, group_size=32 → 8 groups, 272 B):
|
||||
# [0:128) K FP4 codes; [128:136) K UE8M0; [136:264) V FP4; [264:272) V UE8M0
|
||||
# QK MFMA: A=K, B=Q → C[token, query]; PV MFMA: A=V_T, B=P → C[head_dim, query]
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import flydsl.compiler as flyc
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import buffer_ops, vector
|
||||
from flydsl._mlir import ir
|
||||
from flydsl._mlir.dialects import scf as _scf
|
||||
from flydsl.expr import (
|
||||
arith,
|
||||
gpu,
|
||||
rocdl,
|
||||
)
|
||||
from flydsl.expr.primitive import const_expr, range_constexpr
|
||||
from flydsl.expr.typing import T
|
||||
from flydsl.runtime.device import get_rocm_arch as get_hip_arch
|
||||
from flydsl.utils.smem_allocator import SmemAllocator, SmemPtr
|
||||
|
||||
|
||||
def _vector_insert(value, dest, *, static_position):
|
||||
"""Insert a DSL scalar using FlyDSL 0.3's raw vector dialect binding."""
|
||||
return vector.insert(
|
||||
vector.as_ir_value(value),
|
||||
vector.as_ir_value(dest),
|
||||
dynamic_position=[],
|
||||
static_position=static_position,
|
||||
)
|
||||
|
||||
|
||||
# === Constants (256-wide-head ultraquant decode profile) =======================
|
||||
HEAD_SIZE = 256
|
||||
KV_BLOCK_SIZE = 16 # default; overridable via build(kv_block_size=...)
|
||||
TILE_SIZE = 16 # MFMA tile = 16 tokens
|
||||
N_CENTROIDS = 16 # FP4 E2M1 value table (16 entries indexed by nibble)
|
||||
QUERY_GROUP_SIZE = 16 # default for tests; overridable via build(...)
|
||||
WARP_SIZE = 64
|
||||
BLOCK_THREADS = WARP_SIZE
|
||||
KV_COMPUTE_BLOCK = 256 # 16 K-tiles × 16 tokens
|
||||
|
||||
# ultraquant AoS slot layout (per (slot, head), D=256, group_size=32 => 8 groups)
|
||||
FP8_GROUP_SIZE = 32
|
||||
N_GROUPS = HEAD_SIZE // FP8_GROUP_SIZE # 8 UE8M0 scale bytes per head
|
||||
KEY_CODE_BYTES = HEAD_SIZE // 2 # 128
|
||||
VAL_CODE_BYTES = HEAD_SIZE // 2 # 128
|
||||
K_SCALES_OFFSET = KEY_CODE_BYTES # 128
|
||||
V_CODES_OFFSET = KEY_CODE_BYTES + N_GROUPS # 136
|
||||
V_SCALES_OFFSET = V_CODES_OFFSET + VAL_CODE_BYTES # 264
|
||||
SLOT_CONTENT_BYTES = V_SCALES_OFFSET + N_GROUPS # 272
|
||||
|
||||
# MFMA
|
||||
MFMA_N = 16
|
||||
PV_N_CHUNKS = HEAD_SIZE // MFMA_N # 16 for HEAD_SIZE=256
|
||||
|
||||
# --- Per-lane data-movement geometry (generalized from the 128 kernel) ------
|
||||
# The K/V dequant lane assignment splits each token's packed code bytes across
|
||||
# the 4 ``chunk_in_tok`` sub-lanes (lane % 4). Each lane processes
|
||||
# ``KEY_CODE_BYTES // 4`` code bytes = 32 B at 256. buffer_load maxes at 16 B
|
||||
# (dwordx4) so a lane issues ``HALVES``=2 loads; each 16-B half covers exactly
|
||||
# one UE8M0 group of 32 head-dims.
|
||||
# HEAD_SIZE=128: LANE_CODE_BYTES=16, HALVES=1, SUBCHUNK_HDIMS=32
|
||||
# HEAD_SIZE=256: LANE_CODE_BYTES=32, HALVES=2, SUBCHUNK_HDIMS=64
|
||||
LANE_CODE_BYTES = KEY_CODE_BYTES // 4 # 32 code bytes per lane per K-tile
|
||||
HALVES = LANE_CODE_BYTES // 16 # 2 for HEAD_SIZE=256
|
||||
HALF_HDIMS = 32 # head-dims per half (= one group)
|
||||
SUBCHUNK_HDIMS = HEAD_SIZE // 4 # head-dims per chunk_in_tok sub-lane (64)
|
||||
# Q row-major LDS load geometry (STEP B): each lane loads 8 bf16 (=4 i32),
|
||||
# so a row spans ``HEAD_SIZE // 8`` column-groups.
|
||||
Q_GROUPS_PER_ROW = HEAD_SIZE // 8 # 32 for 256
|
||||
Q_ROW_SHIFT = Q_GROUPS_PER_ROW.bit_length() - 1 # 5 for 256
|
||||
Q_COL_MASK = Q_GROUPS_PER_ROW - 1 # 31 for 256
|
||||
|
||||
# LDS regions
|
||||
CENTROID_LDS_BYTES = N_CENTROIDS * 4 # 64
|
||||
|
||||
# KV LDS rows are padded by 16 B to reduce bank conflicts while preserving
|
||||
# ds_read/ds_write_b128 alignment.
|
||||
KV_ROW_PAD_ELEMS = 8
|
||||
assert KV_ROW_PAD_ELEMS % 8 == 0, "KV row pad must be a multiple of 8 bf16 (16 B)"
|
||||
KV_ROW_ELEMS = HEAD_SIZE + KV_ROW_PAD_ELEMS # 264 @ pad 8
|
||||
KV_ROW_BYTES = KV_ROW_ELEMS * 2 # 528
|
||||
KFP4_ROW_PAD_I32 = 4
|
||||
assert KFP4_ROW_PAD_I32 % 4 == 0, "FP4-K row pad must be a multiple of 4 i32 (16 B)"
|
||||
KFP4_ROW_I32 = HEAD_SIZE // 8 + KFP4_ROW_PAD_I32
|
||||
KV_TILE_LDS_BYTES_PADDED = TILE_SIZE * KV_ROW_BYTES # 8448 (V dominates)
|
||||
SCALE_LDS_BYTES = TILE_SIZE * N_GROUPS * 4 # 16*8*4 = 512 for 256
|
||||
|
||||
LOG2E = 1.4426950408889634
|
||||
NEG_INF_VAL = float("-inf")
|
||||
|
||||
|
||||
def _vsplat_mul(vec, scalar):
|
||||
s = scalar.ir_value() if hasattr(scalar, "ir_value") else scalar
|
||||
return vec * vector.broadcast(T.f32x4, s)
|
||||
|
||||
|
||||
allocator = None
|
||||
|
||||
|
||||
def build_ultraquant_decode_hd256_module(
|
||||
num_seqs: int,
|
||||
num_kv_heads: int,
|
||||
num_partitions: int,
|
||||
padded_slot: int,
|
||||
max_blocks_per_seq: int = 512,
|
||||
softmax_scale: float | None = None,
|
||||
query_group_size: int = QUERY_GROUP_SIZE,
|
||||
kv_block_size: int = KV_BLOCK_SIZE,
|
||||
tile_groups_per_partition: int = 1,
|
||||
fuse_qrot: bool = True,
|
||||
stride_q_seq: int | None = None,
|
||||
stride_q_head: int | None = None,
|
||||
):
|
||||
"""Build the production UltraQuant D=256 decode kernel.
|
||||
|
||||
``padded_slot`` is the per-(token, head) slot stride in bytes
|
||||
(``kv_cache.shape[3]``, >= 272). Cache layout is
|
||||
``[num_blocks, block_size, num_kv_heads, padded_slot]`` uint8 AoS.
|
||||
|
||||
QK is always scaled FP4×E4M3 MFMA; V always uses native CVT + HW
|
||||
transpose; tile-groups are always strided. ``fuse_qrot`` selects the
|
||||
in-kernel WHT when the query layout is eligible.
|
||||
"""
|
||||
# QG=6 (Qwen3.6 full-attention layers) reuses the 8-row WHT lane map with
|
||||
# two inactive rows masked. STEP C may read rows QG..15 from adjacent LDS,
|
||||
# but those MFMA rows are discarded by the `mfma_row < QG` output gate.
|
||||
assert query_group_size in (6, 8, 16), (
|
||||
f"query_group_size must be 6, 8 or 16; got {query_group_size}"
|
||||
)
|
||||
# head_dim=256 hybrid models (Qwen3.6/3.8) force a larger mamba-aligned KV
|
||||
# block; the tiled inner loop (_TILES_PER_BLOCK = kv_block_size // TILE_SIZE)
|
||||
# handles any multiple of TILE_SIZE, so accept 16/32/64/128/256.
|
||||
assert kv_block_size in (16, 32, 64, 128, 256), (
|
||||
f"kv_block_size must be 16/32/64/128/256; got {kv_block_size}"
|
||||
)
|
||||
assert kv_block_size % TILE_SIZE == 0
|
||||
assert int(tile_groups_per_partition) >= 1
|
||||
|
||||
TGPP = int(tile_groups_per_partition)
|
||||
# Partition p owns tile-groups p, p+P, p+2P, ... This keeps the graph-fixed
|
||||
# grid wide for short sequences; the reducer is agnostic to the mapping.
|
||||
NUM_PARTS_C = int(num_partitions)
|
||||
MFMA_SCALED_K = 128
|
||||
MFMA_ISSUES = HEAD_SIZE // MFMA_SCALED_K # 2 for HEAD_SIZE=256
|
||||
GRPS_PER_ISSUE = MFMA_SCALED_K // FP8_GROUP_SIZE # 4
|
||||
|
||||
QG = int(query_group_size)
|
||||
QG_LOAD_ITERS = (QG * Q_GROUPS_PER_ROW) // WARP_SIZE # 8@QG16, 4@QG8, 3@QG6
|
||||
|
||||
_BLOCK_THREADS = WARP_SIZE
|
||||
|
||||
# Fuse Q rotation (q_rot = Q @ PiT) into this kernel as an in-register WHT.
|
||||
# The 8-row lane map masks its two inactive rows for GQA-6.
|
||||
_FUSE_QROT = bool(fuse_qrot)
|
||||
|
||||
# --- Cache policy on the KV code stream ----------------------------------
|
||||
# KV codes are read EXACTLY ONCE per decode (81.6 MB at B=4/seq=75k) and
|
||||
# never reused, so caching them only evicts Q / block-table / scale lines
|
||||
# that ARE reused. gfx9 buffer aux bits: bit0=sc0, bit1=nt, bit4=sc1.
|
||||
# 0 = default (what we ship today: no hint at all)
|
||||
# 2 = nt (non-temporal: stream, do not pollute)
|
||||
# 18 = nt|sc1 (also bypass the far cache level)
|
||||
_KV_AUX = 0
|
||||
|
||||
# --- Occupancy: size the Q LDS to the ACTUAL query group -------------------
|
||||
# STEP B only writes rows [0, QG) and STEP C/F only read those rows. For
|
||||
# QG=8 the upper 4 KB is allocated-but-never-touched, so shrinking it to
|
||||
# QG*HEAD_SIZE*2 is bitwise identical and frees LDS/workgroup (17 KB -> 13 KB
|
||||
# at QG=8), lifting waves/CU (~3 -> ~5) since the kernel is LDS-occupancy
|
||||
# bound. The launcher pads segm pools to QG=16 regardless, so this is purely
|
||||
# an in-kernel LDS footprint reduction.
|
||||
_Q_LDS_BYTES = QG * HEAD_SIZE * 2
|
||||
|
||||
_BS = int(kv_block_size)
|
||||
_TILES_PER_BLOCK = _BS // TILE_SIZE
|
||||
_PADDED_SLOT = int(padded_slot)
|
||||
assert _PADDED_SLOT >= SLOT_CONTENT_BYTES, (
|
||||
f"padded_slot={_PADDED_SLOT} < {SLOT_CONTENT_BYTES} (ultraquant D=256 content)"
|
||||
)
|
||||
assert _PADDED_SLOT % 4 == 0, f"padded_slot={_PADDED_SLOT} must be 4-byte aligned"
|
||||
|
||||
arch = get_hip_arch()
|
||||
|
||||
if softmax_scale is None:
|
||||
softmax_scale = 1.0 / (HEAD_SIZE**0.5)
|
||||
_qk_scale = float(softmax_scale)
|
||||
|
||||
# --- Strides ---
|
||||
_Hq = num_kv_heads * QG
|
||||
# Defaults describe the pooled [B, Hq, D] q_rot buffer. When the rotation is
|
||||
# fused in (STEP B'), Q is read straight from the model's query tensor,
|
||||
# which is a SPLIT OF THE FUSED QKV PROJECTION -- so its row stride is
|
||||
# (Hq + 2*Hk)*D, not Hq*D. Those strides are constant for a given model and
|
||||
# TP rank, so the caller passes them in and they bake in like any other
|
||||
# compile-time constant (they are part of the kernel cache key).
|
||||
_stride_q_seq = int(stride_q_seq) if stride_q_seq is not None else _Hq * HEAD_SIZE
|
||||
_stride_q_head = int(stride_q_head) if stride_q_head is not None else HEAD_SIZE
|
||||
_stride_bt_seq = max_blocks_per_seq
|
||||
|
||||
# ultraquant AoS cache: [num_blocks, block_size, num_kv_heads, padded_slot].
|
||||
_stride_cache_head = _PADDED_SLOT
|
||||
_stride_cache_pos = num_kv_heads * _PADDED_SLOT
|
||||
_stride_cache_block = _BS * _stride_cache_pos
|
||||
|
||||
_stride_out_part = QG * HEAD_SIZE
|
||||
_stride_out_head = num_partitions * QG * HEAD_SIZE
|
||||
_stride_out_seq = num_kv_heads * num_partitions * QG * HEAD_SIZE
|
||||
_stride_es_seq = num_kv_heads * num_partitions * QG
|
||||
_stride_ml_seq = _stride_es_seq
|
||||
|
||||
# --- LDS layout ---
|
||||
# `allocator` is intentionally a BUILD-LOCAL captured by the kernel closure
|
||||
# (a freevar), NOT a module global. The @flyc.jit drift guard
|
||||
# (_check_globals_drift) snapshots every module global referenced by the JIT
|
||||
# launcher's dependency tree (which includes this kernel) and ABORTS if one
|
||||
# changes between compiles. When more than one partition variant is
|
||||
# JIT-resident at once (e.g. a ragged/adaptive batch mixes num_partitions
|
||||
# 256 and 512, or fixed cap=512 yields 294@75k and 512@131k), each build
|
||||
# created a fresh SmemAllocator; as a reassigned MODULE GLOBAL that tripped
|
||||
# "global 'allocator' changed since first compile" and crashed the worker.
|
||||
# Keeping it local + a per-variant symbol name lets the variants coexist.
|
||||
# The build-local is handed to the launcher via a module attribute set AFTER
|
||||
# the kernel is built (see end of function); that assignment is safe because
|
||||
# no @flyc.jit function references the module global anymore.
|
||||
allocator = SmemAllocator(
|
||||
None,
|
||||
arch=arch,
|
||||
global_sym_name=(f"ultraquant_hd256_smem_p{int(num_partitions)}"),
|
||||
)
|
||||
centroid_off = 0
|
||||
allocator.ptr = CENTROID_LDS_BYTES
|
||||
q_off = allocator.ptr
|
||||
allocator.ptr += _Q_LDS_BYTES
|
||||
kv_off = allocator.ptr
|
||||
allocator.ptr += KV_TILE_LDS_BYTES_PADDED
|
||||
scale_off = allocator.ptr
|
||||
allocator.ptr += SCALE_LDS_BYTES
|
||||
|
||||
@flyc.kernel
|
||||
def ultraquant_decode_hd256_kernel(
|
||||
out_ptr: fx.Tensor,
|
||||
exp_sums_ptr: fx.Tensor,
|
||||
max_logits_ptr: fx.Tensor,
|
||||
query_ptr: fx.Tensor,
|
||||
kv_cache_ptr: fx.Tensor,
|
||||
centroids_ptr: fx.Tensor,
|
||||
block_tables_ptr: fx.Tensor,
|
||||
seq_lens_ptr: fx.Tensor,
|
||||
):
|
||||
# ---- IDs ---------------------------------------------------------
|
||||
tid = gpu.thread_idx.x
|
||||
seq = gpu.block_idx.x
|
||||
kv_h = gpu.block_idx.y
|
||||
part = gpu.block_idx.z
|
||||
lane = tid # 0..63
|
||||
mfma_row = lane & fx.Int32(15)
|
||||
mfma_col_grp = lane >> fx.Int32(4) # 0..3, K-group dim
|
||||
|
||||
# ---- Buffer resources -------------------------------------------
|
||||
q_rsrc = buffer_ops.create_buffer_resource(query_ptr, max_size=True)
|
||||
bt_rsrc = buffer_ops.create_buffer_resource(block_tables_ptr, max_size=True)
|
||||
sl_rsrc = buffer_ops.create_buffer_resource(seq_lens_ptr, max_size=True)
|
||||
cent_rsrc = buffer_ops.create_buffer_resource(centroids_ptr, max_size=True)
|
||||
out_rsrc = buffer_ops.create_buffer_resource(out_ptr, max_size=True)
|
||||
es_rsrc = buffer_ops.create_buffer_resource(exp_sums_ptr, max_size=True)
|
||||
ml_rsrc = buffer_ops.create_buffer_resource(max_logits_ptr, max_size=True)
|
||||
|
||||
# ---- LDS pointers -----------------------------------------------
|
||||
base = allocator.get_base()
|
||||
cent_lds = SmemPtr(base, centroid_off, T.f32, shape=(N_CENTROIDS,))
|
||||
q_lds_i32 = SmemPtr(base, q_off, T.i32, shape=(_Q_LDS_BYTES // 4,)).get()
|
||||
q_lds_i64 = SmemPtr(base, q_off, T.i64, shape=(_Q_LDS_BYTES // 8,)).get()
|
||||
# KV / scale views span ALL warps' private regions (W*stride); the
|
||||
# per-warp base offset is folded into the element indices below via
|
||||
# _kvi*/_sci (a literal 0 for W=1, so the single-warp path is unchanged).
|
||||
kv_lds_i32 = SmemPtr(
|
||||
base, kv_off, T.i32, shape=(KV_TILE_LDS_BYTES_PADDED // 4,)
|
||||
).get()
|
||||
kv_lds_i64 = SmemPtr(
|
||||
base, kv_off, T.i64, shape=(KV_TILE_LDS_BYTES_PADDED // 8,)
|
||||
).get()
|
||||
scale_lds_i32 = SmemPtr(base, scale_off, T.i32, shape=(TILE_SIZE * N_GROUPS,))
|
||||
|
||||
def _kvi32(e):
|
||||
return e
|
||||
|
||||
def _kvi64(e):
|
||||
return e
|
||||
|
||||
def _sci(e):
|
||||
return e
|
||||
|
||||
# ---- Constants ---------------------------------------------------
|
||||
c_sq = fx.Int32(_stride_q_seq)
|
||||
c_qh = fx.Int32(_stride_q_head)
|
||||
c_qg = fx.Int32(QG)
|
||||
c_bt = fx.Int32(_stride_bt_seq)
|
||||
c_stride_pos = fx.Int32(_stride_cache_pos)
|
||||
c_stride_head = fx.Int32(_stride_cache_head)
|
||||
c_kscale_off = fx.Int32(K_SCALES_OFFSET) # 128
|
||||
c_vcode_off = fx.Int32(V_CODES_OFFSET) # 136
|
||||
c_vscale_off = fx.Int32(V_SCALES_OFFSET) # 264
|
||||
c_w = fx.Int32(WARP_SIZE)
|
||||
|
||||
NEG_INF = arith.constant(NEG_INF_VAL, type=T.f32)
|
||||
ZERO_F = fx.Float32(0.0)
|
||||
ONE_F = fx.Float32(1.0)
|
||||
LOG2E_C = arith.constant(LOG2E, type=T.f32)
|
||||
QK_SCALE = arith.constant(_qk_scale, type=T.f32)
|
||||
|
||||
def _ival(v):
|
||||
return v.ir_value() if hasattr(v, "ir_value") else v
|
||||
|
||||
c_zero_i32 = arith.constant(0, type=T.i32)
|
||||
|
||||
def _f32x8_to_fp8_i64(f):
|
||||
# 8 f32 -> 8 E4M3 bytes packed as i64 (MFMA fp8 operand).
|
||||
w0 = rocdl.cvt_pk_fp8_f32(T.i32, f[0], f[1], c_zero_i32, 0)
|
||||
w0 = rocdl.cvt_pk_fp8_f32(T.i32, f[2], f[3], w0, 1)
|
||||
w1 = rocdl.cvt_pk_fp8_f32(T.i32, f[4], f[5], c_zero_i32, 0)
|
||||
w1 = rocdl.cvt_pk_fp8_f32(T.i32, f[6], f[7], w1, 1)
|
||||
pv = vector.from_elements(T.vec(2, T.i32), [w0, w1])
|
||||
return vector.extract(
|
||||
vector.bitcast(T.vec(1, T.i64), pv), static_position=[0]
|
||||
)
|
||||
|
||||
def _bf16x8_to_fp8_i64(v_bf16):
|
||||
f = [
|
||||
arith.extf(T.f32, vector.extract(v_bf16, static_position=[i]))
|
||||
for i in range(8)
|
||||
]
|
||||
return _f32x8_to_fp8_i64(f)
|
||||
|
||||
# ===== STEP A: Load centroids → LDS (cooperative, race-safe) =====
|
||||
c_idx_safe = lane & fx.Int32(N_CENTROIDS - 1)
|
||||
c_val = buffer_ops.buffer_load(cent_rsrc, c_idx_safe, vec_width=1, dtype=T.f32)
|
||||
cent_lds.store(c_val, [arith.index_cast(T.index, c_idx_safe)])
|
||||
gpu.barrier()
|
||||
|
||||
# ===== STEP B': in-kernel Q rotation (fused WHT) ==================
|
||||
# The shipped path needs a SEPARATE prologue kernel to compute
|
||||
# q_rot = Q @ PiT. That kernel is ~7 us at B=4 -- essentially ALL fixed
|
||||
# per-dispatch cost (measured: 2.1M MACs, and sweeping its grid 4x moves
|
||||
# it 0%). PiT is the orthonormal Sylvester Hadamard with a column
|
||||
# permutation folded in on the host, so Q @ PiT is a WALSH-HADAMARD
|
||||
# TRANSFORM: 8 butterfly stages (2048 add/sub per row) instead of a
|
||||
# 256x256 matmul (65536 MACs) -- 32x less work, cheap enough to redo
|
||||
# redundantly in every partition and delete the dispatch entirely.
|
||||
#
|
||||
# Verified offline: butterfly vs matmul differs by <=1.8e-6 in fp32, and
|
||||
# after the E4M3 haircut the two agree EXACTLY (0/2048 elements differ),
|
||||
# because e4m3 keeps only 3 mantissa bits. So this is bit-exact, not an
|
||||
# approximation.
|
||||
#
|
||||
# Lane map: lane L owns row r = L>>3 and the 32 CONSECUTIVE head-dims
|
||||
# d = (L&7)*32 + t. That puts butterfly stages 0..4 (strides 1..16)
|
||||
# entirely inside one lane's registers, and leaves only stages 5..7
|
||||
# (strides 32/64/128 == chunk XOR 1/2/4) needing a cross-lane exchange.
|
||||
if const_expr(_FUSE_QROT):
|
||||
_wrow = lane >> fx.Int32(3) # 0..7 row within the group
|
||||
_wchk = lane & fx.Int32(7) # 0..7 32-dim chunk
|
||||
# inv(qperm) is a pure bit permutation of the head-dim index:
|
||||
# dest = (d & 0x8F) | ((d>>2)&0x10) | ((d<<1)&0x60)
|
||||
# which splits additively into a compile-time part in t and a
|
||||
# per-lane part in the chunk index (verified exhaustively):
|
||||
# ct(t) = (t & 0x0F) | ((t & 0x10) << 1)
|
||||
# rt(c) = ((c&2)<<3) | ((c&1)<<6) | ((c&4)<<5)
|
||||
_perm_base = (
|
||||
((_wchk & fx.Int32(2)) << fx.Int32(3))
|
||||
| ((_wchk & fx.Int32(1)) << fx.Int32(6))
|
||||
| ((_wchk & fx.Int32(4)) << fx.Int32(5))
|
||||
)
|
||||
_inv_sqrtD = arith.constant(1.0 / math.sqrt(HEAD_SIZE), type=T.f32)
|
||||
for _qit in range_constexpr((QG + 7) // 8):
|
||||
_qrow = _wrow + fx.Int32(_qit * 8)
|
||||
if _qrow < fx.Int32(QG):
|
||||
_qb = (
|
||||
seq * c_sq + (kv_h * c_qg + _qrow) * c_qh + _wchk * fx.Int32(32)
|
||||
)
|
||||
# ---- load this lane's 32 raw-Q head-dims (4 x dwordx4) ----
|
||||
_v = []
|
||||
for _j in range_constexpr(4):
|
||||
_raw = buffer_ops.buffer_load(
|
||||
q_rsrc,
|
||||
(_qb + fx.Int32(_j * 8)) // fx.Int32(2),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
_rb = vector.bitcast(T.vec(8, T.bf16), _raw)
|
||||
for _e in range_constexpr(8):
|
||||
_v.append(
|
||||
arith.extf(
|
||||
T.f32, vector.extract(_rb, static_position=[_e])
|
||||
)
|
||||
)
|
||||
# ---- stages 0..4: in-register, no LDS, no cross-lane ----
|
||||
for _s in range_constexpr(5):
|
||||
_st = 1 << _s
|
||||
_nx = list(_v)
|
||||
for _g in range_constexpr(32 // (2 * _st)):
|
||||
for _k in range_constexpr(_st):
|
||||
_i0 = _g * 2 * _st + _k
|
||||
_i1 = _i0 + _st
|
||||
_a, _b = _v[_i0], _v[_i1]
|
||||
_nx[_i0] = _a + _b
|
||||
_nx[_i1] = _a - _b
|
||||
_v = _nx
|
||||
# ---- stages 5..7: chunk XOR 1/2/4 via ds_swizzle ----
|
||||
# ds_swizzle_b32 offset (bit15=0, 32-lane groups):
|
||||
# [4:0]=and_mask [9:5]=or_mask [14:10]=xor_mask
|
||||
# pure XOR k -> and_mask=0x1F, or_mask=0, xor_mask=k. The
|
||||
# partner is always within the same 32-lane group (k<=4).
|
||||
for _s in range_constexpr(3):
|
||||
_k = 1 << _s
|
||||
_swz = arith.constant((_k << 10) | 0x1F, type=T.i32)
|
||||
# High half of the pair computes (partner - mine), the low
|
||||
# half (mine + partner). Fold that into a single sign
|
||||
# computed ONCE per stage, so each element costs one
|
||||
# multiply-add rather than a branch or a per-element select.
|
||||
_chk_bit = _wchk & fx.Int32(_k)
|
||||
_is_hi = arith.cmpi(
|
||||
arith.CmpIPredicate.ne,
|
||||
_chk_bit.ir_value()
|
||||
if hasattr(_chk_bit, "ir_value")
|
||||
else _chk_bit,
|
||||
fx.Int32(0).ir_value(),
|
||||
)
|
||||
_sf = arith.select(
|
||||
_is_hi,
|
||||
arith.constant(-1.0, type=T.f32),
|
||||
arith.constant(1.0, type=T.f32),
|
||||
)
|
||||
_nx = []
|
||||
for _t in range_constexpr(32):
|
||||
_mi = arith.bitcast(T.i32, _v[_t])
|
||||
_pt = arith.bitcast(
|
||||
T.f32, rocdl.ds_swizzle(T.i32, _mi, _swz)
|
||||
)
|
||||
_nx.append(_v[_t] * _sf + _pt)
|
||||
_v = _nx
|
||||
# ---- 1/sqrt(D) normalization + E4M3 haircut -> bf16 ----
|
||||
# cvt_pk_fp8_f32 is the SAME instruction the QK operand build
|
||||
# uses, so this reproduces the host haircut exactly. E4M3 is a
|
||||
# subset of bf16, so the bf16 store below is lossless and every
|
||||
# downstream consumer stays byte-identical.
|
||||
_hc = []
|
||||
for _p in range_constexpr(16):
|
||||
_w = rocdl.cvt_pk_fp8_f32(
|
||||
T.i32,
|
||||
_v[2 * _p] * _inv_sqrtD,
|
||||
_v[2 * _p + 1] * _inv_sqrtD,
|
||||
fx.Int32(0),
|
||||
0,
|
||||
)
|
||||
_u = rocdl.cvt_pk_f32_fp8(T.vec(2, T.f32), _w, 0)
|
||||
_hc.append(
|
||||
arith.truncf(
|
||||
T.bf16, vector.extract(_u, static_position=[0])
|
||||
)
|
||||
)
|
||||
_hc.append(
|
||||
arith.truncf(
|
||||
T.bf16, vector.extract(_u, static_position=[1])
|
||||
)
|
||||
)
|
||||
# ---- permuted store ----
|
||||
# ct(t) maps t=0..15 -> 0..15 and t=16..31 -> 32..47, i.e. TWO
|
||||
# contiguous 16-element runs, so the scatter is still 4 aligned
|
||||
# 16-byte vector stores rather than 32 scalar writes.
|
||||
_qrow_elem = _qrow * fx.Int32(HEAD_SIZE)
|
||||
for _half in range_constexpr(2):
|
||||
for _q4 in range_constexpr(2):
|
||||
_elems = [_hc[_half * 16 + _q4 * 8 + _e] for _e in range(8)]
|
||||
_vec = vector.from_elements(T.vec(8, T.bf16), _elems)
|
||||
_dst = (
|
||||
_qrow_elem + _perm_base + fx.Int32(_half * 32 + _q4 * 8)
|
||||
)
|
||||
vector.store(
|
||||
vector.bitcast(T.vec(4, T.i32), _vec),
|
||||
q_lds_i32,
|
||||
[arith.index_cast(T.index, _dst // fx.Int32(2))],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# ===== STEP B: Load pre-rotated q_rot → row-major Q LDS ==========
|
||||
for c in range_constexpr(0 if _FUSE_QROT else QG_LOAD_ITERS):
|
||||
row_chunk = lane + fx.Int32(c * WARP_SIZE)
|
||||
row = row_chunk >> fx.Int32(Q_ROW_SHIFT) # 0..QG-1
|
||||
col_b = row_chunk & fx.Int32(Q_COL_MASK) # 0..Q_GROUPS_PER_ROW-1
|
||||
col_elem = col_b * fx.Int32(8) # bf16 elem
|
||||
q_off_elem = seq * c_sq + (kv_h * c_qg + row) * c_qh + col_elem
|
||||
q_v = buffer_ops.buffer_load(
|
||||
q_rsrc,
|
||||
q_off_elem // fx.Int32(2),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
q_lds_byte = row * fx.Int32(HEAD_SIZE * 2) + col_elem * fx.Int32(2)
|
||||
vector.store(
|
||||
q_v,
|
||||
q_lds_i32,
|
||||
[arith.index_cast(T.index, q_lds_byte // fx.Int32(4))],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# Scaled-MFMA Q operands (loop-invariant): one per MFMA issue.
|
||||
q_op_hoisted = []
|
||||
for h in range_constexpr(MFMA_ISSUES):
|
||||
_q_hoist_words = []
|
||||
for j in range_constexpr(4):
|
||||
_q_hoist_idx = (
|
||||
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
+ (fx.Int32(h * GRPS_PER_ISSUE) + mfma_col_grp) * fx.Int32(8)
|
||||
+ fx.Int32(j * 2)
|
||||
)
|
||||
_q_hoist_v = vector.load_op(
|
||||
T.vec(2, T.i64),
|
||||
q_lds_i64,
|
||||
[arith.index_cast(T.index, _q_hoist_idx)],
|
||||
)
|
||||
_q_hoist_words.append(
|
||||
_bf16x8_to_fp8_i64(vector.bitcast(T.vec(8, T.bf16), _q_hoist_v))
|
||||
)
|
||||
q_op_hoisted.append(
|
||||
vector.bitcast(
|
||||
T.vec(8, T.i32),
|
||||
vector.from_elements(T.vec(4, T.i64), _q_hoist_words),
|
||||
)
|
||||
)
|
||||
|
||||
# ===== STEP D: Online softmax + PV state =========================
|
||||
running_max = NEG_INF
|
||||
running_sum = ZERO_F
|
||||
zero_v4 = arith.constant_vector(0.0, T.f32x4)
|
||||
acc_pv = [zero_v4 for _ in range(PV_N_CHUNKS)]
|
||||
|
||||
# ===== STEP E: Sequence-len + partition base ====================
|
||||
seq_len = buffer_ops.buffer_load(sl_rsrc, seq, vec_width=1, dtype=T.i32)
|
||||
|
||||
# Per-K-tile dequant lane assignment: lane t -> token = t/4,
|
||||
# chunk_in_tok = t%4 (each chunk = SUBCHUNK_HDIMS=64 head-dims = 2 groups).
|
||||
tok_in_tile = lane >> fx.Int32(2)
|
||||
chunk_in_tok = lane & fx.Int32(3)
|
||||
# Both UE8M0 group bytes for this lane's chunk (groups 2c, 2c+1) live in
|
||||
# the SAME 4-byte scale word at offset (chunk_in_tok>>1)*4 within the
|
||||
# 8-byte scale region. Load once; extract per-half below.
|
||||
c_scale_word_off = (chunk_in_tok >> fx.Int32(1)) * fx.Int32(4)
|
||||
|
||||
# ===== STEP F: K-tile loop =======================================
|
||||
c_kcb = fx.Int32(KV_COMPUTE_BLOCK)
|
||||
c_tgpp = fx.Int32(TGPP)
|
||||
c_zero_i32 = fx.Int32(0)
|
||||
c_one_i32 = fx.Int32(1)
|
||||
c_nparts = fx.Int32(NUM_PARTS_C)
|
||||
total_tgs = (seq_len + c_kcb - c_one_i32) // c_kcb
|
||||
avail = total_tgs - part
|
||||
in_range = avail > c_zero_i32
|
||||
trip_raw = (avail + c_nparts - c_one_i32) // c_nparts
|
||||
trip_clamped = (trip_raw > c_tgpp).select(c_tgpp, trip_raw)
|
||||
trip_or_zero = in_range.select(trip_clamped, c_zero_i32)
|
||||
c_zero_idx = arith.constant(0, index=True)
|
||||
c_one_idx = arith.constant(1, index=True)
|
||||
trip_idx = arith.index_cast(
|
||||
T.index,
|
||||
trip_or_zero.ir_value()
|
||||
if hasattr(trip_or_zero, "ir_value")
|
||||
else trip_or_zero,
|
||||
)
|
||||
_init_iter = [
|
||||
_ival(running_max),
|
||||
_ival(running_sum),
|
||||
*[_ival(p) for p in acc_pv],
|
||||
]
|
||||
_for_op = _scf.ForOp(
|
||||
c_zero_idx,
|
||||
trip_idx,
|
||||
c_one_idx,
|
||||
_init_iter,
|
||||
)
|
||||
_for_ip = ir.InsertionPoint(_for_op.body)
|
||||
_for_ip.__enter__()
|
||||
try:
|
||||
tg_idx = _for_op.induction_variable
|
||||
tg_i32 = fx.Int32(arith.index_cast(T.i32, tg_idx))
|
||||
partition_start = (part + tg_i32 * c_nparts) * c_kcb
|
||||
bt_seq_base = seq * c_bt + (partition_start // fx.Int32(_BS))
|
||||
running_max = _for_op.inner_iter_args[0]
|
||||
running_sum = _for_op.inner_iter_args[1]
|
||||
acc_pv = list(_for_op.inner_iter_args[2:])
|
||||
|
||||
# ==== SOFTWARE PREFETCH =====================================
|
||||
# Emit ONLY the HBM buffer_loads for one tile (K codes, V codes,
|
||||
# K/V UE8M0 scale words) and return the loaded register values.
|
||||
# Emitting tile N+1's loads BEFORE tile N's dequant+MFMA lets the
|
||||
# vmem fetches overlap compute: the compiler's s_waitcnt for these
|
||||
# values lands at their first use one iteration later, hiding HBM
|
||||
# latency (the decode kernel is ~85% HBM-stalled at low batch).
|
||||
# This is bit-identical to the pre-prefetch path -- only the
|
||||
# *emission order* of independent loads changes, never any
|
||||
# arithmetic. All addressing locals (slot_base_byte, blk_rsrc,
|
||||
# ...) are consumed only inside this closure; the dequant below
|
||||
# needs just the four returned register values.
|
||||
def _emit_tile_loads(n_tile):
|
||||
# n_tile is a compile-time int on the single-warp path and a
|
||||
# runtime fx.Int32 on the multi-warp path (each warp strides the
|
||||
# 16 tiles). Compute the block/tile/slot geometry accordingly.
|
||||
if const_expr(isinstance(n_tile, int)):
|
||||
block_in_part = fx.Int32(n_tile // _TILES_PER_BLOCK)
|
||||
tile_in_block = n_tile % _TILES_PER_BLOCK
|
||||
tile_start_tok = partition_start + fx.Int32(n_tile * TILE_SIZE)
|
||||
slot = fx.Int32(tile_in_block * TILE_SIZE) + tok_in_tile
|
||||
else:
|
||||
block_in_part = n_tile // fx.Int32(_TILES_PER_BLOCK)
|
||||
tile_in_block = n_tile - block_in_part * fx.Int32(_TILES_PER_BLOCK)
|
||||
tile_start_tok = partition_start + n_tile * fx.Int32(TILE_SIZE)
|
||||
slot = tile_in_block * fx.Int32(TILE_SIZE) + tok_in_tile
|
||||
tile_in_seq = tile_start_tok < seq_len
|
||||
bt_off = bt_seq_base + block_in_part
|
||||
bt_off_safe = tile_in_seq.select(bt_off, seq * c_bt)
|
||||
phys_block = buffer_ops.buffer_load(
|
||||
bt_rsrc,
|
||||
bt_off_safe,
|
||||
vec_width=1,
|
||||
dtype=T.i32,
|
||||
)
|
||||
# A buffer descriptor addresses with a 32-bit voffset, so
|
||||
# ``phys_block * c_block`` wraps once the cache view exceeds
|
||||
# 4 GiB. Fold the block base into the descriptor's 64-bit
|
||||
# base pointer and keep only in-block offsets below.
|
||||
blk_off_i64 = arith.extui(T.i64, phys_block) * fx.Int64(
|
||||
_stride_cache_block
|
||||
)
|
||||
blk_rsrc = buffer_ops.create_buffer_resource(
|
||||
kv_cache_ptr,
|
||||
max_size=True,
|
||||
base_byte_offset=blk_off_i64,
|
||||
)
|
||||
block_base = fx.Int32(0)
|
||||
|
||||
slot_base_byte = block_base + slot * c_stride_pos + kv_h * c_stride_head
|
||||
# K codes: LANE_CODE_BYTES (=32) at slot_base + chunk*32, in
|
||||
# HALVES x 16-byte buffer_loads.
|
||||
k_byte0 = slot_base_byte + chunk_in_tok * fx.Int32(LANE_CODE_BYTES)
|
||||
_kpl = []
|
||||
for hf in range_constexpr(HALVES):
|
||||
_kpl.append(
|
||||
buffer_ops.buffer_load(
|
||||
blk_rsrc,
|
||||
(k_byte0 + fx.Int32(hf * 16)) // fx.Int32(4),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
cache_modifier=_KV_AUX,
|
||||
)
|
||||
)
|
||||
|
||||
# ---- V codes + K/V scale HBM loads ------------------------
|
||||
v_byte0 = (
|
||||
slot_base_byte
|
||||
+ c_vcode_off
|
||||
+ chunk_in_tok * fx.Int32(LANE_CODE_BYTES)
|
||||
)
|
||||
_vpl = []
|
||||
for hf in range_constexpr(HALVES):
|
||||
_vpl.append(
|
||||
buffer_ops.buffer_load(
|
||||
blk_rsrc,
|
||||
(v_byte0 + fx.Int32(hf * 16)) // fx.Int32(4),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
cache_modifier=_KV_AUX,
|
||||
)
|
||||
)
|
||||
# UE8M0 group scale words (one 4-byte word covers this lane's
|
||||
# two groups). scale = 2^(byte-127) = bitcast_f32(byte<<23);
|
||||
# byte==0 -> +0.0 (zero sentinel).
|
||||
_ksw = buffer_ops.buffer_load(
|
||||
blk_rsrc,
|
||||
(slot_base_byte + c_kscale_off + c_scale_word_off) // fx.Int32(4),
|
||||
vec_width=1,
|
||||
dtype=T.i32,
|
||||
)
|
||||
_vsw = buffer_ops.buffer_load(
|
||||
blk_rsrc,
|
||||
(slot_base_byte + c_vscale_off + c_scale_word_off) // fx.Int32(4),
|
||||
vec_width=1,
|
||||
dtype=T.i32,
|
||||
)
|
||||
return (_kpl, _vpl, _ksw, _vsw)
|
||||
|
||||
# Depth-1 software prefetch: emit the next tile's HBM loads before
|
||||
# consuming the current tile without changing arithmetic order.
|
||||
_PF_DEPTH = 1
|
||||
# Prime the pipeline with the first _PF_DEPTH tiles' loads.
|
||||
_pf_queue = []
|
||||
for _pi in range_constexpr(min(_PF_DEPTH, 16)):
|
||||
_pf_queue.append(_emit_tile_loads(_pi))
|
||||
_tile_range = range_constexpr(16)
|
||||
for _jt in _tile_range:
|
||||
n_tile = _jt
|
||||
_ntile_tok = fx.Int32(n_tile * TILE_SIZE)
|
||||
(k_packed_list, v_packed_list, kscale_word, vscale_word) = (
|
||||
_pf_queue.pop(0)
|
||||
)
|
||||
_pf_nxt = n_tile + _PF_DEPTH
|
||||
if const_expr(_pf_nxt < 16):
|
||||
_pf_queue.append(_emit_tile_loads(_pf_nxt))
|
||||
# Store raw FP4 codes (natural contiguous order,
|
||||
# 16 B = 32 nibbles per UE8M0 group) straight to KV LDS as
|
||||
# the scaled-MFMA A operand — no dequant here.
|
||||
for hf in range_constexpr(HALVES):
|
||||
grp = chunk_in_tok * fx.Int32(2) + fx.Int32(hf)
|
||||
grp_shift = (grp & fx.Int32(3)) * fx.Int32(8)
|
||||
kscale_byte = (kscale_word >> grp_shift) & fx.Int32(0xFF)
|
||||
k_packed = k_packed_list[hf]
|
||||
vector.store(
|
||||
k_packed,
|
||||
kv_lds_i32,
|
||||
[
|
||||
arith.index_cast(
|
||||
T.index,
|
||||
_kvi32(
|
||||
tok_in_tile * fx.Int32(KFP4_ROW_I32)
|
||||
+ grp * fx.Int32(4)
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
scale_lds_i32.store(
|
||||
kscale_byte,
|
||||
[
|
||||
arith.index_cast(
|
||||
T.index,
|
||||
_sci(tok_in_tile * fx.Int32(N_GROUPS) + grp),
|
||||
)
|
||||
],
|
||||
)
|
||||
gpu.barrier()
|
||||
# QK: MFMA_ISSUES native scaled MFMAs over K=128,
|
||||
# chained through the accumulator.
|
||||
# A = K raw FP4 codes (vec4 i32 = 16 B) per (token, group)
|
||||
# B = Q E4M3 (vec8 i32 = 32 B) for the same group
|
||||
# scaleA = per-(token,group) UE8M0 byte at op_sel byte 0
|
||||
# scaleB = 0x7F identity (Q carries no scale)
|
||||
BLGP_E2M1 = 4 # A operand type code = fp4 e2m1
|
||||
CBSZ_E4M3 = 0 # B operand type code = fp8 e4m3
|
||||
IDENT = fx.Int32(0x7F)
|
||||
qk_acc = zero_v4
|
||||
for h in range_constexpr(MFMA_ISSUES):
|
||||
grp = fx.Int32(h * GRPS_PER_ISSUE) + mfma_col_grp
|
||||
k_op = vector.load_op(
|
||||
T.vec(4, T.i32),
|
||||
kv_lds_i32,
|
||||
[
|
||||
arith.index_cast(
|
||||
T.index,
|
||||
_kvi32(
|
||||
mfma_row * fx.Int32(KFP4_ROW_I32)
|
||||
+ grp * fx.Int32(4)
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
q_op = q_op_hoisted[h]
|
||||
scbyte = fx.Int32(
|
||||
scale_lds_i32.load(
|
||||
[
|
||||
arith.index_cast(
|
||||
T.index,
|
||||
_sci(mfma_row * fx.Int32(N_GROUPS) + grp),
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
kscale = fx.Int32(0x7F7F7F00) | scbyte
|
||||
qk_acc = rocdl.mfma_scale_f32_16x16x128_f8f6f4(
|
||||
T.vec(4, T.f32),
|
||||
[
|
||||
k_op,
|
||||
q_op,
|
||||
qk_acc,
|
||||
BLGP_E2M1,
|
||||
CBSZ_E4M3,
|
||||
0,
|
||||
kscale,
|
||||
0,
|
||||
IDENT,
|
||||
],
|
||||
)
|
||||
|
||||
# Scale + mask out-of-context tokens.
|
||||
qk_acc = _vsplat_mul(qk_acc, QK_SCALE)
|
||||
for elem in range_constexpr(4):
|
||||
kv_tok = (
|
||||
partition_start
|
||||
+ _ntile_tok
|
||||
+ mfma_col_grp * fx.Int32(4)
|
||||
+ fx.Int32(elem)
|
||||
)
|
||||
in_b = kv_tok < seq_len
|
||||
v = vector.extract(qk_acc, static_position=[elem])
|
||||
qk_acc = _vector_insert(
|
||||
in_b.select(v, NEG_INF),
|
||||
qk_acc,
|
||||
static_position=[elem],
|
||||
)
|
||||
|
||||
# FA2 online softmax: per-query-row reduce.
|
||||
local_max = vector.reduction(T.f32, "maxnumf", qk_acc)
|
||||
r1 = local_max.shuffle_xor(fx.Int32(16), c_w)
|
||||
local_max = local_max.maximumf(r1)
|
||||
r2 = local_max.shuffle_xor(fx.Int32(32), c_w)
|
||||
tile_max = local_max.maximumf(r2)
|
||||
|
||||
new_max = running_max.maximumf(tile_max)
|
||||
max_diff = running_max - new_max
|
||||
safe_diff = (running_max > NEG_INF).select(max_diff, ZERO_F)
|
||||
scale = (safe_diff * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
|
||||
running_sum = running_sum * scale
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
acc_pv[h] = _vsplat_mul(acc_pv[h], scale)
|
||||
running_max = new_max
|
||||
|
||||
tile_sum = ZERO_F
|
||||
for elem in range_constexpr(4):
|
||||
s = vector.extract(qk_acc, static_position=[elem])
|
||||
d = s - new_max
|
||||
d = (new_max > NEG_INF).select(d, NEG_INF)
|
||||
p = (d * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
|
||||
tile_sum = tile_sum + p
|
||||
qk_acc = _vector_insert(p, qk_acc, static_position=[elem])
|
||||
|
||||
ts1 = tile_sum.shuffle_xor(fx.Int32(16), c_w)
|
||||
tile_sum = tile_sum + ts1
|
||||
ts2 = tile_sum.shuffle_xor(fx.Int32(32), c_w)
|
||||
tile_sum = tile_sum + ts2
|
||||
running_sum = running_sum + tile_sum
|
||||
# V dequant → LDS [token][head_dim] (ROW-MAJOR, no transpose)
|
||||
# written as ds_write_b128 (vec(2,i64)); read back via the
|
||||
# ds_read_tr16_b64 HW transpose in the PV block below.
|
||||
v_lds_elem_base = tok_in_tile * fx.Int32(
|
||||
KV_ROW_ELEMS
|
||||
) + chunk_in_tok * fx.Int32(SUBCHUNK_HDIMS)
|
||||
for hf in range_constexpr(HALVES):
|
||||
v_packed = v_packed_list[hf]
|
||||
half_hd = fx.Int32(hf * HALF_HDIMS)
|
||||
grp_shift = (
|
||||
(chunk_in_tok * fx.Int32(2) + fx.Int32(hf)) & fx.Int32(3)
|
||||
) * fx.Int32(8)
|
||||
vscale_byte = (vscale_word >> grp_shift) & fx.Int32(0xFF)
|
||||
vscale_f32 = arith.bitcast(
|
||||
T.f32, _ival(vscale_byte << fx.Int32(23))
|
||||
)
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(v_packed, static_position=[w])
|
||||
# Native CDNA4 scaled convert: 2-wide
|
||||
# cvt_scalef32_pk_bf16_fp4. srcSel s picks byte s
|
||||
# of the word (nibbles 2s, 2s+1) → 2 bf16 with the
|
||||
# UE8M0 group scale fused. 4 cvt calls/word.
|
||||
bf16_elems = []
|
||||
for s in range_constexpr(4):
|
||||
v2 = rocdl.cvt_scalef32_pk_bf16_fp4(
|
||||
T.vec(2, T.bf16),
|
||||
_ival(word_i32),
|
||||
_ival(vscale_f32),
|
||||
int(s),
|
||||
)
|
||||
bf16_elems.append(vector.extract(v2, static_position=[0]))
|
||||
bf16_elems.append(vector.extract(v2, static_position=[1]))
|
||||
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
|
||||
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
|
||||
v_lds_i64_idx = (
|
||||
v_lds_elem_base + half_hd + fx.Int32(w * 8)
|
||||
) // fx.Int32(4)
|
||||
vector.store(
|
||||
v_i64,
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, _kvi64(v_lds_i64_idx))],
|
||||
)
|
||||
# HW V transpose cross-lane LDS fence (see 128 kernel).
|
||||
rocdl.sched_barrier(0)
|
||||
rocdl.s_waitcnt(0xC07F)
|
||||
gpu.barrier()
|
||||
|
||||
# ---- PV MFMA: A=V[head_dim, token], B=P (=qk_acc bf16) -----
|
||||
p_bf16 = arith.trunc_f(T.vec(4, T.bf16), qk_acc)
|
||||
p_op = vector.bitcast(T.vec(4, T.i16), p_bf16)
|
||||
# HW-transpose PV: V_lds row-major V[token][head_dim].
|
||||
token_idx = lane >> fx.Int32(2)
|
||||
hd_sub = (lane & fx.Int32(3)) * fx.Int32(4)
|
||||
v_lane_byte = (
|
||||
fx.Int32(kv_off)
|
||||
+ token_idx * fx.Int32(KV_ROW_BYTES)
|
||||
+ hd_sub * fx.Int32(2)
|
||||
)
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
v_byte_off = v_lane_byte + fx.Int32(h * 32)
|
||||
v_byte_i64 = fx.Int64(v_byte_off)
|
||||
v_ptr = buffer_ops.create_llvm_ptr(
|
||||
v_byte_i64,
|
||||
address_space=3,
|
||||
)
|
||||
v_op_raw = rocdl.ds_read_tr16_b64(
|
||||
T.vec(4, T.i16),
|
||||
v_ptr,
|
||||
).result
|
||||
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
|
||||
T.f32x4, [v_op_raw, p_op, acc_pv[h], 0, 0, 0]
|
||||
)
|
||||
|
||||
_scf.YieldOp(
|
||||
[
|
||||
_ival(running_max),
|
||||
_ival(running_sum),
|
||||
*[_ival(p) for p in acc_pv],
|
||||
]
|
||||
)
|
||||
finally:
|
||||
_for_ip.__exit__(None, None, None)
|
||||
running_max = _for_op.results[0]
|
||||
running_sum = _for_op.results[1]
|
||||
acc_pv = list(_for_op.results[2:])
|
||||
|
||||
# ===== STEP G: Output ===========================================
|
||||
safe_sum = (running_sum > ZERO_F).select(running_sum, ONE_F)
|
||||
rcp = ONE_F / safe_sum
|
||||
|
||||
c_os = fx.Int32(_stride_out_seq)
|
||||
c_oh = fx.Int32(_stride_out_head)
|
||||
c_op_ = fx.Int32(_stride_out_part)
|
||||
out_base = seq * c_os + kv_h * c_oh + part * c_op_
|
||||
|
||||
valid_row_pred = arith.cmpi(
|
||||
arith.CmpIPredicate.ult,
|
||||
mfma_row.ir_value() if hasattr(mfma_row, "ir_value") else mfma_row,
|
||||
arith.constant(QG, type=T.i32),
|
||||
)
|
||||
_if = _scf.IfOp(valid_row_pred)
|
||||
with ir.InsertionPoint(_if.then_block):
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
pv_norm = _vsplat_mul(acc_pv[h], rcp)
|
||||
pv_bf16 = arith.trunc_f(T.vec(4, T.bf16), pv_norm)
|
||||
pv_i32x2 = vector.bitcast(T.vec(2, T.i32), pv_bf16)
|
||||
head_dim_start = fx.Int32(h * 16) + mfma_col_grp * fx.Int32(4)
|
||||
out_off_elem = (
|
||||
out_base + mfma_row * fx.Int32(HEAD_SIZE) + head_dim_start
|
||||
)
|
||||
buffer_ops.buffer_store(
|
||||
pv_i32x2,
|
||||
out_rsrc,
|
||||
out_off_elem * fx.Int32(2),
|
||||
offset_is_bytes=True,
|
||||
)
|
||||
|
||||
c_npq = fx.Int32(num_partitions * QG)
|
||||
ml_off = (
|
||||
seq * fx.Int32(_stride_ml_seq) + kv_h * c_npq + part * c_qg + mfma_row
|
||||
)
|
||||
es_off = (
|
||||
seq * fx.Int32(_stride_es_seq) + kv_h * c_npq + part * c_qg + mfma_row
|
||||
)
|
||||
buffer_ops.buffer_store(running_max, ml_rsrc, ml_off)
|
||||
buffer_ops.buffer_store(running_sum, es_rsrc, es_off)
|
||||
_scf.YieldOp([])
|
||||
|
||||
# Hand the build-local allocator to the launcher. `_get_kernel` reads
|
||||
# `kmod.allocator` immediately after this returns (before the next build can
|
||||
# run), so each launcher captures its own variant's allocator. This does NOT
|
||||
# reintroduce the drift bug: the kernel above closes over the LOCAL
|
||||
# `allocator`, so no @flyc.jit function references module-global `allocator`
|
||||
# and it is never snapshotted by _check_globals_drift.
|
||||
globals()["allocator"] = allocator
|
||||
return ultraquant_decode_hd256_kernel
|
||||
@@ -1,874 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FlyDSL TurboQuant decode launcher (vLLM-side).
|
||||
|
||||
Drop-in replacement for ``triton_turboquant_decode_attention_soa`` for the
|
||||
TQ decode profile HEAD_SIZE=128, MSE_BITS=4 K, VQB=4 V, N_CENTROIDS=16,
|
||||
BLOCK_SIZE in {16, 32}.
|
||||
|
||||
Per-GQA kernel dispatch:
|
||||
* GQA group ∈ {8, 16} → canonical kernels.tq_decode (Qwen-class)
|
||||
* GQA group == 6 → kernels.tq_decode_gqa6 sibling (MiniMax-M2.5)
|
||||
|
||||
Auto-selected on gfx950 when FlyDSL is importable; falls back to the SoA Triton
|
||||
decode otherwise (wrong arch, missing build tree). The GQA-6 sibling is imported
|
||||
best-effort: missing it does not affect Qwen GQA-{8,16} paths.
|
||||
|
||||
Architecture:
|
||||
1. Q rotation: ``q_rot = (query.float() @ PiT).bfloat16()`` — the same
|
||||
launcher-side rocBLAS GEMM the SoA Triton decode uses.
|
||||
2. FlyDSL kernel writes per-partition outputs into
|
||||
``[N, Hk, P, QG, D]`` bf16 + ``[N, Hk, P, QG]`` fp32 max/sum buffers.
|
||||
3. A small Triton reducer combines partitions in the kernel's native layout
|
||||
(no permute/cast), writing the final ``[N, Hq, D]`` output.
|
||||
|
||||
Kernel module is built once per ``(num_kv_heads, num_partitions,
|
||||
max_blocks_per_seq, scale)`` shape and cached.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# -- FlyDSL import bootstrap --------------------------------------------------
|
||||
# FlyDSL is imported directly and must be importable (pip-installed, or on
|
||||
# PYTHONPATH). Import failures are caught below and fall back to the SoA Triton
|
||||
# decode, so vLLM startup is unaffected on hosts without FlyDSL.
|
||||
|
||||
|
||||
_FLYDSL_AVAILABLE: bool | None = None
|
||||
_TQ_MOD: Any = None # kernels.tq_decode module (Qwen GQA-{8,16})
|
||||
_TQ_MOD_GQA6: Any = None # kernels.tq_decode_gqa6 module (MiniMax GQA-6, optional)
|
||||
_FLYC: Any = None # flydsl.compiler
|
||||
_FX: Any = None # flydsl.expr
|
||||
_TYPING_T: Any = None
|
||||
_CC: Any = None # CompilationContext
|
||||
_IR: Any = None
|
||||
|
||||
|
||||
def is_flydsl_available() -> bool:
|
||||
"""Return True iff on gfx950 and FlyDSL imports + kernel module load.
|
||||
|
||||
The GQA-6 sibling kernel (``kernels.tq_decode_gqa6``) is imported
|
||||
best-effort: if it's missing (older FlyDSL checkout that pre-dates
|
||||
MiniMax support) the canonical Qwen path stays fully functional and
|
||||
only GQA-6 dispatches will fail with a clear error at launch time.
|
||||
"""
|
||||
global _FLYDSL_AVAILABLE, _TQ_MOD, _TQ_MOD_GQA6
|
||||
global _FLYC, _FX, _TYPING_T, _CC, _IR
|
||||
if _FLYDSL_AVAILABLE is not None:
|
||||
return _FLYDSL_AVAILABLE
|
||||
try:
|
||||
# gfx950-only kernel (CDNA4 intrinsics); gate before importing FlyDSL.
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if not on_gfx950():
|
||||
_FLYDSL_AVAILABLE = False
|
||||
return _FLYDSL_AVAILABLE
|
||||
import flydsl.compiler as flyc # noqa: F401
|
||||
import flydsl.expr as fx # noqa: F401
|
||||
from flydsl._mlir import ir
|
||||
from flydsl.compiler.kernel_function import CompilationContext
|
||||
from flydsl.expr.typing import T # noqa: F401
|
||||
|
||||
# Kernel is vendored in-tree (ships with vLLM); FlyDSL provides only
|
||||
# the compiler/runtime framework (imported above).
|
||||
from vllm.v1.attention.ops.flydsl_kernels import tq_decode as tq_mod
|
||||
|
||||
_FLYC = flyc
|
||||
_FX = fx
|
||||
_TYPING_T = T
|
||||
_CC = CompilationContext
|
||||
_IR = ir
|
||||
_TQ_MOD = tq_mod
|
||||
_FLYDSL_AVAILABLE = True
|
||||
logger.info_once("FlyDSL TQ decode launcher: available")
|
||||
except Exception as ex: # noqa: BLE001
|
||||
_FLYDSL_AVAILABLE = False
|
||||
logger.warning_once(
|
||||
"FlyDSL TQ decode launcher: unavailable (%s). "
|
||||
"Falling back to SoA Triton decode.",
|
||||
ex,
|
||||
)
|
||||
return _FLYDSL_AVAILABLE
|
||||
# Best-effort GQA-6 sibling import (does NOT gate Qwen availability).
|
||||
try:
|
||||
from vllm.v1.attention.ops.flydsl_kernels import (
|
||||
tq_decode_gqa6 as tq_mod_gqa6,
|
||||
)
|
||||
|
||||
_TQ_MOD_GQA6 = tq_mod_gqa6
|
||||
logger.info_once("FlyDSL TQ decode GQA-6 sibling: available (MiniMax-class)")
|
||||
except Exception as ex: # noqa: BLE001
|
||||
_TQ_MOD_GQA6 = None
|
||||
logger.info_once(
|
||||
"FlyDSL TQ decode GQA-6 sibling: not available (%s); "
|
||||
"GQA-6 models will fall back to SoA Triton decode.",
|
||||
ex,
|
||||
)
|
||||
return _FLYDSL_AVAILABLE
|
||||
|
||||
|
||||
def is_flydsl_gqa6_available() -> bool:
|
||||
"""True iff the optional GQA-6 sibling kernel module loaded.
|
||||
|
||||
Used by the eligibility gate in turboquant_attn.py to decide whether
|
||||
a layer with num_kv_groups==6 can run on FlyDSL or must fall back
|
||||
to the SoA Triton decode.
|
||||
"""
|
||||
if _FLYDSL_AVAILABLE is None:
|
||||
is_flydsl_available()
|
||||
return _TQ_MOD_GQA6 is not None
|
||||
|
||||
|
||||
# -- Kernel module cache -------------------------------------------------------
|
||||
# Compiling a FlyDSL kernel is expensive; cache by the constexprs that
|
||||
# parameterize stride math: num_kv_heads, num_partitions, max_blocks_per_seq,
|
||||
# scale. ``num_seqs`` is irrelevant to the kernel body so we omit it.
|
||||
_KERN_CACHE: dict[tuple, Any] = {}
|
||||
|
||||
|
||||
# -- One-shot log guards ------------------------------------------------------
|
||||
# Replaces vllm `logger.{info,warning}_once` (which hashes the format string
|
||||
# every call to dedup) with a true zero-overhead bool guard.
|
||||
_LOG_INVOKED_ONCE: bool = False
|
||||
_LOG_SINKS_WARNED: bool = False
|
||||
_LOG_NORM_WARNED: bool = False
|
||||
|
||||
|
||||
def _detect_max_capture_B() -> int:
|
||||
"""Probe vLLM compilation config for the largest cudagraph capture size.
|
||||
|
||||
Returns max(cudagraph_capture_sizes) if available, else falls back to
|
||||
512 (vLLM's default cap). The segm-pool auto-grows if a larger batch is
|
||||
seen (see ``_SegmBufPool.get``).
|
||||
|
||||
Used to size the segm-pool bucket once, so that all distinct B's
|
||||
(whether captured in a graph, or hit eagerly in mixed batches) share
|
||||
a single allocation per shape — eliminating the per-B growth that
|
||||
previously dominated dense-capture VRAM cost.
|
||||
"""
|
||||
try:
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
cfg = get_current_vllm_config()
|
||||
sizes = cfg.compilation_config.cudagraph_capture_sizes
|
||||
if sizes:
|
||||
return int(max(sizes))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return 512
|
||||
|
||||
|
||||
class _SegmBufPool:
|
||||
"""Single-bucket buffer pool for segm_out/segm_max/segm_sum/output.
|
||||
|
||||
Each unique shape signature (Hk, Hq, num_partitions, QG, D, device,
|
||||
dtype) is backed by ONE allocation sized at ``max_B`` (= max captured
|
||||
cudagraph batch size, or env override). Per-call ``get(B, ...)``
|
||||
returns ``[:B]`` views — same data_ptr base, narrower N dim. The
|
||||
kernel grid is ``(B, ...)`` so it only ever writes the first B rows
|
||||
of segm_out / segm_max / segm_sum; downstream readers (Triton
|
||||
reducer + caller) only consume the first B rows of output.
|
||||
|
||||
Why a single bucket: cudagraph records launch pointers at capture
|
||||
time. With per-B allocations, every captured size triggered a fresh
|
||||
allocation, so total VRAM scaled with the number of captured sizes.
|
||||
With single-bucket, the pool tops out at one max_B-sized buffer per
|
||||
shape, shared across ALL captured sizes and ALL eager mixed-batch
|
||||
calls. The data_ptr is stable from first allocation onward, so
|
||||
cudagraph capture is happy across all captured B's.
|
||||
|
||||
Eliminates per-decode-step ``cudaMalloc`` (× 4) plus the device-side
|
||||
memset kernels behind ``torch.full(-inf)`` and ``torch.zeros``.
|
||||
The kernel always writes the FULL ``[B, Hk, P, QG, D]`` slice and
|
||||
every (n, kv_h, p, qg) position of segm_max / segm_sum (it stores
|
||||
``-inf`` and ``0`` for empty partitions itself), so uninitialized
|
||||
bytes from the pool are safe to reuse for any B ≤ B_bucket.
|
||||
"""
|
||||
|
||||
__slots__ = ("_bufs", "_max_B")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._bufs: dict[tuple, dict[str, torch.Tensor]] = {}
|
||||
self._max_B: int | None = None # lazy-init on first get()
|
||||
|
||||
def get(
|
||||
self,
|
||||
B: int,
|
||||
Hk: int,
|
||||
Hq: int,
|
||||
num_partitions: int,
|
||||
QG: int,
|
||||
D: int,
|
||||
device: torch.device,
|
||||
q_dtype: torch.dtype,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
if self._max_B is None:
|
||||
self._max_B = _detect_max_capture_B()
|
||||
# B may exceed the detected max if user set max-num-seqs higher
|
||||
# than the largest captured size — still safe, we'll grow once.
|
||||
# Note: growing AFTER cudagraph capture would invalidate captured
|
||||
# pointers, so this should only happen during warmup or in
|
||||
# always-eager configs.
|
||||
B_bucket = max(self._max_B, int(B))
|
||||
key = (
|
||||
int(Hk),
|
||||
int(Hq),
|
||||
int(num_partitions),
|
||||
int(QG),
|
||||
int(D),
|
||||
str(device),
|
||||
q_dtype,
|
||||
)
|
||||
bufs = self._bufs.get(key)
|
||||
if bufs is None or bufs["segm_out"].shape[0] < B_bucket:
|
||||
if bufs is not None and bufs["segm_out"].shape[0] < B_bucket:
|
||||
# First-time grow: warn so user knows cudagraphs may be
|
||||
# invalidated. This should only happen during warmup or in
|
||||
# always-eager configs (a batch larger than any captured
|
||||
# cudagraph size).
|
||||
logger.warning_once(
|
||||
"FlyDSL _SegmBufPool: growing bucket from %d to %d "
|
||||
"(B=%d). If you see this AFTER cudagraph warmup, the "
|
||||
"previously-captured graphs hold stale pointers and "
|
||||
"will GPU-fault (max cudagraph capture size < B).",
|
||||
bufs["segm_out"].shape[0],
|
||||
B_bucket,
|
||||
B,
|
||||
)
|
||||
bufs = {
|
||||
"segm_out": torch.empty(
|
||||
(B_bucket, Hk, num_partitions, QG, D),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
),
|
||||
"segm_max": torch.empty(
|
||||
(B_bucket, Hk, num_partitions, QG),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
"segm_sum": torch.empty(
|
||||
(B_bucket, Hk, num_partitions, QG),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
"output": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=q_dtype,
|
||||
device=device,
|
||||
),
|
||||
# Stable pre-allocated buffer for the rotated query
|
||||
# tensor. On ROCm, fresh torch.empty / matmul-result allocations
|
||||
# after HIP graph capture can land in the graph memory pool and
|
||||
# cause GPU memory faults when passed as kernel arguments in the
|
||||
# eager mixed-batch path. By pooling q_rot here (same pattern as
|
||||
# segm_out/output above), the data_ptr is stable from first
|
||||
# allocation onward — same for all captured B's and eager calls.
|
||||
"q_rot": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=q_dtype,
|
||||
device=device,
|
||||
),
|
||||
# q_float: fp32 copy of query (bf16→fp32 for mm input).
|
||||
# Pooled to avoid fresh allocations post-capture.
|
||||
"q_float": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
# q_rot_fp32: fp32 output of the rotation mm (before bf16 cast).
|
||||
# Using torch.mm(..., out=this) eliminates the fresh mm-result
|
||||
# allocation that would otherwise land in the HIP graph pool.
|
||||
"q_rot_fp32": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
}
|
||||
self._bufs[key] = bufs
|
||||
self._max_B = B_bucket
|
||||
logger.info_once(
|
||||
"FlyDSL _SegmBufPool: allocated single-bucket "
|
||||
"shape=(Hk=%d, Hq=%d, P=%d, QG=%d, D=%d, dtype=%s) "
|
||||
"B_bucket=%d (covers all captured + eager B's). "
|
||||
"VRAM = %.1f MiB / shape.",
|
||||
Hk,
|
||||
Hq,
|
||||
num_partitions,
|
||||
QG,
|
||||
D,
|
||||
q_dtype,
|
||||
B_bucket,
|
||||
sum(t.numel() * t.element_size() for t in bufs.values()) / (1 << 20),
|
||||
)
|
||||
# Return [:B] views — same data_ptr base, narrower N dim.
|
||||
return {
|
||||
"segm_out": bufs["segm_out"][:B],
|
||||
"segm_max": bufs["segm_max"][:B],
|
||||
"segm_sum": bufs["segm_sum"][:B],
|
||||
"output": bufs["output"][:B],
|
||||
"q_rot": bufs["q_rot"][:B],
|
||||
"q_float": bufs["q_float"][:B],
|
||||
"q_rot_fp32": bufs["q_rot_fp32"][:B],
|
||||
}
|
||||
|
||||
def stats(self) -> dict[str, int]:
|
||||
return {
|
||||
"shapes": len(self._bufs),
|
||||
"max_B": int(self._max_B or 0),
|
||||
"bytes": sum(
|
||||
sum(t.numel() * t.element_size() for t in d.values())
|
||||
for d in self._bufs.values()
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
_SEGM_POOL = _SegmBufPool()
|
||||
|
||||
|
||||
_HW_TR_CACHED: bool | None = None
|
||||
|
||||
|
||||
def _hw_tr_enabled() -> bool:
|
||||
"""Resolve the HW V-transpose build flag.
|
||||
|
||||
ON for gfx950+ (ds_read_tr16_b64 is bit-exact vs baseline and faster
|
||||
on Qwen-class shapes), off elsewhere. Resolved from the GPU arch.
|
||||
"""
|
||||
global _HW_TR_CACHED
|
||||
if _HW_TR_CACHED is not None:
|
||||
return _HW_TR_CACHED
|
||||
try:
|
||||
from flydsl.runtime.device import get_rocm_arch as _arch
|
||||
|
||||
a = str(_arch() or "")
|
||||
_HW_TR_CACHED = a.startswith("gfx950")
|
||||
except Exception: # noqa: BLE001
|
||||
_HW_TR_CACHED = False
|
||||
if _HW_TR_CACHED:
|
||||
logger.info_once("FlyDSL TQ: HW V transpose ON (default for gfx950+)")
|
||||
else:
|
||||
logger.info_once("FlyDSL TQ: HW V transpose OFF")
|
||||
return _HW_TR_CACHED
|
||||
|
||||
|
||||
_GET_KERNEL_STATS = {"hits": 0, "misses": 0, "build_total_s": 0.0}
|
||||
|
||||
|
||||
def _get_kernel(
|
||||
num_kv_heads: int,
|
||||
num_partitions: int,
|
||||
max_blocks_per_seq: int,
|
||||
scale: float,
|
||||
query_group_size: int,
|
||||
kv_block_size: int,
|
||||
use_hw_v_transpose: bool = False,
|
||||
num_seqs_hint: int = 1,
|
||||
tile_groups_per_partition: int = 1,
|
||||
):
|
||||
# ``num_seqs_hint`` (= runtime B) is forwarded to the build for shape
|
||||
# awareness; it does NOT participate in the cache key because the
|
||||
# kernel body uses gpu.block_idx.x (= seq index at runtime) and is
|
||||
# GQA-symmetric across B at build time.
|
||||
#
|
||||
# ``tile_groups_per_partition`` IS in the cache key — each
|
||||
# value compiles a different unrolled K-tile loop body.
|
||||
key = (
|
||||
num_kv_heads,
|
||||
int(num_partitions),
|
||||
int(max_blocks_per_seq),
|
||||
round(float(scale), 8),
|
||||
int(query_group_size),
|
||||
int(kv_block_size),
|
||||
bool(use_hw_v_transpose),
|
||||
int(tile_groups_per_partition),
|
||||
)
|
||||
cached = _KERN_CACHE.get(key)
|
||||
if cached is not None:
|
||||
_GET_KERNEL_STATS["hits"] += 1
|
||||
return cached
|
||||
assert is_flydsl_available()
|
||||
# Time the FlyDSL build path so we can quantify cudagraph capture cost.
|
||||
import time as _t
|
||||
|
||||
_build_t0 = _t.perf_counter()
|
||||
|
||||
# Per-GQA dispatch: GQA-6 (MiniMax-M2.5) lives in the sibling module
|
||||
# tq_decode_gqa6 to keep the Qwen kernel's invariants untouched.
|
||||
# GQA-{8,16} (Qwen) keep using the canonical tq_decode kernel.
|
||||
qg = int(query_group_size)
|
||||
if qg == 6:
|
||||
if _TQ_MOD_GQA6 is None:
|
||||
raise RuntimeError(
|
||||
"FlyDSL TQ GQA-6 sibling kernel "
|
||||
"(vllm.v1.attention.ops.flydsl_kernels.tq_decode_gqa6) "
|
||||
"failed to import; check that FlyDSL is installed/importable."
|
||||
)
|
||||
kmod = _TQ_MOD_GQA6
|
||||
kfn = kmod.build_tq_decode_gqa6_module(
|
||||
num_seqs=int(num_seqs_hint),
|
||||
num_kv_heads=num_kv_heads,
|
||||
num_partitions=num_partitions,
|
||||
max_blocks_per_seq=max_blocks_per_seq,
|
||||
softmax_scale=float(scale),
|
||||
query_group_size=qg,
|
||||
kv_block_size=int(kv_block_size),
|
||||
use_hw_v_transpose=bool(use_hw_v_transpose),
|
||||
tile_groups_per_partition=int(tile_groups_per_partition),
|
||||
)
|
||||
else:
|
||||
kmod = _TQ_MOD
|
||||
kfn = kmod.build_tq_decode_module(
|
||||
num_seqs=int(num_seqs_hint),
|
||||
num_kv_heads=num_kv_heads,
|
||||
num_partitions=num_partitions,
|
||||
max_blocks_per_seq=max_blocks_per_seq,
|
||||
softmax_scale=float(scale),
|
||||
query_group_size=qg,
|
||||
kv_block_size=int(kv_block_size),
|
||||
use_hw_v_transpose=bool(use_hw_v_transpose),
|
||||
tile_groups_per_partition=int(tile_groups_per_partition),
|
||||
)
|
||||
al = kmod.allocator
|
||||
block_threads = kmod.BLOCK_THREADS
|
||||
|
||||
flyc = _FLYC
|
||||
fx = _FX
|
||||
T = _TYPING_T
|
||||
CompilationContext = _CC
|
||||
ir_mod = _IR
|
||||
|
||||
@flyc.jit
|
||||
def _launch(
|
||||
out,
|
||||
es,
|
||||
ml,
|
||||
q,
|
||||
kvc,
|
||||
cents,
|
||||
bt,
|
||||
sl,
|
||||
gx: fx.Int32, # type: ignore[name-defined]
|
||||
gy: fx.Int32, # type: ignore[name-defined]
|
||||
gz: fx.Int32, # type: ignore[name-defined]
|
||||
stream: fx.Stream, # type: ignore[name-defined]
|
||||
):
|
||||
# Re-finalize the LDS allocator for this launch (idempotent per build).
|
||||
al.finalized = False
|
||||
ctx = CompilationContext.get_current()
|
||||
with ir_mod.InsertionPoint(ctx.gpu_module_body):
|
||||
al.finalize()
|
||||
from flydsl.expr import arith
|
||||
|
||||
grid_x = arith.index_cast(T.index, gx.ir_value())
|
||||
grid_y = arith.index_cast(T.index, gy.ir_value())
|
||||
grid_z = arith.index_cast(T.index, gz.ir_value())
|
||||
kfn(out, es, ml, q, kvc, cents, bt, sl).launch(
|
||||
grid=(grid_x, grid_y, grid_z),
|
||||
block=(block_threads, 1, 1),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
_KERN_CACHE[key] = _launch
|
||||
_build_dt = _t.perf_counter() - _build_t0
|
||||
_GET_KERNEL_STATS["misses"] += 1
|
||||
_GET_KERNEL_STATS["build_total_s"] += _build_dt
|
||||
logger.info(
|
||||
"FlyDSL _get_kernel BUILD #%d dt=%.2fs (cumulative=%.1fs) key=%s",
|
||||
_GET_KERNEL_STATS["misses"],
|
||||
_build_dt,
|
||||
_GET_KERNEL_STATS["build_total_s"],
|
||||
key,
|
||||
)
|
||||
return _launch
|
||||
|
||||
|
||||
# -- Partition reducer (Triton, native kernel layout) -------------------------
|
||||
# Reduces the FA2 split-KV partials produced by the FlyDSL kernel.
|
||||
# segm_out [N, Hk, P, QG, D] bf16
|
||||
# segm_max [N, Hk, P, QG] fp32
|
||||
# segm_sum [N, Hk, P, QG] fp32
|
||||
# → output [N, Hq=Hk*QG, D] of any (bf16/fp16/fp32) dtype.
|
||||
@triton.jit
|
||||
def _reduce_partitions(
|
||||
output_ptr, # [N, Hq, D] in OUT_DTYPE
|
||||
segm_out_ptr, # [N, Hk, P, QG, D] bf16
|
||||
segm_max_ptr, # [N, Hk, P, QG] fp32
|
||||
segm_sum_ptr, # [N, Hk, P, QG] fp32
|
||||
out_stride_n: tl.int64, # stride on N axis (in elems of OUT_DTYPE)
|
||||
out_stride_h: tl.int64, # stride on Hq axis
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
QG: tl.constexpr,
|
||||
NUM_PARTS: tl.constexpr,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
):
|
||||
# grid = (N, Hq)
|
||||
n = tl.program_id(0)
|
||||
hq = tl.program_id(1)
|
||||
kv_h = hq // QG
|
||||
qg = hq % QG
|
||||
|
||||
# Per-partition pointers for this (n, kv_h, qg) row.
|
||||
# Strides in elems of source dtype.
|
||||
msum_base = n * (NUM_KV_HEADS * NUM_PARTS * QG) + kv_h * (NUM_PARTS * QG) + qg
|
||||
so_base = (
|
||||
n * (NUM_KV_HEADS * NUM_PARTS * QG * HEAD_SIZE)
|
||||
+ kv_h * (NUM_PARTS * QG * HEAD_SIZE)
|
||||
+ qg * HEAD_SIZE
|
||||
)
|
||||
|
||||
p_off = tl.arange(0, NUM_PARTS)
|
||||
d_off = tl.arange(0, HEAD_SIZE)
|
||||
|
||||
# Load per-partition max/sum.
|
||||
m_idx = msum_base + p_off * QG
|
||||
seg_max = tl.load(segm_max_ptr + m_idx)
|
||||
seg_sum = tl.load(segm_sum_ptr + m_idx)
|
||||
|
||||
# Mask out empty partitions (max == -inf).
|
||||
valid = seg_max > float("-inf")
|
||||
overall_max = tl.max(tl.where(valid, seg_max, float("-inf")))
|
||||
|
||||
# Rescale exp sums.
|
||||
rescale = tl.where(valid, tl.exp(seg_max - overall_max), 0.0)
|
||||
seg_sum_rescaled = seg_sum * rescale
|
||||
overall_sum = tl.sum(seg_sum_rescaled)
|
||||
|
||||
# Load segment outputs and combine.
|
||||
so_idx = so_base + p_off[:, None] * (QG * HEAD_SIZE) + d_off[None, :]
|
||||
seg_out = tl.load(segm_out_ptr + so_idx).to(tl.float32)
|
||||
# acc = sum_p( seg_out_p * (seg_sum_p * rescale_p) ) / overall_sum
|
||||
weighted = seg_out * seg_sum_rescaled[:, None]
|
||||
acc_sum = tl.sum(weighted, axis=0)
|
||||
acc = tl.where(overall_sum > 0.0, acc_sum / overall_sum, 0.0)
|
||||
|
||||
# Store.
|
||||
out_off = n * out_stride_n + hq * out_stride_h + d_off
|
||||
tl.store(output_ptr + out_off, acc.to(output_ptr.dtype.element_ty))
|
||||
|
||||
|
||||
# -- Public launcher ----------------------------------------------------------
|
||||
def flydsl_turboquant_decode_attention(
|
||||
query: torch.Tensor, # [B, Hq, D] bf16/fp16
|
||||
kv_cache: torch.Tensor, # [num_blocks, BS, Hk, slot_size_aligned]
|
||||
block_table: torch.Tensor, # [B, max_blocks_per_seq] int32
|
||||
seq_lens: torch.Tensor, # [B] int32
|
||||
Pi: torch.Tensor, # [D, D] fp32
|
||||
centroids: torch.Tensor, # [N_CENTROIDS] fp32
|
||||
scale: float,
|
||||
mse_bits: int,
|
||||
key_packed_size: int,
|
||||
value_quant_bits: int,
|
||||
value_packed_size: int,
|
||||
key_fp8: bool = False,
|
||||
norm_correction: bool = False,
|
||||
PiT: torch.Tensor | None = None,
|
||||
max_seq_len: int = 0,
|
||||
mid_o_buf: Any = None,
|
||||
output_buf: torch.Tensor | None = None,
|
||||
lse_buf: Any = None,
|
||||
buf_holder: Any = None,
|
||||
max_num_kv_splits: int = 32,
|
||||
sinks: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""SoA-decode-compatible launcher backed by the FlyDSL decode kernel.
|
||||
|
||||
Constraints:
|
||||
* key_fp8 == False
|
||||
* mse_bits == 4
|
||||
* value_quant_bits == 4
|
||||
* centroids.numel() == 16
|
||||
* D == 128
|
||||
* block_size in {16, 32}
|
||||
* Hq // Hk in {6, 8, 16}
|
||||
- 8/16 → canonical tq_decode kernel (Qwen2.5-72B / Qwen3-32B)
|
||||
- 6 → tq_decode_gqa6 sibling kernel (MiniMax-M2.5)
|
||||
|
||||
Sinks are NYI and silently ignored if set. norm_correction is honored
|
||||
implicitly via the pre-folded stored K-norm (see footer comment).
|
||||
"""
|
||||
del mid_o_buf, lse_buf, key_packed_size, value_packed_size
|
||||
if not is_flydsl_available():
|
||||
raise RuntimeError(
|
||||
"FlyDSL decode requested but FlyDSL is not available (needs gfx950 "
|
||||
"+ importable FlyDSL); the SoA Triton decode is the fallback."
|
||||
)
|
||||
assert not key_fp8, "FlyDSL supports MSE-key path only"
|
||||
assert mse_bits == 4, f"FlyDSL expects mse_bits=4, got {mse_bits}"
|
||||
assert value_quant_bits == 4, (
|
||||
f"FlyDSL expects value_quant_bits=4, got {value_quant_bits}"
|
||||
)
|
||||
|
||||
B, Hq, D = query.shape
|
||||
Hk = kv_cache.shape[2]
|
||||
block_size = kv_cache.shape[1]
|
||||
QG = Hq // Hk
|
||||
assert D == _TQ_MOD.HEAD_SIZE
|
||||
assert block_size in (16, 32), (
|
||||
f"FlyDSL supports kv_block_size 16 or 32, got {block_size}"
|
||||
)
|
||||
assert QG in (6, 8, 16), f"FlyDSL supports GQA factor 6, 8 or 16, got {QG}"
|
||||
if QG == 6 and _TQ_MOD_GQA6 is None:
|
||||
raise RuntimeError(
|
||||
"FlyDSL launcher: GQA-6 requested (MiniMax-class) but the "
|
||||
"tq_decode_gqa6 sibling module is not available. Update your "
|
||||
"FlyDSL checkout (must include kernels/tq_decode_gqa6.py); "
|
||||
"otherwise the SoA Triton decode is the fallback."
|
||||
)
|
||||
assert centroids.numel() == _TQ_MOD.N_CENTROIDS, (
|
||||
f"centroids.numel={centroids.numel()} != {_TQ_MOD.N_CENTROIDS}"
|
||||
)
|
||||
|
||||
# ---- Per-layer cache for PiT_f32 + contiguous centroids ---
|
||||
# PiT and centroids are model constants (set once at layer warmup).
|
||||
# Avoid the per-decode-step transpose+cast and `.contiguous()` no-op
|
||||
# check by stashing the pre-cooked tensors on ``buf_holder`` (= layer).
|
||||
PiT_f32: torch.Tensor
|
||||
centroids_c: torch.Tensor
|
||||
if buf_holder is not None:
|
||||
PiT_f32 = getattr(buf_holder, "_tq_PiT_f32", None)
|
||||
if PiT_f32 is None:
|
||||
_PiT_src = PiT if PiT is not None else Pi.T.contiguous()
|
||||
PiT_f32 = (
|
||||
_PiT_src
|
||||
if _PiT_src.dtype == torch.float32
|
||||
else _PiT_src.to(torch.float32)
|
||||
)
|
||||
buf_holder._tq_PiT_f32 = PiT_f32
|
||||
centroids_c = getattr(buf_holder, "_tq_centroids_c", None)
|
||||
if centroids_c is None:
|
||||
centroids_c = centroids.contiguous()
|
||||
buf_holder._tq_centroids_c = centroids_c
|
||||
else:
|
||||
# Defensive fallback (unit tests may pass ``buf_holder=None``).
|
||||
_PiT_src = PiT if PiT is not None else Pi.T.contiguous()
|
||||
PiT_f32 = (
|
||||
_PiT_src if _PiT_src.dtype == torch.float32 else _PiT_src.to(torch.float32)
|
||||
)
|
||||
centroids_c = centroids.contiguous()
|
||||
|
||||
# ---- Pooled q_rot (stable data_ptr post-capture) ------
|
||||
# On ROCm, fresh tensor allocations made AFTER HIP graph capture can land
|
||||
# in the graph memory pool and cause GPU memory faults when passed as kernel
|
||||
# pointer arguments in the eager mixed-batch path. We pool q_rot and the
|
||||
# fp32 intermediate the same way segm_out/output are pooled: one max-B
|
||||
# allocation per shape, reused across all B's.
|
||||
#
|
||||
# Note: pool_bufs is computed BELOW after num_partitions is determined;
|
||||
# q_rot needs pool_bufs["q_float"] and pool_bufs["q_rot"]. We compute the
|
||||
# rotation inline after the pool is fetched (see the pooled-buffers
|
||||
# section below).
|
||||
|
||||
# ---- Partition count (FA2 split-KV) ----------------------------------
|
||||
#
|
||||
# FlyDSL runs INSIDE FULL cudagraph (TurboQuantMetadataBuilder._cudagraph_
|
||||
# support = UNIFORM_BATCH). That means the gridDim baked at capture
|
||||
# time MUST equal the gridDim at replay — the kernel launch parameters
|
||||
# are recorded into the captured graph. So `num_partitions` (which
|
||||
# becomes gridDim.z) MUST be derived from a stable source that produces
|
||||
# the same value at capture and at runtime.
|
||||
#
|
||||
# We use the worst-case context length implied by the block table's
|
||||
# allocation (block_table.shape[1] * block_size) as that source. It's
|
||||
# bounded by `max_model_len` (vLLM allocates the block table for the
|
||||
# configured max), and it's identical at capture and at runtime
|
||||
# because `block_table.shape[1]` is fixed once the model is loaded.
|
||||
# Per-tile OOB redirects + the masked FA-2 reducer ensure that for
|
||||
# short sequences the extra partitions contribute zero to the output.
|
||||
#
|
||||
# We always size from the worst case (block_table.shape[1] * block_size),
|
||||
# never from the per-step actual max_seq_len: a per-step size would make
|
||||
# the captured gridDim mismatch runtime and GPU-fault under cudagraph.
|
||||
kv_compute_block = _TQ_MOD.KV_COMPUTE_BLOCK
|
||||
worst_case_max_seq_len = int(block_table.shape[1]) * int(block_size)
|
||||
if max_seq_len <= 0:
|
||||
max_seq_len = worst_case_max_seq_len
|
||||
sizing_max_seq_len = worst_case_max_seq_len
|
||||
|
||||
# ── Bounded num_partitions + internal tile-group looping ──
|
||||
# The kernel previously hardcoded one partition = 256 tokens (=
|
||||
# KV_COMPUTE_BLOCK), forcing num_partitions to scale linearly with
|
||||
# max_model_len, which made the cudagraph capture cost grow
|
||||
# prohibitively at long context. The new kernel takes
|
||||
# ``tile_groups_per_partition`` (TGPP); each CTA processes
|
||||
# ``TGPP * 16`` K-tiles = ``TGPP * KV_COMPUTE_BLOCK`` tokens with the
|
||||
# FA-2 online-softmax state accumulating across all of them. This
|
||||
# mirrors what the SoA Triton decode / HIP SoA-fusion already do.
|
||||
#
|
||||
# Strategy: cap num_partitions at ``MAX_PARTITIONS`` (default 32),
|
||||
# then derive TGPP so that ``num_partitions * TGPP * KV_COMPUTE_BLOCK
|
||||
# >= sizing_max_seq_len``.
|
||||
#
|
||||
# Examples (block_size=32 → worst_case max_bps*32):
|
||||
# max_model_len=8K: required=32, parts=32, TGPP=1 (no waste)
|
||||
# max_model_len=16K: required=64, parts=32, TGPP=2 (no waste)
|
||||
# max_model_len=32K: required=128, parts=32, TGPP=4 (no waste)
|
||||
# max_model_len=64K: required=256, parts=32, TGPP=8 (no waste)
|
||||
# max_model_len=128K: required=512, parts=32, TGPP=16
|
||||
MAX_PARTITIONS = 32
|
||||
required_num_partitions = (
|
||||
sizing_max_seq_len + kv_compute_block - 1
|
||||
) // kv_compute_block
|
||||
# max_num_kv_splits acts as a parallelism floor (ensure at least this
|
||||
# many CTAs along grid.z), but is itself capped by MAX_PARTITIONS.
|
||||
parallelism_floor = min(MAX_PARTITIONS, max(1, max_num_kv_splits))
|
||||
# Cap num_partitions at MAX_PARTITIONS, but never go below the floor.
|
||||
num_partitions_actual = max(
|
||||
parallelism_floor,
|
||||
min(MAX_PARTITIONS, required_num_partitions),
|
||||
)
|
||||
# Round up to next power of 2 for the Triton reducer's tl.arange(0, N)
|
||||
# constraint (Triton requires power-of-2 ≥ 2).
|
||||
num_partitions = max(2, triton.next_power_of_2(num_partitions_actual))
|
||||
# Now derive tile_groups_per_partition (TGPP) so total coverage
|
||||
# ``num_partitions * TGPP * KV_COMPUTE_BLOCK`` is >= sizing_max_seq_len.
|
||||
# This guarantees no work is dropped at runtime regardless of seq_len.
|
||||
# Round TGPP up to the next power-of-2 so the JIT cache key is bounded
|
||||
# to a small set of values (e.g. {1, 2, 4, 8, 16}) — limits the number
|
||||
# of distinct kernel binaries that need to be compiled across requests.
|
||||
_tgpp_required = max(
|
||||
1,
|
||||
(required_num_partitions + num_partitions - 1) // num_partitions,
|
||||
)
|
||||
tile_groups_per_partition = int(triton.next_power_of_2(_tgpp_required))
|
||||
|
||||
# ---- Pooled buffers (no per-call cudaMalloc / memset) ----------
|
||||
# The FlyDSL kernel writes the FULL [B, Hk, P, QG, D] segm_out and the
|
||||
# FULL [B, Hk, P, QG] segm_max / segm_sum (running_max=-inf, sum=0 for
|
||||
# empty partitions are stored by the kernel itself), so uninitialized
|
||||
# buffers from the pool are safe to reuse.
|
||||
device = query.device
|
||||
pool_bufs = _SEGM_POOL.get(
|
||||
B,
|
||||
Hk,
|
||||
Hq,
|
||||
num_partitions,
|
||||
QG,
|
||||
D,
|
||||
device,
|
||||
query.dtype,
|
||||
)
|
||||
segm_out = pool_bufs["segm_out"]
|
||||
segm_max = pool_bufs["segm_max"]
|
||||
segm_sum = pool_bufs["segm_sum"]
|
||||
if output_buf is None:
|
||||
output = pool_bufs["output"]
|
||||
else:
|
||||
output = output_buf[:B] if output_buf.shape[0] != B else output_buf
|
||||
|
||||
# ---- q rotation: GEMM q @ PiT via stable pooled buffers -------------
|
||||
_q_float = pool_bufs["q_float"] # [B, Hq, D] fp32, stable
|
||||
_q_rot_f32 = pool_bufs["q_rot_fp32"] # [B, Hq, D] fp32, stable mm output
|
||||
_q_rot_out = pool_bufs["q_rot"] # [B, Hq, D] query.dtype, stable
|
||||
_q_float.copy_(query) # bf16 → fp32 in-place, no alloc
|
||||
# mm into stable fp32 buffer via out= to avoid fresh allocations.
|
||||
# PiT_f32 is cached on buf_holder (stable ptr since first layer warmup).
|
||||
torch.mm(
|
||||
_q_float.view(B * Hq, D),
|
||||
PiT_f32,
|
||||
out=_q_rot_f32.view(B * Hq, D), # in-place into pool buffer
|
||||
)
|
||||
_q_rot_out.copy_(_q_rot_f32) # fp32 → bf16, into stable buf
|
||||
q_for_kernel = _q_rot_out # stable ptr for kernel launch
|
||||
|
||||
# ---- FlyDSL kernel launch -------------------------------------------
|
||||
max_bps = int(block_table.shape[1])
|
||||
use_hw_tr = _hw_tr_enabled()
|
||||
launch = _get_kernel(
|
||||
Hk,
|
||||
num_partitions,
|
||||
max_bps,
|
||||
scale,
|
||||
QG,
|
||||
block_size,
|
||||
use_hw_v_transpose=use_hw_tr,
|
||||
num_seqs_hint=int(B),
|
||||
tile_groups_per_partition=int(tile_groups_per_partition),
|
||||
)
|
||||
# Zero-overhead one-shot info log (replaces logger.info_once which
|
||||
# hashes its format string on every call to dedup).
|
||||
global _LOG_INVOKED_ONCE, _LOG_SINKS_WARNED, _LOG_NORM_WARNED
|
||||
if not _LOG_INVOKED_ONCE:
|
||||
_LOG_INVOKED_ONCE = True
|
||||
logger.info(
|
||||
"FlyDSL launcher invoked (UNIFORM_BATCH cudagraph): "
|
||||
"B=%d Hk=%d Hq=%d D=%d QG=%d num_partitions=%d (actual=%d, "
|
||||
"cap=%d) TGPP=%d max_bps=%d block_size=%d max_seq_len=%d "
|
||||
"hw_v_transpose=%s (coverage=%d tokens, worst_case=%d tokens)",
|
||||
B,
|
||||
Hk,
|
||||
Hq,
|
||||
D,
|
||||
QG,
|
||||
num_partitions,
|
||||
num_partitions_actual,
|
||||
MAX_PARTITIONS,
|
||||
tile_groups_per_partition,
|
||||
max_bps,
|
||||
int(block_size),
|
||||
int(max_seq_len),
|
||||
use_hw_tr,
|
||||
num_partitions * tile_groups_per_partition * kv_compute_block,
|
||||
worst_case_max_seq_len,
|
||||
)
|
||||
launch(
|
||||
segm_out,
|
||||
segm_sum,
|
||||
segm_max,
|
||||
q_for_kernel,
|
||||
kv_cache,
|
||||
centroids_c,
|
||||
block_table,
|
||||
seq_lens,
|
||||
B,
|
||||
Hk,
|
||||
num_partitions,
|
||||
torch.cuda.current_stream(),
|
||||
)
|
||||
|
||||
# ---- Reduce partitions -> [B, Hq, D] --------------------------------
|
||||
_reduce_partitions[(B, Hq)](
|
||||
output_ptr=output,
|
||||
segm_out_ptr=segm_out,
|
||||
segm_max_ptr=segm_max,
|
||||
segm_sum_ptr=segm_sum,
|
||||
out_stride_n=output.stride(0),
|
||||
out_stride_h=output.stride(1),
|
||||
NUM_KV_HEADS=Hk,
|
||||
QG=QG,
|
||||
NUM_PARTS=num_partitions,
|
||||
HEAD_SIZE=D,
|
||||
)
|
||||
if sinks is not None and not _LOG_SINKS_WARNED:
|
||||
_LOG_SINKS_WARNED = True
|
||||
logger.warning(
|
||||
"FlyDSL launcher: sinks ignored (NYI). Disable sinks if they "
|
||||
"are required for correctness."
|
||||
)
|
||||
# ── norm_correction is honored IMPLICITLY ───────────────────────────
|
||||
# When the model was stored with norm_correction=True (the *_nc presets
|
||||
# turboquant_4bit_nc / k3v4_nc / 3bit_nc), the per-token K-norm scalar
|
||||
# was pre-folded to ||k_t|| / ||c_t|| at store time by the TurboQuant
|
||||
# store path. The decode kernel just multiplies `c_vals * stored_knorm`,
|
||||
# which then equals `(c_vals / ||c_t||) * ||k_t||` — exactly the
|
||||
# unit-norm-renormalized centroid times the original key norm. The
|
||||
# Triton decode does the identical multiply, so both decoders honor
|
||||
# norm_correction equivalently and there is nothing to "do" at decode
|
||||
# time. The launcher's `norm_correction` arg is only kept for API
|
||||
# parity; we emit a one-shot INFO log to make the contract explicit.
|
||||
if norm_correction and not _LOG_NORM_WARNED:
|
||||
_LOG_NORM_WARNED = True
|
||||
logger.info(
|
||||
"FlyDSL launcher: norm_correction honored implicitly via "
|
||||
"pre-folded stored K-norm (cf. triton_turboquant_store step 3); "
|
||||
"no decode-time work required, identical to the SoA decode behavior."
|
||||
)
|
||||
return output
|
||||
@@ -1,834 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FlyDSL launcher for the optimized UltraQuant 4-bit D=256 decode kernel.
|
||||
|
||||
The production kernel uses scaled FP4×E4M3 QK MFMA, native V conversion,
|
||||
query hoisting, in-kernel Walsh-Hadamard rotation for GQA-6/8/16, and strided
|
||||
tile-group scheduling. These choices are fixed and are not environment-tunable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_FLYDSL_AVAILABLE: bool | None = None
|
||||
_ULTRAQUANT_MOD: Any = None
|
||||
_FLYC: Any = None
|
||||
_FX: Any = None
|
||||
_TYPING_T: Any = None
|
||||
_CC: Any = None
|
||||
_IR: Any = None
|
||||
|
||||
|
||||
def is_ultraquant_flydsl_available() -> bool:
|
||||
"""Return whether the gfx950 UltraQuant FlyDSL kernel can be loaded."""
|
||||
global _FLYDSL_AVAILABLE, _ULTRAQUANT_MOD
|
||||
global _FLYC, _FX, _TYPING_T, _CC, _IR
|
||||
if _FLYDSL_AVAILABLE is not None:
|
||||
return _FLYDSL_AVAILABLE
|
||||
try:
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if not on_gfx950():
|
||||
_FLYDSL_AVAILABLE = False
|
||||
return False
|
||||
|
||||
import flydsl.compiler as flyc
|
||||
import flydsl.expr as fx
|
||||
from flydsl._mlir import ir
|
||||
from flydsl.compiler.kernel_function import CompilationContext
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
from vllm.v1.attention.ops.flydsl_kernels import (
|
||||
ultraquant_decode_hd256 as ultraquant_mod,
|
||||
)
|
||||
|
||||
_FLYC = flyc
|
||||
_FX = fx
|
||||
_TYPING_T = T
|
||||
_CC = CompilationContext
|
||||
_IR = ir
|
||||
_ULTRAQUANT_MOD = ultraquant_mod
|
||||
_FLYDSL_AVAILABLE = True
|
||||
logger.info_once("FlyDSL UltraQuant D=256 decode is available")
|
||||
except Exception as ex: # noqa: BLE001
|
||||
_FLYDSL_AVAILABLE = False
|
||||
logger.warning_once(
|
||||
"FlyDSL UltraQuant decode is unavailable (%s); using Triton fallback.",
|
||||
ex,
|
||||
)
|
||||
return _FLYDSL_AVAILABLE
|
||||
|
||||
|
||||
_ULTRAQUANT_FLYDSL_GQA = (6, 8, 16)
|
||||
|
||||
|
||||
def ultraquant_flydsl_decode_eligible(
|
||||
*,
|
||||
head_size: int,
|
||||
num_kv_groups: int,
|
||||
has_sinks: bool,
|
||||
sliding_window: int | None,
|
||||
flydsl_loaded: bool,
|
||||
) -> bool:
|
||||
"""Return whether decode should use the gfx950 FlyDSL kernel.
|
||||
|
||||
Production FlyDSL covers D=256 and GQA in {6, 8, 16}. Sinks and sliding
|
||||
window fall back to unified Triton. ``flydsl_loaded`` is the process-level
|
||||
compiler/hardware probe so this policy can be unit-tested without a GPU.
|
||||
"""
|
||||
return (
|
||||
flydsl_loaded
|
||||
and head_size == 256
|
||||
and num_kv_groups in _ULTRAQUANT_FLYDSL_GQA
|
||||
and not has_sinks
|
||||
and not (sliding_window and sliding_window > 0)
|
||||
)
|
||||
|
||||
|
||||
# -- Kernel module cache -------------------------------------------------------
|
||||
# Compiling a FlyDSL kernel is expensive; cache by the constexprs that
|
||||
# parameterize stride math: num_kv_heads, num_partitions, max_blocks_per_seq,
|
||||
# scale, query_group_size, kv_block_size, padded_slot, hw_v_transpose,
|
||||
# tile_groups_per_partition.
|
||||
_KERN_CACHE: dict[tuple, Any] = {}
|
||||
|
||||
_LOG_INVOKED_ONCE: bool = False
|
||||
|
||||
# Per-device cached FP4 E2M1 value table (used as the kernel "centroids" LUT).
|
||||
_FP4_LUT_CACHE: dict[tuple[str, torch.dtype], torch.Tensor] = {}
|
||||
|
||||
|
||||
def _fp4_value_lut(device: torch.device) -> torch.Tensor:
|
||||
"""Return the fixed 16-entry FP4 E2M1 value table as fp32 [16] on device.
|
||||
|
||||
Indexed by the stored 4-bit E2M1 code, so ``lut[nibble]`` is the dequant
|
||||
value directly (matches ``format.FP4_BITS_TO_VALUE``).
|
||||
"""
|
||||
from vllm.v1.attention.ops.ultraquant.format import FP4_BITS_TO_VALUE
|
||||
|
||||
key = (str(device), torch.float32)
|
||||
t = _FP4_LUT_CACHE.get(key)
|
||||
if t is None:
|
||||
t = torch.tensor(
|
||||
FP4_BITS_TO_VALUE, dtype=torch.float32, device=device
|
||||
).contiguous()
|
||||
_FP4_LUT_CACHE[key] = t
|
||||
return t
|
||||
|
||||
# Per-device cached qperm head-dim permutation for scaled QK MFMA.
|
||||
|
||||
|
||||
# qperm[phys] = the natural head-dim placed at physical operand position phys.
|
||||
# It realigns the fixed FP4xFP8 scaled-MFMA contraction so that, with K fed
|
||||
# native-contiguous, Q's columns pair correctly. Folded into PiT on the host
|
||||
# (PiT[:, qperm]) so the Q rotation matmul emits already-permuted q_rot at zero
|
||||
# extra runtime cost. Derived + validated in tmp_qk_scaled_mfma.py (contigK).
|
||||
_QPERM_CACHE: dict[str, torch.Tensor] = {}
|
||||
|
||||
|
||||
def _kmap_phys(group: int, s: int) -> int:
|
||||
# Physical operand position of group's element s for the K=128 fp4/fp8
|
||||
# scaled MFMA (matches tmp_qk_scaled_mfma.py::_kmap).
|
||||
return 32 * (s >> 4) + 64 * (group % 2) + 16 * (group // 2) + (s & 15)
|
||||
|
||||
|
||||
def _qperm_index(
|
||||
device: torch.device, D: int = 128, group_size: int = 32
|
||||
) -> torch.Tensor:
|
||||
# _kmap_phys describes ONE K=128 contraction (4 groups of 32). A head dim
|
||||
# wider than 128 is covered by ceil(D/128) back-to-back MFMA issues, each
|
||||
# contracting its own independent 128-wide slice, so the same map repeats
|
||||
# per slice: qperm = [qperm128, qperm128 + 128, ...]. Applying _kmap_phys
|
||||
# with group >= 4 instead yields positions outside the slice and is not a
|
||||
# permutation at all.
|
||||
key = f"{device}:{D}:{group_size}"
|
||||
t = _QPERM_CACHE.get(key)
|
||||
if t is None:
|
||||
k_per_mfma = 128
|
||||
assert D % k_per_mfma == 0, (
|
||||
f"scaled QK MFMA needs D to be a multiple of {k_per_mfma}, got {D}"
|
||||
)
|
||||
groups_per_mfma = k_per_mfma // group_size
|
||||
qperm = [0] * D
|
||||
for half in range(D // k_per_mfma):
|
||||
base = half * k_per_mfma
|
||||
for g in range(groups_per_mfma):
|
||||
for s in range(group_size):
|
||||
qperm[base + _kmap_phys(g, s)] = base + g * group_size + s
|
||||
assert sorted(qperm) == list(range(D)), "qperm is not a permutation"
|
||||
t = torch.tensor(qperm, dtype=torch.long, device=device).contiguous()
|
||||
_QPERM_CACHE[key] = t
|
||||
return t
|
||||
|
||||
|
||||
def _detect_max_capture_B() -> int:
|
||||
try:
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
cfg = get_current_vllm_config()
|
||||
candidates: list[int] = []
|
||||
sizes = cfg.compilation_config.cudagraph_capture_sizes
|
||||
if sizes:
|
||||
candidates.append(int(max(sizes)))
|
||||
sched = getattr(cfg, "scheduler_config", None)
|
||||
if sched is not None and getattr(sched, "max_num_seqs", None):
|
||||
candidates.append(int(sched.max_num_seqs))
|
||||
if candidates:
|
||||
return max(candidates)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return 512
|
||||
|
||||
|
||||
class _SegmBufPool:
|
||||
"""Single-bucket buffer pool for segm_out/segm_max/segm_sum/output + the
|
||||
pooled Q-rotation intermediates. Identical to the TQ pool except that
|
||||
ultraquant needs an extra ``q_fp8`` (float8_e4m3fn) slot for the Q precision
|
||||
haircut (``q_rot_fp32 -> fp8_e4m3 -> bf16``) so that no fresh allocation
|
||||
lands in the HIP graph memory pool post-capture.
|
||||
"""
|
||||
|
||||
__slots__ = ("_bufs", "_max_B")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._bufs: dict[tuple, dict[str, torch.Tensor]] = {}
|
||||
self._max_B: int | None = None
|
||||
|
||||
def get(
|
||||
self,
|
||||
B: int,
|
||||
Hk: int,
|
||||
Hq: int,
|
||||
num_partitions: int,
|
||||
QG: int,
|
||||
D: int,
|
||||
device: torch.device,
|
||||
q_dtype: torch.dtype,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
if self._max_B is None:
|
||||
self._max_B = _detect_max_capture_B()
|
||||
B_bucket = max(self._max_B, int(B))
|
||||
key = (
|
||||
int(Hk),
|
||||
int(Hq),
|
||||
int(num_partitions),
|
||||
int(QG),
|
||||
int(D),
|
||||
str(device),
|
||||
q_dtype,
|
||||
)
|
||||
bufs = self._bufs.get(key)
|
||||
if bufs is None or bufs["segm_out"].shape[0] < B_bucket:
|
||||
if bufs is not None and bufs["segm_out"].shape[0] < B_bucket:
|
||||
# First-time grow: warn so user knows cudagraphs may be
|
||||
# invalidated. This should only happen during warmup or in
|
||||
# always-eager configs (a batch larger than any captured
|
||||
# cudagraph size).
|
||||
logger.warning_once(
|
||||
"FlyDSL UltraQuant buffer pool growing from %d "
|
||||
"to %d (B=%d). If this happens AFTER cudagraph warmup the "
|
||||
"captured graphs hold stale pointers and will GPU-fault.",
|
||||
bufs["segm_out"].shape[0],
|
||||
B_bucket,
|
||||
B,
|
||||
)
|
||||
bufs = {
|
||||
"segm_out": torch.empty(
|
||||
(B_bucket, Hk, num_partitions, QG, D),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
),
|
||||
"segm_max": torch.empty(
|
||||
(B_bucket, Hk, num_partitions, QG),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
"segm_sum": torch.empty(
|
||||
(B_bucket, Hk, num_partitions, QG),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
"output": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=q_dtype,
|
||||
device=device,
|
||||
),
|
||||
"q_rot": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=q_dtype,
|
||||
device=device,
|
||||
),
|
||||
"q_float": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
"q_rot_fp32": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
# ultraquant-specific: E4M3 haircut intermediate (pooled so the
|
||||
# cast does not allocate a fresh tensor post-capture).
|
||||
"q_fp8": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=device,
|
||||
),
|
||||
}
|
||||
self._bufs[key] = bufs
|
||||
self._max_B = B_bucket
|
||||
logger.info_once(
|
||||
"FlyDSL UltraQuant buffer pool allocated "
|
||||
"shape=(Hk=%d, Hq=%d, P=%d, QG=%d, D=%d, dtype=%s) "
|
||||
"B_bucket=%d. VRAM = %.1f MiB / shape.",
|
||||
Hk,
|
||||
Hq,
|
||||
num_partitions,
|
||||
QG,
|
||||
D,
|
||||
q_dtype,
|
||||
B_bucket,
|
||||
sum(t.numel() * t.element_size() for t in bufs.values()) / (1 << 20),
|
||||
)
|
||||
return {
|
||||
"segm_out": bufs["segm_out"][:B],
|
||||
"segm_max": bufs["segm_max"][:B],
|
||||
"segm_sum": bufs["segm_sum"][:B],
|
||||
"output": bufs["output"][:B],
|
||||
"q_rot": bufs["q_rot"][:B],
|
||||
"q_float": bufs["q_float"][:B],
|
||||
"q_rot_fp32": bufs["q_rot_fp32"][:B],
|
||||
"q_fp8": bufs["q_fp8"][:B],
|
||||
}
|
||||
|
||||
|
||||
_SEGM_POOL = _SegmBufPool()
|
||||
|
||||
_GET_KERNEL_STATS = {"hits": 0, "misses": 0, "build_total_s": 0.0}
|
||||
|
||||
|
||||
def _get_kernel(
|
||||
num_kv_heads: int,
|
||||
num_partitions: int,
|
||||
max_blocks_per_seq: int,
|
||||
scale: float,
|
||||
query_group_size: int,
|
||||
kv_block_size: int,
|
||||
padded_slot: int,
|
||||
fuse_qrot: bool,
|
||||
num_seqs_hint: int,
|
||||
tile_groups_per_partition: int,
|
||||
stride_q_seq: int,
|
||||
stride_q_head: int,
|
||||
):
|
||||
key = (
|
||||
num_kv_heads,
|
||||
num_partitions,
|
||||
max_blocks_per_seq,
|
||||
round(scale, 8),
|
||||
query_group_size,
|
||||
kv_block_size,
|
||||
padded_slot,
|
||||
fuse_qrot,
|
||||
tile_groups_per_partition,
|
||||
stride_q_seq,
|
||||
stride_q_head,
|
||||
)
|
||||
cached = _KERN_CACHE.get(key)
|
||||
if cached is not None:
|
||||
_GET_KERNEL_STATS["hits"] += 1
|
||||
return cached
|
||||
|
||||
assert is_ultraquant_flydsl_available()
|
||||
import time
|
||||
|
||||
build_start = time.perf_counter()
|
||||
kmod = _ULTRAQUANT_MOD
|
||||
assert kmod is not None
|
||||
kfn = kmod.build_ultraquant_decode_hd256_module(
|
||||
num_seqs=num_seqs_hint,
|
||||
num_kv_heads=num_kv_heads,
|
||||
num_partitions=num_partitions,
|
||||
padded_slot=padded_slot,
|
||||
max_blocks_per_seq=max_blocks_per_seq,
|
||||
softmax_scale=scale,
|
||||
query_group_size=query_group_size,
|
||||
kv_block_size=kv_block_size,
|
||||
tile_groups_per_partition=tile_groups_per_partition,
|
||||
fuse_qrot=fuse_qrot,
|
||||
stride_q_seq=stride_q_seq,
|
||||
stride_q_head=stride_q_head,
|
||||
)
|
||||
allocator = kmod.allocator
|
||||
flyc = _FLYC
|
||||
fx = _FX
|
||||
T = _TYPING_T
|
||||
CompilationContext = _CC
|
||||
ir_mod = _IR
|
||||
|
||||
@flyc.jit
|
||||
def _launch(
|
||||
out,
|
||||
es,
|
||||
ml,
|
||||
q,
|
||||
kvc,
|
||||
cents,
|
||||
bt,
|
||||
sl,
|
||||
gx: fx.Int32, # type: ignore[name-defined]
|
||||
gy: fx.Int32, # type: ignore[name-defined]
|
||||
gz: fx.Int32, # type: ignore[name-defined]
|
||||
stream: fx.Stream, # type: ignore[name-defined]
|
||||
):
|
||||
allocator.finalized = False
|
||||
ctx = CompilationContext.get_current()
|
||||
with ir_mod.InsertionPoint(ctx.gpu_module_body):
|
||||
allocator.finalize()
|
||||
from flydsl.expr import arith
|
||||
|
||||
grid_x = arith.index_cast(T.index, gx.ir_value())
|
||||
grid_y = arith.index_cast(T.index, gy.ir_value())
|
||||
grid_z = arith.index_cast(T.index, gz.ir_value())
|
||||
kfn(out, es, ml, q, kvc, cents, bt, sl).launch(
|
||||
grid=(grid_x, grid_y, grid_z),
|
||||
block=(kmod.BLOCK_THREADS, 1, 1),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
_KERN_CACHE[key] = _launch
|
||||
build_time = time.perf_counter() - build_start
|
||||
_GET_KERNEL_STATS["misses"] += 1
|
||||
_GET_KERNEL_STATS["build_total_s"] += build_time
|
||||
logger.info(
|
||||
"Built FlyDSL UltraQuant kernel #%d in %.2fs (total %.1fs), key=%s",
|
||||
_GET_KERNEL_STATS["misses"],
|
||||
build_time,
|
||||
_GET_KERNEL_STATS["build_total_s"],
|
||||
key,
|
||||
)
|
||||
return _launch
|
||||
|
||||
|
||||
# -- Partition reducer ---------------------------------------------------------
|
||||
from vllm.triton_utils import tl # noqa: E402
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _reduce_partitions(
|
||||
output_ptr, # [N, Hq, D] in OUT_DTYPE
|
||||
segm_out_ptr, # [N, Hk, P, QG, D] bf16
|
||||
segm_max_ptr, # [N, Hk, P, QG] fp32
|
||||
segm_sum_ptr, # [N, Hk, P, QG] fp32
|
||||
out_stride_n: tl.int64,
|
||||
out_stride_h: tl.int64,
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
QG: tl.constexpr,
|
||||
NUM_PARTS: tl.constexpr,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
n = tl.program_id(0)
|
||||
hq = tl.program_id(1)
|
||||
kv_h = hq // QG
|
||||
qg = hq % QG
|
||||
# Splitting the head-dim across programs is what gives this kernel enough
|
||||
# parallelism to use the GPU: the reduction runs over PARTITIONS, so every
|
||||
# head-dim column is independent. A (B, Hq) grid is only 32 workgroups at
|
||||
# B=4 -- ~12% of a 256-CU part -- which is why it moved 8.4 MB at just
|
||||
# ~995 GB/s. The partition reduction itself is untouched (same values, same
|
||||
# order), so the result is bit-identical; only the columns each program
|
||||
# owns change. The [P] max/sum vectors get re-read per D-block, which is
|
||||
# 4 KB of duplicated traffic against a 262 KB tile.
|
||||
|
||||
msum_base = n * (NUM_KV_HEADS * NUM_PARTS * QG) + kv_h * (NUM_PARTS * QG) + qg
|
||||
so_base = (
|
||||
n * (NUM_KV_HEADS * NUM_PARTS * QG * HEAD_SIZE)
|
||||
+ kv_h * (NUM_PARTS * QG * HEAD_SIZE)
|
||||
+ qg * HEAD_SIZE
|
||||
)
|
||||
|
||||
p_off = tl.arange(0, NUM_PARTS)
|
||||
d_off = tl.program_id(2) * BLOCK_D + tl.arange(0, BLOCK_D)
|
||||
|
||||
m_idx = msum_base + p_off * QG
|
||||
seg_max = tl.load(segm_max_ptr + m_idx)
|
||||
seg_sum = tl.load(segm_sum_ptr + m_idx)
|
||||
|
||||
valid = seg_max > float("-inf")
|
||||
overall_max = tl.max(tl.where(valid, seg_max, float("-inf")))
|
||||
|
||||
rescale = tl.where(valid, tl.exp(seg_max - overall_max), 0.0)
|
||||
seg_sum_rescaled = seg_sum * rescale
|
||||
overall_sum = tl.sum(seg_sum_rescaled)
|
||||
|
||||
so_idx = so_base + p_off[:, None] * (QG * HEAD_SIZE) + d_off[None, :]
|
||||
seg_out = tl.load(segm_out_ptr + so_idx).to(tl.float32)
|
||||
weighted = seg_out * seg_sum_rescaled[:, None]
|
||||
acc_sum = tl.sum(weighted, axis=0)
|
||||
acc = tl.where(overall_sum > 0.0, acc_sum / overall_sum, 0.0)
|
||||
|
||||
out_off = n * out_stride_n + hq * out_stride_h + d_off
|
||||
tl.store(output_ptr + out_off, acc.to(output_ptr.dtype.element_ty))
|
||||
|
||||
|
||||
# -- Fused Q rotation + E4M3 haircut (Triton) ---------------------------------
|
||||
# Replaces the 4-op torch prologue (copy bf16->fp32, mm by PiT, copy fp32->e4m3,
|
||||
# copy e4m3->bf16) with ONE kernel. The win is kernel COUNT, not bytes: each of
|
||||
# those small ops costs ~2-4us of fixed GPU-side cost here largely independent of
|
||||
# element count (verified: sweeping the pool's B_bucket 512->1, i.e. 512x fewer
|
||||
# elements in the copies, moved the floor 0%). Under CUDA graphs the host launch
|
||||
# cost is already gone, so only per-kernel GPU cost remains and collapsing 4
|
||||
# kernels into 1 is the lever that is left.
|
||||
@triton.jit
|
||||
def _fused_q_rot_haircut(
|
||||
q_ptr, # [B, Hq, D] query dtype (bf16/fp16), strided
|
||||
pit_ptr, # [D, D] fp32 contiguous (already qperm-permuted)
|
||||
out_ptr, # [B_bucket, Hq, D] out dtype, contiguous
|
||||
M, # B * Hq (runtime)
|
||||
q_stride_n: tl.int64,
|
||||
q_stride_h: tl.int64,
|
||||
HQ: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
HAIRCUT: tl.constexpr, # round through E4M3 (fp8-consuming QK MFMA paths)
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0)
|
||||
pid_d = tl.program_id(1)
|
||||
m_off = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
m_mask = m_off < M
|
||||
n = m_off // HQ
|
||||
h = m_off % HQ
|
||||
|
||||
# Splitting the OUTPUT columns across programs (not just the rows) is what
|
||||
# gives this kernel parallelism at small batch: M = B*Hq is only 8 at B=1,
|
||||
# so a row-only grid launches ONE workgroup and serializes the whole
|
||||
# 256x256 rotation on a single CU. The K reduction is untouched by the
|
||||
# split, so the result stays bit-identical.
|
||||
d_off = pid_d * BLOCK_D + tl.arange(0, BLOCK_D)
|
||||
acc = tl.zeros([BLOCK_M, BLOCK_D], dtype=tl.float32)
|
||||
for k0 in range(0, D, BLOCK_K):
|
||||
k_off = k0 + tl.arange(0, BLOCK_K)
|
||||
q_idx = n[:, None] * q_stride_n + h[:, None] * q_stride_h + k_off[None, :]
|
||||
q_blk = tl.load(q_ptr + q_idx, mask=m_mask[:, None], other=0.0)
|
||||
p_blk = tl.load(pit_ptr + k_off[:, None] * D + d_off[None, :])
|
||||
acc = tl.dot(q_blk.to(tl.float32), p_blk, acc)
|
||||
|
||||
if HAIRCUT:
|
||||
# fp32 -> E4M3 -> fp32. Mirrors the torch path's
|
||||
# q_rot_fp32 -> float8_e4m3fn -> bf16 round trip (e4m3 is a subset of
|
||||
# bf16, so the second hop is exact and folds away here).
|
||||
acc = acc.to(tl.float8e4nv).to(tl.float32)
|
||||
|
||||
out_idx = m_off[:, None] * D + d_off[None, :]
|
||||
tl.store(out_ptr + out_idx, acc.to(out_ptr.dtype.element_ty), mask=m_mask[:, None])
|
||||
|
||||
|
||||
# Prologue Q-rotation fusion (fallback for non-hd256 / in-kernel-WHT-off paths).
|
||||
# Cleared the accuracy ladder (offline parity -> autoregressive drift -> paired
|
||||
# GSM8K); pinned ON as part of the validated production config.
|
||||
_FUSE_Q_ROT = True
|
||||
|
||||
# Stronger form: do the rotation INSIDE the decode kernel as a Walsh-Hadamard
|
||||
# butterfly and drop the prologue dispatch entirely. The prologue is ~7us at
|
||||
# B=4 and that is almost all fixed per-dispatch cost (sweeping its grid 4x moves
|
||||
# it 0%), so deleting the launch -- not optimizing it -- is the win.
|
||||
# Default ON: verified bit-exact (torch.equal) against the prologue path across
|
||||
# B={1,2,3,4,8,16} x seq={4k,8k,16k,32k,50021,75k}, including non-power-of-2
|
||||
# batch and seq.
|
||||
_FUSE_QROT_INKERNEL = True
|
||||
|
||||
|
||||
def _run_fused_q_rot(query, PiT_used, out, B, Hq, D, haircut):
|
||||
"""Launch the fused prologue. ``out`` is the pooled [B_bucket, Hq, D] buf."""
|
||||
M = B * Hq
|
||||
BLOCK_M = 16
|
||||
BLOCK_K = 32
|
||||
BLOCK_D = 64
|
||||
grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(D, BLOCK_D))
|
||||
_fused_q_rot_haircut[grid](
|
||||
query,
|
||||
PiT_used,
|
||||
out,
|
||||
M,
|
||||
query.stride(0),
|
||||
query.stride(1),
|
||||
HQ=int(Hq),
|
||||
D=int(D),
|
||||
HAIRCUT=bool(haircut),
|
||||
BLOCK_M=BLOCK_M,
|
||||
BLOCK_K=BLOCK_K,
|
||||
BLOCK_D=BLOCK_D,
|
||||
)
|
||||
|
||||
|
||||
# -- Public launcher ----------------------------------------------------------
|
||||
def flydsl_ultraquant_decode_attention(
|
||||
query: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
scale: float,
|
||||
PiT: torch.Tensor | None = None,
|
||||
max_seq_len: int = 0,
|
||||
output_buf: torch.Tensor | None = None,
|
||||
buf_holder: Any = None,
|
||||
max_num_kv_splits: int = 32,
|
||||
sinks: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Launch the optimized FlyDSL UltraQuant D=256 decode."""
|
||||
if not is_ultraquant_flydsl_available():
|
||||
raise RuntimeError(
|
||||
"FlyDSL UltraQuant decode requires gfx950 and importable FlyDSL."
|
||||
)
|
||||
if sinks is not None:
|
||||
raise NotImplementedError("FlyDSL UltraQuant decode does not support sinks")
|
||||
|
||||
B, Hq, D = query.shape
|
||||
Hk = kv_cache.shape[2]
|
||||
block_size = kv_cache.shape[1]
|
||||
padded_slot = int(kv_cache.shape[3])
|
||||
QG = Hq // Hk
|
||||
assert D == 256, f"UltraQuant FlyDSL requires head_dim=256, got {D}"
|
||||
assert block_size in (16, 32, 64, 128, 256), (
|
||||
f"UltraQuant FlyDSL supports kv_block_size 16/32/64/128/256, got {block_size}"
|
||||
)
|
||||
assert QG in _ULTRAQUANT_FLYDSL_GQA, (
|
||||
f"UltraQuant FlyDSL supports GQA factor "
|
||||
f"{', '.join(map(str, _ULTRAQUANT_FLYDSL_GQA))}, got {QG}"
|
||||
)
|
||||
|
||||
device = query.device
|
||||
|
||||
# ---- PiT (Hadamard) — cache the contiguous fp32 form on the layer -----
|
||||
PiT_f32: torch.Tensor
|
||||
if buf_holder is not None:
|
||||
PiT_f32 = getattr(buf_holder, "_ultraquant_PiT_f32", None)
|
||||
if PiT_f32 is None:
|
||||
assert PiT is not None, (
|
||||
"UltraQuant launcher requires PiT (the Hadamard rotation)"
|
||||
)
|
||||
PiT_f32 = PiT if PiT.dtype == torch.float32 else PiT.to(torch.float32)
|
||||
if not PiT_f32.is_contiguous():
|
||||
PiT_f32 = PiT_f32.contiguous()
|
||||
buf_holder._ultraquant_PiT_f32 = PiT_f32
|
||||
else:
|
||||
assert PiT is not None, (
|
||||
"UltraQuant launcher requires PiT (the Hadamard rotation)"
|
||||
)
|
||||
PiT_f32 = PiT if PiT.dtype == torch.float32 else PiT.to(torch.float32)
|
||||
if not PiT_f32.is_contiguous():
|
||||
PiT_f32 = PiT_f32.contiguous()
|
||||
|
||||
centroids_c = _fp4_value_lut(device)
|
||||
|
||||
# ---- Partition count (FA2 split-KV) -----------------------------------
|
||||
kv_compute_block = _ULTRAQUANT_MOD.KV_COMPUTE_BLOCK
|
||||
worst_case_max_seq_len = int(block_table.shape[1]) * int(block_size)
|
||||
if max_seq_len <= 0:
|
||||
max_seq_len = worst_case_max_seq_len
|
||||
sizing_max_seq_len = worst_case_max_seq_len
|
||||
|
||||
# Split-KV parallelism cap. The old default (32) silently serialized long
|
||||
# contexts: once required_num_partitions exceeds the cap, TGPP grows and each
|
||||
# workgroup walks MORE tile-groups instead of the grid getting wider. The
|
||||
# decode kernel is latency-bound at ~1-2 waves/CU (grid = B*Hk*P workgroups
|
||||
# of a single wavefront), so that serialization dominated long-context decode.
|
||||
# Raising the cap to 256 keeps TGPP ~1 and measured (B=16, rocprof, per step
|
||||
# decode+reduce+store): 16k 82.7->40.0us, 32k 152.0->55.1us, 64k 339.4->95.7us,
|
||||
# 131k 671.3->179.0us (2.1x - 3.8x, growing with context). 512 regresses
|
||||
# because _reduce_partitions cost grows with partition count.
|
||||
# Cost: the segm pool scales with P (~8x vs P=32).
|
||||
# BATCH-ADAPTIVE split-KV cap (low-concurrency occupancy fix).
|
||||
# rocprof (B=4, 75k): the decode kernel is occupancy/latency-bound, not
|
||||
# compute-bound -- OccupancyPercent 5.8%, MfmaUtil 4.5%, VALUBusy 14.5%
|
||||
# (stalled ~85% on HBM), VALUUtilization 90% (no divergence). The grid is
|
||||
# B*Hk*num_partitions single-wavefront workgroups; at B=4/Hk=1/P=256 that's
|
||||
# only 1024 wg = 4/CU, too few to hide memory latency. Splitting MORE at low
|
||||
# batch adds resident wavefronts and cuts ITL; at high batch the grid is
|
||||
# already full so extra partitions only inflate _reduce_partitions + empty-
|
||||
# partition overhead. Measured end-to-end (graphs on, TP=8, uniform 75k,
|
||||
# median ITL vs kv8): cap512 vs cap256 -> C4 +9.0%->+4.9%, C8 +6.6%->+2.5%
|
||||
# (better), C16 +2.8%->+4.3% (worse). So use 512 at low batch, 256 at high.
|
||||
# This is only SAFE because the build-local-allocator fix in
|
||||
# ultraquant_decode_hd256.py lets the two num_partitions variants (256 & 512)
|
||||
# coexist across CUDA-graph batch buckets without the @flyc.jit
|
||||
# "global 'allocator' changed since first compile" drift crash.
|
||||
MAX_PARTITIONS = 512 if B <= 8 else 256
|
||||
required_num_partitions = (
|
||||
sizing_max_seq_len + kv_compute_block - 1
|
||||
) // kv_compute_block
|
||||
parallelism_floor = min(MAX_PARTITIONS, max(1, max_num_kv_splits))
|
||||
num_partitions_actual = max(
|
||||
parallelism_floor,
|
||||
min(MAX_PARTITIONS, required_num_partitions),
|
||||
)
|
||||
# Cap partitions by how much work there actually IS to split. Nothing above
|
||||
# forces a partition to hold a useful number of tokens -- max_num_kv_splits
|
||||
# sets a parallelism FLOOR -- so a short context gets split into partitions
|
||||
# of a handful of tokens each, where every partition is almost entirely
|
||||
# fixed cost (and each one still writes a full QG x D partial for the
|
||||
# reduce to read back). Measured sweet spot is ~130-147 tokens/partition,
|
||||
# roughly flat in between; next_power_of_2 below rounds this target up, so
|
||||
# the effective range lands at ~96-192 tokens. At B=4 this is worth 27.9 ->
|
||||
# 24.4 us at seq=2048 and 27.0 -> 24.8 us at seq=16384, and is inert once
|
||||
# the context is long enough to want every partition (seq >= ~64k).
|
||||
_tok_per_part = 192
|
||||
if _tok_per_part > 0:
|
||||
_useful_parts = max(
|
||||
1, (int(sizing_max_seq_len) + _tok_per_part - 1) // _tok_per_part
|
||||
)
|
||||
num_partitions_actual = min(num_partitions_actual, _useful_parts)
|
||||
num_partitions = max(2, triton.next_power_of_2(num_partitions_actual))
|
||||
_tgpp_required = max(
|
||||
1,
|
||||
(required_num_partitions + num_partitions - 1) // num_partitions,
|
||||
)
|
||||
tile_groups_per_partition = int(triton.next_power_of_2(_tgpp_required))
|
||||
|
||||
# ---- Pooled buffers ---------------------------------------------------
|
||||
pool_bufs = _SEGM_POOL.get(
|
||||
B,
|
||||
Hk,
|
||||
Hq,
|
||||
num_partitions,
|
||||
QG,
|
||||
D,
|
||||
device,
|
||||
query.dtype,
|
||||
)
|
||||
segm_out = pool_bufs["segm_out"]
|
||||
segm_max = pool_bufs["segm_max"]
|
||||
segm_sum = pool_bufs["segm_sum"]
|
||||
if output_buf is None:
|
||||
output = pool_bufs["output"]
|
||||
else:
|
||||
output = output_buf[:B] if output_buf.shape[0] != B else output_buf
|
||||
|
||||
# Fold the scaled-MFMA head-dim permutation into PiT's columns so the
|
||||
# WHT / prologue produces native operand order. Always apply the E4M3
|
||||
# haircut: scaled QK consumes Q as E4M3.
|
||||
PiT_used = PiT_f32
|
||||
qperm_idx = _qperm_index(device, D=D)
|
||||
if buf_holder is not None:
|
||||
PiT_used = getattr(buf_holder, "_ultraquant_PiT_perm_f32", None)
|
||||
if PiT_used is None:
|
||||
PiT_used = PiT_f32.index_select(1, qperm_idx).contiguous()
|
||||
buf_holder._ultraquant_PiT_perm_f32 = PiT_used
|
||||
else:
|
||||
PiT_used = PiT_f32.index_select(1, qperm_idx).contiguous()
|
||||
_q_float = pool_bufs["q_float"]
|
||||
_q_rot_f32 = pool_bufs["q_rot_fp32"]
|
||||
_q_fp8 = pool_bufs["q_fp8"]
|
||||
_q_rot_out = pool_bufs["q_rot"]
|
||||
|
||||
_inkernel_qrot = (
|
||||
_FUSE_QROT_INKERNEL
|
||||
and int(D) == 256
|
||||
and QG in _ULTRAQUANT_FLYDSL_GQA
|
||||
and query.dim() == 3
|
||||
and query.stride(2) == 1
|
||||
and query.stride(1) == int(D)
|
||||
)
|
||||
if _inkernel_qrot:
|
||||
q_for_kernel = query
|
||||
# The fused kernel indexes the head-dim contiguously; fall back if not.
|
||||
elif _FUSE_Q_ROT and query.stride(-1) == 1:
|
||||
# One kernel for the whole prologue instead of four ops.
|
||||
_run_fused_q_rot(query, PiT_used, _q_rot_out, B, Hq, D, haircut=True)
|
||||
else:
|
||||
_q_float.copy_(query)
|
||||
torch.mm(
|
||||
_q_float.view(B * Hq, D),
|
||||
PiT_used,
|
||||
out=_q_rot_f32.view(B * Hq, D),
|
||||
)
|
||||
_q_fp8.copy_(_q_rot_f32) # fp32 -> e4m3
|
||||
_q_rot_out.copy_(_q_fp8) # e4m3 -> bf16
|
||||
if not _inkernel_qrot:
|
||||
q_for_kernel = _q_rot_out
|
||||
|
||||
# ---- FlyDSL kernel launch --------------------------------------------
|
||||
max_bps = int(block_table.shape[1])
|
||||
launch = _get_kernel(
|
||||
Hk,
|
||||
num_partitions,
|
||||
max_bps,
|
||||
scale,
|
||||
QG,
|
||||
block_size,
|
||||
padded_slot,
|
||||
fuse_qrot=_inkernel_qrot,
|
||||
num_seqs_hint=int(B),
|
||||
tile_groups_per_partition=int(tile_groups_per_partition),
|
||||
stride_q_seq=int(q_for_kernel.stride(0)),
|
||||
stride_q_head=int(q_for_kernel.stride(1)),
|
||||
)
|
||||
global _LOG_INVOKED_ONCE
|
||||
if not _LOG_INVOKED_ONCE:
|
||||
_LOG_INVOKED_ONCE = True
|
||||
logger.info(
|
||||
"FlyDSL UltraQuant launcher invoked: B=%d Hk=%d Hq=%d D=%d QG=%d "
|
||||
"num_partitions=%d (actual=%d, cap=%d) TGPP=%d max_bps=%d "
|
||||
"block_size=%d padded_slot=%d max_seq_len=%d "
|
||||
"(coverage=%d tokens, worst_case=%d tokens)",
|
||||
B,
|
||||
Hk,
|
||||
Hq,
|
||||
D,
|
||||
QG,
|
||||
num_partitions,
|
||||
num_partitions_actual,
|
||||
MAX_PARTITIONS,
|
||||
tile_groups_per_partition,
|
||||
max_bps,
|
||||
int(block_size),
|
||||
padded_slot,
|
||||
int(max_seq_len),
|
||||
num_partitions * tile_groups_per_partition * kv_compute_block,
|
||||
worst_case_max_seq_len,
|
||||
)
|
||||
launch(
|
||||
segm_out,
|
||||
segm_sum,
|
||||
segm_max,
|
||||
q_for_kernel,
|
||||
kv_cache,
|
||||
centroids_c,
|
||||
block_table,
|
||||
seq_lens,
|
||||
B,
|
||||
Hk,
|
||||
num_partitions,
|
||||
torch.cuda.current_stream(),
|
||||
)
|
||||
|
||||
# ---- Reduce partitions -> [B, Hq, D] ---------------------------------
|
||||
# 64 splits the head-dim 4 ways, taking the grid from 32 to 128 workgroups.
|
||||
# Bit-identical by construction (the partition reduction is per-column).
|
||||
_red_block_d = min(int(D), 64)
|
||||
_reduce_partitions[(B, Hq, triton.cdiv(int(D), _red_block_d))](
|
||||
output_ptr=output,
|
||||
segm_out_ptr=segm_out,
|
||||
segm_max_ptr=segm_max,
|
||||
segm_sum_ptr=segm_sum,
|
||||
out_stride_n=output.stride(0),
|
||||
out_stride_h=output.stride(1),
|
||||
NUM_KV_HEADS=Hk,
|
||||
QG=QG,
|
||||
NUM_PARTS=num_partitions,
|
||||
HEAD_SIZE=D,
|
||||
BLOCK_D=_red_block_d,
|
||||
)
|
||||
return output
|
||||
@@ -1,136 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Merge AITER segmented MLA split-K partials into output plus natural-log LSE.
|
||||
|
||||
This mirrors AITER's own segment reduction, including its
|
||||
``tiles_per_segment = cdiv(seq_len, NUM_SEGMENTS * TILE_SIZE)`` partitioning, so
|
||||
it has to move together with the ``skip_reduce=True`` call in the AITER MLA
|
||||
backend. It is a rank-local split-K merge and unrelated to any collective
|
||||
reduce; the natural-log LSE it returns is what the cross-rank DCP merge
|
||||
consumes.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import LOGE2, tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _merge_mla_segments_kernel(
|
||||
out_ptr,
|
||||
lse_ptr,
|
||||
segm_output_ptr,
|
||||
segm_max_ptr,
|
||||
segm_expsum_ptr,
|
||||
seq_lens_ptr,
|
||||
num_query_heads: tl.constexpr,
|
||||
out_stride0: tl.int64,
|
||||
out_stride1: tl.int64,
|
||||
lse_stride0: tl.int64,
|
||||
TILE_SIZE: tl.constexpr,
|
||||
KV_LORA_RANK: tl.constexpr,
|
||||
NUM_SEGMENTS_PER_SEQ: tl.constexpr,
|
||||
LOGE2: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
head_idx = tl.program_id(1)
|
||||
|
||||
seq_len = tl.load(seq_lens_ptr + token_idx)
|
||||
tiles_per_segment = tl.maximum(
|
||||
1, tl.cdiv(seq_len, NUM_SEGMENTS_PER_SEQ * TILE_SIZE)
|
||||
)
|
||||
active_segments = tl.where(
|
||||
seq_len > 0,
|
||||
tl.cdiv(seq_len, tiles_per_segment * TILE_SIZE),
|
||||
0,
|
||||
)
|
||||
segment_offsets = tl.arange(0, NUM_SEGMENTS_PER_SEQ)
|
||||
segment_mask = segment_offsets < active_segments
|
||||
|
||||
segment_base = (
|
||||
token_idx.to(tl.int64) * (num_query_heads * NUM_SEGMENTS_PER_SEQ)
|
||||
+ head_idx * NUM_SEGMENTS_PER_SEQ
|
||||
)
|
||||
segment_indices = segment_base + segment_offsets
|
||||
segment_max = tl.load(
|
||||
segm_max_ptr + segment_indices,
|
||||
mask=segment_mask,
|
||||
other=float("-inf"),
|
||||
)
|
||||
overall_max = tl.max(segment_max)
|
||||
|
||||
segment_expsum = tl.load(
|
||||
segm_expsum_ptr + segment_indices,
|
||||
mask=segment_mask,
|
||||
other=0.0,
|
||||
)
|
||||
segment_weight = tl.where(
|
||||
segment_mask,
|
||||
tl.exp2(segment_max - overall_max),
|
||||
0.0,
|
||||
)
|
||||
overall_expsum = tl.sum(segment_expsum * segment_weight)
|
||||
|
||||
output_offsets = (
|
||||
token_idx.to(tl.int64) * (num_query_heads * NUM_SEGMENTS_PER_SEQ * KV_LORA_RANK)
|
||||
+ head_idx * (NUM_SEGMENTS_PER_SEQ * KV_LORA_RANK)
|
||||
+ segment_offsets[:, None] * KV_LORA_RANK
|
||||
+ tl.arange(0, KV_LORA_RANK)[None, :]
|
||||
)
|
||||
segment_output = tl.load(
|
||||
segm_output_ptr + output_offsets,
|
||||
mask=segment_mask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
accumulator = tl.sum(segment_output * segment_weight[:, None], axis=0)
|
||||
accumulator = tl.where(overall_expsum == 0.0, 0.0, accumulator / overall_expsum)
|
||||
lse = tl.where(
|
||||
overall_expsum == 0.0,
|
||||
float("-inf"),
|
||||
(overall_max + tl.log2(overall_expsum)) * LOGE2,
|
||||
)
|
||||
|
||||
output_indices = (
|
||||
token_idx * out_stride0 + head_idx * out_stride1 + tl.arange(0, KV_LORA_RANK)
|
||||
)
|
||||
tl.store(out_ptr + output_indices, accumulator)
|
||||
tl.store(lse_ptr + token_idx * lse_stride0 + head_idx, lse)
|
||||
|
||||
|
||||
def merge_mla_segments_triton(
|
||||
segm_output: torch.Tensor,
|
||||
segm_max: torch.Tensor,
|
||||
segm_expsum: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
tile_size: int,
|
||||
out_dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Merge AITER base-2 segment partials into output and natural-log LSE."""
|
||||
num_tokens, num_heads, num_segments, kv_lora_rank = segm_output.shape
|
||||
output = torch.empty(
|
||||
(num_tokens, num_heads, kv_lora_rank),
|
||||
dtype=out_dtype,
|
||||
device=segm_output.device,
|
||||
)
|
||||
lse = torch.empty(
|
||||
(num_tokens, num_heads),
|
||||
dtype=torch.float32,
|
||||
device=segm_output.device,
|
||||
)
|
||||
_merge_mla_segments_kernel[(num_tokens, num_heads)](
|
||||
output,
|
||||
lse,
|
||||
segm_output,
|
||||
segm_max,
|
||||
segm_expsum,
|
||||
seq_lens,
|
||||
num_query_heads=num_heads,
|
||||
out_stride0=output.stride(0),
|
||||
out_stride1=output.stride(1),
|
||||
lse_stride0=lse.stride(0),
|
||||
TILE_SIZE=tile_size,
|
||||
KV_LORA_RANK=kv_lora_rank,
|
||||
NUM_SEGMENTS_PER_SEQ=num_segments,
|
||||
LOGE2=LOGE2,
|
||||
)
|
||||
return output, lse
|
||||
@@ -1,630 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton fused TurboQuant decode attention.
|
||||
|
||||
Decode path: Triton stage1 (split-KV tiled attention scoring + value
|
||||
accumulation) + stage2 (log-sum-exp reduction across splits).
|
||||
|
||||
Supports FP8 (E4M3) keys, 3-bit and 4-bit uniform quantized values.
|
||||
"""
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.ops.triton_decode_attention import (
|
||||
_fwd_kernel_stage2,
|
||||
)
|
||||
|
||||
_FP8_E4B15: dict[int, int] = {}
|
||||
|
||||
|
||||
def _use_fp8_e4b15(device: int = 0) -> int:
|
||||
"""Return 1 if device needs fp8e4b15 (Ampere/Ada, SM < 8.9), else 0.
|
||||
On non-CUDA platforms (e.g. XPU), always returns 0 (use e4nv format).
|
||||
"""
|
||||
if device not in _FP8_E4B15:
|
||||
if current_platform.is_cuda_alike():
|
||||
cap = torch.cuda.get_device_capability(device)
|
||||
_FP8_E4B15[device] = 1 if cap < (8, 9) else 0
|
||||
else:
|
||||
_FP8_E4B15[device] = 0
|
||||
return _FP8_E4B15[device]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 1: Fused TQ score + value accumulation (BLOCK_KV tiled)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_decode_stage1(
|
||||
# Precomputed query projection
|
||||
Q_rot_ptr, # [B, Hq, D] float32
|
||||
# Compressed KV cache (combined K+V)
|
||||
KV_cache_ptr, # [num_blocks, block_size, Hk, padded_slot] uint8
|
||||
# Block table and sequence info
|
||||
Block_table_ptr, # [B, max_num_blocks] int32
|
||||
Seq_lens_ptr, # [B] int32
|
||||
# TQ parameters
|
||||
Centroids_ptr, # [n_centroids] float32
|
||||
# Output (intermediate for stage2)
|
||||
Mid_o_ptr, # [B, Hq, NUM_KV_SPLITS, D+1] float32
|
||||
# Strides
|
||||
stride_qb,
|
||||
stride_qh, # Q strides: [B, Hq, D]
|
||||
stride_cache_block,
|
||||
stride_cache_pos,
|
||||
stride_cache_head, # KV cache
|
||||
stride_bt_b, # block_table stride per batch
|
||||
stride_mid_b,
|
||||
stride_mid_h,
|
||||
stride_mid_s, # mid_o strides
|
||||
# Constexpr dims
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr, # KV cache block_size (pages)
|
||||
NUM_KV_SPLITS: tl.constexpr,
|
||||
KV_GROUP_SIZE: tl.constexpr, # Hq // Hk
|
||||
# TQ layout constants
|
||||
MSE_BITS: tl.constexpr, # 3 or 4
|
||||
MSE_BYTES: tl.constexpr, # ceil(D * mse_bits / 8)
|
||||
KPS: tl.constexpr, # key_packed_size
|
||||
VQB: tl.constexpr, # value_quant_bits (4 or 8=FP8)
|
||||
VAL_DATA_BYTES: tl.constexpr, # ceil(D * vqb / 8) or D for FP8
|
||||
# Score constants
|
||||
ATTN_SCALE: tl.constexpr, # 1/sqrt(D)
|
||||
# Block tile sizes
|
||||
BLOCK_D: tl.constexpr, # next_power_of_2(HEAD_DIM)
|
||||
BLOCK_KV: tl.constexpr, # tokens per tile (16)
|
||||
KEY_FP8: tl.constexpr, # 1 if K is stored as FP8
|
||||
NORM_CORRECTION: tl.constexpr = 0, # 1 = re-normalize centroids
|
||||
FP8_E4B15: tl.constexpr = 0, # 1 = use e4b15 (Ampere/Ada), 0 = e4nv (Hopper+)
|
||||
):
|
||||
bid = tl.program_id(0) # batch index
|
||||
hid = tl.program_id(1) # q_head index
|
||||
sid = tl.program_id(2) # kv_split index
|
||||
|
||||
kv_head = hid // KV_GROUP_SIZE
|
||||
|
||||
# Sequence length for this batch
|
||||
seq_len = tl.load(Seq_lens_ptr + bid)
|
||||
|
||||
# KV split range
|
||||
split_len = tl.cdiv(seq_len, NUM_KV_SPLITS)
|
||||
split_start = split_len * sid
|
||||
split_end = tl.minimum(split_start + split_len, seq_len)
|
||||
|
||||
if split_start >= split_end:
|
||||
return
|
||||
|
||||
# Dimension offsets
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < HEAD_DIM
|
||||
kv_range = tl.arange(0, BLOCK_KV)
|
||||
|
||||
# Load query vector: q_rot — [BLOCK_D] float32
|
||||
q_base = bid * stride_qb + hid * stride_qh
|
||||
q_rot = tl.load(Q_rot_ptr + q_base + d_offs, mask=d_mask, other=0.0).to(tl.float32)
|
||||
|
||||
# Precompute byte/bit index vectors for MSE gather loads
|
||||
if not KEY_FP8:
|
||||
mse_bit_off = d_offs * MSE_BITS
|
||||
mse_byte_idx = mse_bit_off // 8
|
||||
mse_bit_shift = mse_bit_off % 8
|
||||
mse_mask = (1 << MSE_BITS) - 1
|
||||
|
||||
# Precompute value bit/byte index vectors (loop-invariant)
|
||||
if VQB == 3:
|
||||
val_bit_off = d_offs * 3
|
||||
val_byte_idx = val_bit_off // 8
|
||||
val_bit_shift = val_bit_off % 8
|
||||
|
||||
# Online softmax accumulators
|
||||
m_prev = -float("inf")
|
||||
l_prev = 0.0
|
||||
acc = tl.zeros([BLOCK_D], dtype=tl.float32)
|
||||
|
||||
bt_base = bid * stride_bt_b
|
||||
|
||||
# ================================================================
|
||||
# TILED LOOP: process BLOCK_KV tokens per iteration
|
||||
# ================================================================
|
||||
for start_n in range(split_start, split_end, BLOCK_KV):
|
||||
kv_offs = start_n + kv_range
|
||||
kv_mask = kv_offs < split_end
|
||||
|
||||
page_idx = kv_offs // BLOCK_SIZE
|
||||
page_off = kv_offs % BLOCK_SIZE
|
||||
block_nums = tl.load(
|
||||
Block_table_ptr + bt_base + page_idx,
|
||||
mask=kv_mask,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
|
||||
slot_bases = (
|
||||
block_nums * stride_cache_block
|
||||
+ page_off.to(tl.int64) * stride_cache_pos
|
||||
+ tl.cast(kv_head, tl.int64) * stride_cache_head
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# COMPUTE ATTENTION SCORES: [BLOCK_KV]
|
||||
# ============================================================
|
||||
if KEY_FP8:
|
||||
k_addrs = slot_bases[:, None] + d_offs[None, :]
|
||||
k_raw = tl.load(
|
||||
KV_cache_ptr + k_addrs,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
)
|
||||
if FP8_E4B15:
|
||||
k_float = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32)
|
||||
else:
|
||||
k_float = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32)
|
||||
scores = (
|
||||
tl.sum(
|
||||
tl.where(d_mask[None, :], q_rot[None, :] * k_float, 0.0),
|
||||
axis=1,
|
||||
)
|
||||
* ATTN_SCALE
|
||||
)
|
||||
scores = tl.where(kv_mask, scores, -float("inf"))
|
||||
else:
|
||||
# MSE unpack + norms
|
||||
mse_addrs0 = slot_bases[:, None] + mse_byte_idx[None, :]
|
||||
mse_raw0 = tl.load(
|
||||
KV_cache_ptr + mse_addrs0,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
mse_raw1 = tl.load(
|
||||
KV_cache_ptr + mse_addrs0 + 1,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
raw16 = mse_raw0 | (mse_raw1 << 8)
|
||||
mse_idx = (raw16 >> mse_bit_shift[None, :]) & mse_mask
|
||||
|
||||
# Centroid gather + dot product
|
||||
c_vals = tl.load(
|
||||
Centroids_ptr + mse_idx,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
# Norm correction: re-normalize centroid vector to unit norm
|
||||
if NORM_CORRECTION:
|
||||
c_norm_sq = tl.sum(
|
||||
tl.where(d_mask[None, :], c_vals * c_vals, 0.0),
|
||||
axis=1,
|
||||
)
|
||||
c_inv_norm = 1.0 / tl.sqrt(c_norm_sq + 1e-16)
|
||||
c_vals = c_vals * c_inv_norm[:, None]
|
||||
|
||||
term1 = tl.sum(
|
||||
tl.where(d_mask[None, :], q_rot[None, :] * c_vals, 0.0),
|
||||
axis=1,
|
||||
)
|
||||
|
||||
# Load norms (fp16 -> fp32): norms are at MSE_BYTES offset
|
||||
norm_bases = slot_bases + MSE_BYTES
|
||||
n_lo = tl.load(KV_cache_ptr + norm_bases, mask=kv_mask, other=0).to(
|
||||
tl.uint16
|
||||
)
|
||||
n_hi = tl.load(KV_cache_ptr + norm_bases + 1, mask=kv_mask, other=0).to(
|
||||
tl.uint16
|
||||
)
|
||||
vec_norms = (n_lo | (n_hi << 8)).to(tl.float16, bitcast=True).to(tl.float32)
|
||||
|
||||
scores = vec_norms * term1 * ATTN_SCALE
|
||||
scores = tl.where(kv_mask, scores, -float("inf"))
|
||||
|
||||
# ============================================================
|
||||
# ONLINE SOFTMAX UPDATE (block-level)
|
||||
# ============================================================
|
||||
n_e_max = tl.maximum(tl.max(scores, 0), m_prev)
|
||||
re_scale = tl.exp(m_prev - n_e_max)
|
||||
p = tl.exp(scores - n_e_max)
|
||||
|
||||
# ============================================================
|
||||
# VALUE LOAD + DEQUANTIZE: [BLOCK_KV, BLOCK_D]
|
||||
# ============================================================
|
||||
val_bases = slot_bases + KPS
|
||||
|
||||
if VQB == 3:
|
||||
val_addrs0 = val_bases[:, None] + val_byte_idx[None, :]
|
||||
val_raw0 = tl.load(
|
||||
KV_cache_ptr + val_addrs0,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
val_raw1 = tl.load(
|
||||
KV_cache_ptr + val_addrs0 + 1,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
raw16 = val_raw0 | (val_raw1 << 8)
|
||||
v_idx = ((raw16 >> val_bit_shift[None, :]) & 0x7).to(tl.float32)
|
||||
|
||||
sc_bases = val_bases + VAL_DATA_BYTES
|
||||
sc_lo = tl.load(KV_cache_ptr + sc_bases, mask=kv_mask, other=0).to(
|
||||
tl.uint16
|
||||
)
|
||||
sc_hi = tl.load(KV_cache_ptr + sc_bases + 1, mask=kv_mask, other=0).to(
|
||||
tl.uint16
|
||||
)
|
||||
v_scales = (
|
||||
(sc_lo | (sc_hi << 8)).to(tl.float16, bitcast=True).to(tl.float32)
|
||||
)
|
||||
zr_lo = tl.load(KV_cache_ptr + sc_bases + 2, mask=kv_mask, other=0).to(
|
||||
tl.uint16
|
||||
)
|
||||
zr_hi = tl.load(KV_cache_ptr + sc_bases + 3, mask=kv_mask, other=0).to(
|
||||
tl.uint16
|
||||
)
|
||||
v_zeros = (zr_lo | (zr_hi << 8)).to(tl.float16, bitcast=True).to(tl.float32)
|
||||
values = v_idx * v_scales[:, None] + v_zeros[:, None]
|
||||
else: # VQB == 4
|
||||
vb_idx = d_offs // 2
|
||||
vb_shift = (d_offs % 2) * 4
|
||||
val_addrs = val_bases[:, None] + vb_idx[None, :]
|
||||
val_raw = tl.load(
|
||||
KV_cache_ptr + val_addrs,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
v_idx = ((val_raw >> vb_shift[None, :]) & 0xF).to(tl.float32)
|
||||
|
||||
sc_bases = val_bases + VAL_DATA_BYTES
|
||||
sc_lo = tl.load(KV_cache_ptr + sc_bases, mask=kv_mask, other=0).to(
|
||||
tl.uint16
|
||||
)
|
||||
sc_hi = tl.load(KV_cache_ptr + sc_bases + 1, mask=kv_mask, other=0).to(
|
||||
tl.uint16
|
||||
)
|
||||
v_scales = (
|
||||
(sc_lo | (sc_hi << 8)).to(tl.float16, bitcast=True).to(tl.float32)
|
||||
)
|
||||
zr_lo = tl.load(KV_cache_ptr + sc_bases + 2, mask=kv_mask, other=0).to(
|
||||
tl.uint16
|
||||
)
|
||||
zr_hi = tl.load(KV_cache_ptr + sc_bases + 3, mask=kv_mask, other=0).to(
|
||||
tl.uint16
|
||||
)
|
||||
v_zeros = (zr_lo | (zr_hi << 8)).to(tl.float16, bitcast=True).to(tl.float32)
|
||||
values = v_idx * v_scales[:, None] + v_zeros[:, None]
|
||||
|
||||
# ============================================================
|
||||
# WEIGHTED VALUE ACCUMULATION
|
||||
# ============================================================
|
||||
acc = acc * re_scale + tl.sum(p[:, None] * values, 0)
|
||||
l_prev = l_prev * re_scale + tl.sum(p, 0)
|
||||
m_prev = n_e_max
|
||||
|
||||
# Store partial result
|
||||
out_base = bid * stride_mid_b + hid * stride_mid_h + sid * stride_mid_s
|
||||
safe_l = tl.where(l_prev > 0.0, l_prev, 1.0)
|
||||
tl.store(Mid_o_ptr + out_base + d_offs, acc / safe_l, mask=d_mask)
|
||||
lse = m_prev + tl.log(safe_l)
|
||||
tl.store(Mid_o_ptr + out_base + HEAD_DIM, lse)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-dequant kernel: Bulk dequant K (MSE+norms) and V to fp16
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_full_dequant_kv(
|
||||
KV_cache_ptr,
|
||||
Block_table_ptr,
|
||||
Centroids_ptr,
|
||||
K_out_ptr, # [B, Hk, max_seq, D] float16
|
||||
V_out_ptr, # [B, Hk, max_seq, D] float16
|
||||
stride_ko_b,
|
||||
stride_ko_h,
|
||||
stride_ko_s,
|
||||
stride_vo_b,
|
||||
stride_vo_h,
|
||||
stride_vo_s,
|
||||
stride_cache_block,
|
||||
stride_cache_pos,
|
||||
stride_cache_head,
|
||||
stride_bt_b,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
MSE_BYTES: tl.constexpr,
|
||||
KPS: tl.constexpr,
|
||||
VQB: tl.constexpr,
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
MSE_BITS: tl.constexpr,
|
||||
KEY_FP8: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
NORM_CORRECTION: tl.constexpr = 0,
|
||||
FP8_E4B15: tl.constexpr = 0, # 1 = use e4b15 (Ampere/Ada), 0 = e4nv (Hopper+)
|
||||
):
|
||||
"""Full dequant: reconstruct K (MSE centroids * norm or FP8) and V to fp16."""
|
||||
pos = tl.program_id(0)
|
||||
bh = tl.program_id(1)
|
||||
bid = bh // NUM_KV_HEADS
|
||||
hid = bh % NUM_KV_HEADS
|
||||
|
||||
page_idx = pos // BLOCK_SIZE
|
||||
page_off = pos % BLOCK_SIZE
|
||||
block_num = tl.load(Block_table_ptr + bid * stride_bt_b + page_idx).to(tl.int64)
|
||||
slot_base = (
|
||||
block_num * stride_cache_block
|
||||
+ tl.cast(page_off, tl.int64) * stride_cache_pos
|
||||
+ tl.cast(hid, tl.int64) * stride_cache_head
|
||||
)
|
||||
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < HEAD_DIM
|
||||
|
||||
# === K dequant ===
|
||||
ko_base = bid * stride_ko_b + hid * stride_ko_h + pos * stride_ko_s
|
||||
if KEY_FP8:
|
||||
k_raw = tl.load(KV_cache_ptr + slot_base + d_offs, mask=d_mask, other=0)
|
||||
if FP8_E4B15:
|
||||
k_recon = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32)
|
||||
else:
|
||||
k_recon = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32)
|
||||
tl.store(K_out_ptr + ko_base + d_offs, k_recon.to(tl.float16), mask=d_mask)
|
||||
else:
|
||||
# MSE unpack (3-bit or 4-bit) + norms
|
||||
mse_bit_off = d_offs * MSE_BITS
|
||||
mse_byte_idx = mse_bit_off // 8
|
||||
mse_bit_shift = mse_bit_off % 8
|
||||
mse_umask = (1 << MSE_BITS) - 1
|
||||
|
||||
mse_raw0 = tl.load(
|
||||
KV_cache_ptr + slot_base + mse_byte_idx, mask=d_mask, other=0
|
||||
).to(tl.int32)
|
||||
mse_raw1 = tl.load(
|
||||
KV_cache_ptr + slot_base + mse_byte_idx + 1, mask=d_mask, other=0
|
||||
).to(tl.int32)
|
||||
raw16_key = mse_raw0 | (mse_raw1 << 8)
|
||||
mse_idx = (raw16_key >> mse_bit_shift) & mse_umask
|
||||
|
||||
k_mse = tl.load(Centroids_ptr + mse_idx, mask=d_mask, other=0.0)
|
||||
|
||||
# Norm correction: re-normalize centroid vector to unit norm
|
||||
if NORM_CORRECTION:
|
||||
c_norm_sq = tl.sum(tl.where(d_mask, k_mse * k_mse, 0.0), axis=0)
|
||||
c_inv_norm = 1.0 / tl.sqrt(c_norm_sq + 1e-16)
|
||||
k_mse = k_mse * c_inv_norm
|
||||
|
||||
# Norms at MSE_BYTES offset (no QJL bytes)
|
||||
norm_base = slot_base + MSE_BYTES
|
||||
n_lo = tl.load(KV_cache_ptr + norm_base).to(tl.uint16)
|
||||
n_hi = tl.load(KV_cache_ptr + norm_base + 1).to(tl.uint16)
|
||||
vec_norm = (n_lo | (n_hi << 8)).to(tl.float16, bitcast=True).to(tl.float32)
|
||||
|
||||
k_recon = vec_norm * k_mse
|
||||
tl.store(K_out_ptr + ko_base + d_offs, k_recon.to(tl.float16), mask=d_mask)
|
||||
|
||||
# === V dequant ===
|
||||
val_base = slot_base + KPS
|
||||
if VQB == 4:
|
||||
vb_idx = d_offs // 2
|
||||
vb_shift = (d_offs % 2) * 4
|
||||
val_raw = tl.load(KV_cache_ptr + val_base + vb_idx, mask=d_mask, other=0).to(
|
||||
tl.int32
|
||||
)
|
||||
v_idx = ((val_raw >> vb_shift) & 0xF).to(tl.float32)
|
||||
|
||||
sc_base = val_base + VAL_DATA_BYTES
|
||||
sc_lo = tl.load(KV_cache_ptr + sc_base).to(tl.uint16)
|
||||
sc_hi = tl.load(KV_cache_ptr + sc_base + 1).to(tl.uint16)
|
||||
v_scale = (sc_lo | (sc_hi << 8)).to(tl.float16, bitcast=True).to(tl.float32)
|
||||
zr_lo = tl.load(KV_cache_ptr + sc_base + 2).to(tl.uint16)
|
||||
zr_hi = tl.load(KV_cache_ptr + sc_base + 3).to(tl.uint16)
|
||||
v_zero = (zr_lo | (zr_hi << 8)).to(tl.float16, bitcast=True).to(tl.float32)
|
||||
v_vals = v_idx * v_scale + v_zero
|
||||
elif VQB == 3:
|
||||
# 3-bit value unpack: 8 values per 3 bytes
|
||||
val_bit_off = d_offs * 3
|
||||
val_byte_idx = val_bit_off // 8
|
||||
val_bit_shift = val_bit_off % 8
|
||||
val_raw0 = tl.load(
|
||||
KV_cache_ptr + val_base + val_byte_idx, mask=d_mask, other=0
|
||||
).to(tl.int32)
|
||||
val_raw1 = tl.load(
|
||||
KV_cache_ptr + val_base + val_byte_idx + 1, mask=d_mask, other=0
|
||||
).to(tl.int32)
|
||||
raw16_val = val_raw0 | (val_raw1 << 8)
|
||||
v_idx = ((raw16_val >> val_bit_shift) & 0x7).to(tl.float32)
|
||||
|
||||
sc_base = val_base + VAL_DATA_BYTES
|
||||
sc_lo = tl.load(KV_cache_ptr + sc_base).to(tl.uint16)
|
||||
sc_hi = tl.load(KV_cache_ptr + sc_base + 1).to(tl.uint16)
|
||||
v_scale = (sc_lo | (sc_hi << 8)).to(tl.float16, bitcast=True).to(tl.float32)
|
||||
zr_lo = tl.load(KV_cache_ptr + sc_base + 2).to(tl.uint16)
|
||||
zr_hi = tl.load(KV_cache_ptr + sc_base + 3).to(tl.uint16)
|
||||
v_zero = (zr_lo | (zr_hi << 8)).to(tl.float16, bitcast=True).to(tl.float32)
|
||||
v_vals = v_idx * v_scale + v_zero
|
||||
else:
|
||||
v_vals = tl.zeros([BLOCK_D], dtype=tl.float32)
|
||||
|
||||
vo_base = bid * stride_vo_b + hid * stride_vo_h + pos * stride_vo_s
|
||||
tl.store(V_out_ptr + vo_base + d_offs, v_vals.to(tl.float16), mask=d_mask)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 2: Reuse from triton_decode_attention.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Launcher — cached constants + fused GEMM
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_layout_cache: dict = {}
|
||||
|
||||
|
||||
def _get_layout(D, mse_bits, value_quant_bits, key_packed_size):
|
||||
"""Get cached layout constants."""
|
||||
key = (D, mse_bits, value_quant_bits, key_packed_size)
|
||||
cfg = _layout_cache.get(key)
|
||||
if cfg is None:
|
||||
val_data_bytes = math.ceil(D * value_quant_bits / 8)
|
||||
cfg = {
|
||||
"mse_bytes": math.ceil(D * mse_bits / 8),
|
||||
"val_data_bytes": val_data_bytes,
|
||||
"mse_bits": mse_bits,
|
||||
"n_centroids": 2**mse_bits,
|
||||
"BLOCK_D": triton.next_power_of_2(D),
|
||||
}
|
||||
_layout_cache[key] = cfg
|
||||
return cfg
|
||||
|
||||
|
||||
def triton_turboquant_decode_attention(
|
||||
query: torch.Tensor, # [B, Hq, D] — original query
|
||||
kv_cache: torch.Tensor, # [num_blocks, block_size, Hk, padded_slot] uint8
|
||||
block_table: torch.Tensor, # [B, max_num_blocks] int32
|
||||
seq_lens: torch.Tensor, # [B] int32
|
||||
Pi: torch.Tensor, # [D, D] float32
|
||||
centroids: torch.Tensor, # [n_centroids] float32
|
||||
scale: float,
|
||||
mse_bits: int,
|
||||
key_packed_size: int,
|
||||
value_quant_bits: int,
|
||||
key_fp8: bool = False,
|
||||
norm_correction: bool = False,
|
||||
PiT: torch.Tensor | None = None, # [D, D] pre-computed Pi.T contiguous
|
||||
# Pre-allocated buffers (optional, avoids per-call allocation)
|
||||
mid_o_buf: torch.Tensor | None = None,
|
||||
output_buf: torch.Tensor | None = None,
|
||||
lse_buf: torch.Tensor | None = None,
|
||||
buf_holder: Any = None,
|
||||
max_num_kv_splits: int = 32, # fixed split count (must be constant for cudagraph)
|
||||
) -> torch.Tensor:
|
||||
"""Launch fused TQ decode attention (Triton stage1 + stage2).
|
||||
|
||||
Returns: output tensor [B, Hq, D] in query's dtype.
|
||||
"""
|
||||
B, Hq, D = query.shape
|
||||
Hk = kv_cache.shape[2]
|
||||
block_size = kv_cache.shape[1]
|
||||
kv_group_size = Hq // Hk
|
||||
device = query.device
|
||||
|
||||
cfg = _get_layout(D, mse_bits, value_quant_bits, key_packed_size)
|
||||
|
||||
# Compute q_rot = q @ Pi.T (rotated query for MSE key scoring)
|
||||
# FP8 path: pass query directly (float16); kernel casts inline.
|
||||
# MSE path: still needs external GEMM (cuBLAS), so q_rot is float32.
|
||||
if key_fp8:
|
||||
q_rot = query.contiguous()
|
||||
else:
|
||||
q_float = query.float()
|
||||
if PiT is None:
|
||||
PiT = Pi.T.contiguous()
|
||||
q_rot = (q_float @ PiT).contiguous()
|
||||
|
||||
NUM_KV_SPLITS = max_num_kv_splits
|
||||
|
||||
if (
|
||||
mid_o_buf is not None
|
||||
and mid_o_buf.shape[0] >= B
|
||||
and mid_o_buf.shape[2] >= NUM_KV_SPLITS
|
||||
):
|
||||
mid_o = mid_o_buf[:B, :Hq, :NUM_KV_SPLITS, :]
|
||||
else:
|
||||
mid_o = torch.empty(
|
||||
B,
|
||||
Hq,
|
||||
NUM_KV_SPLITS,
|
||||
D + 1,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
if buf_holder is not None:
|
||||
buf_holder._tq_mid_o_buf = mid_o
|
||||
|
||||
# Stage 1: split-KV tiled attention scoring + value accumulation
|
||||
fp8_e4b15 = _use_fp8_e4b15(device.index or 0)
|
||||
BLOCK_KV = 4
|
||||
grid = (B, Hq, NUM_KV_SPLITS)
|
||||
_tq_decode_stage1[grid](
|
||||
q_rot,
|
||||
kv_cache,
|
||||
block_table,
|
||||
seq_lens,
|
||||
centroids,
|
||||
mid_o,
|
||||
q_rot.stride(0),
|
||||
q_rot.stride(1),
|
||||
kv_cache.stride(0),
|
||||
kv_cache.stride(1),
|
||||
kv_cache.stride(2),
|
||||
block_table.stride(0),
|
||||
mid_o.stride(0),
|
||||
mid_o.stride(1),
|
||||
mid_o.stride(2),
|
||||
NUM_KV_HEADS=Hk,
|
||||
HEAD_DIM=D,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_KV_SPLITS=NUM_KV_SPLITS,
|
||||
KV_GROUP_SIZE=kv_group_size,
|
||||
MSE_BITS=mse_bits,
|
||||
MSE_BYTES=cfg["mse_bytes"],
|
||||
KPS=key_packed_size,
|
||||
VQB=value_quant_bits,
|
||||
VAL_DATA_BYTES=cfg["val_data_bytes"],
|
||||
ATTN_SCALE=scale,
|
||||
BLOCK_D=cfg["BLOCK_D"],
|
||||
BLOCK_KV=BLOCK_KV,
|
||||
KEY_FP8=1 if key_fp8 else 0,
|
||||
NORM_CORRECTION=1 if norm_correction else 0,
|
||||
FP8_E4B15=fp8_e4b15,
|
||||
num_warps=1,
|
||||
num_stages=1,
|
||||
)
|
||||
|
||||
# Stage 2: Reduce across KV splits
|
||||
# Output in query dtype — eliminates float16_copy kernel after stage2
|
||||
out_dtype = query.dtype
|
||||
if (
|
||||
output_buf is not None
|
||||
and output_buf.shape[0] >= B
|
||||
and output_buf.dtype == out_dtype
|
||||
):
|
||||
output = output_buf[:B, :Hq, :D]
|
||||
else:
|
||||
output = torch.empty(B, Hq, D, dtype=out_dtype, device=device)
|
||||
if buf_holder is not None:
|
||||
buf_holder._tq_output_buf = output
|
||||
if lse_buf is not None and lse_buf.shape[0] >= B:
|
||||
lse = lse_buf[:B, :Hq]
|
||||
else:
|
||||
lse = torch.empty(B, Hq, dtype=torch.float32, device=device)
|
||||
if buf_holder is not None:
|
||||
buf_holder._tq_lse_buf = lse
|
||||
|
||||
grid2 = (B, Hq)
|
||||
_fwd_kernel_stage2[grid2](
|
||||
mid_o,
|
||||
output,
|
||||
lse,
|
||||
seq_lens,
|
||||
mid_o.stride(0),
|
||||
mid_o.stride(1),
|
||||
mid_o.stride(2),
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
lse.stride(0),
|
||||
NUM_KV_SPLITS=NUM_KV_SPLITS,
|
||||
BLOCK_DV=cfg["BLOCK_D"],
|
||||
Lv=D,
|
||||
OUTPUT_FP16=1 if out_dtype == torch.float16 else 0,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
|
||||
return output # already in query dtype
|
||||
@@ -1,447 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused Triton kernels for TurboQuant KV store.
|
||||
|
||||
Two kernels:
|
||||
1. _tq_fused_store_fp8: FP8 key scatter + value uniform quantization.
|
||||
2. _tq_fused_store_mse: Fused binary-search bucketize + MSE index
|
||||
packing + value quantization.
|
||||
|
||||
The launcher `triton_turboquant_store` selects the appropriate kernel.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.ops.triton_turboquant_decode import _use_fp8_e4b15
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Shared: value uniform quantization + pack + scale/zero store
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_quantized_value(
|
||||
Value_ptr,
|
||||
KV_cache_ptr,
|
||||
base, # pid * D offset into Value_ptr
|
||||
slot_base, # byte offset into KV_cache_ptr for this slot+head
|
||||
d_offs, # tl.arange(0, BLOCK_D)
|
||||
d_mask, # d_offs < D
|
||||
D: tl.constexpr,
|
||||
KPS: tl.constexpr,
|
||||
VQB: tl.constexpr,
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
BLOCK_VAL: tl.constexpr,
|
||||
BLOCK_GRP: tl.constexpr,
|
||||
):
|
||||
"""Uniform quantization of values to VQB bits, pack, and store with scale/zero."""
|
||||
val_cache_offset = KPS
|
||||
|
||||
if VQB == 3:
|
||||
val_vec = tl.load(Value_ptr + base + d_offs, mask=d_mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
val_min = tl.min(tl.where(d_mask, val_vec, float("inf")), axis=0)
|
||||
val_max = tl.max(tl.where(d_mask, val_vec, -float("inf")), axis=0)
|
||||
v_scale = (val_max - val_min) / 7.0
|
||||
v_scale = tl.where(v_scale > 1e-8, v_scale, 1e-8)
|
||||
|
||||
q_vals = tl.minimum(
|
||||
tl.maximum(((val_vec - val_min) / v_scale + 0.5).to(tl.int32), 0), 7
|
||||
)
|
||||
|
||||
grp_offs = tl.arange(0, BLOCK_GRP)
|
||||
grp_mask = grp_offs < (D // 8)
|
||||
q_grp = tl.reshape(q_vals, [BLOCK_GRP, 8])
|
||||
shifts_3bit = tl.arange(0, 8) * 3
|
||||
packed_24 = tl.sum(q_grp << shifts_3bit[None, :], axis=1)
|
||||
b0 = (packed_24 & 0xFF).to(tl.uint8)
|
||||
b1 = ((packed_24 >> 8) & 0xFF).to(tl.uint8)
|
||||
b2 = ((packed_24 >> 16) & 0xFF).to(tl.uint8)
|
||||
tl.store(
|
||||
KV_cache_ptr + slot_base + val_cache_offset + grp_offs * 3,
|
||||
b0,
|
||||
mask=grp_mask,
|
||||
)
|
||||
tl.store(
|
||||
KV_cache_ptr + slot_base + val_cache_offset + grp_offs * 3 + 1,
|
||||
b1,
|
||||
mask=grp_mask,
|
||||
)
|
||||
tl.store(
|
||||
KV_cache_ptr + slot_base + val_cache_offset + grp_offs * 3 + 2,
|
||||
b2,
|
||||
mask=grp_mask,
|
||||
)
|
||||
|
||||
sc_offset = val_cache_offset + VAL_DATA_BYTES
|
||||
sc_f16 = v_scale.to(tl.float16)
|
||||
sc_u16 = sc_f16.to(tl.uint16, bitcast=True)
|
||||
tl.store(KV_cache_ptr + slot_base + sc_offset, (sc_u16 & 0xFF).to(tl.uint8))
|
||||
tl.store(
|
||||
KV_cache_ptr + slot_base + sc_offset + 1,
|
||||
((sc_u16 >> 8) & 0xFF).to(tl.uint8),
|
||||
)
|
||||
zr_f16 = val_min.to(tl.float16)
|
||||
zr_u16 = zr_f16.to(tl.uint16, bitcast=True)
|
||||
tl.store(KV_cache_ptr + slot_base + sc_offset + 2, (zr_u16 & 0xFF).to(tl.uint8))
|
||||
tl.store(
|
||||
KV_cache_ptr + slot_base + sc_offset + 3,
|
||||
((zr_u16 >> 8) & 0xFF).to(tl.uint8),
|
||||
)
|
||||
|
||||
else: # VQB == 4
|
||||
val_vec = tl.load(Value_ptr + base + d_offs, mask=d_mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
val_min = tl.min(tl.where(d_mask, val_vec, float("inf")), axis=0)
|
||||
val_max = tl.max(tl.where(d_mask, val_vec, -float("inf")), axis=0)
|
||||
v_scale = (val_max - val_min) / 15.0
|
||||
v_scale = tl.where(v_scale > 1e-8, v_scale, 1e-8)
|
||||
|
||||
# Quantize all D elements from register (no re-load)
|
||||
q_all = tl.minimum(
|
||||
tl.maximum(((val_vec - val_min) / v_scale + 0.5).to(tl.int32), 0), 15
|
||||
)
|
||||
# Reshape to pairs and pack two 4-bit values per byte
|
||||
q_pairs = tl.reshape(q_all, [BLOCK_D // 2, 2])
|
||||
shifts_4 = tl.arange(0, 2) * 4
|
||||
packed_val = tl.sum((q_pairs & 0xF) << shifts_4[None, :], axis=1).to(tl.uint8)
|
||||
val_offs = tl.arange(0, BLOCK_D // 2)
|
||||
val_mask = val_offs < VAL_DATA_BYTES
|
||||
tl.store(
|
||||
KV_cache_ptr + slot_base + val_cache_offset + val_offs,
|
||||
packed_val,
|
||||
mask=val_mask,
|
||||
)
|
||||
|
||||
sc_offset = val_cache_offset + VAL_DATA_BYTES
|
||||
sc_f16 = v_scale.to(tl.float16)
|
||||
sc_u16 = sc_f16.to(tl.uint16, bitcast=True)
|
||||
tl.store(KV_cache_ptr + slot_base + sc_offset, (sc_u16 & 0xFF).to(tl.uint8))
|
||||
tl.store(
|
||||
KV_cache_ptr + slot_base + sc_offset + 1,
|
||||
((sc_u16 >> 8) & 0xFF).to(tl.uint8),
|
||||
)
|
||||
zr_f16 = val_min.to(tl.float16)
|
||||
zr_u16 = zr_f16.to(tl.uint16, bitcast=True)
|
||||
tl.store(KV_cache_ptr + slot_base + sc_offset + 2, (zr_u16 & 0xFF).to(tl.uint8))
|
||||
tl.store(
|
||||
KV_cache_ptr + slot_base + sc_offset + 3,
|
||||
((zr_u16 >> 8) & 0xFF).to(tl.uint8),
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# FP8 key store + value uniform quantization
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_fused_store_fp8(
|
||||
Key_ptr, # [NH, D] float16/bfloat16 — raw keys
|
||||
Value_ptr, # [NH, D] float16/bfloat16 — raw values
|
||||
KV_cache_ptr, # [total_bytes] uint8 (flattened view)
|
||||
Slot_mapping_ptr, # [N] int32 — per-token slot indices
|
||||
# Cache strides (for computing byte offsets)
|
||||
stride_cache_block: tl.constexpr,
|
||||
stride_cache_pos: tl.constexpr,
|
||||
stride_cache_head: tl.constexpr,
|
||||
# Dimensions
|
||||
D: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
# TQ layout
|
||||
KPS: tl.constexpr,
|
||||
# Value quantization
|
||||
VQB: tl.constexpr,
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
# Packing block sizes
|
||||
BLOCK_VAL: tl.constexpr,
|
||||
BLOCK_GRP: tl.constexpr = 16,
|
||||
FP8_E4B15: tl.constexpr = 0, # 1 = e4b15 (Ampere/Ada), 0 = e4nv (Hopper+)
|
||||
):
|
||||
"""FP8 key cast+scatter + value uniform quantization."""
|
||||
pid = tl.program_id(0)
|
||||
token_idx = pid // H
|
||||
head_idx = pid % H
|
||||
|
||||
slot = tl.load(Slot_mapping_ptr + token_idx)
|
||||
if slot < 0:
|
||||
return
|
||||
blk = (slot // BLOCK_SIZE).to(tl.int64)
|
||||
off = (slot % BLOCK_SIZE).to(tl.int64)
|
||||
head_idx_i64 = tl.cast(head_idx, tl.int64)
|
||||
slot_base = (
|
||||
blk * stride_cache_block
|
||||
+ off * stride_cache_pos
|
||||
+ head_idx_i64 * stride_cache_head
|
||||
)
|
||||
|
||||
base = pid * D
|
||||
|
||||
# ── FP8 KEY: cast to FP8 in-kernel and store ─────────────────
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < D
|
||||
k_vals = tl.load(Key_ptr + base + d_offs, mask=d_mask, other=0.0).to(tl.float32)
|
||||
k_fp8 = k_vals.to(tl.float8e4b15) if FP8_E4B15 else k_vals.to(tl.float8e4nv)
|
||||
k_bytes = k_fp8.to(tl.uint8, bitcast=True)
|
||||
tl.store(KV_cache_ptr + slot_base + d_offs, k_bytes, mask=d_mask)
|
||||
|
||||
# ── VALUE QUANTIZE + PACK ───────────────────────────────────────
|
||||
_store_quantized_value(
|
||||
Value_ptr,
|
||||
KV_cache_ptr,
|
||||
base,
|
||||
slot_base,
|
||||
d_offs,
|
||||
d_mask,
|
||||
D=D,
|
||||
KPS=KPS,
|
||||
VQB=VQB,
|
||||
VAL_DATA_BYTES=VAL_DATA_BYTES,
|
||||
BLOCK_D=BLOCK_D,
|
||||
BLOCK_VAL=BLOCK_VAL,
|
||||
BLOCK_GRP=BLOCK_GRP,
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Fused MSE store: bucketize + MSE index pack + norm store + value pack
|
||||
# (eliminates 4 PyTorch kernel launches per layer vs pack-only kernel)
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_fused_store_mse(
|
||||
# Post-rotation inputs
|
||||
Y_ptr, # [NH, D] float32 — rotated normalized keys (x_hat @ PiT)
|
||||
Norms_ptr, # [NH] float32 — key vector norms (||k||)
|
||||
Value_ptr, # [NH, D] float32 — raw values
|
||||
# Quantization tables
|
||||
Midpoints_ptr, # [n_centroids-1] float32
|
||||
# Cache and indexing
|
||||
KV_cache_ptr, # [total_bytes] uint8 (flattened view)
|
||||
Slot_mapping_ptr, # [N] int32 — per-token slot indices
|
||||
# Cache strides
|
||||
stride_cache_block: tl.constexpr,
|
||||
stride_cache_pos: tl.constexpr,
|
||||
stride_cache_head: tl.constexpr,
|
||||
# Dimensions
|
||||
D: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
# TQ layout
|
||||
MSE_BYTES: tl.constexpr,
|
||||
KPS: tl.constexpr,
|
||||
# Value quantization
|
||||
VQB: tl.constexpr,
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
# Packing block sizes
|
||||
BLOCK_VAL: tl.constexpr,
|
||||
# MSE params
|
||||
MSE_BITS: tl.constexpr,
|
||||
N_CENTROIDS: tl.constexpr,
|
||||
BLOCK_GRP: tl.constexpr = 16,
|
||||
):
|
||||
"""Fused MSE quantize + pack + store.
|
||||
|
||||
Performs binary-search bucketize, MSE index packing, norm storage,
|
||||
and value quantization in one kernel.
|
||||
"""
|
||||
pid = tl.program_id(0)
|
||||
token_idx = pid // H
|
||||
head_idx = pid % H
|
||||
|
||||
slot = tl.load(Slot_mapping_ptr + token_idx)
|
||||
if slot < 0:
|
||||
return
|
||||
blk = (slot // BLOCK_SIZE).to(tl.int64)
|
||||
off = (slot % BLOCK_SIZE).to(tl.int64)
|
||||
head_idx_i64 = tl.cast(head_idx, tl.int64)
|
||||
slot_base = (
|
||||
blk * stride_cache_block
|
||||
+ off * stride_cache_pos
|
||||
+ head_idx_i64 * stride_cache_head
|
||||
)
|
||||
|
||||
base = pid * D
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < D
|
||||
|
||||
# ── 1. BINARY SEARCH BUCKETIZE ───────────────────────────────────
|
||||
# Midpoints are sorted (N_CENTROIDS-1 values); binary search finds
|
||||
# insertion point in MSE_BITS iterations vs N_CENTROIDS-1 for linear.
|
||||
y_vec = tl.load(Y_ptr + base + d_offs, mask=d_mask, other=0.0)
|
||||
lo = tl.zeros([BLOCK_D], dtype=tl.int32)
|
||||
hi = tl.full([BLOCK_D], N_CENTROIDS - 1, dtype=tl.int32)
|
||||
for _ in range(MSE_BITS):
|
||||
mid = (lo + hi) >> 1
|
||||
# Clamp to valid midpoint index [0, N_CENTROIDS-2] for load safety;
|
||||
# the search result (lo) is still correct since converged lanes
|
||||
# don't change.
|
||||
safe_mid = tl.minimum(mid, N_CENTROIDS - 2)
|
||||
mid_val = tl.load(Midpoints_ptr + safe_mid, mask=d_mask, other=0.0)
|
||||
lo = tl.where(y_vec >= mid_val, mid + 1, lo)
|
||||
hi = tl.where(y_vec >= mid_val, hi, mid)
|
||||
idx = tl.minimum(lo, N_CENTROIDS - 1)
|
||||
|
||||
# ── 2. PACK MSE INDICES from register idx ─────────────────────────
|
||||
if MSE_BITS == 4:
|
||||
idx_pairs = tl.reshape(idx, [BLOCK_D // 2, 2])
|
||||
shifts_4 = tl.arange(0, 2) * 4
|
||||
packed = tl.sum((idx_pairs & 0xF) << shifts_4[None, :], axis=1).to(tl.uint8)
|
||||
mse_offs = tl.arange(0, BLOCK_D // 2)
|
||||
mse_mask = mse_offs < MSE_BYTES
|
||||
tl.store(KV_cache_ptr + slot_base + mse_offs, packed, mask=mse_mask)
|
||||
|
||||
elif MSE_BITS == 3:
|
||||
grp_offs = tl.arange(0, BLOCK_GRP)
|
||||
grp_mask = grp_offs < (D // 8)
|
||||
idx_grp = tl.reshape(idx, [BLOCK_GRP, 8])
|
||||
shifts_3 = tl.arange(0, 8) * 3
|
||||
packed_24 = tl.sum((idx_grp & 0x7) << shifts_3[None, :], axis=1)
|
||||
b0 = (packed_24 & 0xFF).to(tl.uint8)
|
||||
b1 = ((packed_24 >> 8) & 0xFF).to(tl.uint8)
|
||||
b2 = ((packed_24 >> 16) & 0xFF).to(tl.uint8)
|
||||
tl.store(KV_cache_ptr + slot_base + grp_offs * 3, b0, mask=grp_mask)
|
||||
tl.store(KV_cache_ptr + slot_base + grp_offs * 3 + 1, b1, mask=grp_mask)
|
||||
tl.store(KV_cache_ptr + slot_base + grp_offs * 3 + 2, b2, mask=grp_mask)
|
||||
|
||||
# ── 3. STORE vec_norm (fp16, 2 bytes) ─────────────────────────────
|
||||
norm_offset = MSE_BYTES
|
||||
|
||||
vn_f16 = tl.load(Norms_ptr + pid).to(tl.float16)
|
||||
vn_u16 = vn_f16.to(tl.uint16, bitcast=True)
|
||||
tl.store(KV_cache_ptr + slot_base + norm_offset, (vn_u16 & 0xFF).to(tl.uint8))
|
||||
tl.store(
|
||||
KV_cache_ptr + slot_base + norm_offset + 1, ((vn_u16 >> 8) & 0xFF).to(tl.uint8)
|
||||
)
|
||||
|
||||
# ── 4. VALUE QUANTIZE + PACK ──────────────────────────────────────
|
||||
_store_quantized_value(
|
||||
Value_ptr,
|
||||
KV_cache_ptr,
|
||||
base,
|
||||
slot_base,
|
||||
d_offs,
|
||||
d_mask,
|
||||
D=D,
|
||||
KPS=KPS,
|
||||
VQB=VQB,
|
||||
VAL_DATA_BYTES=VAL_DATA_BYTES,
|
||||
BLOCK_D=BLOCK_D,
|
||||
BLOCK_VAL=BLOCK_VAL,
|
||||
BLOCK_GRP=BLOCK_GRP,
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Launcher
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def triton_turboquant_store(
|
||||
key: torch.Tensor, # [N, H, D] — raw keys (post-RoPE)
|
||||
value: torch.Tensor, # [N, H, D] — raw values
|
||||
kv_cache: torch.Tensor, # [num_blocks, block_size, Hk, padded_slot] uint8
|
||||
slot_mapping: torch.Tensor, # [N] int32
|
||||
PiT: torch.Tensor, # [D, D] float32
|
||||
midpoints: torch.Tensor, # [n_centroids-1] float32
|
||||
mse_bits: int,
|
||||
key_packed_size: int,
|
||||
value_quant_bits: int,
|
||||
key_fp8: bool = False,
|
||||
):
|
||||
"""Launch TQ store kernel (FP8 or MSE path)."""
|
||||
N, H, D = key.shape
|
||||
NH = N * H
|
||||
block_size = kv_cache.shape[1]
|
||||
BLOCK_D = triton.next_power_of_2(D)
|
||||
mse_bytes = math.ceil(D * mse_bits / 8)
|
||||
n_centroids = 2**mse_bits
|
||||
|
||||
val_data_bytes = math.ceil(D * value_quant_bits / 8)
|
||||
|
||||
BLOCK_VAL = triton.next_power_of_2(val_data_bytes)
|
||||
|
||||
# Cache strides (element_size=1 for uint8, so stride in bytes = stride())
|
||||
stride_block = kv_cache.stride(0)
|
||||
stride_pos = kv_cache.stride(1)
|
||||
stride_head = kv_cache.stride(2)
|
||||
|
||||
block_grp = triton.next_power_of_2(D // 8) if D >= 8 else 1
|
||||
|
||||
# ── FP8 PATH: in-kernel FP8 cast + scatter via fp8 kernel ──
|
||||
if key_fp8:
|
||||
k_flat = key.reshape(NH, D).contiguous()
|
||||
v_flat = value.reshape(NH, D).contiguous()
|
||||
|
||||
fp8_e4b15 = _use_fp8_e4b15(key.device.index or 0)
|
||||
|
||||
grid = (NH,)
|
||||
_tq_fused_store_fp8[grid](
|
||||
k_flat,
|
||||
v_flat,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
stride_cache_block=stride_block,
|
||||
stride_cache_pos=stride_pos,
|
||||
stride_cache_head=stride_head,
|
||||
D=D,
|
||||
H=H,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_D=BLOCK_D,
|
||||
KPS=key_packed_size,
|
||||
VQB=value_quant_bits,
|
||||
VAL_DATA_BYTES=val_data_bytes,
|
||||
BLOCK_VAL=BLOCK_VAL,
|
||||
BLOCK_GRP=block_grp,
|
||||
FP8_E4B15=fp8_e4b15,
|
||||
num_warps=4,
|
||||
num_stages=1,
|
||||
)
|
||||
return
|
||||
|
||||
# ── MSE PATH: external GEMM + fused bucketize/pack kernel ──
|
||||
# Normalize + rotation GEMM externally (cuBLAS is faster than in-kernel)
|
||||
k_flat = key.float().reshape(NH, D)
|
||||
norms = k_flat.norm(dim=1, keepdim=True)
|
||||
x_hat = k_flat / (norms + 1e-8)
|
||||
y = x_hat @ PiT
|
||||
|
||||
v_flat = value.float().reshape(NH, D)
|
||||
|
||||
# Fused kernel: bucketize + MSE index pack + norm store + value pack
|
||||
grid = (NH,)
|
||||
_tq_fused_store_mse[grid](
|
||||
y,
|
||||
norms.squeeze(1),
|
||||
v_flat,
|
||||
midpoints,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
stride_cache_block=stride_block,
|
||||
stride_cache_pos=stride_pos,
|
||||
stride_cache_head=stride_head,
|
||||
D=D,
|
||||
H=H,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_D=BLOCK_D,
|
||||
MSE_BYTES=mse_bytes,
|
||||
KPS=key_packed_size,
|
||||
VQB=value_quant_bits,
|
||||
VAL_DATA_BYTES=val_data_bytes,
|
||||
BLOCK_VAL=BLOCK_VAL,
|
||||
MSE_BITS=mse_bits,
|
||||
N_CENTROIDS=n_centroids,
|
||||
BLOCK_GRP=block_grp,
|
||||
num_warps=4,
|
||||
num_stages=1,
|
||||
)
|
||||
@@ -1,3 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# HIP-free SoA Triton subset for the FlyDSL TurboQuant decode path.
|
||||
@@ -1,643 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton fused TurboQuant decode attention.
|
||||
|
||||
Decode path: Triton stage1 (split-KV tiled attention scoring + value
|
||||
accumulation) + stage2 (log-sum-exp reduction across splits).
|
||||
|
||||
Supports FP8 (E4M3) keys, 3-bit and 4-bit uniform quantized values.
|
||||
"""
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.ops.triton_decode_attention import (
|
||||
_fwd_kernel_stage2,
|
||||
)
|
||||
|
||||
_FP8_E4B15: dict[int, int] = {}
|
||||
|
||||
|
||||
def _use_fp8_e4b15(device: int = 0) -> int:
|
||||
"""Return 1 if device needs fp8e4b15 (Ampere/Ada, SM < 8.9), else 0.
|
||||
On non-CUDA platforms (e.g. XPU), always returns 0 (use e4nv format).
|
||||
"""
|
||||
if device not in _FP8_E4B15:
|
||||
if current_platform.is_cuda_alike():
|
||||
cap = torch.cuda.get_device_capability(device)
|
||||
_FP8_E4B15[device] = 1 if cap < (8, 9) else 0
|
||||
else:
|
||||
_FP8_E4B15[device] = 0
|
||||
return _FP8_E4B15[device]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 1: Fused TQ score + value accumulation (BLOCK_KV tiled)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_decode_stage1(
|
||||
# Precomputed query projection
|
||||
Q_rot_ptr, # [B, Hq, D] float32
|
||||
# Compressed KV cache (combined K+V)
|
||||
KV_cache_ptr, # [num_blocks, block_size, Hk, padded_slot] uint8
|
||||
KV_cache_u16_ptr, # uint16 view of same storage — Opt#3 SoA loads
|
||||
# Block table and sequence info
|
||||
Block_table_ptr, # [B, max_num_blocks] int32
|
||||
Seq_lens_ptr, # [B] int32
|
||||
# TQ parameters
|
||||
Centroids_ptr, # [n_centroids] float32
|
||||
# Output (intermediate for stage2)
|
||||
Mid_o_ptr, # [B, Hq, NUM_KV_SPLITS, D+1] float32
|
||||
# Sink support
|
||||
Sink_ptr, # [Hq] float32, pre-computed sink attention logits per head
|
||||
# Strides
|
||||
stride_qb,
|
||||
stride_qh, # Q strides: [B, Hq, D]
|
||||
stride_cache_block: tl.int64, # bytes per block (bs*H*slot_aligned)
|
||||
stride_bt_b: tl.int64, # block_table stride per batch
|
||||
stride_mid_b,
|
||||
stride_mid_h,
|
||||
stride_mid_s, # mid_o strides
|
||||
# Constexpr dims
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr, # KV cache block_size (pages)
|
||||
NUM_KV_SPLITS: tl.constexpr,
|
||||
KV_GROUP_SIZE: tl.constexpr, # Hq // Hk
|
||||
# TQ layout constants
|
||||
MSE_BITS: tl.constexpr, # 3 or 4
|
||||
MSE_BYTES: tl.constexpr, # ceil(D * mse_bits / 8)
|
||||
VQB: tl.constexpr, # value_quant_bits (4 or 8=FP8)
|
||||
VAL_DATA_BYTES: tl.constexpr, # ceil(D * vqb / 8) or D for FP8
|
||||
# Opt#3 SoA layout
|
||||
KEY_DATA_BYTES: tl.constexpr, # MSE_BYTES (MSE) or HEAD_DIM (FP8)
|
||||
META_REGION_OFFSET: tl.constexpr, # bytes from block start to SoA region
|
||||
NUM_SOA_FIELDS: tl.constexpr, # 3 (MSE) or 2 (FP8)
|
||||
SOA_K_NORM: tl.constexpr, # 0 (MSE); unused for FP8
|
||||
SOA_V_SCALE: tl.constexpr, # 1 (MSE) / 0 (FP8)
|
||||
SOA_V_ZERO: tl.constexpr, # 2 (MSE) / 1 (FP8)
|
||||
# Score constants
|
||||
ATTN_SCALE: tl.constexpr, # 1/sqrt(D)
|
||||
# Block tile sizes
|
||||
BLOCK_D: tl.constexpr, # next_power_of_2(HEAD_DIM)
|
||||
BLOCK_KV: tl.constexpr, # tokens per tile (16)
|
||||
KEY_FP8: tl.constexpr, # 1 if K is stored as FP8
|
||||
NORM_CORRECTION: tl.constexpr = 0, # 1 = re-normalize centroids
|
||||
FP8_E4B15: tl.constexpr = 0, # 1 = use e4b15 (Ampere/Ada), 0 = e4nv (Hopper+)
|
||||
USE_SINKS: tl.constexpr = 0, # 1 = use sink tokens for attention anchoring
|
||||
):
|
||||
bid = tl.program_id(0) # batch index
|
||||
hid = tl.program_id(1) # q_head index
|
||||
sid = tl.program_id(2) # kv_split index
|
||||
|
||||
kv_head = hid // KV_GROUP_SIZE
|
||||
|
||||
# Sequence length for this batch
|
||||
seq_len = tl.load(Seq_lens_ptr + bid)
|
||||
|
||||
# KV split range
|
||||
split_len = tl.cdiv(seq_len, NUM_KV_SPLITS)
|
||||
split_start = split_len * sid
|
||||
split_end = tl.minimum(split_start + split_len, seq_len)
|
||||
|
||||
if split_start >= split_end:
|
||||
return
|
||||
|
||||
# Dimension offsets
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < HEAD_DIM
|
||||
kv_range = tl.arange(0, BLOCK_KV)
|
||||
|
||||
# Load query vector: q_rot — [BLOCK_D] float32
|
||||
q_base = bid * stride_qb + hid * stride_qh
|
||||
q_rot = tl.load(Q_rot_ptr + q_base + d_offs, mask=d_mask, other=0.0).to(tl.float32)
|
||||
|
||||
# Precompute byte/bit index vectors for MSE gather loads
|
||||
if not KEY_FP8:
|
||||
mse_bit_off = d_offs * MSE_BITS
|
||||
mse_byte_idx = mse_bit_off // 8
|
||||
mse_bit_shift = mse_bit_off % 8
|
||||
mse_mask = (1 << MSE_BITS) - 1
|
||||
|
||||
# Precompute value bit/byte index vectors (loop-invariant)
|
||||
if VQB == 3:
|
||||
val_bit_off = d_offs * 3
|
||||
val_byte_idx = val_bit_off // 8
|
||||
val_bit_shift = val_bit_off % 8
|
||||
|
||||
# Online softmax accumulators
|
||||
# Sink tokens provide a pre-computed attention bias that should be
|
||||
# included in the softmax normalization. For the first split (sid==0),
|
||||
# we initialize m_prev with the sink logit. L is always 1.0 for all
|
||||
# splits to match unified attention semantics.
|
||||
if USE_SINKS:
|
||||
if sid == 0:
|
||||
m_prev = tl.load(Sink_ptr + hid).to(tl.float32)
|
||||
l_prev = 1.0
|
||||
else:
|
||||
m_prev = -float("inf")
|
||||
l_prev = 0.0
|
||||
else:
|
||||
m_prev = -float("inf")
|
||||
l_prev = 0.0
|
||||
acc = tl.zeros([BLOCK_D], dtype=tl.float32)
|
||||
|
||||
bt_base = bid * stride_bt_b
|
||||
|
||||
# ================================================================
|
||||
# TILED LOOP: process BLOCK_KV tokens per iteration
|
||||
# ================================================================
|
||||
for start_n in range(split_start, split_end, BLOCK_KV):
|
||||
kv_offs = start_n + kv_range
|
||||
kv_mask = kv_offs < split_end
|
||||
|
||||
page_idx = kv_offs // BLOCK_SIZE
|
||||
page_off = kv_offs % BLOCK_SIZE
|
||||
block_nums = tl.load(
|
||||
Block_table_ptr + bt_base + page_idx,
|
||||
mask=kv_mask,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
|
||||
# Opt#3 SoA addressing: data region then metadata region per block.
|
||||
slot_within_block = page_off.to(tl.int64)
|
||||
block_base = block_nums * stride_cache_block
|
||||
DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES
|
||||
data_bases = (
|
||||
block_base
|
||||
+ slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT)
|
||||
+ tl.cast(kv_head, tl.int64) * DATA_BYTES_PER_SLOT
|
||||
)
|
||||
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast(
|
||||
kv_head, tl.int64
|
||||
) * (NUM_SOA_FIELDS * BLOCK_SIZE)
|
||||
knorm_u16_addrs = (
|
||||
head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block
|
||||
)
|
||||
vscale_u16_addrs = (
|
||||
head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block
|
||||
)
|
||||
vzero_u16_addrs = (
|
||||
head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# COMPUTE ATTENTION SCORES: [BLOCK_KV]
|
||||
# ============================================================
|
||||
if KEY_FP8:
|
||||
k_addrs = data_bases[:, None] + d_offs[None, :]
|
||||
k_raw = tl.load(
|
||||
KV_cache_ptr + k_addrs,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
)
|
||||
if FP8_E4B15:
|
||||
k_float = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32)
|
||||
else:
|
||||
k_float = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32)
|
||||
scores = (
|
||||
tl.sum(
|
||||
tl.where(d_mask[None, :], q_rot[None, :] * k_float, 0.0),
|
||||
axis=1,
|
||||
)
|
||||
* ATTN_SCALE
|
||||
)
|
||||
scores = tl.where(kv_mask, scores, -float("inf"))
|
||||
else:
|
||||
# MSE unpack + norms
|
||||
mse_addrs0 = data_bases[:, None] + mse_byte_idx[None, :]
|
||||
mse_raw0 = tl.load(
|
||||
KV_cache_ptr + mse_addrs0,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
mse_raw1 = tl.load(
|
||||
KV_cache_ptr + mse_addrs0 + 1,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
raw16 = mse_raw0 | (mse_raw1 << 8)
|
||||
mse_idx = (raw16 >> mse_bit_shift[None, :]) & mse_mask
|
||||
|
||||
# Centroid gather + dot product
|
||||
c_vals = tl.load(
|
||||
Centroids_ptr + mse_idx,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
# Opt#1: norm-correction (1/||c_vec||) is pre-folded into the
|
||||
# stored per-token scalar by the MSE store kernel when
|
||||
# NORM_CORRECTION=1, so we skip the per-tile sum+sqrt+divide.
|
||||
# NORM_CORRECTION is kept in the signature for API parity; it is
|
||||
# implicit in the stored norm value now.
|
||||
|
||||
term1 = tl.sum(
|
||||
tl.where(d_mask[None, :], q_rot[None, :] * c_vals, 0.0),
|
||||
axis=1,
|
||||
)
|
||||
|
||||
# Opt#3: K-norms live in the per-block SoA region. Single u16
|
||||
# load per token; contiguous across tokens within a block.
|
||||
norm_u16 = tl.load(
|
||||
KV_cache_u16_ptr + knorm_u16_addrs, mask=kv_mask, other=0
|
||||
)
|
||||
vec_norms = norm_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
|
||||
scores = vec_norms * term1 * ATTN_SCALE
|
||||
scores = tl.where(kv_mask, scores, -float("inf"))
|
||||
|
||||
# ============================================================
|
||||
# ONLINE SOFTMAX UPDATE (block-level)
|
||||
# ============================================================
|
||||
n_e_max = tl.maximum(tl.max(scores, 0), m_prev)
|
||||
re_scale = tl.exp(m_prev - n_e_max)
|
||||
p = tl.exp(scores - n_e_max)
|
||||
|
||||
# ============================================================
|
||||
# VALUE LOAD + DEQUANTIZE: [BLOCK_KV, BLOCK_D]
|
||||
# Opt#3: V data immediately follows K data in the slot (no
|
||||
# interleaved metadata). Scale/zero come from the SoA metadata
|
||||
# region as single u16 loads per field per token.
|
||||
# ============================================================
|
||||
val_bases = data_bases + KEY_DATA_BYTES
|
||||
|
||||
if VQB == 3:
|
||||
val_addrs0 = val_bases[:, None] + val_byte_idx[None, :]
|
||||
val_raw0 = tl.load(
|
||||
KV_cache_ptr + val_addrs0,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
val_raw1 = tl.load(
|
||||
KV_cache_ptr + val_addrs0 + 1,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
raw16 = val_raw0 | (val_raw1 << 8)
|
||||
v_idx = ((raw16 >> val_bit_shift[None, :]) & 0x7).to(tl.float32)
|
||||
else: # VQB == 4
|
||||
vb_idx = d_offs // 2
|
||||
vb_shift = (d_offs % 2) * 4
|
||||
val_addrs = val_bases[:, None] + vb_idx[None, :]
|
||||
val_raw = tl.load(
|
||||
KV_cache_ptr + val_addrs,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
v_idx = ((val_raw >> vb_shift[None, :]) & 0xF).to(tl.float32)
|
||||
|
||||
scale_u16 = tl.load(KV_cache_u16_ptr + vscale_u16_addrs, mask=kv_mask, other=0)
|
||||
zero_u16 = tl.load(KV_cache_u16_ptr + vzero_u16_addrs, mask=kv_mask, other=0)
|
||||
v_scales = scale_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
v_zeros = zero_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
values = v_idx * v_scales[:, None] + v_zeros[:, None]
|
||||
|
||||
# ============================================================
|
||||
# WEIGHTED VALUE ACCUMULATION
|
||||
# ============================================================
|
||||
acc = acc * re_scale + tl.sum(p[:, None] * values, 0)
|
||||
l_prev = l_prev * re_scale + tl.sum(p, 0)
|
||||
m_prev = n_e_max
|
||||
|
||||
# Store partial result
|
||||
out_base = bid * stride_mid_b + hid * stride_mid_h + sid * stride_mid_s
|
||||
safe_l = tl.where(l_prev > 0.0, l_prev, 1.0)
|
||||
tl.store(Mid_o_ptr + out_base + d_offs, acc / safe_l, mask=d_mask)
|
||||
lse = m_prev + tl.log(safe_l)
|
||||
tl.store(Mid_o_ptr + out_base + HEAD_DIM, lse)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-dequant kernel: Bulk dequant K (MSE+norms) and V to fp16
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_full_dequant_kv(
|
||||
KV_cache_ptr,
|
||||
KV_cache_u16_ptr,
|
||||
Block_table_ptr,
|
||||
Centroids_ptr,
|
||||
K_out_ptr, # [B, Hk, max_seq, D] float16
|
||||
V_out_ptr, # [B, Hk, max_seq, D] float16
|
||||
stride_ko_b: tl.int64,
|
||||
stride_ko_h: tl.int64,
|
||||
stride_ko_s: tl.int64,
|
||||
stride_vo_b: tl.int64,
|
||||
stride_vo_h: tl.int64,
|
||||
stride_vo_s: tl.int64,
|
||||
stride_cache_block: tl.int64,
|
||||
stride_bt_b: tl.int64,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
MSE_BYTES: tl.constexpr,
|
||||
VQB: tl.constexpr,
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
MSE_BITS: tl.constexpr,
|
||||
KEY_FP8: tl.constexpr,
|
||||
# Opt#3 SoA layout
|
||||
KEY_DATA_BYTES: tl.constexpr,
|
||||
META_REGION_OFFSET: tl.constexpr,
|
||||
NUM_SOA_FIELDS: tl.constexpr,
|
||||
SOA_K_NORM: tl.constexpr,
|
||||
SOA_V_SCALE: tl.constexpr,
|
||||
SOA_V_ZERO: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
NORM_CORRECTION: tl.constexpr = 0,
|
||||
FP8_E4B15: tl.constexpr = 0, # 1 = use e4b15 (Ampere/Ada), 0 = e4nv (Hopper+)
|
||||
):
|
||||
"""Full dequant: reconstruct K (MSE centroids * norm or FP8) and V to fp16.
|
||||
|
||||
Opt#3 SoA layout: data region [bs, H, KD+VD] + per-block SoA metadata.
|
||||
"""
|
||||
pos = tl.program_id(0)
|
||||
bh = tl.program_id(1)
|
||||
bid = bh // NUM_KV_HEADS
|
||||
hid = bh % NUM_KV_HEADS
|
||||
|
||||
page_idx = pos // BLOCK_SIZE
|
||||
page_off = pos % BLOCK_SIZE
|
||||
block_num = tl.load(Block_table_ptr + bid * stride_bt_b + page_idx).to(tl.int64)
|
||||
|
||||
block_base = block_num * stride_cache_block
|
||||
slot_within_block = tl.cast(page_off, tl.int64)
|
||||
DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES
|
||||
data_base = (
|
||||
block_base
|
||||
+ slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT)
|
||||
+ tl.cast(hid, tl.int64) * DATA_BYTES_PER_SLOT
|
||||
)
|
||||
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast(
|
||||
hid, tl.int64
|
||||
) * (NUM_SOA_FIELDS * BLOCK_SIZE)
|
||||
knorm_u16_addr = head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block
|
||||
vscale_u16_addr = head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block
|
||||
vzero_u16_addr = head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block
|
||||
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < HEAD_DIM
|
||||
|
||||
# === K dequant ===
|
||||
ko_base = bid * stride_ko_b + hid * stride_ko_h + pos * stride_ko_s
|
||||
if KEY_FP8:
|
||||
k_raw = tl.load(KV_cache_ptr + data_base + d_offs, mask=d_mask, other=0)
|
||||
if FP8_E4B15:
|
||||
k_recon = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32)
|
||||
else:
|
||||
k_recon = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32)
|
||||
tl.store(K_out_ptr + ko_base + d_offs, k_recon.to(tl.float16), mask=d_mask)
|
||||
else:
|
||||
mse_bit_off = d_offs * MSE_BITS
|
||||
mse_byte_idx = mse_bit_off // 8
|
||||
mse_bit_shift = mse_bit_off % 8
|
||||
mse_umask = (1 << MSE_BITS) - 1
|
||||
|
||||
mse_raw0 = tl.load(
|
||||
KV_cache_ptr + data_base + mse_byte_idx, mask=d_mask, other=0
|
||||
).to(tl.int32)
|
||||
mse_raw1 = tl.load(
|
||||
KV_cache_ptr + data_base + mse_byte_idx + 1, mask=d_mask, other=0
|
||||
).to(tl.int32)
|
||||
raw16_key = mse_raw0 | (mse_raw1 << 8)
|
||||
mse_idx = (raw16_key >> mse_bit_shift) & mse_umask
|
||||
|
||||
k_mse = tl.load(Centroids_ptr + mse_idx, mask=d_mask, other=0.0)
|
||||
|
||||
# Opt#1: norm-correction is pre-folded into the stored scalar.
|
||||
# Opt#3: K-norm lives in the SoA region — single u16 load.
|
||||
norm_u16 = tl.load(KV_cache_u16_ptr + knorm_u16_addr)
|
||||
vec_norm = norm_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
|
||||
k_recon = vec_norm * k_mse
|
||||
tl.store(K_out_ptr + ko_base + d_offs, k_recon.to(tl.float16), mask=d_mask)
|
||||
|
||||
# === V dequant ===
|
||||
val_base = data_base + KEY_DATA_BYTES
|
||||
if VQB == 4:
|
||||
vb_idx = d_offs // 2
|
||||
vb_shift = (d_offs % 2) * 4
|
||||
val_raw = tl.load(KV_cache_ptr + val_base + vb_idx, mask=d_mask, other=0).to(
|
||||
tl.int32
|
||||
)
|
||||
v_idx = ((val_raw >> vb_shift) & 0xF).to(tl.float32)
|
||||
elif VQB == 3:
|
||||
val_bit_off = d_offs * 3
|
||||
val_byte_idx = val_bit_off // 8
|
||||
val_bit_shift = val_bit_off % 8
|
||||
val_raw0 = tl.load(
|
||||
KV_cache_ptr + val_base + val_byte_idx, mask=d_mask, other=0
|
||||
).to(tl.int32)
|
||||
val_raw1 = tl.load(
|
||||
KV_cache_ptr + val_base + val_byte_idx + 1, mask=d_mask, other=0
|
||||
).to(tl.int32)
|
||||
raw16_val = val_raw0 | (val_raw1 << 8)
|
||||
v_idx = ((raw16_val >> val_bit_shift) & 0x7).to(tl.float32)
|
||||
else:
|
||||
v_idx = tl.zeros([BLOCK_D], dtype=tl.float32)
|
||||
|
||||
# Opt#3: V scale/zero single-u16 loads from SoA region.
|
||||
sc_u16 = tl.load(KV_cache_u16_ptr + vscale_u16_addr)
|
||||
zr_u16 = tl.load(KV_cache_u16_ptr + vzero_u16_addr)
|
||||
v_scale = sc_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
v_zero = zr_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
v_vals = v_idx * v_scale + v_zero
|
||||
|
||||
vo_base = bid * stride_vo_b + hid * stride_vo_h + pos * stride_vo_s
|
||||
tl.store(V_out_ptr + vo_base + d_offs, v_vals.to(tl.float16), mask=d_mask)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 2: Reuse from triton_decode_attention.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Launcher — cached constants + fused GEMM
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_layout_cache: dict = {}
|
||||
|
||||
|
||||
def _get_layout(D, mse_bits, value_quant_bits, key_packed_size):
|
||||
"""Get cached layout constants."""
|
||||
key = (D, mse_bits, value_quant_bits, key_packed_size)
|
||||
cfg = _layout_cache.get(key)
|
||||
if cfg is None:
|
||||
val_data_bytes = math.ceil(D * value_quant_bits / 8)
|
||||
cfg = {
|
||||
"mse_bytes": math.ceil(D * mse_bits / 8),
|
||||
"val_data_bytes": val_data_bytes,
|
||||
"mse_bits": mse_bits,
|
||||
"n_centroids": 2**mse_bits,
|
||||
"BLOCK_D": triton.next_power_of_2(D),
|
||||
}
|
||||
_layout_cache[key] = cfg
|
||||
return cfg
|
||||
|
||||
|
||||
def triton_turboquant_decode_attention(
|
||||
query: torch.Tensor, # [B, Hq, D] — original query
|
||||
kv_cache: torch.Tensor, # [num_blocks, block_size, Hk, padded_slot] uint8
|
||||
block_table: torch.Tensor, # [B, max_num_blocks] int32
|
||||
seq_lens: torch.Tensor, # [B] int32
|
||||
Pi: torch.Tensor, # [D, D] float32
|
||||
centroids: torch.Tensor, # [n_centroids] float32
|
||||
scale: float,
|
||||
mse_bits: int,
|
||||
key_packed_size: int,
|
||||
value_quant_bits: int,
|
||||
key_fp8: bool = False,
|
||||
norm_correction: bool = False,
|
||||
PiT: torch.Tensor | None = None, # [D, D] pre-computed Pi.T contiguous
|
||||
# Pre-allocated buffers (optional, avoids per-call allocation)
|
||||
mid_o_buf: torch.Tensor | None = None,
|
||||
output_buf: torch.Tensor | None = None,
|
||||
lse_buf: torch.Tensor | None = None,
|
||||
buf_holder: Any = None,
|
||||
max_num_kv_splits: int = 32, # fixed split count (must be constant for cudagraph)
|
||||
sinks: torch.Tensor | None = None, # [Hq] float32, pre-computed sink logits
|
||||
) -> torch.Tensor:
|
||||
"""Launch fused TQ decode attention (Triton stage1 + stage2).
|
||||
|
||||
Returns: output tensor [B, Hq, D] in query's dtype.
|
||||
"""
|
||||
B, Hq, D = query.shape
|
||||
Hk = kv_cache.shape[2]
|
||||
block_size = kv_cache.shape[1]
|
||||
kv_group_size = Hq // Hk
|
||||
device = query.device
|
||||
|
||||
cfg = _get_layout(D, mse_bits, value_quant_bits, key_packed_size)
|
||||
|
||||
# Opt#3 SoA layout constants (derived locally; match store-side values).
|
||||
key_data_bytes = D if key_fp8 else cfg["mse_bytes"]
|
||||
data_bytes_per_slot = key_data_bytes + cfg["val_data_bytes"]
|
||||
meta_region_offset = block_size * Hk * data_bytes_per_slot
|
||||
num_soa_fields = 2 if key_fp8 else 3
|
||||
soa_k_norm = 0
|
||||
soa_v_scale = 0 if key_fp8 else 1
|
||||
soa_v_zero = 1 if key_fp8 else 2
|
||||
kv_cache_u16 = kv_cache.view(torch.uint16)
|
||||
|
||||
# Compute q_rot = q @ Pi.T (rotated query for MSE key scoring)
|
||||
# FP8 path: pass query directly (float16); kernel casts inline.
|
||||
# MSE path: still needs external GEMM (cuBLAS), so q_rot is float32.
|
||||
if key_fp8:
|
||||
q_rot = query.contiguous()
|
||||
else:
|
||||
q_float = query.float()
|
||||
if PiT is None:
|
||||
PiT = Pi.T.contiguous()
|
||||
q_rot = (q_float @ PiT).contiguous()
|
||||
|
||||
NUM_KV_SPLITS = max_num_kv_splits
|
||||
|
||||
if (
|
||||
mid_o_buf is not None
|
||||
and mid_o_buf.shape[0] >= B
|
||||
and mid_o_buf.shape[2] >= NUM_KV_SPLITS
|
||||
):
|
||||
mid_o = mid_o_buf[:B, :Hq, :NUM_KV_SPLITS, :]
|
||||
else:
|
||||
mid_o = torch.empty(
|
||||
B,
|
||||
Hq,
|
||||
NUM_KV_SPLITS,
|
||||
D + 1,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
if buf_holder is not None:
|
||||
buf_holder._tq_mid_o_buf = mid_o
|
||||
|
||||
# Stage 1: split-KV tiled attention scoring + value accumulation
|
||||
fp8_e4b15 = _use_fp8_e4b15(device.index or 0)
|
||||
BLOCK_KV = 4
|
||||
grid = (B, Hq, NUM_KV_SPLITS)
|
||||
|
||||
# Prepare sink pointer (convert to float32 on device if provided)
|
||||
use_sinks = sinks is not None
|
||||
|
||||
_tq_decode_stage1[grid](
|
||||
q_rot,
|
||||
kv_cache,
|
||||
kv_cache_u16,
|
||||
block_table,
|
||||
seq_lens,
|
||||
centroids,
|
||||
mid_o,
|
||||
sinks,
|
||||
q_rot.stride(0),
|
||||
q_rot.stride(1),
|
||||
kv_cache.stride(0),
|
||||
block_table.stride(0),
|
||||
mid_o.stride(0),
|
||||
mid_o.stride(1),
|
||||
mid_o.stride(2),
|
||||
NUM_KV_HEADS=Hk,
|
||||
HEAD_DIM=D,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_KV_SPLITS=NUM_KV_SPLITS,
|
||||
KV_GROUP_SIZE=kv_group_size,
|
||||
MSE_BITS=mse_bits,
|
||||
MSE_BYTES=cfg["mse_bytes"],
|
||||
VQB=value_quant_bits,
|
||||
VAL_DATA_BYTES=cfg["val_data_bytes"],
|
||||
KEY_DATA_BYTES=key_data_bytes,
|
||||
META_REGION_OFFSET=meta_region_offset,
|
||||
NUM_SOA_FIELDS=num_soa_fields,
|
||||
SOA_K_NORM=soa_k_norm,
|
||||
SOA_V_SCALE=soa_v_scale,
|
||||
SOA_V_ZERO=soa_v_zero,
|
||||
ATTN_SCALE=scale,
|
||||
BLOCK_D=cfg["BLOCK_D"],
|
||||
BLOCK_KV=BLOCK_KV,
|
||||
KEY_FP8=1 if key_fp8 else 0,
|
||||
NORM_CORRECTION=1 if norm_correction else 0,
|
||||
FP8_E4B15=fp8_e4b15,
|
||||
USE_SINKS=1 if use_sinks else 0,
|
||||
num_warps=1,
|
||||
num_stages=1,
|
||||
)
|
||||
|
||||
# Stage 2: Reduce across KV splits
|
||||
if output_buf is not None and output_buf.shape[0] >= B:
|
||||
output = output_buf[:B, :Hq, :D]
|
||||
else:
|
||||
output = torch.empty(B, Hq, D, dtype=torch.float32, device=device)
|
||||
if buf_holder is not None:
|
||||
buf_holder._tq_output_buf = output
|
||||
if lse_buf is not None and lse_buf.shape[0] >= B:
|
||||
lse = lse_buf[:B, :Hq]
|
||||
else:
|
||||
lse = torch.empty(B, Hq, dtype=torch.float32, device=device)
|
||||
if buf_holder is not None:
|
||||
buf_holder._tq_lse_buf = lse
|
||||
|
||||
grid2 = (B, Hq)
|
||||
_fwd_kernel_stage2[grid2](
|
||||
mid_o,
|
||||
output,
|
||||
lse,
|
||||
seq_lens,
|
||||
mid_o.stride(0),
|
||||
mid_o.stride(1),
|
||||
mid_o.stride(2),
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
lse.stride(0),
|
||||
NUM_KV_SPLITS=NUM_KV_SPLITS,
|
||||
BLOCK_DV=cfg["BLOCK_D"],
|
||||
Lv=D,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
|
||||
return output.to(query.dtype)
|
||||
@@ -1,97 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Optimized Triton TurboQuant decode attention (v2).
|
||||
|
||||
FLUTE-paper optimizations applied:
|
||||
1. Grouped Q heads: grid over (B, Hk, splits) instead of (B, Hq, splits).
|
||||
Each program loads BLOCK_M Q heads sharing a KV head into a 2D tile,
|
||||
enabling tl.dot on tensor cores (MFMA/WMMA) for both Q·K and P·V.
|
||||
2. Vectorized pair LUT: precompute `pair_table[i][j] = (T[i], T[j])`
|
||||
offline. At runtime, extract adjacent index pairs and fetch two
|
||||
dequantized centroids with a single gather, halving LUT lookups.
|
||||
3. exp2 instead of exp: scores pre-scaled by log2(e) so the hardware-
|
||||
native exp2 instruction replaces the more expensive exp.
|
||||
4. Wider index extraction: for 4-bit MSE, two adjacent 4-bit indices share
|
||||
a byte. One byte load yields both, eliminating redundant loads.
|
||||
5. Centroids pre-warmed in L1 at kernel start.
|
||||
6. BLOCK_KV = TILE_SIZE raised to 16-32 (from 4), reducing loop iterations
|
||||
and softmax rescaling overhead.
|
||||
|
||||
Stage 2 is reused unchanged from triton_decode_attention.py.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
# ROCm prefers num_stages=1 in attention-like kernels to reduce shared-memory
|
||||
# pressure (mirrors the pattern used in triton_decode_attention.py and
|
||||
# triton_turboquant_decode.py).
|
||||
_is_hip = current_platform.is_rocm()
|
||||
|
||||
# On ROCm, bf16 has the same MFMA throughput as fp16 but wider dynamic range
|
||||
# (8-bit exponent vs 5-bit), which is safer for attention scores. On CUDA,
|
||||
# fp16 tensor cores may be faster than bf16 for some shapes.
|
||||
_DOT_DTYPE = tl.bfloat16 if _is_hip else tl.float16
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pair LUT construction (FLUTE) — called once at launcher time
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_pair_lut(centroids: torch.Tensor) -> torch.Tensor:
|
||||
"""Build vectorized pair lookup table.
|
||||
|
||||
For N centroids, returns a [N, N, 2] float32 tensor where
|
||||
pair_lut[i, j] = (centroids[i], centroids[j]).
|
||||
Flattened to [N*N, 2] for kernel indexing: pair_lut[i*N + j].
|
||||
|
||||
For 4-bit MSE (N=16): 16*16*2*4 = 2048 bytes — fits in L1/smem.
|
||||
For 3-bit MSE (N=8): 8*8*2*4 = 512 bytes.
|
||||
"""
|
||||
N = centroids.shape[0]
|
||||
# pair_lut[i,j,0] = centroids[i], pair_lut[i,j,1] = centroids[j]
|
||||
c = centroids.float()
|
||||
lut = torch.empty(N, N, 2, dtype=torch.float32, device=centroids.device)
|
||||
lut[:, :, 0] = c[:, None]
|
||||
lut[:, :, 1] = c[None, :]
|
||||
return lut.reshape(N * N, 2).contiguous()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Launcher v2
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_layout_cache: dict = {}
|
||||
|
||||
|
||||
def _get_layout(D, mse_bits, value_quant_bits, key_packed_size):
|
||||
key = (D, mse_bits, value_quant_bits, key_packed_size)
|
||||
cfg = _layout_cache.get(key)
|
||||
if cfg is None:
|
||||
val_data_bytes = math.ceil(D * value_quant_bits / 8)
|
||||
cfg = {
|
||||
"mse_bytes": math.ceil(D * mse_bits / 8),
|
||||
"val_data_bytes": val_data_bytes,
|
||||
"mse_bits": mse_bits,
|
||||
"n_centroids": 2**mse_bits,
|
||||
"BLOCK_D": triton.next_power_of_2(D),
|
||||
}
|
||||
_layout_cache[key] = cfg
|
||||
return cfg
|
||||
|
||||
|
||||
def _get_pair_lut(centroids: torch.Tensor) -> torch.Tensor:
|
||||
"""Return a fresh pair-LUT for ``centroids`` on each call.
|
||||
|
||||
The LUT is tiny (N*N*2 fp32, e.g. 2KB for 4-bit MSE) so the build cost
|
||||
is negligible compared to attention. We avoid caching by data_ptr()
|
||||
because CUDA allocator memory reuse across different centroid tensors
|
||||
can silently return a stale LUT (subtle correctness bug). If this ever
|
||||
shows up on a profile, cache by a hash-of-values fingerprint instead.
|
||||
"""
|
||||
return build_pair_lut(centroids)
|
||||
@@ -1,524 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused Triton kernels for TurboQuant KV store.
|
||||
|
||||
Two kernels:
|
||||
1. _tq_fused_store_fp8: FP8 key scatter + value uniform quantization.
|
||||
2. _tq_fused_store_mse: Fused binary-search bucketize + MSE index
|
||||
packing + value quantization.
|
||||
|
||||
The launcher `triton_turboquant_store` selects the appropriate kernel.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from .triton_turboquant_decode import _use_fp8_e4b15
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Shared: value uniform quantization + pack + SoA scale/zero store
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_quantized_value(
|
||||
Value_ptr,
|
||||
KV_cache_ptr,
|
||||
KV_cache_u16_ptr, # Opt#3: u16 view for SoA scale/zero stores
|
||||
base, # pid * D offset into Value_ptr
|
||||
data_base, # byte offset into KV_cache_ptr for this slot+head's DATA region
|
||||
vscale_u16_addr, # u16 element index for V-scale in the SoA region
|
||||
vzero_u16_addr, # u16 element index for V-zero in the SoA region
|
||||
d_offs, # tl.arange(0, BLOCK_D)
|
||||
d_mask, # d_offs < D
|
||||
D: tl.constexpr,
|
||||
# Offset (bytes) from data_base to V-data region within the slot's data.
|
||||
# For MSE keys this equals MSE_BYTES; for FP8 keys it equals D.
|
||||
V_DATA_OFFSET: tl.constexpr,
|
||||
VQB: tl.constexpr,
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
BLOCK_VAL: tl.constexpr,
|
||||
BLOCK_GRP: tl.constexpr,
|
||||
):
|
||||
"""Uniform quantization of values to VQB bits, pack, and store with
|
||||
scale/zero in the per-block SoA metadata region (Opt#3 layout)."""
|
||||
if VQB == 3:
|
||||
val_vec = tl.load(Value_ptr + base + d_offs, mask=d_mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
val_min = tl.min(tl.where(d_mask, val_vec, float("inf")), axis=0)
|
||||
val_max = tl.max(tl.where(d_mask, val_vec, -float("inf")), axis=0)
|
||||
v_scale = (val_max - val_min) / 7.0
|
||||
v_scale = tl.where(v_scale > 1e-8, v_scale, 1e-8)
|
||||
|
||||
q_vals = tl.minimum(
|
||||
tl.maximum(((val_vec - val_min) / v_scale + 0.5).to(tl.int32), 0), 7
|
||||
)
|
||||
|
||||
grp_offs = tl.arange(0, BLOCK_GRP)
|
||||
grp_mask = grp_offs < (D // 8)
|
||||
q_grp = tl.reshape(q_vals, [BLOCK_GRP, 8])
|
||||
shifts_3bit = tl.arange(0, 8) * 3
|
||||
packed_24 = tl.sum(q_grp << shifts_3bit[None, :], axis=1)
|
||||
b0 = (packed_24 & 0xFF).to(tl.uint8)
|
||||
b1 = ((packed_24 >> 8) & 0xFF).to(tl.uint8)
|
||||
b2 = ((packed_24 >> 16) & 0xFF).to(tl.uint8)
|
||||
tl.store(
|
||||
KV_cache_ptr + data_base + V_DATA_OFFSET + grp_offs * 3,
|
||||
b0,
|
||||
mask=grp_mask,
|
||||
)
|
||||
tl.store(
|
||||
KV_cache_ptr + data_base + V_DATA_OFFSET + grp_offs * 3 + 1,
|
||||
b1,
|
||||
mask=grp_mask,
|
||||
)
|
||||
tl.store(
|
||||
KV_cache_ptr + data_base + V_DATA_OFFSET + grp_offs * 3 + 2,
|
||||
b2,
|
||||
mask=grp_mask,
|
||||
)
|
||||
else: # VQB == 4
|
||||
val_vec = tl.load(Value_ptr + base + d_offs, mask=d_mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
val_min = tl.min(tl.where(d_mask, val_vec, float("inf")), axis=0)
|
||||
val_max = tl.max(tl.where(d_mask, val_vec, -float("inf")), axis=0)
|
||||
v_scale = (val_max - val_min) / 15.0
|
||||
v_scale = tl.where(v_scale > 1e-8, v_scale, 1e-8)
|
||||
|
||||
# Quantize all D elements from register (no re-load)
|
||||
q_all = tl.minimum(
|
||||
tl.maximum(((val_vec - val_min) / v_scale + 0.5).to(tl.int32), 0), 15
|
||||
)
|
||||
q_pairs = tl.reshape(q_all, [BLOCK_D // 2, 2])
|
||||
shifts_4 = tl.arange(0, 2) * 4
|
||||
packed_val = tl.sum((q_pairs & 0xF) << shifts_4[None, :], axis=1).to(tl.uint8)
|
||||
val_offs = tl.arange(0, BLOCK_D // 2)
|
||||
val_mask = val_offs < VAL_DATA_BYTES
|
||||
tl.store(
|
||||
KV_cache_ptr + data_base + V_DATA_OFFSET + val_offs,
|
||||
packed_val,
|
||||
mask=val_mask,
|
||||
)
|
||||
|
||||
# ── SoA scale/zero store (Opt#3) ──────────────────────────────────
|
||||
# Scale and zero each get one fp16 stored at their token's slot index
|
||||
# within the per-head SoA region. Single u16 store per field vs the
|
||||
# previous 2×u8 writes per field.
|
||||
sc_u16 = v_scale.to(tl.float16).to(tl.uint16, bitcast=True)
|
||||
zr_u16 = val_min.to(tl.float16).to(tl.uint16, bitcast=True)
|
||||
tl.store(KV_cache_u16_ptr + vscale_u16_addr, sc_u16)
|
||||
tl.store(KV_cache_u16_ptr + vzero_u16_addr, zr_u16)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# FP8 key store + value uniform quantization
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_fused_store_fp8(
|
||||
Key_ptr, # [NH, D] float16/bfloat16 — raw keys
|
||||
Value_ptr, # [NH, D] float16/bfloat16 — raw values
|
||||
KV_cache_ptr, # [total_bytes] uint8 (flattened view)
|
||||
KV_cache_u16_ptr, # uint16-aliased view of the same storage (Opt#3)
|
||||
Slot_mapping_ptr, # [N] int32 — per-token slot indices
|
||||
# Cache strides (stride_cache_block = bytes per block = bs*H*slot_aligned)
|
||||
stride_cache_block: tl.constexpr,
|
||||
# Dimensions
|
||||
D: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
# SoA layout (Opt#3): data region is [bs, H, KEY_DATA_BYTES+VAL_DATA_BYTES],
|
||||
# followed by metadata region [H, NUM_SOA_FIELDS, bs] of fp16.
|
||||
KEY_DATA_BYTES: tl.constexpr, # FP8: = D
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
META_REGION_OFFSET: tl.constexpr, # bytes from block start to SoA region
|
||||
NUM_SOA_FIELDS: tl.constexpr, # FP8: 2 (V-scale, V-zero)
|
||||
SOA_V_SCALE: tl.constexpr, # FP8: 0
|
||||
SOA_V_ZERO: tl.constexpr, # FP8: 1
|
||||
# Value quantization
|
||||
VQB: tl.constexpr,
|
||||
# Packing block sizes
|
||||
BLOCK_VAL: tl.constexpr,
|
||||
BLOCK_GRP: tl.constexpr = 16,
|
||||
FP8_E4B15: tl.constexpr = 0, # 1 = e4b15 (Ampere/Ada), 0 = e4nv (Hopper+)
|
||||
):
|
||||
"""FP8 key cast+scatter + value uniform quantization (SoA layout)."""
|
||||
pid = tl.program_id(0)
|
||||
token_idx = pid // H
|
||||
head_idx = pid % H
|
||||
|
||||
slot = tl.load(Slot_mapping_ptr + token_idx)
|
||||
if slot < 0:
|
||||
return
|
||||
blk = (slot // BLOCK_SIZE).to(tl.int64)
|
||||
off = (slot % BLOCK_SIZE).to(tl.int64)
|
||||
head_idx_i64 = tl.cast(head_idx, tl.int64)
|
||||
|
||||
# Block start byte address — stride_cache_block is unchanged (same total
|
||||
# bytes per block as the AoS layout).
|
||||
block_base = blk * stride_cache_block
|
||||
|
||||
# Data region: per-slot stride = H * DATA_BYTES_PER_SLOT; per-head stride
|
||||
# within a slot = DATA_BYTES_PER_SLOT.
|
||||
DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES
|
||||
data_base = (
|
||||
block_base
|
||||
+ off * (H * DATA_BYTES_PER_SLOT)
|
||||
+ head_idx_i64 * DATA_BYTES_PER_SLOT
|
||||
)
|
||||
|
||||
# SoA metadata region (u16 element indexing): head strip = NUM_SOA_FIELDS
|
||||
# consecutive bs-long fp16 arrays.
|
||||
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + head_idx_i64 * (
|
||||
NUM_SOA_FIELDS * BLOCK_SIZE
|
||||
)
|
||||
vscale_u16_addr = head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + off
|
||||
vzero_u16_addr = head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + off
|
||||
|
||||
base = pid * D
|
||||
|
||||
# ── FP8 KEY: cast to FP8 in-kernel and store at data_base ────────
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < D
|
||||
k_vals = tl.load(Key_ptr + base + d_offs, mask=d_mask, other=0.0)
|
||||
k_fp8 = k_vals.to(tl.float8e4b15) if FP8_E4B15 else k_vals.to(tl.float8e4nv)
|
||||
k_bytes = k_fp8.to(tl.uint8, bitcast=True)
|
||||
tl.store(KV_cache_ptr + data_base + d_offs, k_bytes, mask=d_mask)
|
||||
|
||||
# ── VALUE QUANTIZE + PACK + SoA scale/zero store ─────────────────
|
||||
_store_quantized_value(
|
||||
Value_ptr,
|
||||
KV_cache_ptr,
|
||||
KV_cache_u16_ptr,
|
||||
base,
|
||||
data_base,
|
||||
vscale_u16_addr,
|
||||
vzero_u16_addr,
|
||||
d_offs,
|
||||
d_mask,
|
||||
D=D,
|
||||
V_DATA_OFFSET=KEY_DATA_BYTES, # V follows K in the data region
|
||||
VQB=VQB,
|
||||
VAL_DATA_BYTES=VAL_DATA_BYTES,
|
||||
BLOCK_D=BLOCK_D,
|
||||
BLOCK_VAL=BLOCK_VAL,
|
||||
BLOCK_GRP=BLOCK_GRP,
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Fused MSE store: bucketize + MSE index pack + norm store + value pack
|
||||
# (eliminates 4 PyTorch kernel launches per layer vs pack-only kernel)
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_fused_store_mse(
|
||||
# Post-rotation inputs
|
||||
Y_ptr, # [NH, D] float32 — rotated normalized keys (x_hat @ PiT)
|
||||
Norms_ptr, # [NH] float32 — key vector norms (||k||)
|
||||
Value_ptr, # [NH, D] float32 — raw values
|
||||
# Quantization tables
|
||||
Midpoints_ptr, # [n_centroids-1] float32
|
||||
Centroids_ptr, # [n_centroids] float32 — used only when NORM_CORRECTION=1
|
||||
# Cache and indexing
|
||||
KV_cache_ptr, # [total_bytes] uint8 (flattened view)
|
||||
KV_cache_u16_ptr, # uint16-aliased view of the same storage (Opt#3)
|
||||
Slot_mapping_ptr, # [N] int32 — per-token slot indices
|
||||
# Cache strides (stride_cache_block = bs*H*slot_aligned, unchanged)
|
||||
stride_cache_block: tl.constexpr,
|
||||
# Dimensions
|
||||
D: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
# SoA layout (Opt#3): data region [bs, H, MSE_BYTES+VAL_DATA_BYTES],
|
||||
# then metadata region [H, NUM_SOA_FIELDS, bs] of fp16.
|
||||
MSE_BYTES: tl.constexpr,
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
META_REGION_OFFSET: tl.constexpr, # bytes
|
||||
NUM_SOA_FIELDS: tl.constexpr, # MSE: 3 (k_norm, v_scale, v_zero)
|
||||
SOA_K_NORM: tl.constexpr, # MSE: 0
|
||||
SOA_V_SCALE: tl.constexpr, # MSE: 1
|
||||
SOA_V_ZERO: tl.constexpr, # MSE: 2
|
||||
# Value quantization
|
||||
VQB: tl.constexpr,
|
||||
# Packing block sizes
|
||||
BLOCK_VAL: tl.constexpr,
|
||||
# MSE params
|
||||
MSE_BITS: tl.constexpr,
|
||||
N_CENTROIDS: tl.constexpr,
|
||||
# When 1, fold the per-token centroid-vector norm ||c_t|| into the
|
||||
# stored scalar: store ||k_t|| / ||c_t|| instead of ||k_t||. This
|
||||
# removes the per-tile sum+sqrt+divide cost from the attention kernel's
|
||||
# K-load path. Load-side math becomes a plain multiply K = c * stored.
|
||||
NORM_CORRECTION: tl.constexpr = 0,
|
||||
BLOCK_GRP: tl.constexpr = 16,
|
||||
):
|
||||
"""Fused MSE quantize + pack + SoA-metadata store (Opt#3 layout).
|
||||
|
||||
Binary-search bucketize → MSE-index pack → K-norm store (SoA) → value
|
||||
uniform quantize + pack → V scale/zero store (SoA), in one kernel.
|
||||
"""
|
||||
pid = tl.program_id(0)
|
||||
token_idx = pid // H
|
||||
head_idx = pid % H
|
||||
|
||||
slot = tl.load(Slot_mapping_ptr + token_idx)
|
||||
if slot < 0:
|
||||
return
|
||||
blk = (slot // BLOCK_SIZE).to(tl.int64)
|
||||
off = (slot % BLOCK_SIZE).to(tl.int64)
|
||||
head_idx_i64 = tl.cast(head_idx, tl.int64)
|
||||
|
||||
# Block start byte address — stride unchanged from AoS (same total
|
||||
# bytes per block); layout inside the block differs.
|
||||
block_base = blk * stride_cache_block
|
||||
|
||||
# Data region offset (packed K indices + packed V data).
|
||||
DATA_BYTES_PER_SLOT: tl.constexpr = MSE_BYTES + VAL_DATA_BYTES
|
||||
data_base = (
|
||||
block_base
|
||||
+ off * (H * DATA_BYTES_PER_SLOT)
|
||||
+ head_idx_i64 * DATA_BYTES_PER_SLOT
|
||||
)
|
||||
|
||||
# SoA metadata addresses (u16 element indexing).
|
||||
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + head_idx_i64 * (
|
||||
NUM_SOA_FIELDS * BLOCK_SIZE
|
||||
)
|
||||
knorm_u16_addr = head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + off
|
||||
vscale_u16_addr = head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + off
|
||||
vzero_u16_addr = head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + off
|
||||
|
||||
base = pid * D
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < D
|
||||
|
||||
# ── 1. BINARY SEARCH BUCKETIZE ────────────────────────────────────
|
||||
y_vec = tl.load(Y_ptr + base + d_offs, mask=d_mask, other=0.0)
|
||||
lo = tl.zeros([BLOCK_D], dtype=tl.int32)
|
||||
hi = tl.full([BLOCK_D], N_CENTROIDS - 1, dtype=tl.int32)
|
||||
for _ in range(MSE_BITS):
|
||||
mid = (lo + hi) >> 1
|
||||
safe_mid = tl.minimum(mid, N_CENTROIDS - 2)
|
||||
mid_val = tl.load(Midpoints_ptr + safe_mid, mask=d_mask, other=0.0)
|
||||
lo = tl.where(y_vec >= mid_val, mid + 1, lo)
|
||||
hi = tl.where(y_vec >= mid_val, hi, mid)
|
||||
idx = tl.minimum(lo, N_CENTROIDS - 1)
|
||||
|
||||
# ── 2. PACK MSE INDICES into data region at data_base[0:MSE_BYTES] ──
|
||||
if MSE_BITS == 4:
|
||||
idx_pairs = tl.reshape(idx, [BLOCK_D // 2, 2])
|
||||
shifts_4 = tl.arange(0, 2) * 4
|
||||
packed = tl.sum((idx_pairs & 0xF) << shifts_4[None, :], axis=1).to(tl.uint8)
|
||||
mse_offs = tl.arange(0, BLOCK_D // 2)
|
||||
mse_mask = mse_offs < MSE_BYTES
|
||||
tl.store(KV_cache_ptr + data_base + mse_offs, packed, mask=mse_mask)
|
||||
|
||||
elif MSE_BITS == 3:
|
||||
grp_offs = tl.arange(0, BLOCK_GRP)
|
||||
grp_mask = grp_offs < (D // 8)
|
||||
idx_grp = tl.reshape(idx, [BLOCK_GRP, 8])
|
||||
shifts_3 = tl.arange(0, 8) * 3
|
||||
packed_24 = tl.sum((idx_grp & 0x7) << shifts_3[None, :], axis=1)
|
||||
b0 = (packed_24 & 0xFF).to(tl.uint8)
|
||||
b1 = ((packed_24 >> 8) & 0xFF).to(tl.uint8)
|
||||
b2 = ((packed_24 >> 16) & 0xFF).to(tl.uint8)
|
||||
tl.store(KV_cache_ptr + data_base + grp_offs * 3, b0, mask=grp_mask)
|
||||
tl.store(KV_cache_ptr + data_base + grp_offs * 3 + 1, b1, mask=grp_mask)
|
||||
tl.store(KV_cache_ptr + data_base + grp_offs * 3 + 2, b2, mask=grp_mask)
|
||||
|
||||
# ── 3. STORE K-norm in SoA region (single u16 store) ─────────────
|
||||
# Norm-correction (Opt#1) still folds 1/||c_vec|| into the stored value;
|
||||
# only the destination moved to the SoA K-norm array.
|
||||
vn_f32 = tl.load(Norms_ptr + pid) # ||k_t||, fp32
|
||||
|
||||
if NORM_CORRECTION:
|
||||
safe_idx = tl.minimum(tl.maximum(idx, 0), N_CENTROIDS - 1)
|
||||
c_vals = tl.load(Centroids_ptr + safe_idx, mask=d_mask, other=0.0)
|
||||
c_norm_sq = tl.sum(tl.where(d_mask, c_vals * c_vals, 0.0), axis=0)
|
||||
c_inv_norm = 1.0 / tl.sqrt(c_norm_sq + 1e-16)
|
||||
vn_f32 = vn_f32 * c_inv_norm
|
||||
|
||||
vn_u16 = vn_f32.to(tl.float16).to(tl.uint16, bitcast=True)
|
||||
tl.store(KV_cache_u16_ptr + knorm_u16_addr, vn_u16)
|
||||
|
||||
# ── 4. VALUE QUANTIZE + PACK + SoA scale/zero store ──────────────
|
||||
_store_quantized_value(
|
||||
Value_ptr,
|
||||
KV_cache_ptr,
|
||||
KV_cache_u16_ptr,
|
||||
base,
|
||||
data_base,
|
||||
vscale_u16_addr,
|
||||
vzero_u16_addr,
|
||||
d_offs,
|
||||
d_mask,
|
||||
D=D,
|
||||
V_DATA_OFFSET=MSE_BYTES, # V follows K indices in the data region
|
||||
VQB=VQB,
|
||||
VAL_DATA_BYTES=VAL_DATA_BYTES,
|
||||
BLOCK_D=BLOCK_D,
|
||||
BLOCK_VAL=BLOCK_VAL,
|
||||
BLOCK_GRP=BLOCK_GRP,
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Launcher
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def triton_turboquant_store(
|
||||
key: torch.Tensor, # [N, H, D] — raw keys (post-RoPE)
|
||||
value: torch.Tensor, # [N, H, D] — raw values
|
||||
kv_cache: torch.Tensor, # [num_blocks, block_size, Hk, padded_slot] uint8
|
||||
slot_mapping: torch.Tensor, # [N] int32
|
||||
PiT: torch.Tensor, # [D, D] float32
|
||||
midpoints: torch.Tensor, # [n_centroids-1] float32
|
||||
mse_bits: int,
|
||||
key_packed_size: int,
|
||||
value_quant_bits: int,
|
||||
key_fp8: bool = False,
|
||||
centroids: torch.Tensor
|
||||
| None = None, # [n_centroids] float32 — required when norm_correction=True
|
||||
norm_correction: bool = False,
|
||||
):
|
||||
"""Launch TQ store kernel (FP8 or MSE path).
|
||||
|
||||
When ``norm_correction=True`` (default off; set by MSE norm-correction
|
||||
presets), the centroid-vector norm ||c_t|| is folded into the stored
|
||||
per-token scalar at store time (stored value = ||k_t|| / ||c_t||). The
|
||||
load-side attention kernel no longer computes per-tile sum+sqrt+divide,
|
||||
saving ~26% of TQ overhead on gpt-oss decode (ctx=4096). ``centroids``
|
||||
must be provided in this mode.
|
||||
"""
|
||||
if norm_correction and not key_fp8:
|
||||
assert centroids is not None, (
|
||||
"norm_correction=True requires centroids tensor; got None. "
|
||||
"Pass the same [n_centroids] fp32 tensor used to derive midpoints."
|
||||
)
|
||||
N, H, D = key.shape
|
||||
NH = N * H
|
||||
block_size = kv_cache.shape[1]
|
||||
BLOCK_D = triton.next_power_of_2(D)
|
||||
mse_bytes = math.ceil(D * mse_bits / 8)
|
||||
n_centroids = 2**mse_bits
|
||||
|
||||
val_data_bytes = math.ceil(D * value_quant_bits / 8)
|
||||
|
||||
BLOCK_VAL = triton.next_power_of_2(val_data_bytes)
|
||||
|
||||
# Cache strides (element_size=1 for uint8, so stride in bytes = stride()).
|
||||
# Only stride_cache_block (== bs*H*slot_aligned) is used under Opt#3;
|
||||
# per-slot and per-head strides are computed from DATA_BYTES_PER_SLOT.
|
||||
stride_block = kv_cache.stride(0)
|
||||
|
||||
block_grp = triton.next_power_of_2(D // 8) if D >= 8 else 1
|
||||
|
||||
# ── Opt#3 SoA layout constants (derived locally so the launcher API
|
||||
# stays unchanged). Invariant: key_packed_size + value_packed_size
|
||||
# == slot_size_aligned == (key_data + val_data) + (k_meta + v_meta).
|
||||
key_data_bytes = D if key_fp8 else mse_bytes
|
||||
data_bytes_per_slot = key_data_bytes + val_data_bytes
|
||||
# Data region occupies [0, block_size * H * data_bytes_per_slot) bytes
|
||||
# at the start of each block; metadata region starts at that offset.
|
||||
meta_region_offset = block_size * H * data_bytes_per_slot
|
||||
num_soa_fields = 2 if key_fp8 else 3
|
||||
soa_v_scale = 0 if key_fp8 else 1
|
||||
soa_v_zero = 1 if key_fp8 else 2
|
||||
|
||||
# uint16-aliased view of the same storage — enables single-instruction
|
||||
# u16 writes for the SoA K-norm / V-scale / V-zero stores. The cache
|
||||
# bytes are always contiguous and 2-byte aligned under the Opt#3 layout
|
||||
# (MSE_BYTES, VAL_DATA_BYTES, meta_region_offset are all even).
|
||||
kv_cache_u16 = kv_cache.view(torch.uint16)
|
||||
|
||||
# ── FP8 PATH: in-kernel FP8 cast + scatter via fp8 kernel ──
|
||||
if key_fp8:
|
||||
k_flat = key.reshape(NH, D).contiguous()
|
||||
v_flat = value.reshape(NH, D).contiguous()
|
||||
|
||||
fp8_e4b15 = _use_fp8_e4b15(key.device.index or 0)
|
||||
|
||||
grid = (NH,)
|
||||
_tq_fused_store_fp8[grid](
|
||||
k_flat,
|
||||
v_flat,
|
||||
kv_cache.view(-1),
|
||||
kv_cache_u16.view(-1),
|
||||
slot_mapping,
|
||||
stride_cache_block=stride_block,
|
||||
D=D,
|
||||
H=H,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_D=BLOCK_D,
|
||||
KEY_DATA_BYTES=key_data_bytes,
|
||||
VAL_DATA_BYTES=val_data_bytes,
|
||||
META_REGION_OFFSET=meta_region_offset,
|
||||
NUM_SOA_FIELDS=num_soa_fields,
|
||||
SOA_V_SCALE=soa_v_scale,
|
||||
SOA_V_ZERO=soa_v_zero,
|
||||
VQB=value_quant_bits,
|
||||
BLOCK_VAL=BLOCK_VAL,
|
||||
BLOCK_GRP=block_grp,
|
||||
FP8_E4B15=fp8_e4b15,
|
||||
num_warps=4,
|
||||
num_stages=1,
|
||||
)
|
||||
return
|
||||
|
||||
# ── MSE PATH: external GEMM + fused bucketize/pack kernel ──
|
||||
# Normalize + rotation GEMM externally (cuBLAS is faster than in-kernel)
|
||||
k_flat = key.float().reshape(NH, D)
|
||||
norms = k_flat.norm(dim=1, keepdim=True)
|
||||
x_hat = k_flat / (norms + 1e-8)
|
||||
y = x_hat @ PiT
|
||||
|
||||
v_flat = value.float().reshape(NH, D)
|
||||
|
||||
# When norm_correction=True, the kernel folds 1/||c_t|| into the
|
||||
# stored per-token K-norm so the load kernel can skip per-tile norm
|
||||
# math (Opt#1). centroids is only dereferenced inside the kernel when
|
||||
# NORM_CORRECTION=1; pass midpoints as a harmless placeholder otherwise.
|
||||
centroids_ptr = centroids if centroids is not None else midpoints
|
||||
grid = (NH,)
|
||||
_tq_fused_store_mse[grid](
|
||||
y,
|
||||
norms.squeeze(1),
|
||||
v_flat,
|
||||
midpoints,
|
||||
centroids_ptr,
|
||||
kv_cache.view(-1),
|
||||
kv_cache_u16.view(-1),
|
||||
slot_mapping,
|
||||
stride_cache_block=stride_block,
|
||||
D=D,
|
||||
H=H,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_D=BLOCK_D,
|
||||
MSE_BYTES=mse_bytes,
|
||||
VAL_DATA_BYTES=val_data_bytes,
|
||||
META_REGION_OFFSET=meta_region_offset,
|
||||
NUM_SOA_FIELDS=num_soa_fields,
|
||||
SOA_K_NORM=0,
|
||||
SOA_V_SCALE=soa_v_scale,
|
||||
SOA_V_ZERO=soa_v_zero,
|
||||
VQB=value_quant_bits,
|
||||
BLOCK_VAL=BLOCK_VAL,
|
||||
MSE_BITS=mse_bits,
|
||||
N_CENTROIDS=n_centroids,
|
||||
NORM_CORRECTION=1 if norm_correction else 0,
|
||||
BLOCK_GRP=block_grp,
|
||||
num_warps=4,
|
||||
num_stages=1,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,19 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""UltraQuant 4-bit KV cache: FP4 E2M1 codes + UE8M0 group-of-32 scales.
|
||||
|
||||
Production decode uses the FlyDSL D=256 kernel on gfx950, with Triton
|
||||
unified attention as the fallback. Slot size is ``slot_size(head_dim)``
|
||||
(272 B at D=256). Format helpers live in ``format``; import them from
|
||||
there, not this package root.
|
||||
"""
|
||||
|
||||
__all__ = ["ultraquant_store"]
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name == "ultraquant_store":
|
||||
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
|
||||
|
||||
return ultraquant_store
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -1,218 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FP4 codepoint + UE8M0 scale constants for the UltraQuant KV cache.
|
||||
|
||||
K/V codes are FP4 E2M1. Per-group scales are UE8M0 (one byte, a power of
|
||||
two): ``s = 2^round(log2(c · absmax))`` with ``c = 0.156``. Decode
|
||||
consumes Q as FP8 E4M3 so scaled F8F6F4 MFMA can run natively on CDNA4.
|
||||
|
||||
AoS slot layout (group_size=32), per (token, head):
|
||||
bytes [0 .. D/2) : K codes (FP4 nibbles, 2/byte)
|
||||
bytes [D/2 .. D/2 + Gk) : K scales (UE8M0, 1 byte × Gk groups)
|
||||
bytes [D/2 + Gk .. D + Gk) : V codes
|
||||
bytes [D + Gk .. D + 2*Gk) : V scales
|
||||
D=256 → Gk=8 → 272 B/slot. D=128 → Gk=4 → 136 B/slot.
|
||||
|
||||
No per-token norm-fold. No V rotation. K is Hadamard-rotated at store.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
# ── 15 unique FP4 E2M1 levels, sorted ascending ────
|
||||
FP4_LEVELS_SORTED: tuple[float, ...] = (
|
||||
-6.0,
|
||||
-4.0,
|
||||
-3.0,
|
||||
-2.0,
|
||||
-1.5,
|
||||
-1.0,
|
||||
-0.5,
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
)
|
||||
assert len(FP4_LEVELS_SORTED) == 15
|
||||
|
||||
# Midpoints between consecutive sorted levels (14 boundaries).
|
||||
MIDPOINTS_SORTED: tuple[float, ...] = tuple(
|
||||
(FP4_LEVELS_SORTED[i] + FP4_LEVELS_SORTED[i + 1]) / 2.0
|
||||
for i in range(len(FP4_LEVELS_SORTED) - 1)
|
||||
)
|
||||
assert len(MIDPOINTS_SORTED) == 14
|
||||
|
||||
# FP4 E2M1 bit-pattern decode (16 entries indexed by 4-bit code).
|
||||
# Sign bit (MSB) | 2 exp bits | 1 mantissa bit (LSB).
|
||||
FP4_BITS_TO_VALUE: tuple[float, ...] = (
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
-0.0,
|
||||
-0.5,
|
||||
-1.0,
|
||||
-1.5,
|
||||
-2.0,
|
||||
-3.0,
|
||||
-4.0,
|
||||
-6.0,
|
||||
)
|
||||
assert len(FP4_BITS_TO_VALUE) == 16
|
||||
|
||||
# Sorted index → FP4 E2M1 bit pattern (uint4).
|
||||
SORTED_TO_BITS: tuple[int, ...] = (
|
||||
0b1111,
|
||||
0b1110,
|
||||
0b1101,
|
||||
0b1100,
|
||||
0b1011,
|
||||
0b1010,
|
||||
0b1001,
|
||||
0b0000,
|
||||
0b0001,
|
||||
0b0010,
|
||||
0b0011,
|
||||
0b0100,
|
||||
0b0101,
|
||||
0b0110,
|
||||
0b0111,
|
||||
)
|
||||
assert len(SORTED_TO_BITS) == 15
|
||||
for _i, _bits in enumerate(SORTED_TO_BITS):
|
||||
_v = FP4_BITS_TO_VALUE[_bits]
|
||||
_v = 0.0 if _v == 0.0 else _v
|
||||
assert _v == FP4_LEVELS_SORTED[_i]
|
||||
|
||||
# ── Format constants ──────────────────────────────────────────────────────
|
||||
GROUP_SIZE: int = 32
|
||||
"""Elements per group; one E8M0 scale per group. Fixed at 32 because the
|
||||
AMD scaled F8F6F4 MFMA instruction also consumes one E8M0 scale per 32
|
||||
elements along K. Other group sizes would force a fallback to plain MFMA
|
||||
+ accumulator-side scale multiply (no hardware fast-path)."""
|
||||
|
||||
FP4_MAX: float = 6.0
|
||||
"""Largest representable FP4 magnitude."""
|
||||
|
||||
DEFAULT_CONSTANT_C: float = 0.156
|
||||
"""MSE-optimal scale constant; ``s_raw = c * absmax``, then UE8M0-snapped."""
|
||||
|
||||
|
||||
def get_constant_c() -> float:
|
||||
"""Return the fixed UltraQuant scale constant."""
|
||||
return DEFAULT_CONSTANT_C
|
||||
|
||||
|
||||
def get_group_size() -> int:
|
||||
"""Return the fixed group size required by scaled MFMA."""
|
||||
return GROUP_SIZE
|
||||
|
||||
|
||||
# ── Scale encoding ────────────────────────────────────────────────────────
|
||||
# Per-group scale `s = pow2_round(c · absmax)` stored as the E8M0 byte;
|
||||
# codebook is the raw FP4 grid {0, ±0.5, ..., ±6}; dequant = code · s.
|
||||
|
||||
|
||||
# ── UE8M0 helpers ──────────────────────────────────────────────────────────
|
||||
# E8M0 = 8 exponent bits, 0 mantissa bits, no sign. The byte value `e`
|
||||
# represents `2^(e - 127)` for e in [1, 254]. e=0 and e=255 are reserved
|
||||
# (zero and NaN respectively in the OCP spec). We use e=0 as our zero
|
||||
# sentinel for sink-zero groups.
|
||||
|
||||
UE8M0_BIAS: int = 127
|
||||
|
||||
# fp32 minimum positive normal = 2^-126 ≈ 1.175e-38. Anything smaller
|
||||
# clamps to the zero sentinel.
|
||||
_UE8M0_MIN_EXP: int = -126
|
||||
_UE8M0_MAX_EXP: int = 127
|
||||
|
||||
|
||||
def ue8m0_encode(s: float) -> int:
|
||||
"""Snap a positive fp32 scale `s` to the nearest power of 2 and
|
||||
encode as a UE8M0 byte. `s <= 0` encodes as 0 (zero sentinel)."""
|
||||
if s <= 0.0 or not math.isfinite(s):
|
||||
return 0
|
||||
exp = int(round(math.log2(s)))
|
||||
if exp < _UE8M0_MIN_EXP:
|
||||
return 0
|
||||
if exp > _UE8M0_MAX_EXP:
|
||||
exp = _UE8M0_MAX_EXP
|
||||
return exp + UE8M0_BIAS
|
||||
|
||||
|
||||
def ue8m0_decode(byte: int) -> float:
|
||||
"""Decode a UE8M0 byte back to fp32. 0 → 0.0 (zero sentinel)."""
|
||||
if byte == 0:
|
||||
return 0.0
|
||||
return 2.0 ** (byte - UE8M0_BIAS)
|
||||
|
||||
|
||||
# ── Per-slot AoS byte layout ──────────────────────────────────────────────
|
||||
# Per slot, per head (group_size=GS=32, Gk = D/GS):
|
||||
# bytes [0 .. D/2) : K codes (2 nibbles/byte)
|
||||
# bytes [D/2 .. D/2 + Gk) : K scales (E8M0, 1 byte each)
|
||||
# bytes [D/2 + Gk .. D + Gk) : V codes
|
||||
# bytes [D + Gk .. D + 2*Gk) : V scales (E8M0, 1 byte each)
|
||||
# D=256 → 272 B/slot. D=128 → 136 B/slot.
|
||||
|
||||
|
||||
def k_codes_bytes(head_dim: int) -> int:
|
||||
"""Bytes for one head's packed K codes."""
|
||||
if head_dim % 2 != 0:
|
||||
raise ValueError(f"head_dim must be even, got {head_dim}")
|
||||
return head_dim // 2
|
||||
|
||||
|
||||
def n_groups(head_dim: int, group_size: int | None = None) -> int:
|
||||
gs = group_size if group_size is not None else get_group_size()
|
||||
if head_dim % gs != 0:
|
||||
raise ValueError(f"head_dim={head_dim} must be a multiple of group_size={gs}")
|
||||
return head_dim // gs
|
||||
|
||||
|
||||
def k_scales_bytes(head_dim: int, group_size: int | None = None) -> int:
|
||||
"""Bytes for one head's K scales (one E8M0 byte per group)."""
|
||||
return n_groups(head_dim, group_size)
|
||||
|
||||
|
||||
def v_codes_bytes(head_dim: int) -> int:
|
||||
return k_codes_bytes(head_dim)
|
||||
|
||||
|
||||
def v_scales_bytes(head_dim: int, group_size: int | None = None) -> int:
|
||||
return k_scales_bytes(head_dim, group_size)
|
||||
|
||||
|
||||
def slot_size(head_dim: int, group_size: int | None = None) -> int:
|
||||
"""Bytes per (token, head) slot."""
|
||||
return (
|
||||
k_codes_bytes(head_dim)
|
||||
+ k_scales_bytes(head_dim, group_size)
|
||||
+ v_codes_bytes(head_dim)
|
||||
+ v_scales_bytes(head_dim, group_size)
|
||||
)
|
||||
|
||||
|
||||
def k_codes_offset(head_dim: int, group_size: int | None = None) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def k_scales_offset(head_dim: int, group_size: int | None = None) -> int:
|
||||
return k_codes_bytes(head_dim)
|
||||
|
||||
|
||||
def v_codes_offset(head_dim: int, group_size: int | None = None) -> int:
|
||||
return k_codes_bytes(head_dim) + k_scales_bytes(head_dim, group_size)
|
||||
|
||||
|
||||
def v_scales_offset(head_dim: int, group_size: int | None = None) -> int:
|
||||
return v_codes_offset(head_dim, group_size) + v_codes_bytes(head_dim)
|
||||
@@ -1,349 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""PyTorch reference for the UltraQuant KV cache format.
|
||||
|
||||
Scale is UE8M0 (power of two): after ``s = c * absmax`` snap
|
||||
``s = 2^round(log2(s))`` and store one byte. Q is Hadamard-rotated then
|
||||
cast to FP8 E4M3 before the QK matmul, matching the kernel launcher.
|
||||
|
||||
Same FP4 E2M1 grid. ``c = 0.156``. No per-token L2 norm. No V rotation.
|
||||
K is rotated at encode; Q arrives pre-rotated at decode.
|
||||
|
||||
This module is the bit-comparison ground truth for the kernels.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.ops.ultraquant.format import (
|
||||
FP4_LEVELS_SORTED,
|
||||
GROUP_SIZE,
|
||||
MIDPOINTS_SORTED,
|
||||
SORTED_TO_BITS,
|
||||
UE8M0_BIAS,
|
||||
get_constant_c,
|
||||
)
|
||||
|
||||
_LEVELS_CACHE: dict[tuple[torch.device, torch.dtype], torch.Tensor] = {}
|
||||
_MIDPOINTS_CACHE: dict[tuple[torch.device, torch.dtype], torch.Tensor] = {}
|
||||
_SORTED_TO_BITS_CACHE: dict[torch.device, torch.Tensor] = {}
|
||||
|
||||
|
||||
def _get_levels(device: torch.device, dtype: torch.dtype) -> torch.Tensor:
|
||||
key = (device, dtype)
|
||||
t = _LEVELS_CACHE.get(key)
|
||||
if t is None:
|
||||
t = torch.tensor(FP4_LEVELS_SORTED, device=device, dtype=dtype)
|
||||
_LEVELS_CACHE[key] = t
|
||||
return t
|
||||
|
||||
|
||||
def _get_midpoints(device: torch.device, dtype: torch.dtype) -> torch.Tensor:
|
||||
key = (device, dtype)
|
||||
t = _MIDPOINTS_CACHE.get(key)
|
||||
if t is None:
|
||||
t = torch.tensor(MIDPOINTS_SORTED, device=device, dtype=dtype)
|
||||
_MIDPOINTS_CACHE[key] = t
|
||||
return t
|
||||
|
||||
|
||||
def _get_sorted_to_bits(device: torch.device) -> torch.Tensor:
|
||||
t = _SORTED_TO_BITS_CACHE.get(device)
|
||||
if t is None:
|
||||
t = torch.tensor(SORTED_TO_BITS, device=device, dtype=torch.uint8)
|
||||
_SORTED_TO_BITS_CACHE[device] = t
|
||||
return t
|
||||
|
||||
|
||||
_HADAMARD_CACHE: dict[tuple[int, torch.device, torch.dtype], torch.Tensor] = {}
|
||||
|
||||
|
||||
def hadamard_matrix(
|
||||
dim: int, device: torch.device, dtype: torch.dtype = torch.float32
|
||||
) -> torch.Tensor:
|
||||
"""Sylvester Hadamard, normalised so H @ H.T = I."""
|
||||
if dim <= 0 or (dim & (dim - 1)) != 0:
|
||||
raise ValueError(f"hadamard_matrix requires power-of-two dim, got {dim}")
|
||||
key = (dim, device, dtype)
|
||||
cached = _HADAMARD_CACHE.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
H = torch.tensor([[1.0]], dtype=torch.float64)
|
||||
while H.shape[0] < dim:
|
||||
H = torch.cat([torch.cat([H, H], dim=1), torch.cat([H, -H], dim=1)], dim=0)
|
||||
H = (H / (dim**0.5)).to(device=device, dtype=dtype).contiguous()
|
||||
_HADAMARD_CACHE[key] = H
|
||||
return H
|
||||
|
||||
|
||||
# ── Core quantization primitives ───────────────────────────────────────────
|
||||
def _snap_to_sorted_idx(
|
||||
x_norm: torch.Tensor, *, dtype: torch.dtype = torch.float32
|
||||
) -> torch.Tensor:
|
||||
"""Snap normalised values to nearest FP4 level, returning sorted index
|
||||
in [0, 14]. Same semantic as `torch.bucketize` against midpoints
|
||||
(reproduced by 14 `tl.where` in the kernel)."""
|
||||
boundaries = _get_midpoints(x_norm.device, dtype)
|
||||
x = x_norm.to(dtype)
|
||||
# ROCm's bucketize uses int32 element indexing and raises
|
||||
# hipErrorInvalidConfiguration once numel >= 2**31 (e.g. a [B, Hk, N, D]
|
||||
# K tensor at B=128, N=16384, Hk=8, D=128 is exactly 2**31). Chunk the
|
||||
# flattened input so each call stays under the limit (bit-identical output).
|
||||
_INT32_LIMIT = 2**31
|
||||
if x.numel() < _INT32_LIMIT:
|
||||
return torch.bucketize(x, boundaries)
|
||||
flat = x.reshape(-1)
|
||||
out = torch.empty_like(flat, dtype=torch.long)
|
||||
chunk = _INT32_LIMIT // 2
|
||||
for i in range(0, flat.numel(), chunk):
|
||||
out[i : i + chunk] = torch.bucketize(flat[i : i + chunk], boundaries)
|
||||
return out.reshape(x.shape)
|
||||
|
||||
|
||||
def _pack_nibbles_last_dim(codes: torch.Tensor) -> torch.Tensor:
|
||||
if codes.shape[-1] % 2 != 0:
|
||||
raise ValueError(f"pack_nibbles last dim must be even, got {codes.shape[-1]}")
|
||||
lo = codes[..., 0::2] & 0xF
|
||||
hi = codes[..., 1::2] & 0xF
|
||||
return (lo | (hi << 4)).to(torch.uint8)
|
||||
|
||||
|
||||
def _unpack_nibbles_last_dim(packed: torch.Tensor, out_dim: int) -> torch.Tensor:
|
||||
if out_dim != 2 * packed.shape[-1]:
|
||||
packed_dim = packed.shape[-1]
|
||||
raise ValueError(
|
||||
f"unpack_nibbles: out_dim={out_dim} must be 2*packed.last={2 * packed_dim}"
|
||||
)
|
||||
p = packed.to(torch.int32)
|
||||
lo = (p & 0xF).to(torch.uint8)
|
||||
hi = ((p >> 4) & 0xF).to(torch.uint8)
|
||||
out_shape = packed.shape[:-1] + (out_dim,)
|
||||
out = torch.empty(out_shape, dtype=torch.uint8, device=packed.device)
|
||||
out[..., 0::2] = lo
|
||||
out[..., 1::2] = hi
|
||||
return out
|
||||
|
||||
|
||||
def _ue8m0_snap_tensor(
|
||||
s_raw: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Snap a positive-scale tensor `s_raw` to a power of two and return
|
||||
``(s_snapped_fp32, byte_uint8)``.
|
||||
|
||||
Matches the kernel: zero / non-finite inputs map to byte=0 / value=0.0.
|
||||
Round-half-up via ``floor(log2(s) + 0.5)``, matching Triton's
|
||||
``tl.floor(... + 0.5)`` idiom.
|
||||
"""
|
||||
is_zero = (s_raw <= 0) | ~torch.isfinite(s_raw)
|
||||
s_safe = torch.where(is_zero, torch.ones_like(s_raw), s_raw)
|
||||
log2_s = torch.log2(s_safe)
|
||||
exp = torch.floor(log2_s + 0.5).to(torch.int32)
|
||||
# Clamp to UE8M0's representable exponent range (signed pre-bias).
|
||||
exp = torch.clamp(exp, min=-126, max=127)
|
||||
byte = (exp + UE8M0_BIAS).to(torch.int32)
|
||||
byte = torch.where(is_zero, torch.zeros_like(byte), byte).to(torch.uint8)
|
||||
s_snapped = torch.where(
|
||||
is_zero, torch.zeros_like(s_raw), torch.exp2(exp.to(torch.float32))
|
||||
)
|
||||
return s_snapped, byte
|
||||
|
||||
|
||||
@dataclass
|
||||
class UltraQuantEncoded:
|
||||
"""Encoded ultraquant representation of one tensor along its last axis.
|
||||
|
||||
Shapes assume input shape `[..., D]`:
|
||||
- codes_packed: `[..., D // 2]` uint8 — 2 FP4 nibbles per byte
|
||||
- scale_bytes: `[..., D // GROUP_SIZE]` uint8 — E8M0 byte per group
|
||||
Dequant is ``code · 2^(byte - 127)`` (`c` is folded into the byte).
|
||||
"""
|
||||
|
||||
codes_packed: torch.Tensor # uint8, [..., D//2]
|
||||
scale_bytes: torch.Tensor # uint8, [..., D//GROUP_SIZE]
|
||||
|
||||
@property
|
||||
def scales_fp32(self) -> torch.Tensor:
|
||||
"""Decode the E8M0 bytes back to fp32 (for dequant / inspection)."""
|
||||
b = self.scale_bytes.to(torch.int32)
|
||||
exp = b - UE8M0_BIAS
|
||||
is_zero = b == 0
|
||||
s = torch.where(
|
||||
is_zero,
|
||||
torch.zeros_like(b, dtype=torch.float32),
|
||||
torch.exp2(exp.to(torch.float32)),
|
||||
)
|
||||
return s
|
||||
|
||||
|
||||
def ultraquant_encode(
|
||||
x: torch.Tensor,
|
||||
*,
|
||||
rotate: bool = True,
|
||||
constant_c: float | None = None,
|
||||
) -> UltraQuantEncoded:
|
||||
"""Encode `x` (shape [..., D]) to FP4 codes + E8M0 per-group-of-32 scales.
|
||||
|
||||
Steps:
|
||||
1. Optional Hadamard rotation: `x_rot = x @ H.T`
|
||||
2. Reshape to `[..., G, GROUP_SIZE]`.
|
||||
3. `absmax = max(|x_rot|)` per group.
|
||||
4. `s_raw = c · absmax`; `s_snapped = 2^round(log2(s_raw))`.
|
||||
Zero-amax groups encode byte=0.
|
||||
5. `sorted_idx = bucketize(x_g / s_snapped, midpoints)` ∈ [0, 14].
|
||||
6. Remap sorted_idx → FP4 E2M1 bit pattern via SORTED_TO_BITS.
|
||||
7. Pack pairs of 4-bit codes into bytes.
|
||||
"""
|
||||
if x.shape[-1] % GROUP_SIZE != 0:
|
||||
raise ValueError(
|
||||
f"ultraquant_encode: last dim {x.shape[-1]} must be a multiple of "
|
||||
f"GROUP_SIZE={GROUP_SIZE}"
|
||||
)
|
||||
D = x.shape[-1]
|
||||
G = D // GROUP_SIZE
|
||||
c = constant_c if constant_c is not None else get_constant_c()
|
||||
|
||||
x_f32 = x.to(torch.float32)
|
||||
if rotate:
|
||||
H = hadamard_matrix(D, x.device, torch.float32)
|
||||
x_rot = x_f32 @ H.T
|
||||
else:
|
||||
x_rot = x_f32
|
||||
|
||||
x_g = x_rot.reshape(*x.shape[:-1], G, GROUP_SIZE)
|
||||
absmax = x_g.abs().amax(dim=-1, keepdim=True) # [..., G, 1]
|
||||
s_raw = absmax * c
|
||||
s_snapped, scale_bytes = _ue8m0_snap_tensor(s_raw.squeeze(-1))
|
||||
s_snapped = s_snapped.unsqueeze(-1) # [..., G, 1]
|
||||
s_for_div = torch.where(s_snapped == 0, torch.ones_like(s_snapped), s_snapped)
|
||||
|
||||
sorted_idx = _snap_to_sorted_idx(x_g / s_for_div)
|
||||
sorted_idx = torch.where(
|
||||
(s_snapped == 0).expand_as(sorted_idx),
|
||||
torch.full_like(sorted_idx, 7), # sorted idx 7 → +0.0
|
||||
sorted_idx,
|
||||
)
|
||||
|
||||
sorted_to_bits = _get_sorted_to_bits(x.device)
|
||||
fp4_bits = sorted_to_bits[sorted_idx] # [..., G, GS]
|
||||
fp4_bits_flat = fp4_bits.reshape(*x.shape[:-1], D)
|
||||
codes_packed = _pack_nibbles_last_dim(fp4_bits_flat) # [..., D//2]
|
||||
|
||||
return UltraQuantEncoded(
|
||||
codes_packed=codes_packed,
|
||||
scale_bytes=scale_bytes,
|
||||
)
|
||||
|
||||
|
||||
def ultraquant_dequant(
|
||||
encoded: UltraQuantEncoded,
|
||||
head_dim: int,
|
||||
) -> torch.Tensor:
|
||||
"""Decode `encoded` to fp32 (no inverse rotation — result in the
|
||||
rotated basis if rotation was applied at encode).
|
||||
|
||||
Dequant is ``code · 2^(byte - 127)`` (`c` is already folded into the byte).
|
||||
"""
|
||||
G = head_dim // GROUP_SIZE
|
||||
codes = _unpack_nibbles_last_dim(encoded.codes_packed, head_dim)
|
||||
bits_to_val = torch.tensor(
|
||||
[
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
-0.0,
|
||||
-0.5,
|
||||
-1.0,
|
||||
-1.5,
|
||||
-2.0,
|
||||
-3.0,
|
||||
-4.0,
|
||||
-6.0,
|
||||
],
|
||||
device=encoded.codes_packed.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
decoded = bits_to_val[codes.to(torch.long)] # [..., D] fp32
|
||||
decoded_g = decoded.reshape(*decoded.shape[:-1], G, GROUP_SIZE)
|
||||
scales = encoded.scales_fp32.unsqueeze(-1) # [..., G, 1]
|
||||
return (decoded_g * scales).reshape(*decoded.shape[:-1], head_dim)
|
||||
|
||||
|
||||
def ultraquant_encode_decode(
|
||||
x: torch.Tensor,
|
||||
*,
|
||||
rotate: bool = True,
|
||||
constant_c: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Full encode→decode round-trip in the rotated basis. Returns fp32."""
|
||||
enc = ultraquant_encode(
|
||||
x,
|
||||
rotate=rotate,
|
||||
constant_c=constant_c,
|
||||
)
|
||||
return ultraquant_dequant(enc, x.shape[-1])
|
||||
|
||||
|
||||
# ── Reference attention against the ultraquant encoded cache ──────────────────
|
||||
def reference_ultraquant_attention(
|
||||
query: torch.Tensor, # [B, Hq, D] bf16 — RAW (will be rotated + cast to FP8 here)
|
||||
key: torch.Tensor, # [B, N, Hk, D] bf16
|
||||
value: torch.Tensor, # [B, N, Hk, D] bf16
|
||||
*,
|
||||
scale: float,
|
||||
sinks: torch.Tensor | None = None,
|
||||
constant_c: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""End-to-end reference attention with ultraquant K/V encode→decode in
|
||||
the rotated basis. Q is Hadamard-rotated AND cast through FP8 E4M3
|
||||
(scale = 1) before the QK matmul — mirrors what the launcher does.
|
||||
|
||||
Output shape: [B, Hq, D] in query.dtype.
|
||||
"""
|
||||
B, Hq, D = query.shape
|
||||
B2, N, Hk, D2 = key.shape
|
||||
assert B == B2 and D == D2, (B, B2, D, D2)
|
||||
assert value.shape == key.shape
|
||||
assert Hq % Hk == 0, f"Hq={Hq} must be a multiple of Hk={Hk}"
|
||||
g = Hq // Hk
|
||||
|
||||
H = hadamard_matrix(D, query.device, torch.float32)
|
||||
q_rot = query.to(torch.float32) @ H.T
|
||||
# Round-trip Q through FP8 E4M3 with scale=1 — same precision haircut
|
||||
# as the launcher (`q_rot.to(torch.float8_e4m3fn)`).
|
||||
q_fp8 = q_rot.to(torch.float8_e4m3fn)
|
||||
q_for_matmul = q_fp8.to(torch.float32)
|
||||
|
||||
k_recon_rot = ultraquant_encode_decode(
|
||||
key.transpose(1, 2).contiguous(),
|
||||
rotate=True,
|
||||
constant_c=constant_c,
|
||||
)
|
||||
v_recon = ultraquant_encode_decode(
|
||||
value.transpose(1, 2).contiguous(),
|
||||
rotate=False,
|
||||
constant_c=constant_c,
|
||||
)
|
||||
|
||||
k_recon_rot = k_recon_rot.unsqueeze(2).expand(B, Hk, g, N, D).reshape(B, Hq, N, D)
|
||||
v_recon = v_recon.unsqueeze(2).expand(B, Hk, g, N, D).reshape(B, Hq, N, D)
|
||||
|
||||
qk = torch.einsum("bhd,bhnd->bhn", q_for_matmul, k_recon_rot) * scale # [B, Hq, N]
|
||||
|
||||
if sinks is not None:
|
||||
sink_log = sinks.to(torch.float32).reshape(1, Hq, 1).expand(B, Hq, 1)
|
||||
qk_full = torch.cat([qk, sink_log], dim=-1)
|
||||
p_full = torch.softmax(qk_full, dim=-1)
|
||||
p = p_full[..., :N]
|
||||
else:
|
||||
p = torch.softmax(qk, dim=-1)
|
||||
|
||||
out = torch.einsum("bhn,bhnd->bhd", p, v_recon)
|
||||
return out.to(query.dtype)
|
||||
@@ -1,190 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Full KV dequant for the UltraQuant cache format.
|
||||
|
||||
Used by continuation prefill: when a chunk brings many new query tokens on
|
||||
top of a long cached prefix, dequanting the prefix once and running a dense
|
||||
prefill kernel beats replaying the decode kernel per query token.
|
||||
|
||||
Reads FP4 codes plus UE8M0 group scales and writes K (Hadamard-rotated, as
|
||||
stored) and V (raw) into pre-allocated fp16/bf16 buffers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.ops.ultraquant.format import (
|
||||
FP4_BITS_TO_VALUE,
|
||||
UE8M0_BIAS,
|
||||
get_group_size,
|
||||
k_scales_offset,
|
||||
n_groups,
|
||||
v_codes_offset,
|
||||
v_scales_offset,
|
||||
)
|
||||
from vllm.v1.attention.ops.ultraquant.triton_store import _kv_cache_flat
|
||||
|
||||
_FP4_DECODE_CACHE: dict[tuple[torch.device, torch.dtype], torch.Tensor] = {}
|
||||
|
||||
|
||||
def _get_fp4_decode_table(device: torch.device, dtype: torch.dtype) -> torch.Tensor:
|
||||
"""16-entry FP4 E2M1 bit-pattern -> value table (cached per device)."""
|
||||
key = (device, dtype)
|
||||
t = _FP4_DECODE_CACHE.get(key)
|
||||
if t is None:
|
||||
t = torch.tensor(FP4_BITS_TO_VALUE, device=device, dtype=dtype).contiguous()
|
||||
_FP4_DECODE_CACHE[key] = t
|
||||
return t
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _ultraquant_full_dequant_kv(
|
||||
KV_cache_ptr, # uint8 view, flat
|
||||
Block_table_ptr, # [B, max_num_blocks] int32
|
||||
Fp4_decode_ptr, # [16] FP4 bit pattern -> value
|
||||
K_out_ptr,
|
||||
V_out_ptr,
|
||||
stride_ko_b: tl.int64,
|
||||
stride_ko_h: tl.int64,
|
||||
stride_ko_s: tl.int64,
|
||||
stride_vo_b: tl.int64,
|
||||
stride_vo_h: tl.int64,
|
||||
stride_vo_s: tl.int64,
|
||||
stride_cache_block: tl.int64,
|
||||
stride_cache_pos: tl.int64,
|
||||
stride_cache_head: tl.int64,
|
||||
stride_bt_b: tl.int64,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
K_SCALES_OFFSET: tl.constexpr,
|
||||
V_CODES_OFFSET: tl.constexpr,
|
||||
V_SCALES_OFFSET: tl.constexpr,
|
||||
GROUP_SIZE_C: tl.constexpr,
|
||||
N_GROUPS_C: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
OUT_BF16: tl.constexpr,
|
||||
UE8M0_BIAS_C: tl.constexpr,
|
||||
):
|
||||
pos = tl.program_id(0)
|
||||
bh = tl.program_id(1)
|
||||
bid = bh // NUM_KV_HEADS
|
||||
hid = bh % NUM_KV_HEADS
|
||||
|
||||
page_idx = pos // BLOCK_SIZE
|
||||
page_off = pos % BLOCK_SIZE
|
||||
block_num = tl.load(Block_table_ptr + bid * stride_bt_b + page_idx).to(tl.int64)
|
||||
slot_base = (
|
||||
block_num * stride_cache_block
|
||||
+ page_off * stride_cache_pos
|
||||
+ tl.cast(hid, tl.int64) * stride_cache_head
|
||||
)
|
||||
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < HEAD_DIM
|
||||
byte_idx = (d_offs // 2).to(tl.int32)
|
||||
nibble_shift = ((d_offs % 2) * 4).to(tl.int32)
|
||||
g_offs = tl.arange(0, N_GROUPS_C)
|
||||
|
||||
out_dtype = tl.bfloat16 if OUT_BF16 else tl.float16
|
||||
|
||||
# K side: codes are packed nibbles, one UE8M0 byte per group of 32.
|
||||
k_byte_raw = tl.load(KV_cache_ptr + slot_base + byte_idx, mask=d_mask, other=0).to(
|
||||
tl.int32
|
||||
)
|
||||
k_codes = (k_byte_raw >> nibble_shift) & 0xF
|
||||
k_dec = tl.load(Fp4_decode_ptr + k_codes).to(tl.float32)
|
||||
k_dec = tl.where(d_mask, k_dec, 0.0)
|
||||
|
||||
# UE8M0 scale: value = 2^(byte - bias); byte == 0 encodes exact zero.
|
||||
k_scale_bytes = tl.load(KV_cache_ptr + slot_base + K_SCALES_OFFSET + g_offs).to(
|
||||
tl.int32
|
||||
)
|
||||
k_scales = tl.where(
|
||||
k_scale_bytes == 0,
|
||||
0.0,
|
||||
tl.exp2(tl.cast(k_scale_bytes - UE8M0_BIAS_C, tl.float32)),
|
||||
)
|
||||
|
||||
k_g = tl.reshape(k_dec, [N_GROUPS_C, GROUP_SIZE_C])
|
||||
k_recon = tl.reshape(k_g * k_scales[:, None], [BLOCK_D])
|
||||
k_recon = tl.where(d_mask, k_recon, 0.0)
|
||||
|
||||
ko_base = bid * stride_ko_b + hid * stride_ko_h + pos * stride_ko_s
|
||||
tl.store(K_out_ptr + ko_base + d_offs, k_recon.to(out_dtype), mask=d_mask)
|
||||
|
||||
# V side mirrors K but is stored unrotated.
|
||||
v_byte_raw = tl.load(
|
||||
KV_cache_ptr + slot_base + V_CODES_OFFSET + byte_idx,
|
||||
mask=d_mask,
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
v_codes = (v_byte_raw >> nibble_shift) & 0xF
|
||||
v_dec = tl.load(Fp4_decode_ptr + v_codes).to(tl.float32)
|
||||
v_dec = tl.where(d_mask, v_dec, 0.0)
|
||||
|
||||
v_scale_bytes = tl.load(KV_cache_ptr + slot_base + V_SCALES_OFFSET + g_offs).to(
|
||||
tl.int32
|
||||
)
|
||||
v_scales = tl.where(
|
||||
v_scale_bytes == 0,
|
||||
0.0,
|
||||
tl.exp2(tl.cast(v_scale_bytes - UE8M0_BIAS_C, tl.float32)),
|
||||
)
|
||||
|
||||
v_g = tl.reshape(v_dec, [N_GROUPS_C, GROUP_SIZE_C])
|
||||
v_recon = tl.reshape(v_g * v_scales[:, None], [BLOCK_D])
|
||||
v_recon = tl.where(d_mask, v_recon, 0.0)
|
||||
|
||||
vo_base = bid * stride_vo_b + hid * stride_vo_h + pos * stride_vo_s
|
||||
tl.store(V_out_ptr + vo_base + d_offs, v_recon.to(out_dtype), mask=d_mask)
|
||||
|
||||
|
||||
def ultraquant_full_dequant_kv(
|
||||
kv_cache: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
k_out: torch.Tensor,
|
||||
v_out: torch.Tensor,
|
||||
alloc_len: int,
|
||||
) -> None:
|
||||
"""Dequant ``alloc_len`` cached positions into ``k_out`` / ``v_out``.
|
||||
|
||||
``k_out`` / ``v_out`` are ``[B, Hk, alloc_len, D]`` in fp16 or bf16.
|
||||
"""
|
||||
B = block_table.shape[0]
|
||||
Hk = kv_cache.shape[2]
|
||||
D = k_out.shape[3]
|
||||
block_size = kv_cache.shape[1]
|
||||
|
||||
gs = get_group_size()
|
||||
_ultraquant_full_dequant_kv[(alloc_len, B * Hk)](
|
||||
_kv_cache_flat(kv_cache),
|
||||
block_table,
|
||||
_get_fp4_decode_table(kv_cache.device, k_out.dtype),
|
||||
k_out,
|
||||
v_out,
|
||||
k_out.stride(0),
|
||||
k_out.stride(1),
|
||||
k_out.stride(2),
|
||||
v_out.stride(0),
|
||||
v_out.stride(1),
|
||||
v_out.stride(2),
|
||||
kv_cache.stride(0),
|
||||
kv_cache.stride(1),
|
||||
kv_cache.stride(2),
|
||||
block_table.stride(0),
|
||||
HEAD_DIM=D,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_KV_HEADS=Hk,
|
||||
K_SCALES_OFFSET=k_scales_offset(D, gs),
|
||||
V_CODES_OFFSET=v_codes_offset(D, gs),
|
||||
V_SCALES_OFFSET=v_scales_offset(D, gs),
|
||||
GROUP_SIZE_C=gs,
|
||||
N_GROUPS_C=n_groups(D, gs),
|
||||
BLOCK_D=triton.next_power_of_2(D),
|
||||
OUT_BF16=1 if k_out.dtype == torch.bfloat16 else 0,
|
||||
UE8M0_BIAS_C=UE8M0_BIAS,
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -1,351 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton store kernel for the ultraquant KV cache format.
|
||||
|
||||
Per-token, per-head, per-group-of-32 the kernel computes:
|
||||
|
||||
s_raw = c * absmax(group) # c = 0.156 by default
|
||||
s = 2^round(log2(s_raw)) # UE8M0 snap (power of 2)
|
||||
code = quantize_to_fp4(group / s) # OCP E2M1, 4 bits/elt
|
||||
|
||||
and stores ``code`` as packed nibbles + ``s`` as a single UE8M0 byte per
|
||||
group. The output layout matches ``format.py`` (272 B/slot for D=256).
|
||||
|
||||
K is Hadamard-rotated in-register; V is not. There is no per-token L2
|
||||
norm. The UE8M0 snap makes ``s`` a power of two so scaled F8F6F4 MFMA
|
||||
can consume it on the read side.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.ops.ultraquant.format import (
|
||||
MIDPOINTS_SORTED,
|
||||
SORTED_TO_BITS,
|
||||
UE8M0_BIAS,
|
||||
get_constant_c,
|
||||
get_group_size,
|
||||
k_scales_offset,
|
||||
n_groups,
|
||||
slot_size,
|
||||
v_codes_offset,
|
||||
v_scales_offset,
|
||||
)
|
||||
|
||||
|
||||
def _kv_cache_flat(kv_cache: torch.Tensor) -> torch.Tensor:
|
||||
"""1-D uint8 view of the raw KV allocation (handles padded layouts)."""
|
||||
if kv_cache.is_contiguous():
|
||||
return kv_cache.reshape(-1)
|
||||
n = kv_cache.untyped_storage().nbytes() // kv_cache.element_size()
|
||||
return torch.empty(0, dtype=kv_cache.dtype, device=kv_cache.device).set_(
|
||||
kv_cache.untyped_storage(), 0, (n,)
|
||||
)
|
||||
|
||||
|
||||
_SUPPORTED_DTYPES = {torch.float16, torch.bfloat16}
|
||||
|
||||
# Built lazily on first launch so importing this module does not require triton.
|
||||
_MIDPOINTS_C: Any = None
|
||||
|
||||
|
||||
def _ensure_const_tables() -> None:
|
||||
global _MIDPOINTS_C
|
||||
if _MIDPOINTS_C is None:
|
||||
_MIDPOINTS_C = tl.constexpr(tuple(float(m) for m in MIDPOINTS_SORTED))
|
||||
|
||||
|
||||
# ── Cached per-device constant tensors ─────────────────────────────────────
|
||||
_MIDPOINTS_CACHE: dict[torch.device, torch.Tensor] = {}
|
||||
_SORTED_TO_BITS_CACHE: dict[torch.device, torch.Tensor] = {}
|
||||
_PIT_CACHE: dict[tuple[int, torch.device, torch.dtype], torch.Tensor] = {}
|
||||
|
||||
|
||||
def _get_midpoints_tensor(device: torch.device) -> torch.Tensor:
|
||||
t = _MIDPOINTS_CACHE.get(device)
|
||||
if t is None:
|
||||
t = torch.tensor(MIDPOINTS_SORTED, device=device, dtype=torch.float32)
|
||||
_MIDPOINTS_CACHE[device] = t
|
||||
return t
|
||||
|
||||
|
||||
def _get_sorted_to_bits_tensor(device: torch.device) -> torch.Tensor:
|
||||
t = _SORTED_TO_BITS_CACHE.get(device)
|
||||
if t is None:
|
||||
t = torch.tensor(SORTED_TO_BITS, device=device, dtype=torch.int32)
|
||||
_SORTED_TO_BITS_CACHE[device] = t
|
||||
return t
|
||||
|
||||
|
||||
def _get_hadamard(
|
||||
dim: int, device: torch.device, dtype: torch.dtype = torch.float32
|
||||
) -> torch.Tensor:
|
||||
"""Sylvester Hadamard in fp32 (cached, normalised)."""
|
||||
key = (dim, device, dtype)
|
||||
cached = _PIT_CACHE.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
if dim <= 0 or (dim & (dim - 1)) != 0:
|
||||
raise ValueError(f"ultraquant store requires power-of-two dim, got {dim}")
|
||||
H = torch.tensor([[1.0]], dtype=torch.float64)
|
||||
while H.shape[0] < dim:
|
||||
H = torch.cat([torch.cat([H, H], dim=1), torch.cat([H, -H], dim=1)], dim=0)
|
||||
H = (H / (dim**0.5)).to(device=device, dtype=dtype).contiguous()
|
||||
_PIT_CACHE[key] = H
|
||||
return H
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Triton kernel
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _ultraquant_store_kernel(
|
||||
Key_ptr, # [N*H, D] in raw dtype (bf16 or fp16)
|
||||
Value_ptr, # [N*H, D] in raw dtype
|
||||
KV_cache_ptr, # raw uint8 view, flat
|
||||
Slot_mapping_ptr, # [N] int64
|
||||
Midpoints_ptr, # [14] fp32
|
||||
SortedToBits_ptr, # [15] int32
|
||||
# Cache strides (in bytes)
|
||||
stride_cache_block: tl.constexpr,
|
||||
stride_cache_pos: tl.constexpr,
|
||||
stride_cache_head: tl.constexpr,
|
||||
# Dimensions
|
||||
HEAD_DIM: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
LOG2_D: tl.constexpr,
|
||||
# Layout
|
||||
GROUP_SIZE_C: tl.constexpr,
|
||||
N_GROUPS_C: tl.constexpr,
|
||||
K_SCALES_OFFSET: tl.constexpr,
|
||||
V_CODES_OFFSET: tl.constexpr,
|
||||
V_SCALES_OFFSET: tl.constexpr,
|
||||
FP4_C: tl.constexpr,
|
||||
UE8M0_BIAS_C: tl.constexpr,
|
||||
# When 1, the FP4 midpoints and the sorted->bits remap are materialised as
|
||||
# immediates instead of being read from `Midpoints_ptr`/`SortedToBits_ptr`.
|
||||
# That removes 28 scalar global loads and two 256-lane divergent gathers
|
||||
# from the critical path of a kernel whose runtime is pure latency (flat at
|
||||
# ~22 us from batch 1 to batch 64). Numerically identical: the immediates
|
||||
# are the same fp32 values, and the remap formula is exact (asserted in
|
||||
# `format`, verified for all 15 entries).
|
||||
CONST_TABLES: tl.constexpr = 1,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
token_idx = pid // H
|
||||
head_idx = pid % H
|
||||
|
||||
slot = tl.load(Slot_mapping_ptr + token_idx)
|
||||
if slot < 0:
|
||||
return
|
||||
blk = slot // BLOCK_SIZE
|
||||
off = slot % BLOCK_SIZE
|
||||
slot_base = (
|
||||
blk * stride_cache_block + off * stride_cache_pos + head_idx * stride_cache_head
|
||||
).to(tl.int64)
|
||||
k_code_base = slot_base
|
||||
k_scale_base = slot_base + K_SCALES_OFFSET
|
||||
v_code_base = slot_base + V_CODES_OFFSET
|
||||
v_scale_base = slot_base + V_SCALES_OFFSET
|
||||
|
||||
base = pid * HEAD_DIM
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < HEAD_DIM
|
||||
shifts_4 = tl.arange(0, 2) * 4 # 0, 4 for nibble packing
|
||||
g_offs = tl.arange(0, N_GROUPS_C)
|
||||
|
||||
# ── K side ─────────────────────────────────────────────────────────
|
||||
# 1. Load + Hadamard-rotate K via WHT butterfly.
|
||||
k_raw = tl.load(Key_ptr + base + d_offs, mask=d_mask, other=0.0).to(tl.float32)
|
||||
k_rot = tl.where(d_mask, k_raw, 0.0)
|
||||
|
||||
INV_SQRT2: tl.constexpr = 0.7071067811865476
|
||||
for _stage in tl.static_range(LOG2_D):
|
||||
_bmask = 1 << _stage
|
||||
_pidx = (d_offs ^ _bmask).to(tl.int32)
|
||||
_pval = tl.gather(k_rot, _pidx, 0)
|
||||
_sign = tl.where((d_offs & _bmask) == 0, 1.0, -1.0).to(tl.float32)
|
||||
k_rot = (_sign * k_rot + _pval) * INV_SQRT2
|
||||
k_rot = tl.where(d_mask, k_rot, 0.0)
|
||||
|
||||
# 2. Per-group absmax.
|
||||
k_g = tl.reshape(k_rot, [N_GROUPS_C, GROUP_SIZE_C])
|
||||
k_absmax = tl.max(tl.abs(k_g), axis=1) # [N_GROUPS]
|
||||
k_is_zero = k_absmax == 0.0
|
||||
|
||||
# 3. UE8M0 snap: s_raw = c * absmax; exp = round(log2(s_raw));
|
||||
# divisor = 2^exp (c folded in). Zero groups write byte=0.
|
||||
k_s_raw = k_absmax * tl.cast(FP4_C, tl.float32)
|
||||
k_s_safe = tl.where(k_is_zero, 1.0, k_s_raw)
|
||||
k_log2 = tl.log2(k_s_safe)
|
||||
k_exp = tl.cast(tl.floor(k_log2 + 0.5), tl.int32) # round-half-up
|
||||
k_s_snapped = tl.exp2(tl.cast(k_exp, tl.float32))
|
||||
k_s_div = tl.where(k_is_zero, 1.0, k_s_snapped)
|
||||
|
||||
# 4. Normalize + snap to sorted FP4 idx via 14 sequential tl.where (`>` for
|
||||
# bucketize(right=False) parity with the reference).
|
||||
k_norm = k_g / k_s_div[:, None]
|
||||
k_sorted = tl.zeros([N_GROUPS_C, GROUP_SIZE_C], dtype=tl.int32)
|
||||
for i in tl.static_range(14):
|
||||
mid = _MIDPOINTS_C.value[i] if CONST_TABLES else tl.load(Midpoints_ptr + i)
|
||||
k_sorted += tl.where(k_norm > mid, 1, 0)
|
||||
k_sorted = tl.where(k_is_zero[:, None], 7, k_sorted)
|
||||
|
||||
# 5. Remap sorted idx → FP4 E2M1 bit pattern.
|
||||
if CONST_TABLES:
|
||||
k_bits = tl.where(k_sorted < 7, 15 - k_sorted, k_sorted - 7)
|
||||
else:
|
||||
k_bits = tl.load(SortedToBits_ptr + k_sorted)
|
||||
|
||||
# 6. Pack pairs of 4-bit codes into uint8 bytes.
|
||||
k_bits_flat = tl.reshape(k_bits, [HEAD_DIM])
|
||||
k_pairs = tl.reshape(k_bits_flat, [HEAD_DIM // 2, 2])
|
||||
k_packed = tl.sum((k_pairs & 0xF) << shifts_4[None, :], axis=1).to(tl.uint8)
|
||||
k_code_addrs = k_code_base + tl.arange(0, HEAD_DIM // 2)
|
||||
tl.store(KV_cache_ptr + k_code_addrs, k_packed)
|
||||
|
||||
# 7. Store K scales as E8M0 bytes (one per group). Zero-amax groups
|
||||
# write byte=0 (zero sentinel).
|
||||
k_byte = tl.where(k_is_zero, 0, k_exp + UE8M0_BIAS_C)
|
||||
k_byte = tl.where((k_byte < 0) | (k_byte > 255), 0, k_byte).to(tl.uint8)
|
||||
k_scale_addrs = k_scale_base + g_offs
|
||||
tl.store(KV_cache_ptr + k_scale_addrs, k_byte)
|
||||
|
||||
# ── V side (mirror K but skip rotation) ────────────────────────────
|
||||
v_raw = tl.load(Value_ptr + base + d_offs, mask=d_mask, other=0.0).to(tl.float32)
|
||||
v_raw = tl.where(d_mask, v_raw, 0.0)
|
||||
|
||||
v_g = tl.reshape(v_raw, [N_GROUPS_C, GROUP_SIZE_C])
|
||||
v_absmax = tl.max(tl.abs(v_g), axis=1)
|
||||
v_is_zero = v_absmax == 0.0
|
||||
|
||||
v_s_raw = v_absmax * tl.cast(FP4_C, tl.float32)
|
||||
v_s_safe = tl.where(v_is_zero, 1.0, v_s_raw)
|
||||
v_log2 = tl.log2(v_s_safe)
|
||||
v_exp = tl.cast(tl.floor(v_log2 + 0.5), tl.int32)
|
||||
v_s_snapped = tl.exp2(tl.cast(v_exp, tl.float32))
|
||||
v_s_div = tl.where(v_is_zero, 1.0, v_s_snapped)
|
||||
|
||||
v_norm = v_g / v_s_div[:, None]
|
||||
v_sorted = tl.zeros([N_GROUPS_C, GROUP_SIZE_C], dtype=tl.int32)
|
||||
for i in tl.static_range(14):
|
||||
mid = _MIDPOINTS_C.value[i] if CONST_TABLES else tl.load(Midpoints_ptr + i)
|
||||
v_sorted += tl.where(v_norm > mid, 1, 0)
|
||||
v_sorted = tl.where(v_is_zero[:, None], 7, v_sorted)
|
||||
|
||||
if CONST_TABLES:
|
||||
v_bits = tl.where(v_sorted < 7, 15 - v_sorted, v_sorted - 7)
|
||||
else:
|
||||
v_bits = tl.load(SortedToBits_ptr + v_sorted)
|
||||
v_bits_flat = tl.reshape(v_bits, [HEAD_DIM])
|
||||
v_pairs = tl.reshape(v_bits_flat, [HEAD_DIM // 2, 2])
|
||||
v_packed = tl.sum((v_pairs & 0xF) << shifts_4[None, :], axis=1).to(tl.uint8)
|
||||
v_code_addrs = v_code_base + tl.arange(0, HEAD_DIM // 2)
|
||||
tl.store(KV_cache_ptr + v_code_addrs, v_packed)
|
||||
|
||||
v_byte = tl.where(v_is_zero, 0, v_exp + UE8M0_BIAS_C)
|
||||
v_byte = tl.where((v_byte < 0) | (v_byte > 255), 0, v_byte).to(tl.uint8)
|
||||
v_scale_addrs = v_scale_base + g_offs
|
||||
tl.store(KV_cache_ptr + v_scale_addrs, v_byte)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Launcher
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def ultraquant_store(
|
||||
key: torch.Tensor, # [N, H, D] bf16 or fp16
|
||||
value: torch.Tensor, # [N, H, D] bf16 or fp16
|
||||
kv_cache: torch.Tensor, # [num_blocks, block_size, Hk, slot_size_aligned] uint8
|
||||
slot_mapping: torch.Tensor, # [N] int64
|
||||
*,
|
||||
PiT: torch.Tensor | None = None, # unused; kept for call-site compatibility
|
||||
constant_c: float | None = None,
|
||||
) -> None:
|
||||
"""Launch the ultraquant store kernel. Writes K/V into `kv_cache`
|
||||
in-place at the slots specified by `slot_mapping`."""
|
||||
if key.dtype not in _SUPPORTED_DTYPES:
|
||||
raise ValueError(
|
||||
f"ultraquant_store: key.dtype must be one of "
|
||||
f"{_SUPPORTED_DTYPES}, got {key.dtype}"
|
||||
)
|
||||
if value.dtype != key.dtype:
|
||||
raise ValueError(
|
||||
f"ultraquant_store: key/value dtype mismatch ({key.dtype} vs {value.dtype})"
|
||||
)
|
||||
if slot_mapping.dtype != torch.int64:
|
||||
slot_mapping = slot_mapping.to(torch.int64)
|
||||
|
||||
N, H, D = key.shape
|
||||
NH = N * H
|
||||
block_size = kv_cache.shape[1]
|
||||
num_kv_heads = kv_cache.shape[2]
|
||||
padded_slot = kv_cache.shape[3]
|
||||
|
||||
expected_slot = slot_size(D)
|
||||
if padded_slot < expected_slot:
|
||||
raise ValueError(
|
||||
f"ultraquant_store: kv_cache slot {padded_slot} < expected "
|
||||
f"{expected_slot} for head_dim={D}"
|
||||
)
|
||||
if num_kv_heads != H:
|
||||
raise ValueError(
|
||||
f"ultraquant_store: kv_cache num_kv_heads {num_kv_heads} != key heads {H}"
|
||||
)
|
||||
|
||||
midpoints = _get_midpoints_tensor(key.device)
|
||||
sorted_to_bits = _get_sorted_to_bits_tensor(key.device)
|
||||
|
||||
k_flat = key.reshape(NH, D).contiguous()
|
||||
v_flat = value.reshape(NH, D).contiguous()
|
||||
|
||||
gs = get_group_size()
|
||||
BLOCK_D = triton.next_power_of_2(D)
|
||||
N_GROUPS_C = n_groups(D, gs)
|
||||
K_SCALES_OFFSET = k_scales_offset(D, gs)
|
||||
V_CODES_OFFSET = v_codes_offset(D, gs)
|
||||
V_SCALES_OFFSET = v_scales_offset(D, gs)
|
||||
|
||||
stride_block = kv_cache.stride(0)
|
||||
stride_pos = kv_cache.stride(1)
|
||||
stride_head = kv_cache.stride(2)
|
||||
|
||||
c = constant_c if constant_c is not None else get_constant_c()
|
||||
|
||||
_ensure_const_tables()
|
||||
grid = (NH,)
|
||||
_ultraquant_store_kernel[grid](
|
||||
k_flat,
|
||||
v_flat,
|
||||
_kv_cache_flat(kv_cache),
|
||||
slot_mapping,
|
||||
midpoints,
|
||||
sorted_to_bits,
|
||||
stride_cache_block=stride_block,
|
||||
stride_cache_pos=stride_pos,
|
||||
stride_cache_head=stride_head,
|
||||
HEAD_DIM=D,
|
||||
H=H,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_D=BLOCK_D,
|
||||
LOG2_D=int(D).bit_length() - 1,
|
||||
GROUP_SIZE_C=gs,
|
||||
N_GROUPS_C=N_GROUPS_C,
|
||||
K_SCALES_OFFSET=K_SCALES_OFFSET,
|
||||
V_CODES_OFFSET=V_CODES_OFFSET,
|
||||
V_SCALES_OFFSET=V_SCALES_OFFSET,
|
||||
FP4_C=c,
|
||||
UE8M0_BIAS_C=UE8M0_BIAS,
|
||||
CONST_TABLES=1,
|
||||
num_warps=4,
|
||||
num_stages=1,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,279 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import LOG2E, LOGE2, tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _bf16_mla_sparse_kernel(
|
||||
q_buffer,
|
||||
k_buffer,
|
||||
v_buffer,
|
||||
indices_ptr,
|
||||
out_ptr,
|
||||
softmax_lse_ptr,
|
||||
max_logits_ptr,
|
||||
seq_q,
|
||||
seq_kv,
|
||||
h_q,
|
||||
dim_qk,
|
||||
dim_v,
|
||||
stride_q_token,
|
||||
stride_q_head,
|
||||
stride_k_token,
|
||||
stride_k_head,
|
||||
stride_v_token,
|
||||
stride_v_head,
|
||||
stride_out_token,
|
||||
stride_out_head,
|
||||
stride_lse,
|
||||
stride_indices_token,
|
||||
stride_indices_head,
|
||||
sm_scale,
|
||||
kv_group_num: tl.constexpr,
|
||||
index_topk: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr, # block size for num heads
|
||||
BLOCK_M: tl.constexpr, # block size for num tokens
|
||||
BLOCK_N: tl.constexpr, # block size for indices
|
||||
BLOCK_DV: tl.constexpr, # block size for dim_v
|
||||
BLOCK_DMODEL: tl.constexpr, # block size for dim_nope
|
||||
BLOCK_DPE: tl.constexpr, # block size for positional embedding
|
||||
LOGE2: tl.constexpr,
|
||||
):
|
||||
cur_q = tl.program_id(0)
|
||||
cur_head_id = tl.program_id(1)
|
||||
cur_kv_head_id = cur_head_id // tl.cdiv(kv_group_num, BLOCK_H)
|
||||
|
||||
VALID_BLOCK_H: tl.constexpr = BLOCK_H if kv_group_num > BLOCK_H else kv_group_num
|
||||
cur_head = cur_head_id * VALID_BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
mask_h = cur_head < (cur_head_id + 1) * VALID_BLOCK_H
|
||||
mask_h = mask_h & (cur_head < h_q)
|
||||
|
||||
offs_d = tl.arange(0, BLOCK_DMODEL)
|
||||
offs_dv = tl.arange(0, BLOCK_DV)
|
||||
|
||||
off_q = cur_q * stride_q_token + cur_head[:, None] * stride_q_head + offs_d[None, :]
|
||||
mask_dmodel = offs_d < BLOCK_DMODEL
|
||||
q = tl.load(
|
||||
q_buffer + off_q, mask=(mask_h[:, None]) & (mask_dmodel[None, :]), other=0.0
|
||||
)
|
||||
|
||||
if BLOCK_DPE > 0:
|
||||
offs_dpe = BLOCK_DMODEL + tl.arange(0, BLOCK_DPE)
|
||||
off_qpe = (
|
||||
cur_q * stride_q_token
|
||||
+ cur_head[:, None] * stride_q_head
|
||||
+ offs_dpe[None, :]
|
||||
)
|
||||
# assume dim_qk == BLOCK_DMODEL + BLOCK_DPE
|
||||
mask_dpe = offs_dpe < dim_qk
|
||||
qpe = tl.load(
|
||||
q_buffer + off_qpe, mask=(mask_h[:, None]) & (mask_dpe[None, :]), other=0.0
|
||||
)
|
||||
|
||||
# Use a finite sentinel rather than -inf: if every key in a chunk is
|
||||
# masked (e.g. leading -1 padding in `indices`) an -inf running max gives
|
||||
# re_scale = exp2(-inf - -inf) = NaN, permanently poisoning acc / e_sum.
|
||||
e_max = tl.zeros([BLOCK_H], dtype=tl.float32) - 1.0e30
|
||||
e_sum = tl.zeros([BLOCK_H], dtype=tl.float32)
|
||||
acc = tl.zeros([BLOCK_H, BLOCK_DV], dtype=tl.float32)
|
||||
|
||||
for start_indice in range(0, index_topk, BLOCK_N):
|
||||
offs_indice = start_indice + tl.arange(0, BLOCK_N)
|
||||
mask_indice = offs_indice < index_topk
|
||||
indices = tl.load(
|
||||
indices_ptr
|
||||
+ (
|
||||
cur_q * stride_indices_token
|
||||
+ cur_kv_head_id * stride_indices_head
|
||||
+ offs_indice
|
||||
),
|
||||
mask=mask_indice,
|
||||
other=-1,
|
||||
)
|
||||
|
||||
mask_kv = (indices >= 0) & (indices < seq_kv)
|
||||
mask_kv_d = mask_dmodel
|
||||
offs_k = (
|
||||
indices[None, :] * stride_k_token
|
||||
+ cur_kv_head_id * stride_k_head
|
||||
+ offs_d[:, None]
|
||||
)
|
||||
|
||||
# q_nope @ k_nope
|
||||
k = tl.load(
|
||||
k_buffer + offs_k, mask=(mask_kv[None, :]) & (mask_kv_d[:, None]), other=0.0
|
||||
)
|
||||
qk = tl.dot(q, k.to(q.dtype))
|
||||
|
||||
if BLOCK_DPE > 0:
|
||||
# q_rope @ k_rope
|
||||
offs_kpe = (
|
||||
indices[None, :] * stride_k_token
|
||||
+ cur_kv_head_id * stride_k_head
|
||||
+ offs_dpe[:, None]
|
||||
)
|
||||
mask_k_dpe = offs_dpe < dim_qk
|
||||
kpe = tl.load(
|
||||
k_buffer + offs_kpe,
|
||||
mask=(mask_kv[None, :]) & (mask_k_dpe[:, None]),
|
||||
other=0.0,
|
||||
)
|
||||
qk += tl.dot(qpe, kpe.to(q.dtype))
|
||||
|
||||
# apply scaling
|
||||
qk *= sm_scale
|
||||
qk = tl.where((mask_h[:, None]) & (mask_kv[None, :]), qk, -1.0e30)
|
||||
|
||||
# load v
|
||||
mask_v_d = offs_dv < dim_v
|
||||
offs_v = (
|
||||
indices[:, None] * stride_v_token
|
||||
+ cur_kv_head_id * stride_v_head
|
||||
+ offs_dv[None, :]
|
||||
)
|
||||
v = tl.load(
|
||||
v_buffer + offs_v, mask=(mask_kv[:, None]) & (mask_v_d[None, :]), other=0.0
|
||||
)
|
||||
|
||||
# online softmax
|
||||
n_e_max = tl.maximum(tl.max(qk, 1), e_max)
|
||||
re_scale = tl.exp2(e_max - n_e_max)
|
||||
p = tl.exp2(qk - n_e_max[:, None])
|
||||
acc *= re_scale[:, None]
|
||||
|
||||
# score @ v
|
||||
acc += tl.dot(p.to(v.dtype), v)
|
||||
|
||||
# update global sum and max
|
||||
e_sum = e_sum * re_scale + tl.sum(p, 1)
|
||||
e_max = n_e_max
|
||||
|
||||
# rescaling
|
||||
acc /= e_sum[:, None]
|
||||
|
||||
max_logits = e_max * LOGE2
|
||||
# calculate lse
|
||||
lse = max_logits + tl.log2(e_sum) * LOGE2
|
||||
|
||||
# write output
|
||||
offs_o = (
|
||||
cur_q * stride_out_token
|
||||
+ cur_head[:, None] * stride_out_head
|
||||
+ offs_dv[None, :]
|
||||
)
|
||||
mask_out_d = offs_dv < dim_v
|
||||
tl.store(
|
||||
out_ptr + offs_o,
|
||||
acc.to(tl.bfloat16),
|
||||
mask=(mask_h[:, None]) & (mask_out_d[None, :]),
|
||||
)
|
||||
|
||||
offs_lse = cur_q * stride_lse + cur_head
|
||||
tl.store(softmax_lse_ptr + offs_lse, lse, mask=mask_h)
|
||||
tl.store(max_logits_ptr + offs_lse, max_logits, mask=mask_h)
|
||||
|
||||
|
||||
# reference implementation of bf16 sparse prefill kernel
|
||||
def triton_bf16_mla_sparse_interface(
|
||||
q: torch.Tensor, # [num_tokens, num_heads_q, dim_qk]
|
||||
kv: torch.Tensor, # [num_tokens, num_heads_kv, dim_qk]
|
||||
indices: torch.Tensor, # [num_tokens, num_heads_kv, topk]
|
||||
sm_scale: float,
|
||||
d_v: int = 512,
|
||||
block_dpe: int = 64,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Out : [num_tokens, num_heads_q, d_v]
|
||||
max_logits : [num_tokens, num_heads_q]
|
||||
lse : logsumexp, [num_tokens, num_heads_q]
|
||||
|
||||
Args:
|
||||
q: Query tensor, [num_tokens, num_heads_q, dim_qk].
|
||||
kv: Latent KV cache tensor.
|
||||
indices: Sparse KV indices selected for each query token.
|
||||
sm_scale: Softmax scale applied to the QK product.
|
||||
d_v: Size of the value dimension.
|
||||
block_dpe: Size of positional embedding portion of dim_qk.
|
||||
Set to 0 when q/kv contain only the nope latent (e.g. DSv4
|
||||
prefill where RoPE is not split out).
|
||||
|
||||
"""
|
||||
num_tokens, num_heads_q, dim_qk = q.shape
|
||||
_, num_heads_kv, _ = kv.shape
|
||||
assert dim_qk == kv.shape[2], "q and kv have different head dimensions"
|
||||
|
||||
# for deepseek v3.2, index topk should be 2048
|
||||
_, _, index_topk = indices.shape
|
||||
|
||||
BLOCK_H = 16
|
||||
BLOCK_DPE = block_dpe
|
||||
BLOCK_DMODEL = dim_qk - BLOCK_DPE
|
||||
BLOCK_M = 32
|
||||
BLOCK_N = 16
|
||||
BLOCK_DV = 512
|
||||
assert d_v == BLOCK_DV, "only support d_v = 512"
|
||||
|
||||
assert dim_qk == BLOCK_DMODEL + BLOCK_DPE, (
|
||||
"dim_qk does not match BLOCK_DMODEL + BLOCK_DPE"
|
||||
)
|
||||
assert num_heads_kv == 1, "only support kv head = 1 for now"
|
||||
assert index_topk % BLOCK_N == 0, "index_topk must be multiple of BLOCK_N"
|
||||
|
||||
sm_scale *= LOG2E
|
||||
|
||||
kv_group_num = num_heads_q // num_heads_kv
|
||||
grid = (
|
||||
num_tokens,
|
||||
triton.cdiv(num_heads_q, min(BLOCK_H, kv_group_num)),
|
||||
)
|
||||
|
||||
out = torch.zeros((num_tokens, num_heads_q, d_v), dtype=q.dtype, device=q.device)
|
||||
softmax_lse = torch.zeros(
|
||||
(num_tokens, num_heads_q), dtype=torch.float32, device=q.device
|
||||
)
|
||||
max_logits = torch.zeros(
|
||||
(num_tokens, num_heads_q), dtype=torch.float32, device=q.device
|
||||
)
|
||||
|
||||
k = kv
|
||||
v = kv[..., :d_v]
|
||||
|
||||
_bf16_mla_sparse_kernel[grid](
|
||||
q_buffer=q,
|
||||
k_buffer=k,
|
||||
v_buffer=v,
|
||||
indices_ptr=indices,
|
||||
out_ptr=out,
|
||||
softmax_lse_ptr=softmax_lse,
|
||||
max_logits_ptr=max_logits,
|
||||
seq_q=num_tokens,
|
||||
seq_kv=kv.shape[0],
|
||||
h_q=num_heads_q,
|
||||
dim_qk=dim_qk,
|
||||
dim_v=d_v,
|
||||
stride_q_token=q.stride(0),
|
||||
stride_q_head=q.stride(1),
|
||||
stride_k_token=k.stride(0),
|
||||
stride_k_head=k.stride(1),
|
||||
stride_v_token=v.stride(0),
|
||||
stride_v_head=v.stride(1),
|
||||
stride_out_token=out.stride(0),
|
||||
stride_out_head=out.stride(1),
|
||||
stride_lse=softmax_lse.stride(0),
|
||||
stride_indices_token=indices.stride(0),
|
||||
stride_indices_head=indices.stride(1),
|
||||
sm_scale=sm_scale,
|
||||
kv_group_num=kv_group_num,
|
||||
index_topk=index_topk,
|
||||
BLOCK_H=BLOCK_H,
|
||||
BLOCK_M=BLOCK_M,
|
||||
BLOCK_N=BLOCK_N,
|
||||
BLOCK_DV=BLOCK_DV,
|
||||
BLOCK_DMODEL=BLOCK_DMODEL,
|
||||
BLOCK_DPE=BLOCK_DPE,
|
||||
LOGE2=LOGE2,
|
||||
)
|
||||
|
||||
return out, max_logits, softmax_lse
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Canvas-width handling and async read deferral for diffusion requests."""
|
||||
|
||||
from vllm.v1.core.sched.async_scheduler import AsyncScheduler
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.outputs import DraftTokenIds
|
||||
from vllm.v1.request import Request
|
||||
|
||||
|
||||
def diffusion_canvas_width(request: Request, canvas_length: int) -> int:
|
||||
"""The canvas width a diffusion request asked for, else the served one."""
|
||||
params = request.sampling_params
|
||||
extra = params.extra_args if params is not None else None
|
||||
width = extra.get("diffusion_canvas_length") if extra else None
|
||||
return int(width) if width else canvas_length
|
||||
|
||||
|
||||
def _read_in_flight(request: Request, width: int) -> bool:
|
||||
"""True when a read-only request has all its denoise steps in flight.
|
||||
|
||||
The request emits its canvas on the last step and ends, so a further
|
||||
step is discarded.
|
||||
"""
|
||||
params = request.sampling_params
|
||||
extra = params.extra_args if params is not None else None
|
||||
if not extra or not extra.get("diffusion_read_only"):
|
||||
return False
|
||||
steps = extra.get("diffusion_max_steps")
|
||||
if not steps:
|
||||
return False
|
||||
# Each in-flight denoise step holds one canvas of placeholders.
|
||||
return request.num_output_placeholders >= steps * width
|
||||
|
||||
|
||||
class DiffusionScheduler(Scheduler):
|
||||
def update_draft_token_ids(self, draft_token_ids: DraftTokenIds) -> None:
|
||||
for i, (req_id, token_ids) in enumerate(
|
||||
zip(draft_token_ids.req_ids, draft_token_ids.draft_token_ids)
|
||||
):
|
||||
request = self.requests.get(req_id)
|
||||
if request is None:
|
||||
continue
|
||||
width = diffusion_canvas_width(request, self.num_spec_tokens)
|
||||
# Remove padded positions before the base scheduler validates grammar.
|
||||
if len(token_ids) > width:
|
||||
draft_token_ids.draft_token_ids[i] = token_ids[:width]
|
||||
super().update_draft_token_ids(draft_token_ids)
|
||||
|
||||
|
||||
class DiffusionAsyncScheduler(DiffusionScheduler, AsyncScheduler):
|
||||
def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput:
|
||||
for request in self.running:
|
||||
width = diffusion_canvas_width(request, self.num_spec_tokens)
|
||||
# The placeholders are as wide as the served canvas.
|
||||
if len(request.spec_token_ids) > width:
|
||||
request.spec_token_ids = request.spec_token_ids[:width]
|
||||
if _read_in_flight(request, width):
|
||||
# Scheduler.schedule()'s max_tokens guard cannot be reached
|
||||
# from a subclass. schedule() advances current_step before it
|
||||
# checks decode eligibility, so current_step + 2 skips this
|
||||
# call alone. max keeps a longer pipeline-parallel wait.
|
||||
request.next_decode_eligible_step = max(
|
||||
request.next_decode_eligible_step, self.current_step + 2
|
||||
)
|
||||
return super().schedule(throttle_prefills)
|
||||
@@ -1,75 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.utils.torch_utils import supports_xpu_graph
|
||||
from vllm.v1.worker.gpu.model_runner import (
|
||||
GPUModelRunner as GPUModelRunnerV2,
|
||||
)
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
|
||||
|
||||
|
||||
class XPUModelRunner(GPUModelRunner):
|
||||
"""A model runner for XPU devices."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
):
|
||||
with _torch_cuda_wrapper():
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
|
||||
class XPUModelRunnerV2(GPUModelRunnerV2):
|
||||
"""A model runner for XPU devices."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
):
|
||||
with _torch_cuda_wrapper():
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
|
||||
class XPUMMEncoderModelRunner(MMEncoderModelRunner):
|
||||
"""An encoder-only model runner for XPU devices."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
):
|
||||
with _torch_cuda_wrapper():
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _torch_cuda_wrapper():
|
||||
# Replace cuda APIs with xpu APIs. Each callable gets its own functools.partial
|
||||
# so it is not the same object as torch.xpu.* (Torch Dynamo _get_handlers()
|
||||
# asserts on duplicate registration when cuda aliases xpu directly).
|
||||
torch.cuda.Stream = torch.xpu.Stream
|
||||
torch.cuda.default_stream = partial(torch.xpu.current_stream)
|
||||
torch.cuda.current_stream = partial(torch.xpu.current_stream)
|
||||
torch.cuda.stream = partial(torch.xpu.stream)
|
||||
torch.cuda.set_stream = partial(torch.xpu.set_stream)
|
||||
|
||||
# torch.xpu.Event does not accept the ``blocking`` or ``external`` kwargs
|
||||
# that torch.cuda.Event supports, so drop them here.
|
||||
def _xpu_event(*args, blocking=None, external=None, **kwargs):
|
||||
return torch.xpu.Event(*args, **kwargs)
|
||||
|
||||
torch.cuda.Event = _xpu_event
|
||||
if supports_xpu_graph():
|
||||
torch.cuda.graph = partial(torch.xpu.graph)
|
||||
torch.cuda.CUDAGraph = torch.xpu.XPUGraph
|
||||
torch.cuda.graph_pool_handle = partial(torch.xpu.graph_pool_handle)
|
||||
torch.cuda.is_current_stream_capturing = torch.xpu.is_current_stream_capturing
|
||||
yield
|
||||
@@ -1,243 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import gc
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_utils import MemorySnapshot, format_gib
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.gpu_worker import Worker, init_worker_distributed_environment
|
||||
from vllm.v1.worker.xpu_model_runner import (
|
||||
XPUMMEncoderModelRunner,
|
||||
XPUModelRunner,
|
||||
XPUModelRunnerV2,
|
||||
)
|
||||
|
||||
from .utils import request_memory
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class XPUWorker(Worker):
|
||||
"""A XPU worker class."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
local_rank: int,
|
||||
rank: int,
|
||||
distributed_init_method: str,
|
||||
is_driver_worker: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
vllm_config, local_rank, rank, distributed_init_method, is_driver_worker
|
||||
)
|
||||
device_config = self.device_config
|
||||
assert device_config.device_type == "xpu"
|
||||
assert current_platform.is_xpu()
|
||||
|
||||
def init_device(self):
|
||||
# In DP mode, XPU workers see all visible devices.
|
||||
# Offset local_rank by the local DP shard.
|
||||
parallel_config = self.parallel_config
|
||||
if (
|
||||
parallel_config.distributed_executor_backend
|
||||
not in ("ray", "external_launcher")
|
||||
and parallel_config.data_parallel_backend != "ray"
|
||||
and (
|
||||
parallel_config.data_parallel_external_lb
|
||||
or parallel_config.nnodes_within_dp == 1
|
||||
)
|
||||
):
|
||||
dp_local_rank = parallel_config.data_parallel_rank_local
|
||||
if dp_local_rank is None:
|
||||
dp_local_rank = parallel_config.data_parallel_index
|
||||
replica_world_size = parallel_config.world_size
|
||||
visible_device_count = torch.accelerator.device_count()
|
||||
|
||||
if parallel_config.data_parallel_external_lb:
|
||||
if parallel_config.nnodes_within_dp > 1:
|
||||
# The replica spans nodes, so use its node-local shard.
|
||||
replica_world_size = parallel_config.local_world_size
|
||||
if replica_world_size > visible_device_count:
|
||||
raise ValueError(
|
||||
f"Local TP/PP/PCP replica size ({replica_world_size}) "
|
||||
"exceeds the number of visible XPU devices "
|
||||
f"({visible_device_count})."
|
||||
)
|
||||
local_dp_capacity = visible_device_count // replica_world_size
|
||||
elif replica_world_size < visible_device_count:
|
||||
# A node can host multiple complete TP/PP/PCP replicas.
|
||||
local_dp_capacity = visible_device_count // replica_world_size
|
||||
if visible_device_count % replica_world_size != 0:
|
||||
logger.warning_once(
|
||||
"XPU external LB cannot evenly divide "
|
||||
"%d visible devices into TP/PP/PCP replicas of "
|
||||
"size %d. This node can host %d complete DP "
|
||||
"replicas, leaving %d visible devices unused.",
|
||||
visible_device_count,
|
||||
replica_world_size,
|
||||
local_dp_capacity,
|
||||
visible_device_count % replica_world_size,
|
||||
)
|
||||
elif replica_world_size == visible_device_count:
|
||||
# A node hosts exactly one complete TP/PP/PCP replica.
|
||||
local_dp_capacity = 1
|
||||
logger.warning_once(
|
||||
"XPU external LB sees exactly enough devices for one "
|
||||
"TP/PP/PCP replica. This may be the intended "
|
||||
"configuration, but it may also indicate that device "
|
||||
"visibility is misconfigured. Every DP rank must see "
|
||||
"all XPU devices on its node; consider removing "
|
||||
"ZE_AFFINITY_MASK or setting it to expose the complete "
|
||||
"device set."
|
||||
)
|
||||
else:
|
||||
# The topology says single-node, but the replica does not fit.
|
||||
raise ValueError(
|
||||
f"TP/PP/PCP replica size ({replica_world_size}) exceeds "
|
||||
f"the number of visible XPU devices ({visible_device_count}), "
|
||||
"but nnodes_within_dp is 1. Configure the multi-node "
|
||||
"topology, or ensure every DP rank can see all devices "
|
||||
"on its node."
|
||||
)
|
||||
# Strip the node component off the global DP index to get this
|
||||
# engine's slot on its own node. Assumes the launcher assigns
|
||||
# DP ranks to nodes in contiguous blocks (node 0 gets ranks
|
||||
# 0..capacity-1, and so on), which is what the usual sequential
|
||||
# and one-pod-per-rank deployments do. A round-robin or
|
||||
# unbalanced assignment would silently map two engines onto the
|
||||
# same device.
|
||||
dp_local_rank = parallel_config.data_parallel_index % local_dp_capacity
|
||||
|
||||
self.local_rank += dp_local_rank * replica_world_size
|
||||
|
||||
device = self.device_config.device
|
||||
if (
|
||||
isinstance(device, torch.device)
|
||||
and device.type == "xpu"
|
||||
and current_platform.is_xpu()
|
||||
):
|
||||
assigned_physical_gpu_ids = self.parallel_config.assigned_physical_gpu_ids
|
||||
if assigned_physical_gpu_ids is not None:
|
||||
from vllm.platforms.interface import set_assigned_physical_gpu_ids
|
||||
|
||||
set_assigned_physical_gpu_ids(assigned_physical_gpu_ids)
|
||||
assert self.local_rank < len(assigned_physical_gpu_ids), (
|
||||
f"local_rank {self.local_rank} is out of bounds for "
|
||||
f"assigned_physical_gpu_ids {assigned_physical_gpu_ids}"
|
||||
)
|
||||
# NOTE: local_world_size is derived from parallel_config.nnodes,
|
||||
# which is only set for the "mp" multi-node backend. With the
|
||||
# "ray"/"external_launcher" backends nnodes stays 1, so
|
||||
# local_world_size collapses to the full world_size and this
|
||||
# check wrongly fires on cross-node deployments.
|
||||
# assigned_physical_gpu_ids is already per-node and the
|
||||
# local_rank bound above fully validates the mapping for
|
||||
# these backends, so skip the check for them.
|
||||
if parallel_config.distributed_executor_backend not in (
|
||||
"ray",
|
||||
"external_launcher",
|
||||
):
|
||||
assert parallel_config.local_world_size <= len(
|
||||
assigned_physical_gpu_ids
|
||||
), (
|
||||
f"local_world_size ({parallel_config.local_world_size})"
|
||||
" exceeds assigned_physical_gpu_ids count "
|
||||
f"({len(assigned_physical_gpu_ids)})"
|
||||
)
|
||||
else:
|
||||
assert self.local_rank < torch.accelerator.device_count(), (
|
||||
f"DP adjusted local rank {self.local_rank} is out of "
|
||||
f"bounds for {torch.accelerator.device_count()} devices."
|
||||
)
|
||||
|
||||
visible_device_index = (
|
||||
current_platform.logical_device_id_to_visible_device_id(self.local_rank)
|
||||
)
|
||||
self.device = torch.device(f"xpu:{visible_device_index}")
|
||||
torch.accelerator.set_device_index(self.device)
|
||||
current_platform.check_if_supports_dtype(self.model_config.dtype)
|
||||
torch.accelerator.empty_cache()
|
||||
self.init_gpu_memory = torch.xpu.get_device_properties(
|
||||
visible_device_index
|
||||
).total_memory
|
||||
else:
|
||||
raise RuntimeError(f"Unsupported device type: {self.device_config.device}")
|
||||
|
||||
ENV_CCL_ATL_TRANSPORT = os.getenv("CCL_ATL_TRANSPORT", "ofi")
|
||||
ENV_LOCAL_WORLD_SIZE = os.getenv(
|
||||
"LOCAL_WORLD_SIZE", str(self.parallel_config.world_size)
|
||||
)
|
||||
os.environ["CCL_ATL_TRANSPORT"] = ENV_CCL_ATL_TRANSPORT
|
||||
os.environ["LOCAL_WORLD_SIZE"] = ENV_LOCAL_WORLD_SIZE
|
||||
os.environ["LOCAL_RANK"] = str(self.local_rank)
|
||||
|
||||
init_worker_distributed_environment(
|
||||
self.vllm_config,
|
||||
self.rank,
|
||||
self.distributed_init_method,
|
||||
self.local_rank,
|
||||
current_platform.dist_backend,
|
||||
)
|
||||
|
||||
# oneCCL warm-up; only meaningful for multi-device runs. Requiring it
|
||||
# with a single worker breaks platforms where oneCCL cannot enumerate
|
||||
# device topology (e.g. paravirtualized GPUs).
|
||||
if (
|
||||
self.parallel_config.world_size > 1
|
||||
and torch.distributed.is_xccl_available()
|
||||
):
|
||||
torch.distributed.all_reduce(torch.zeros(1).xpu())
|
||||
|
||||
if self.use_v2_model_runner:
|
||||
logger.info_once("Using V2 Model Runner")
|
||||
|
||||
# Set random seed.
|
||||
set_random_seed(self.model_config.seed)
|
||||
|
||||
# Now take memory snapshot after NCCL is initialized
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
# take current memory snapshot
|
||||
self.init_snapshot = init_snapshot = MemorySnapshot(device=self.device)
|
||||
self.requested_memory = request_memory(init_snapshot, self.cache_config)
|
||||
logger.debug("worker init memory snapshot: %r", self.init_snapshot)
|
||||
logger.debug(
|
||||
"worker requested memory: %sGiB", format_gib(self.requested_memory)
|
||||
)
|
||||
|
||||
self._init_workspace_and_model_runner()
|
||||
|
||||
def _make_model_runner(self):
|
||||
if self.use_v2_model_runner:
|
||||
model_runner = (
|
||||
XPUMMEncoderModelRunner
|
||||
if self.vllm_config.is_mm_encoder_only
|
||||
else XPUModelRunnerV2
|
||||
)
|
||||
else:
|
||||
model_runner = XPUModelRunner # type: ignore[assignment]
|
||||
return model_runner(self.vllm_config, self.device) # type: ignore
|
||||
|
||||
def shutdown(self) -> None:
|
||||
logger.info(
|
||||
"XPUWorker shutdown: cleaning up (rank=%d, local_rank=%d)",
|
||||
self.rank,
|
||||
self.local_rank,
|
||||
)
|
||||
super().shutdown()
|
||||
from vllm.device_allocator.xpumem import XpuMemAllocator
|
||||
|
||||
if XpuMemAllocator.instance is not None:
|
||||
XpuMemAllocator.instance.release_pools()
|
||||
logger.info(
|
||||
"XPUWorker shutdown: done (rank=%d, local_rank=%d)",
|
||||
self.rank,
|
||||
self.local_rank,
|
||||
)
|
||||
Reference in New Issue
Block a user