Triton attention: add USE_TD constexpr for tensor descriptor Q/K/V load/store (#40327)
Signed-off-by: Artur Fierka <artur.fierka@intel.com> Co-authored-by: quinnlp <quinnlp@users.noreply.github.com>
This commit is contained in:
@@ -345,3 +345,182 @@ def test_triton_unified_attn_fp16_input_fp8_output(
|
||||
torch.testing.assert_close(output_fp16, ref_output, atol=atol, rtol=rtol),
|
||||
f"{torch.max(torch.abs(output_fp16 - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
# USE_TD path covers two head-size regimes:
|
||||
# - pow2 (HEAD_SIZE == HEAD_SIZE_PADDED): full TD path including Q/O.
|
||||
# - non-pow2 (96, HEAD_SIZE_PADDED=128): gates USE_TD_QO off — Q load
|
||||
# and output store fall back to pointer path, KV tile TD load remains.
|
||||
# The non-pow2 case mirrors real models like Phi-3-mini (head_size=96).
|
||||
HEAD_SIZES_USE_TD = [128, 256, 96]
|
||||
|
||||
|
||||
def _run_use_td_case(
|
||||
seq_lens: list[tuple[int, int]],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
sliding_window: int | None,
|
||||
soft_cap: float | None,
|
||||
seq_threshold_3D: int,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
num_blocks: int = 2048,
|
||||
) -> None:
|
||||
"""Shared driver for the USE_TD test cases.
|
||||
|
||||
Runs ``unified_attention(..., use_td=True)`` and compares against the
|
||||
reference paged-attention implementation that the sibling non-TD
|
||||
tests use.
|
||||
"""
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
set_random_seed(0)
|
||||
|
||||
num_seqs = len(seq_lens)
|
||||
query_lens = [x[0] for x in seq_lens]
|
||||
kv_lens = [x[1] for x in seq_lens]
|
||||
num_query_heads, num_kv_heads = num_heads
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
max_query_len = max(query_lens)
|
||||
max_kv_len = max(kv_lens)
|
||||
window_size = (sliding_window - 1, 0) if sliding_window is not None else (-1, -1)
|
||||
scale = head_size**-0.5
|
||||
|
||||
query = torch.randn(sum(query_lens), num_query_heads, head_size, dtype=dtype)
|
||||
key_cache = torch.randn(
|
||||
num_blocks, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
value_cache = torch.randn_like(key_cache)
|
||||
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
|
||||
dim=0, dtype=torch.int32
|
||||
)
|
||||
kv_lens_tensor = torch.tensor(kv_lens, dtype=torch.int32)
|
||||
|
||||
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, num_blocks, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
output = torch.empty_like(query)
|
||||
|
||||
num_par_softmax_segments = 16
|
||||
head_size_padded = next_power_of_2(head_size)
|
||||
softmax_segm_output = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_max = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_expsum = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
unified_attention(
|
||||
q=query,
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
out=output,
|
||||
cu_seqlens_q=cu_query_lens,
|
||||
seqused_k=kv_lens_tensor,
|
||||
max_seqlen_q=max_query_len,
|
||||
max_seqlen_k=max_kv_len,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
block_table=block_tables,
|
||||
softcap=soft_cap if soft_cap is not None else 0,
|
||||
q_descale=None,
|
||||
k_descale=None,
|
||||
v_descale=None,
|
||||
seq_threshold_3D=seq_threshold_3D,
|
||||
num_par_softmax_segments=num_par_softmax_segments,
|
||||
softmax_segm_output=softmax_segm_output,
|
||||
softmax_segm_max=softmax_segm_max,
|
||||
softmax_segm_expsum=softmax_segm_expsum,
|
||||
use_td=True,
|
||||
)
|
||||
|
||||
ref_output = ref_paged_attn(
|
||||
query=query,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
query_lens=query_lens,
|
||||
kv_lens=kv_lens,
|
||||
block_tables=block_tables,
|
||||
scale=scale,
|
||||
sliding_window=sliding_window,
|
||||
soft_cap=soft_cap,
|
||||
)
|
||||
torch.testing.assert_close(output, ref_output, atol=1.5e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens", [[(1, 1328), (5, 18), (129, 463)], [(1, 523), (1, 37), (1, 2011)]]
|
||||
)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES_USE_TD)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("sliding_window", [None, 128])
|
||||
@pytest.mark.parametrize("soft_cap", [None, 50.0])
|
||||
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
|
||||
@pytest.mark.parametrize("seq_threshold_3D", SEQ_THRESHOLD_3D_VALUES)
|
||||
@torch.inference_mode()
|
||||
def test_triton_unified_attn_use_td(
|
||||
seq_lens: list[tuple[int, int]],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
sliding_window: int | None,
|
||||
block_size: int,
|
||||
soft_cap: float | None,
|
||||
num_blocks: int,
|
||||
seq_threshold_3D: int,
|
||||
) -> None:
|
||||
"""Exercise the USE_TD (tensor-descriptor) Q/K/V load/store path.
|
||||
|
||||
Covers both 2D and 3D kernels via ``seq_threshold_3D``. Two routes
|
||||
to the USE_TD_QO=False fallback (pointer path for Q/O with TD still
|
||||
active for KV tile loads):
|
||||
|
||||
- non-pow2 ``num_queries_per_kv`` via ``NUM_HEADS`` entry ``(5, 1)``,
|
||||
- non-pow2 ``head_size`` via ``HEAD_SIZES_USE_TD`` entry ``96``.
|
||||
"""
|
||||
_run_use_td_case(
|
||||
seq_lens=seq_lens,
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
block_size=block_size,
|
||||
sliding_window=sliding_window,
|
||||
soft_cap=soft_cap,
|
||||
seq_threshold_3D=seq_threshold_3D,
|
||||
num_blocks=num_blocks,
|
||||
)
|
||||
|
||||
|
||||
# Prefill-heavy shape: long query drives the prefill kernel path where
|
||||
# ``_get_tile_size`` returns 32, which exceeds block_size=16 and must be
|
||||
# clamped by the fix in 'clamp TILE_SIZE to block_size when USE_TD'.
|
||||
# Only the prefill launch exercises the clamp, so parameterize only over
|
||||
# the (num_heads, seq_threshold_3D=0) combinations needed to cover it.
|
||||
@pytest.mark.parametrize("num_heads", [(4, 4), (5, 1)])
|
||||
@torch.inference_mode()
|
||||
def test_triton_unified_attn_use_td_tile_clamp(
|
||||
num_heads: tuple[int, int],
|
||||
) -> None:
|
||||
"""Regression guard: ``USE_TD`` needs ``BLOCK_SIZE % TILE_SIZE == 0``.
|
||||
|
||||
With ``block_size=16`` and ``head_size=128`` (non-Gemma3),
|
||||
``_get_tile_size`` returns 32 for prefill, which violates the
|
||||
``USE_TD`` constraint unless clamped to ``block_size``. Without
|
||||
the clamp the triton kernel ``static_assert`` fires at compile time.
|
||||
"""
|
||||
_run_use_td_case(
|
||||
seq_lens=[(256, 256), (128, 128)],
|
||||
num_heads=num_heads,
|
||||
head_size=128,
|
||||
block_size=16,
|
||||
sliding_window=None,
|
||||
soft_cap=None,
|
||||
seq_threshold_3D=0,
|
||||
)
|
||||
|
||||
@@ -82,6 +82,7 @@ if TYPE_CHECKING:
|
||||
VLLM_MAIN_CUDA_VERSION: str = "13.0"
|
||||
VLLM_FLOAT32_MATMUL_PRECISION: Literal["highest", "high", "medium"] = "highest"
|
||||
VLLM_BATCH_INVARIANT: bool = False
|
||||
VLLM_TRITON_ATTN_USE_TD: bool | None = None
|
||||
MAX_JOBS: str | None = None
|
||||
NVCC_THREADS: str | None = None
|
||||
VLLM_USE_PRECOMPILED: bool = False
|
||||
@@ -525,6 +526,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Enable batch-invariant mode: deterministic results regardless of
|
||||
# batch composition. Requires NVIDIA GPU with compute capability >= 9.0.
|
||||
"VLLM_BATCH_INVARIANT": lambda: bool(int(os.getenv("VLLM_BATCH_INVARIANT", "0"))),
|
||||
# Use tensor descriptors for Q/K/V loads and output stores in the
|
||||
# Triton unified-attention kernel. Enables HW 2D block reads on
|
||||
# Intel Xe2/Xe3; the non-TD branch is dead-code-eliminated at Triton
|
||||
# compile time so other platforms see no overhead. Tri-state override:
|
||||
# unset (default) lets the `triton_attn` backend auto-select per
|
||||
# platform (currently auto-enabled on XPU only); ``1`` forces TD on;
|
||||
# ``0`` forces TD off. Useful for A/B benchmarking the TD path.
|
||||
"VLLM_TRITON_ATTN_USE_TD": lambda: {"1": True, "0": False}.get(
|
||||
os.getenv("VLLM_TRITON_ATTN_USE_TD", "").strip()
|
||||
),
|
||||
# Maximum number of compilation jobs to run in parallel.
|
||||
# By default this is the number of CPUs
|
||||
"MAX_JOBS": lambda: os.getenv("MAX_JOBS", None),
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
@@ -502,6 +503,21 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
self._kv_quant_mode = get_kv_quant_mode(kv_cache_dtype)
|
||||
self._is_per_token_head_quant = self._kv_quant_mode.is_per_token_head
|
||||
|
||||
# Enable tensor descriptors for Q/K/V load/store on platforms that
|
||||
# benefit from HW 2D block reads (Intel Xe2/Xe3). The dead branch
|
||||
# is eliminated at Triton compile time, so other platforms see
|
||||
# zero cost when TD is off.
|
||||
#
|
||||
# ``VLLM_TRITON_ATTN_USE_TD`` is tri-state:
|
||||
# - unset (None): auto-select (TD on for XPU, off elsewhere),
|
||||
# - ``1``: force TD on regardless of platform,
|
||||
# - ``0``: force TD off regardless of platform (useful for A/B).
|
||||
td_override = envs.VLLM_TRITON_ATTN_USE_TD
|
||||
if td_override is None:
|
||||
self.use_td = current_platform.is_xpu()
|
||||
else:
|
||||
self.use_td = td_override
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -637,6 +653,7 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
k_scale_cache=k_scale_cache,
|
||||
v_scale_cache=v_scale_cache,
|
||||
chunk_lookback=self.chunk_lookback,
|
||||
use_td=self.use_td,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
@@ -54,16 +54,132 @@ def _cast_kv_tile(data, Q, tensor_scale, KV_QUANT_MODE: tl.constexpr):
|
||||
return data.to(Q.dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tensor-descriptor (TD) helpers
|
||||
#
|
||||
# Used when the caller enables ``USE_TD`` (Intel Xe2/Xe3 HW 2D block reads).
|
||||
# When ``USE_TD`` is False the helpers are dead-code-eliminated at Triton
|
||||
# compile time, leaving the pointer-arithmetic load/store path untouched.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _load_q_td(
|
||||
query_ptr,
|
||||
q_block_local_len,
|
||||
query_stride_0: tl.int64,
|
||||
query_stride_1: tl.int64,
|
||||
cur_batch_in_all_start_index,
|
||||
q_block_local_idx,
|
||||
kv_head_idx,
|
||||
num_queries_per_kv: tl.constexpr,
|
||||
BLOCK_Q: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
HEAD_SIZE_PADDED: tl.constexpr,
|
||||
):
|
||||
"""Load Q via a 2D tensor descriptor.
|
||||
|
||||
Caller guarantees (via the wrapper's ``use_td_qo`` gate):
|
||||
* ``HEAD_SIZE == HEAD_SIZE_PADDED`` (head_size is a power of 2),
|
||||
* ``num_queries_per_kv`` is a power of 2,
|
||||
* the ``num_queries_per_kv`` heads of the current KV group are
|
||||
contiguous in memory (``query_stride_1 == HEAD_SIZE``, which is
|
||||
the default vLLM query layout).
|
||||
|
||||
Under those preconditions the inner two axes are flattened into one
|
||||
row of size ``num_queries_per_kv * HEAD_SIZE`` with stride 1, which
|
||||
avoids the non-power-of-2 ``block_shape`` error from the Triton
|
||||
tensor-descriptor validator. Returns (BLOCK_M, HEAD_SIZE_PADDED).
|
||||
"""
|
||||
q_base = (
|
||||
query_ptr
|
||||
+ (cur_batch_in_all_start_index + q_block_local_idx * BLOCK_Q) * query_stride_0
|
||||
+ (kv_head_idx * num_queries_per_kv) * query_stride_1
|
||||
)
|
||||
q_desc = tl.make_tensor_descriptor(
|
||||
base=q_base,
|
||||
shape=(q_block_local_len, num_queries_per_kv * HEAD_SIZE),
|
||||
strides=(query_stride_0, 1),
|
||||
block_shape=(BLOCK_Q, num_queries_per_kv * HEAD_SIZE_PADDED),
|
||||
)
|
||||
return q_desc.load([0, 0]).reshape(BLOCK_M, HEAD_SIZE_PADDED)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _load_kv_tile_td(
|
||||
cache_ptr,
|
||||
physical_block_idx_scalar,
|
||||
kv_head_idx,
|
||||
offset_in_block,
|
||||
stride_cache_0: tl.int64,
|
||||
stride_cache_1: tl.int64,
|
||||
stride_cache_2: tl.int64,
|
||||
stride_cache_3: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
TILE_SIZE: tl.constexpr,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
HEAD_SIZE_PADDED: tl.constexpr,
|
||||
):
|
||||
"""Load a KV cache tile via tensor descriptor.
|
||||
|
||||
Returns shape (TILE_SIZE, HEAD_SIZE_PADDED). Caller transposes for K.
|
||||
Tensor descriptors zero-pad reads beyond the shape boundary, so
|
||||
``HEAD_SIZE_PADDED > HEAD_SIZE`` is handled correctly.
|
||||
"""
|
||||
base = (
|
||||
cache_ptr
|
||||
+ physical_block_idx_scalar * stride_cache_0
|
||||
+ kv_head_idx * stride_cache_2
|
||||
)
|
||||
desc = tl.make_tensor_descriptor(
|
||||
base=base,
|
||||
shape=(BLOCK_SIZE, HEAD_SIZE),
|
||||
strides=(stride_cache_1, stride_cache_3),
|
||||
block_shape=(TILE_SIZE, HEAD_SIZE_PADDED),
|
||||
)
|
||||
return desc.load([offset_in_block, 0])
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_output_td(
|
||||
base_ptr,
|
||||
acc,
|
||||
q_block_local_len,
|
||||
stride_token: tl.int64,
|
||||
stride_head: tl.int64,
|
||||
num_queries_per_kv: tl.constexpr,
|
||||
BLOCK_Q: tl.constexpr,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
HEAD_SIZE_PADDED: tl.constexpr,
|
||||
):
|
||||
"""Store an output tile via a tensor descriptor.
|
||||
|
||||
The 2D and 3D epilogues differ only in ``base_ptr`` and the
|
||||
``(stride_token, stride_head)`` pair: 2D writes directly to the
|
||||
flat output buffer, 3D writes to a single per-segment slice of
|
||||
``segm_output_ptr``. Descriptor shape / block_shape / reshape
|
||||
are the same in both modes, so share one helper.
|
||||
"""
|
||||
acc = acc.to(base_ptr.dtype.element_ty)
|
||||
output_desc = tl.make_tensor_descriptor(
|
||||
base=base_ptr,
|
||||
shape=(q_block_local_len, num_queries_per_kv, HEAD_SIZE),
|
||||
strides=(stride_token, stride_head, 1),
|
||||
block_shape=(BLOCK_Q, num_queries_per_kv, HEAD_SIZE_PADDED),
|
||||
)
|
||||
output_desc.store(
|
||||
[0, 0, 0],
|
||||
acc.reshape(BLOCK_Q, num_queries_per_kv, HEAD_SIZE_PADDED),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def kernel_unified_attention(
|
||||
# Output destinations. In 2D mode we write the final result into
|
||||
# ``output_ptr``; in 3D mode we write per-segment partials into the
|
||||
# three ``segm_*`` tensors and ``output_ptr`` is unused (callers may
|
||||
# pass any non-null pointer).
|
||||
# Output destination for the 2D path. In 3D mode per-segment partials
|
||||
# go to the ``segm_*`` tensors (see bottom of signature) and
|
||||
# ``output_ptr`` is unused (callers may pass any non-null pointer).
|
||||
output_ptr,
|
||||
segm_output_ptr,
|
||||
segm_max_ptr,
|
||||
segm_expsum_ptr,
|
||||
# Inputs
|
||||
query_ptr,
|
||||
key_cache_ptr,
|
||||
@@ -73,10 +189,6 @@ def kernel_unified_attention(
|
||||
seq_lens_ptr,
|
||||
alibi_slopes_ptr,
|
||||
qq_bias_ptr,
|
||||
# Per-(token, head) scale caches (used iff KV_QUANT_MODE in {2, 3}).
|
||||
# For other modes callers may pass any non-null pointer.
|
||||
k_scale_cache_ptr,
|
||||
v_scale_cache_ptr,
|
||||
# Scalars
|
||||
scale,
|
||||
k_scale,
|
||||
@@ -112,12 +224,6 @@ def kernel_unified_attention(
|
||||
stride_v_cache_1: tl.int64, # int
|
||||
stride_v_cache_2: tl.int64, # int
|
||||
stride_v_cache_3: tl.constexpr, # int
|
||||
stride_ks_blk: tl.int64,
|
||||
stride_ks_slot: tl.int64,
|
||||
stride_ks_head: tl.int64,
|
||||
stride_vs_blk: tl.int64,
|
||||
stride_vs_slot: tl.int64,
|
||||
stride_vs_head: tl.int64,
|
||||
query_start_len_ptr,
|
||||
BLOCK_Q: tl.constexpr,
|
||||
num_seqs: tl.int32,
|
||||
@@ -129,6 +235,23 @@ def kernel_unified_attention(
|
||||
# to ``[segm_idx, segm_idx+1) × tiles_per_segment`` and writes
|
||||
# per-segment partials, finalized by ``reduce_segments``.
|
||||
IS_3D: tl.constexpr,
|
||||
# Parameters below default to None so Triton can skip materialising them
|
||||
# on call sites where the corresponding constexpr branch is dead.
|
||||
# Credit: @quinnlp identified this as a perf regression source in
|
||||
# intel/intel-xpu-backend-for-triton#6758 (review comment r3204641104).
|
||||
# Per-segment outputs: used in 3D mode; unused in 2D (IS_3D=False).
|
||||
segm_output_ptr=None,
|
||||
segm_max_ptr=None,
|
||||
segm_expsum_ptr=None,
|
||||
# Per-(token, head) scale caches: used iff KV_QUANT_MODE in {2, 3}.
|
||||
k_scale_cache_ptr=None,
|
||||
v_scale_cache_ptr=None,
|
||||
stride_ks_blk: tl.int64 = None,
|
||||
stride_ks_slot: tl.int64 = None,
|
||||
stride_ks_head: tl.int64 = None,
|
||||
stride_vs_blk: tl.int64 = None,
|
||||
stride_vs_slot: tl.int64 = None,
|
||||
stride_vs_head: tl.int64 = None,
|
||||
# KV cache quantization mode handled inside this kernel via constexpr
|
||||
# branches: NONE (0), FP8_PER_TENSOR (1), INT8_PER_TOKEN_HEAD (2),
|
||||
# FP8_PER_TOKEN_HEAD (3).
|
||||
@@ -140,9 +263,20 @@ def kernel_unified_attention(
|
||||
# over ``SLIDING_WINDOW`` inside the helpers. ``-1`` disables.
|
||||
CHUNK_LOOKBACK: tl.constexpr = -1,
|
||||
CHUNK_SIZE: tl.constexpr = -1,
|
||||
# Tensor-descriptor load/store for HW 2D block reads on Intel Xe2/Xe3.
|
||||
# ``USE_TD`` gates KV tile loads; ``USE_TD_QO`` separately gates Q/output
|
||||
# (see ``unified_attention`` wrapper for the gating rules).
|
||||
USE_TD: tl.constexpr = False,
|
||||
USE_TD_QO: tl.constexpr = False,
|
||||
):
|
||||
USE_PER_TOKEN_HEAD_SCALES: tl.constexpr = KV_QUANT_MODE >= 2
|
||||
|
||||
if USE_TD:
|
||||
tl.static_assert(
|
||||
BLOCK_SIZE % TILE_SIZE == 0,
|
||||
"USE_TD requires BLOCK_SIZE to be a multiple of TILE_SIZE",
|
||||
)
|
||||
|
||||
q_block_global_idx = tl.program_id(0)
|
||||
kv_head_idx = tl.program_id(1)
|
||||
segm_idx = tl.program_id(2) if IS_3D else 0
|
||||
@@ -167,6 +301,12 @@ def kernel_unified_attention(
|
||||
else:
|
||||
tiles_per_segment = 0
|
||||
|
||||
# Number of valid query rows in this block (used by TD descriptor
|
||||
# shapes, but always computed so the variable stays in scope).
|
||||
q_block_local_len = tl.minimum(
|
||||
BLOCK_Q, cur_batch_query_len - q_block_local_idx * BLOCK_Q
|
||||
)
|
||||
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_d = tl.arange(0, HEAD_SIZE_PADDED)
|
||||
offs_t = tl.arange(0, TILE_SIZE)
|
||||
@@ -185,11 +325,27 @@ def kernel_unified_attention(
|
||||
query_mask_1 = tl.where(query_offset_1 < num_query_heads, 1, 0).to(tl.int1)
|
||||
|
||||
# Q : (BLOCK_M, HEAD_SIZE_PADDED)
|
||||
Q = tl.load(
|
||||
query_ptr + query_offset,
|
||||
mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
if USE_TD_QO:
|
||||
Q = _load_q_td(
|
||||
query_ptr,
|
||||
q_block_local_len,
|
||||
query_stride_0,
|
||||
query_stride_1,
|
||||
cur_batch_in_all_start_index,
|
||||
q_block_local_idx,
|
||||
kv_head_idx,
|
||||
num_queries_per_kv,
|
||||
BLOCK_Q,
|
||||
BLOCK_M,
|
||||
HEAD_SIZE,
|
||||
HEAD_SIZE_PADDED,
|
||||
)
|
||||
else:
|
||||
Q = tl.load(
|
||||
query_ptr + query_offset,
|
||||
mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
block_table_offset = seq_idx * block_table_stride
|
||||
|
||||
@@ -237,31 +393,71 @@ def kernel_unified_attention(
|
||||
block_tables_ptr + block_table_offset + seq_offset // BLOCK_SIZE
|
||||
).to(tl.int64)
|
||||
|
||||
v_offset = (
|
||||
physical_block_idx[:, None] * stride_v_cache_0
|
||||
+ kv_head_idx * stride_v_cache_2
|
||||
+ offs_d[None, :] * stride_v_cache_3
|
||||
+ (seq_offset % BLOCK_SIZE)[:, None] * stride_v_cache_1
|
||||
)
|
||||
k_offset = (
|
||||
physical_block_idx[None, :] * stride_k_cache_0
|
||||
+ kv_head_idx * stride_k_cache_2
|
||||
+ offs_d[:, None] * stride_k_cache_3
|
||||
+ (seq_offset % BLOCK_SIZE)[None, :] * stride_k_cache_1
|
||||
)
|
||||
# K : (HEAD_SIZE, TILE_SIZE)
|
||||
K_load = tl.load(
|
||||
key_cache_ptr + k_offset,
|
||||
mask=dim_mask[:, None] & tile_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if USE_TD:
|
||||
# All TILE_SIZE slots within a single KV tile map to one
|
||||
# physical block (guaranteed by ``BLOCK_SIZE % TILE_SIZE == 0``
|
||||
# from the static_assert above), so load the block index as
|
||||
# a scalar instead of a broadcast reduction.
|
||||
offset_in_block = (j * TILE_SIZE) % BLOCK_SIZE
|
||||
physical_block_scalar = tl.load(
|
||||
block_tables_ptr + block_table_offset + (j * TILE_SIZE) // BLOCK_SIZE
|
||||
).to(tl.int64)
|
||||
# K : (HEAD_SIZE, TILE_SIZE)
|
||||
K_load = _load_kv_tile_td(
|
||||
key_cache_ptr,
|
||||
physical_block_scalar,
|
||||
kv_head_idx,
|
||||
offset_in_block,
|
||||
stride_k_cache_0,
|
||||
stride_k_cache_1,
|
||||
stride_k_cache_2,
|
||||
stride_k_cache_3,
|
||||
BLOCK_SIZE,
|
||||
TILE_SIZE,
|
||||
HEAD_SIZE,
|
||||
HEAD_SIZE_PADDED,
|
||||
).T
|
||||
# V : (TILE_SIZE, HEAD_SIZE)
|
||||
V_load = _load_kv_tile_td(
|
||||
value_cache_ptr,
|
||||
physical_block_scalar,
|
||||
kv_head_idx,
|
||||
offset_in_block,
|
||||
stride_v_cache_0,
|
||||
stride_v_cache_1,
|
||||
stride_v_cache_2,
|
||||
stride_v_cache_3,
|
||||
BLOCK_SIZE,
|
||||
TILE_SIZE,
|
||||
HEAD_SIZE,
|
||||
HEAD_SIZE_PADDED,
|
||||
)
|
||||
else:
|
||||
v_offset = (
|
||||
physical_block_idx[:, None] * stride_v_cache_0
|
||||
+ kv_head_idx * stride_v_cache_2
|
||||
+ offs_d[None, :] * stride_v_cache_3
|
||||
+ (seq_offset % BLOCK_SIZE)[:, None] * stride_v_cache_1
|
||||
)
|
||||
k_offset = (
|
||||
physical_block_idx[None, :] * stride_k_cache_0
|
||||
+ kv_head_idx * stride_k_cache_2
|
||||
+ offs_d[:, None] * stride_k_cache_3
|
||||
+ (seq_offset % BLOCK_SIZE)[None, :] * stride_k_cache_1
|
||||
)
|
||||
# K : (HEAD_SIZE, TILE_SIZE)
|
||||
K_load = tl.load(
|
||||
key_cache_ptr + k_offset,
|
||||
mask=dim_mask[:, None] & tile_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
# V : (TILE_SIZE, HEAD_SIZE)
|
||||
V_load = tl.load(
|
||||
value_cache_ptr + v_offset,
|
||||
mask=dim_mask[None, :] & tile_mask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
K = _cast_kv_tile(K_load, Q, k_scale, KV_QUANT_MODE)
|
||||
# V : (TILE_SIZE, HEAD_SIZE)
|
||||
V_load = tl.load(
|
||||
value_cache_ptr + v_offset,
|
||||
mask=dim_mask[None, :] & tile_mask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
V = _cast_kv_tile(V_load, Q, v_scale, KV_QUANT_MODE)
|
||||
|
||||
# Per-(token, head) scales for INT8 / FP8 per-token-head modes.
|
||||
@@ -342,18 +538,45 @@ def kernel_unified_attention(
|
||||
# ---- Epilogue ---------------------------------------------------------
|
||||
if IS_3D:
|
||||
# Store per-segment partials; finalized by ``reduce_segments``.
|
||||
segm_output_offset = (
|
||||
query_offset_0[:, None].to(tl.int64)
|
||||
* (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED)
|
||||
+ query_offset_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED)
|
||||
+ segm_idx * HEAD_SIZE_PADDED
|
||||
+ tl.arange(0, HEAD_SIZE_PADDED)[None, :]
|
||||
)
|
||||
tl.store(
|
||||
segm_output_ptr + segm_output_offset,
|
||||
acc,
|
||||
mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None],
|
||||
)
|
||||
if USE_TD_QO:
|
||||
# 3D target: segm_output[token, head, segm_idx, :]. Advance
|
||||
# the base to the correct (token-start, head-start, segm)
|
||||
# slice; strides step between tokens / heads of the flattened
|
||||
# (T, H, SEGS, PAD) layout.
|
||||
segm_base = (
|
||||
segm_output_ptr
|
||||
+ (cur_batch_in_all_start_index + q_block_local_idx * BLOCK_Q).to(
|
||||
tl.int64
|
||||
)
|
||||
* (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED)
|
||||
+ (kv_head_idx * num_queries_per_kv)
|
||||
* (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED)
|
||||
+ segm_idx * HEAD_SIZE_PADDED
|
||||
)
|
||||
_store_output_td(
|
||||
segm_base,
|
||||
acc,
|
||||
q_block_local_len,
|
||||
num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED,
|
||||
NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED,
|
||||
num_queries_per_kv,
|
||||
BLOCK_Q,
|
||||
HEAD_SIZE,
|
||||
HEAD_SIZE_PADDED,
|
||||
)
|
||||
else:
|
||||
segm_output_offset = (
|
||||
query_offset_0[:, None].to(tl.int64)
|
||||
* (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED)
|
||||
+ query_offset_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED)
|
||||
+ segm_idx * HEAD_SIZE_PADDED
|
||||
+ tl.arange(0, HEAD_SIZE_PADDED)[None, :]
|
||||
)
|
||||
tl.store(
|
||||
segm_output_ptr + segm_output_offset,
|
||||
acc,
|
||||
mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None],
|
||||
)
|
||||
store_segm_reduce_scalars(
|
||||
segm_max_ptr,
|
||||
segm_expsum_ptr,
|
||||
@@ -372,16 +595,38 @@ def kernel_unified_attention(
|
||||
if USE_FP8:
|
||||
acc = acc * tl.load(out_scale)
|
||||
acc = tl.clamp(acc, FP8_MIN, FP8_MAX)
|
||||
output_offset = (
|
||||
query_offset_0[:, None] * output_stride_0
|
||||
+ query_offset_1[:, None] * output_stride_1
|
||||
+ offs_d[None, :]
|
||||
)
|
||||
tl.store(
|
||||
output_ptr + output_offset,
|
||||
acc,
|
||||
mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None],
|
||||
)
|
||||
if USE_TD_QO:
|
||||
# 2D target: flat output[token, head, :]. Strides come
|
||||
# straight from the caller (``output_stride_0`` per token,
|
||||
# ``output_stride_1`` per head).
|
||||
output_base = (
|
||||
output_ptr
|
||||
+ (cur_batch_in_all_start_index + q_block_local_idx * BLOCK_Q)
|
||||
* output_stride_0
|
||||
+ (kv_head_idx * num_queries_per_kv) * output_stride_1
|
||||
)
|
||||
_store_output_td(
|
||||
output_base,
|
||||
acc,
|
||||
q_block_local_len,
|
||||
output_stride_0,
|
||||
output_stride_1,
|
||||
num_queries_per_kv,
|
||||
BLOCK_Q,
|
||||
HEAD_SIZE,
|
||||
HEAD_SIZE_PADDED,
|
||||
)
|
||||
else:
|
||||
output_offset = (
|
||||
query_offset_0[:, None] * output_stride_0
|
||||
+ query_offset_1[:, None] * output_stride_1
|
||||
+ offs_d[None, :]
|
||||
)
|
||||
tl.store(
|
||||
output_ptr + output_offset,
|
||||
acc,
|
||||
mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -538,6 +783,11 @@ def unified_attention(
|
||||
v_scale_cache=None, # [num_blocks, block_size, num_kv_heads] float32
|
||||
# Chunked attention: restrict attention to aligned blocks with lookback.
|
||||
chunk_lookback=-1,
|
||||
# Tensor-descriptor mode: use ``tl.make_tensor_descriptor`` for Q/K/V
|
||||
# loads and output stores. Enables HW 2D block reads on Intel Xe2/Xe3.
|
||||
# The non-TD branch is dead-code-eliminated at Triton compile time so
|
||||
# disabling this flag costs nothing.
|
||||
use_td: bool = False,
|
||||
):
|
||||
assert causal, "Only causal attention is supported"
|
||||
assert q_descale is None, "Q scales not supported"
|
||||
@@ -608,6 +858,52 @@ def unified_attention(
|
||||
head_size, sliding_window_val, q.element_size(), is_prefill=False
|
||||
)
|
||||
|
||||
# USE_TD requires BLOCK_SIZE % TILE_SIZE == 0 (enforced by a
|
||||
# ``tl.static_assert`` in the kernel). The default prefill tile
|
||||
# size (32) is larger than a common ``block_size=16``, so clamp it
|
||||
# down when TD is enabled. Zero overhead when disabled.
|
||||
if use_td:
|
||||
TILE_SIZE_PREFILL = min(TILE_SIZE_PREFILL, block_size)
|
||||
TILE_SIZE_DECODE = min(TILE_SIZE_DECODE, block_size)
|
||||
|
||||
# Tensor descriptors for Q load / output store require every element
|
||||
# of ``block_shape`` to be a power of 2. ``num_queries_per_kv`` is
|
||||
# not always pow2 (e.g. Qwen2-7B: 28 / 4 = 7), so gate the Q/O paths
|
||||
# separately from the KV tile loads (whose ``block_shape`` does not
|
||||
# include ``num_queries_per_kv``).
|
||||
#
|
||||
# The Q/O descriptors also encode ``HEAD_SIZE_PADDED`` on the inner
|
||||
# axis while the backing buffers (both flat output and per-segment
|
||||
# output) are laid out with ``HEAD_SIZE``. When they differ (e.g.
|
||||
# Phi-3's head_size=96 → HEAD_SIZE_PADDED=128) the store would spill
|
||||
# padded lanes into neighbouring heads because tensor-descriptor
|
||||
# stores don't mask the padded tail. Fall back to the pointer path
|
||||
# for Q/O in that case — KV tile loads are unaffected because their
|
||||
# ``shape`` already matches ``block_shape`` on the inner axis.
|
||||
head_size_padded = triton.next_power_of_2(head_size)
|
||||
_is_pow2_nq = (num_queries_per_kv & (num_queries_per_kv - 1)) == 0
|
||||
_is_pow2_hs = head_size == head_size_padded
|
||||
use_td_qo = use_td and _is_pow2_nq and _is_pow2_hs
|
||||
|
||||
# ``_load_q_td`` / ``_store_output_td`` flatten ``(num_queries_per_kv,
|
||||
# HEAD_SIZE)`` into a single contiguous inner axis. That's only
|
||||
# equivalent to the pointer path when the ``num_queries_per_kv`` heads
|
||||
# for this KV group start at ``kv_head_idx * num_queries_per_kv`` and
|
||||
# lie exactly HEAD_SIZE apart — i.e. ``query_stride_1 == HEAD_SIZE``
|
||||
# and ``output_stride_1 == head_size``. This is the default vLLM
|
||||
# query/output layout; assert it explicitly so we fail fast if a
|
||||
# future caller passes a non-contiguous query tensor.
|
||||
if use_td_qo:
|
||||
assert q.stride(1) == head_size, (
|
||||
f"USE_TD_QO requires contiguous query heads "
|
||||
f"(q.stride(1) = {q.stride(1)} != head_size = {head_size}); "
|
||||
f"set VLLM_TRITON_ATTN_USE_TD=0 or pad the query layout."
|
||||
)
|
||||
assert out.stride(1) == head_size, (
|
||||
f"USE_TD_QO requires contiguous output heads "
|
||||
f"(out.stride(1) = {out.stride(1)} != head_size = {head_size})."
|
||||
)
|
||||
|
||||
# Launch the 2D kernel if
|
||||
# 1. No intermediate tiled softmax buffers for the 3D kernel have been allocated, or
|
||||
# 2. The batch includes at least one prefill request, or
|
||||
@@ -689,7 +985,7 @@ def unified_attention(
|
||||
BLOCK_SIZE=block_size,
|
||||
TILE_SIZE=tile_size,
|
||||
HEAD_SIZE=head_size,
|
||||
HEAD_SIZE_PADDED=triton.next_power_of_2(head_size),
|
||||
HEAD_SIZE_PADDED=head_size_padded,
|
||||
USE_ALIBI_SLOPES=use_alibi_slopes,
|
||||
USE_ALIBI_SQRT=use_alibi_sqrt,
|
||||
USE_QQ_BIAS=use_qq_bias,
|
||||
@@ -723,6 +1019,8 @@ def unified_attention(
|
||||
KV_QUANT_MODE=kv_quant_mode,
|
||||
CHUNK_LOOKBACK=chunk_lookback,
|
||||
CHUNK_SIZE=chunk_size,
|
||||
USE_TD=use_td,
|
||||
USE_TD_QO=use_td_qo,
|
||||
)
|
||||
|
||||
if use_3d:
|
||||
@@ -740,7 +1038,7 @@ def unified_attention(
|
||||
block_table_stride=block_table.stride(0),
|
||||
TILE_SIZE=TILE_SIZE_DECODE,
|
||||
HEAD_SIZE=head_size,
|
||||
HEAD_SIZE_PADDED=triton.next_power_of_2(head_size),
|
||||
HEAD_SIZE_PADDED=head_size_padded,
|
||||
query_start_len_ptr=cu_seqlens_q,
|
||||
BLOCK_Q=BLOCK_Q,
|
||||
NUM_SEGMENTS_PER_SEQ=num_par_softmax_segments,
|
||||
|
||||
Reference in New Issue
Block a user