diff --git a/tests/kernels/attention/test_triton_unified_attention.py b/tests/kernels/attention/test_triton_unified_attention.py index ef52d80929..0d9ba92295 100644 --- a/tests/kernels/attention/test_triton_unified_attention.py +++ b/tests/kernels/attention/test_triton_unified_attention.py @@ -345,3 +345,182 @@ def test_triton_unified_attn_fp16_input_fp8_output( torch.testing.assert_close(output_fp16, ref_output, atol=atol, rtol=rtol), f"{torch.max(torch.abs(output_fp16 - ref_output))}", ) + + +# USE_TD path covers two head-size regimes: +# - pow2 (HEAD_SIZE == HEAD_SIZE_PADDED): full TD path including Q/O. +# - non-pow2 (96, HEAD_SIZE_PADDED=128): gates USE_TD_QO off — Q load +# and output store fall back to pointer path, KV tile TD load remains. +# The non-pow2 case mirrors real models like Phi-3-mini (head_size=96). +HEAD_SIZES_USE_TD = [128, 256, 96] + + +def _run_use_td_case( + seq_lens: list[tuple[int, int]], + num_heads: tuple[int, int], + head_size: int, + block_size: int, + sliding_window: int | None, + soft_cap: float | None, + seq_threshold_3D: int, + dtype: torch.dtype = torch.bfloat16, + num_blocks: int = 2048, +) -> None: + """Shared driver for the USE_TD test cases. + + Runs ``unified_attention(..., use_td=True)`` and compares against the + reference paged-attention implementation that the sibling non-TD + tests use. + """ + torch.set_default_device(DEVICE_TYPE) + set_random_seed(0) + + num_seqs = len(seq_lens) + query_lens = [x[0] for x in seq_lens] + kv_lens = [x[1] for x in seq_lens] + num_query_heads, num_kv_heads = num_heads + assert num_query_heads % num_kv_heads == 0 + max_query_len = max(query_lens) + max_kv_len = max(kv_lens) + window_size = (sliding_window - 1, 0) if sliding_window is not None else (-1, -1) + scale = head_size**-0.5 + + query = torch.randn(sum(query_lens), num_query_heads, head_size, dtype=dtype) + key_cache = torch.randn( + num_blocks, block_size, num_kv_heads, head_size, dtype=dtype + ) + value_cache = torch.randn_like(key_cache) + cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum( + dim=0, dtype=torch.int32 + ) + kv_lens_tensor = torch.tensor(kv_lens, dtype=torch.int32) + + max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size + block_tables = torch.randint( + 0, num_blocks, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32 + ) + + output = torch.empty_like(query) + + num_par_softmax_segments = 16 + head_size_padded = next_power_of_2(head_size) + softmax_segm_output = torch.empty( + (seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded), + dtype=torch.float32, + ) + softmax_segm_max = torch.empty( + (seq_threshold_3D, num_query_heads, num_par_softmax_segments), + dtype=torch.float32, + ) + softmax_segm_expsum = torch.empty( + (seq_threshold_3D, num_query_heads, num_par_softmax_segments), + dtype=torch.float32, + ) + + unified_attention( + q=query, + k=key_cache, + v=value_cache, + out=output, + cu_seqlens_q=cu_query_lens, + seqused_k=kv_lens_tensor, + max_seqlen_q=max_query_len, + max_seqlen_k=max_kv_len, + softmax_scale=scale, + causal=True, + window_size=window_size, + block_table=block_tables, + softcap=soft_cap if soft_cap is not None else 0, + q_descale=None, + k_descale=None, + v_descale=None, + seq_threshold_3D=seq_threshold_3D, + num_par_softmax_segments=num_par_softmax_segments, + softmax_segm_output=softmax_segm_output, + softmax_segm_max=softmax_segm_max, + softmax_segm_expsum=softmax_segm_expsum, + use_td=True, + ) + + ref_output = ref_paged_attn( + query=query, + key_cache=key_cache, + value_cache=value_cache, + query_lens=query_lens, + kv_lens=kv_lens, + block_tables=block_tables, + scale=scale, + sliding_window=sliding_window, + soft_cap=soft_cap, + ) + torch.testing.assert_close(output, ref_output, atol=1.5e-2, rtol=1e-2) + + +@pytest.mark.parametrize( + "seq_lens", [[(1, 1328), (5, 18), (129, 463)], [(1, 523), (1, 37), (1, 2011)]] +) +@pytest.mark.parametrize("num_heads", NUM_HEADS) +@pytest.mark.parametrize("head_size", HEAD_SIZES_USE_TD) +@pytest.mark.parametrize("block_size", BLOCK_SIZES) +@pytest.mark.parametrize("sliding_window", [None, 128]) +@pytest.mark.parametrize("soft_cap", [None, 50.0]) +@pytest.mark.parametrize("num_blocks", NUM_BLOCKS) +@pytest.mark.parametrize("seq_threshold_3D", SEQ_THRESHOLD_3D_VALUES) +@torch.inference_mode() +def test_triton_unified_attn_use_td( + seq_lens: list[tuple[int, int]], + num_heads: tuple[int, int], + head_size: int, + sliding_window: int | None, + block_size: int, + soft_cap: float | None, + num_blocks: int, + seq_threshold_3D: int, +) -> None: + """Exercise the USE_TD (tensor-descriptor) Q/K/V load/store path. + + Covers both 2D and 3D kernels via ``seq_threshold_3D``. Two routes + to the USE_TD_QO=False fallback (pointer path for Q/O with TD still + active for KV tile loads): + + - non-pow2 ``num_queries_per_kv`` via ``NUM_HEADS`` entry ``(5, 1)``, + - non-pow2 ``head_size`` via ``HEAD_SIZES_USE_TD`` entry ``96``. + """ + _run_use_td_case( + seq_lens=seq_lens, + num_heads=num_heads, + head_size=head_size, + block_size=block_size, + sliding_window=sliding_window, + soft_cap=soft_cap, + seq_threshold_3D=seq_threshold_3D, + num_blocks=num_blocks, + ) + + +# Prefill-heavy shape: long query drives the prefill kernel path where +# ``_get_tile_size`` returns 32, which exceeds block_size=16 and must be +# clamped by the fix in 'clamp TILE_SIZE to block_size when USE_TD'. +# Only the prefill launch exercises the clamp, so parameterize only over +# the (num_heads, seq_threshold_3D=0) combinations needed to cover it. +@pytest.mark.parametrize("num_heads", [(4, 4), (5, 1)]) +@torch.inference_mode() +def test_triton_unified_attn_use_td_tile_clamp( + num_heads: tuple[int, int], +) -> None: + """Regression guard: ``USE_TD`` needs ``BLOCK_SIZE % TILE_SIZE == 0``. + + With ``block_size=16`` and ``head_size=128`` (non-Gemma3), + ``_get_tile_size`` returns 32 for prefill, which violates the + ``USE_TD`` constraint unless clamped to ``block_size``. Without + the clamp the triton kernel ``static_assert`` fires at compile time. + """ + _run_use_td_case( + seq_lens=[(256, 256), (128, 128)], + num_heads=num_heads, + head_size=128, + block_size=16, + sliding_window=None, + soft_cap=None, + seq_threshold_3D=0, + ) diff --git a/vllm/envs.py b/vllm/envs.py index 03230eed06..851f32a095 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -82,6 +82,7 @@ if TYPE_CHECKING: VLLM_MAIN_CUDA_VERSION: str = "13.0" VLLM_FLOAT32_MATMUL_PRECISION: Literal["highest", "high", "medium"] = "highest" VLLM_BATCH_INVARIANT: bool = False + VLLM_TRITON_ATTN_USE_TD: bool | None = None MAX_JOBS: str | None = None NVCC_THREADS: str | None = None VLLM_USE_PRECOMPILED: bool = False @@ -525,6 +526,16 @@ environment_variables: dict[str, Callable[[], Any]] = { # Enable batch-invariant mode: deterministic results regardless of # batch composition. Requires NVIDIA GPU with compute capability >= 9.0. "VLLM_BATCH_INVARIANT": lambda: bool(int(os.getenv("VLLM_BATCH_INVARIANT", "0"))), + # Use tensor descriptors for Q/K/V loads and output stores in the + # Triton unified-attention kernel. Enables HW 2D block reads on + # Intel Xe2/Xe3; the non-TD branch is dead-code-eliminated at Triton + # compile time so other platforms see no overhead. Tri-state override: + # unset (default) lets the `triton_attn` backend auto-select per + # platform (currently auto-enabled on XPU only); ``1`` forces TD on; + # ``0`` forces TD off. Useful for A/B benchmarking the TD path. + "VLLM_TRITON_ATTN_USE_TD": lambda: {"1": True, "0": False}.get( + os.getenv("VLLM_TRITON_ATTN_USE_TD", "").strip() + ), # Maximum number of compilation jobs to run in parallel. # By default this is the number of CPUs "MAX_JOBS": lambda: os.getenv("MAX_JOBS", None), diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index 0aeb33ae6f..96784ca1fe 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -7,6 +7,7 @@ from typing import ClassVar import torch +import vllm.envs as envs from vllm._aiter_ops import rocm_aiter_ops from vllm.config import CUDAGraphMode, VllmConfig from vllm.config.cache import CacheDType @@ -502,6 +503,21 @@ class TritonAttentionImpl(AttentionImpl): self._kv_quant_mode = get_kv_quant_mode(kv_cache_dtype) self._is_per_token_head_quant = self._kv_quant_mode.is_per_token_head + # Enable tensor descriptors for Q/K/V load/store on platforms that + # benefit from HW 2D block reads (Intel Xe2/Xe3). The dead branch + # is eliminated at Triton compile time, so other platforms see + # zero cost when TD is off. + # + # ``VLLM_TRITON_ATTN_USE_TD`` is tri-state: + # - unset (None): auto-select (TD on for XPU, off elsewhere), + # - ``1``: force TD on regardless of platform, + # - ``0``: force TD off regardless of platform (useful for A/B). + td_override = envs.VLLM_TRITON_ATTN_USE_TD + if td_override is None: + self.use_td = current_platform.is_xpu() + else: + self.use_td = td_override + def forward( self, layer: torch.nn.Module, @@ -637,6 +653,7 @@ class TritonAttentionImpl(AttentionImpl): k_scale_cache=k_scale_cache, v_scale_cache=v_scale_cache, chunk_lookback=self.chunk_lookback, + use_td=self.use_td, ) return output diff --git a/vllm/v1/attention/ops/triton_unified_attention.py b/vllm/v1/attention/ops/triton_unified_attention.py index 1774f1b305..c3435b0822 100644 --- a/vllm/v1/attention/ops/triton_unified_attention.py +++ b/vllm/v1/attention/ops/triton_unified_attention.py @@ -54,16 +54,132 @@ def _cast_kv_tile(data, Q, tensor_scale, KV_QUANT_MODE: tl.constexpr): return data.to(Q.dtype) +# --------------------------------------------------------------------------- +# Tensor-descriptor (TD) helpers +# +# Used when the caller enables ``USE_TD`` (Intel Xe2/Xe3 HW 2D block reads). +# When ``USE_TD`` is False the helpers are dead-code-eliminated at Triton +# compile time, leaving the pointer-arithmetic load/store path untouched. +# --------------------------------------------------------------------------- + + +@triton.jit +def _load_q_td( + query_ptr, + q_block_local_len, + query_stride_0: tl.int64, + query_stride_1: tl.int64, + cur_batch_in_all_start_index, + q_block_local_idx, + kv_head_idx, + num_queries_per_kv: tl.constexpr, + BLOCK_Q: tl.constexpr, + BLOCK_M: tl.constexpr, + HEAD_SIZE: tl.constexpr, + HEAD_SIZE_PADDED: tl.constexpr, +): + """Load Q via a 2D tensor descriptor. + + Caller guarantees (via the wrapper's ``use_td_qo`` gate): + * ``HEAD_SIZE == HEAD_SIZE_PADDED`` (head_size is a power of 2), + * ``num_queries_per_kv`` is a power of 2, + * the ``num_queries_per_kv`` heads of the current KV group are + contiguous in memory (``query_stride_1 == HEAD_SIZE``, which is + the default vLLM query layout). + + Under those preconditions the inner two axes are flattened into one + row of size ``num_queries_per_kv * HEAD_SIZE`` with stride 1, which + avoids the non-power-of-2 ``block_shape`` error from the Triton + tensor-descriptor validator. Returns (BLOCK_M, HEAD_SIZE_PADDED). + """ + q_base = ( + query_ptr + + (cur_batch_in_all_start_index + q_block_local_idx * BLOCK_Q) * query_stride_0 + + (kv_head_idx * num_queries_per_kv) * query_stride_1 + ) + q_desc = tl.make_tensor_descriptor( + base=q_base, + shape=(q_block_local_len, num_queries_per_kv * HEAD_SIZE), + strides=(query_stride_0, 1), + block_shape=(BLOCK_Q, num_queries_per_kv * HEAD_SIZE_PADDED), + ) + return q_desc.load([0, 0]).reshape(BLOCK_M, HEAD_SIZE_PADDED) + + +@triton.jit +def _load_kv_tile_td( + cache_ptr, + physical_block_idx_scalar, + kv_head_idx, + offset_in_block, + stride_cache_0: tl.int64, + stride_cache_1: tl.int64, + stride_cache_2: tl.int64, + stride_cache_3: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + TILE_SIZE: tl.constexpr, + HEAD_SIZE: tl.constexpr, + HEAD_SIZE_PADDED: tl.constexpr, +): + """Load a KV cache tile via tensor descriptor. + + Returns shape (TILE_SIZE, HEAD_SIZE_PADDED). Caller transposes for K. + Tensor descriptors zero-pad reads beyond the shape boundary, so + ``HEAD_SIZE_PADDED > HEAD_SIZE`` is handled correctly. + """ + base = ( + cache_ptr + + physical_block_idx_scalar * stride_cache_0 + + kv_head_idx * stride_cache_2 + ) + desc = tl.make_tensor_descriptor( + base=base, + shape=(BLOCK_SIZE, HEAD_SIZE), + strides=(stride_cache_1, stride_cache_3), + block_shape=(TILE_SIZE, HEAD_SIZE_PADDED), + ) + return desc.load([offset_in_block, 0]) + + +@triton.jit +def _store_output_td( + base_ptr, + acc, + q_block_local_len, + stride_token: tl.int64, + stride_head: tl.int64, + num_queries_per_kv: tl.constexpr, + BLOCK_Q: tl.constexpr, + HEAD_SIZE: tl.constexpr, + HEAD_SIZE_PADDED: tl.constexpr, +): + """Store an output tile via a tensor descriptor. + + The 2D and 3D epilogues differ only in ``base_ptr`` and the + ``(stride_token, stride_head)`` pair: 2D writes directly to the + flat output buffer, 3D writes to a single per-segment slice of + ``segm_output_ptr``. Descriptor shape / block_shape / reshape + are the same in both modes, so share one helper. + """ + acc = acc.to(base_ptr.dtype.element_ty) + output_desc = tl.make_tensor_descriptor( + base=base_ptr, + shape=(q_block_local_len, num_queries_per_kv, HEAD_SIZE), + strides=(stride_token, stride_head, 1), + block_shape=(BLOCK_Q, num_queries_per_kv, HEAD_SIZE_PADDED), + ) + output_desc.store( + [0, 0, 0], + acc.reshape(BLOCK_Q, num_queries_per_kv, HEAD_SIZE_PADDED), + ) + + @triton.jit def kernel_unified_attention( - # Output destinations. In 2D mode we write the final result into - # ``output_ptr``; in 3D mode we write per-segment partials into the - # three ``segm_*`` tensors and ``output_ptr`` is unused (callers may - # pass any non-null pointer). + # Output destination for the 2D path. In 3D mode per-segment partials + # go to the ``segm_*`` tensors (see bottom of signature) and + # ``output_ptr`` is unused (callers may pass any non-null pointer). output_ptr, - segm_output_ptr, - segm_max_ptr, - segm_expsum_ptr, # Inputs query_ptr, key_cache_ptr, @@ -73,10 +189,6 @@ def kernel_unified_attention( seq_lens_ptr, alibi_slopes_ptr, qq_bias_ptr, - # Per-(token, head) scale caches (used iff KV_QUANT_MODE in {2, 3}). - # For other modes callers may pass any non-null pointer. - k_scale_cache_ptr, - v_scale_cache_ptr, # Scalars scale, k_scale, @@ -112,12 +224,6 @@ def kernel_unified_attention( stride_v_cache_1: tl.int64, # int stride_v_cache_2: tl.int64, # int stride_v_cache_3: tl.constexpr, # int - stride_ks_blk: tl.int64, - stride_ks_slot: tl.int64, - stride_ks_head: tl.int64, - stride_vs_blk: tl.int64, - stride_vs_slot: tl.int64, - stride_vs_head: tl.int64, query_start_len_ptr, BLOCK_Q: tl.constexpr, num_seqs: tl.int32, @@ -129,6 +235,23 @@ def kernel_unified_attention( # to ``[segm_idx, segm_idx+1) × tiles_per_segment`` and writes # per-segment partials, finalized by ``reduce_segments``. IS_3D: tl.constexpr, + # Parameters below default to None so Triton can skip materialising them + # on call sites where the corresponding constexpr branch is dead. + # Credit: @quinnlp identified this as a perf regression source in + # intel/intel-xpu-backend-for-triton#6758 (review comment r3204641104). + # Per-segment outputs: used in 3D mode; unused in 2D (IS_3D=False). + segm_output_ptr=None, + segm_max_ptr=None, + segm_expsum_ptr=None, + # Per-(token, head) scale caches: used iff KV_QUANT_MODE in {2, 3}. + k_scale_cache_ptr=None, + v_scale_cache_ptr=None, + stride_ks_blk: tl.int64 = None, + stride_ks_slot: tl.int64 = None, + stride_ks_head: tl.int64 = None, + stride_vs_blk: tl.int64 = None, + stride_vs_slot: tl.int64 = None, + stride_vs_head: tl.int64 = None, # KV cache quantization mode handled inside this kernel via constexpr # branches: NONE (0), FP8_PER_TENSOR (1), INT8_PER_TOKEN_HEAD (2), # FP8_PER_TOKEN_HEAD (3). @@ -140,9 +263,20 @@ def kernel_unified_attention( # over ``SLIDING_WINDOW`` inside the helpers. ``-1`` disables. CHUNK_LOOKBACK: tl.constexpr = -1, CHUNK_SIZE: tl.constexpr = -1, + # Tensor-descriptor load/store for HW 2D block reads on Intel Xe2/Xe3. + # ``USE_TD`` gates KV tile loads; ``USE_TD_QO`` separately gates Q/output + # (see ``unified_attention`` wrapper for the gating rules). + USE_TD: tl.constexpr = False, + USE_TD_QO: tl.constexpr = False, ): USE_PER_TOKEN_HEAD_SCALES: tl.constexpr = KV_QUANT_MODE >= 2 + if USE_TD: + tl.static_assert( + BLOCK_SIZE % TILE_SIZE == 0, + "USE_TD requires BLOCK_SIZE to be a multiple of TILE_SIZE", + ) + q_block_global_idx = tl.program_id(0) kv_head_idx = tl.program_id(1) segm_idx = tl.program_id(2) if IS_3D else 0 @@ -167,6 +301,12 @@ def kernel_unified_attention( else: tiles_per_segment = 0 + # Number of valid query rows in this block (used by TD descriptor + # shapes, but always computed so the variable stays in scope). + q_block_local_len = tl.minimum( + BLOCK_Q, cur_batch_query_len - q_block_local_idx * BLOCK_Q + ) + offs_m = tl.arange(0, BLOCK_M) offs_d = tl.arange(0, HEAD_SIZE_PADDED) offs_t = tl.arange(0, TILE_SIZE) @@ -185,11 +325,27 @@ def kernel_unified_attention( query_mask_1 = tl.where(query_offset_1 < num_query_heads, 1, 0).to(tl.int1) # Q : (BLOCK_M, HEAD_SIZE_PADDED) - Q = tl.load( - query_ptr + query_offset, - mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], - other=0.0, - ) + if USE_TD_QO: + Q = _load_q_td( + query_ptr, + q_block_local_len, + query_stride_0, + query_stride_1, + cur_batch_in_all_start_index, + q_block_local_idx, + kv_head_idx, + num_queries_per_kv, + BLOCK_Q, + BLOCK_M, + HEAD_SIZE, + HEAD_SIZE_PADDED, + ) + else: + Q = tl.load( + query_ptr + query_offset, + mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], + other=0.0, + ) block_table_offset = seq_idx * block_table_stride @@ -237,31 +393,71 @@ def kernel_unified_attention( block_tables_ptr + block_table_offset + seq_offset // BLOCK_SIZE ).to(tl.int64) - v_offset = ( - physical_block_idx[:, None] * stride_v_cache_0 - + kv_head_idx * stride_v_cache_2 - + offs_d[None, :] * stride_v_cache_3 - + (seq_offset % BLOCK_SIZE)[:, None] * stride_v_cache_1 - ) - k_offset = ( - physical_block_idx[None, :] * stride_k_cache_0 - + kv_head_idx * stride_k_cache_2 - + offs_d[:, None] * stride_k_cache_3 - + (seq_offset % BLOCK_SIZE)[None, :] * stride_k_cache_1 - ) - # K : (HEAD_SIZE, TILE_SIZE) - K_load = tl.load( - key_cache_ptr + k_offset, - mask=dim_mask[:, None] & tile_mask[None, :], - other=0.0, - ) + if USE_TD: + # All TILE_SIZE slots within a single KV tile map to one + # physical block (guaranteed by ``BLOCK_SIZE % TILE_SIZE == 0`` + # from the static_assert above), so load the block index as + # a scalar instead of a broadcast reduction. + offset_in_block = (j * TILE_SIZE) % BLOCK_SIZE + physical_block_scalar = tl.load( + block_tables_ptr + block_table_offset + (j * TILE_SIZE) // BLOCK_SIZE + ).to(tl.int64) + # K : (HEAD_SIZE, TILE_SIZE) + K_load = _load_kv_tile_td( + key_cache_ptr, + physical_block_scalar, + kv_head_idx, + offset_in_block, + stride_k_cache_0, + stride_k_cache_1, + stride_k_cache_2, + stride_k_cache_3, + BLOCK_SIZE, + TILE_SIZE, + HEAD_SIZE, + HEAD_SIZE_PADDED, + ).T + # V : (TILE_SIZE, HEAD_SIZE) + V_load = _load_kv_tile_td( + value_cache_ptr, + physical_block_scalar, + kv_head_idx, + offset_in_block, + stride_v_cache_0, + stride_v_cache_1, + stride_v_cache_2, + stride_v_cache_3, + BLOCK_SIZE, + TILE_SIZE, + HEAD_SIZE, + HEAD_SIZE_PADDED, + ) + else: + v_offset = ( + physical_block_idx[:, None] * stride_v_cache_0 + + kv_head_idx * stride_v_cache_2 + + offs_d[None, :] * stride_v_cache_3 + + (seq_offset % BLOCK_SIZE)[:, None] * stride_v_cache_1 + ) + k_offset = ( + physical_block_idx[None, :] * stride_k_cache_0 + + kv_head_idx * stride_k_cache_2 + + offs_d[:, None] * stride_k_cache_3 + + (seq_offset % BLOCK_SIZE)[None, :] * stride_k_cache_1 + ) + # K : (HEAD_SIZE, TILE_SIZE) + K_load = tl.load( + key_cache_ptr + k_offset, + mask=dim_mask[:, None] & tile_mask[None, :], + other=0.0, + ) + # V : (TILE_SIZE, HEAD_SIZE) + V_load = tl.load( + value_cache_ptr + v_offset, + mask=dim_mask[None, :] & tile_mask[:, None], + other=0.0, + ) K = _cast_kv_tile(K_load, Q, k_scale, KV_QUANT_MODE) - # V : (TILE_SIZE, HEAD_SIZE) - V_load = tl.load( - value_cache_ptr + v_offset, - mask=dim_mask[None, :] & tile_mask[:, None], - other=0.0, - ) V = _cast_kv_tile(V_load, Q, v_scale, KV_QUANT_MODE) # Per-(token, head) scales for INT8 / FP8 per-token-head modes. @@ -342,18 +538,45 @@ def kernel_unified_attention( # ---- Epilogue --------------------------------------------------------- if IS_3D: # Store per-segment partials; finalized by ``reduce_segments``. - segm_output_offset = ( - query_offset_0[:, None].to(tl.int64) - * (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) - + query_offset_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) - + segm_idx * HEAD_SIZE_PADDED - + tl.arange(0, HEAD_SIZE_PADDED)[None, :] - ) - tl.store( - segm_output_ptr + segm_output_offset, - acc, - mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], - ) + if USE_TD_QO: + # 3D target: segm_output[token, head, segm_idx, :]. Advance + # the base to the correct (token-start, head-start, segm) + # slice; strides step between tokens / heads of the flattened + # (T, H, SEGS, PAD) layout. + segm_base = ( + segm_output_ptr + + (cur_batch_in_all_start_index + q_block_local_idx * BLOCK_Q).to( + tl.int64 + ) + * (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + + (kv_head_idx * num_queries_per_kv) + * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + + segm_idx * HEAD_SIZE_PADDED + ) + _store_output_td( + segm_base, + acc, + q_block_local_len, + num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED, + NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED, + num_queries_per_kv, + BLOCK_Q, + HEAD_SIZE, + HEAD_SIZE_PADDED, + ) + else: + segm_output_offset = ( + query_offset_0[:, None].to(tl.int64) + * (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + + query_offset_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + + segm_idx * HEAD_SIZE_PADDED + + tl.arange(0, HEAD_SIZE_PADDED)[None, :] + ) + tl.store( + segm_output_ptr + segm_output_offset, + acc, + mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], + ) store_segm_reduce_scalars( segm_max_ptr, segm_expsum_ptr, @@ -372,16 +595,38 @@ def kernel_unified_attention( if USE_FP8: acc = acc * tl.load(out_scale) acc = tl.clamp(acc, FP8_MIN, FP8_MAX) - output_offset = ( - query_offset_0[:, None] * output_stride_0 - + query_offset_1[:, None] * output_stride_1 - + offs_d[None, :] - ) - tl.store( - output_ptr + output_offset, - acc, - mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], - ) + if USE_TD_QO: + # 2D target: flat output[token, head, :]. Strides come + # straight from the caller (``output_stride_0`` per token, + # ``output_stride_1`` per head). + output_base = ( + output_ptr + + (cur_batch_in_all_start_index + q_block_local_idx * BLOCK_Q) + * output_stride_0 + + (kv_head_idx * num_queries_per_kv) * output_stride_1 + ) + _store_output_td( + output_base, + acc, + q_block_local_len, + output_stride_0, + output_stride_1, + num_queries_per_kv, + BLOCK_Q, + HEAD_SIZE, + HEAD_SIZE_PADDED, + ) + else: + output_offset = ( + query_offset_0[:, None] * output_stride_0 + + query_offset_1[:, None] * output_stride_1 + + offs_d[None, :] + ) + tl.store( + output_ptr + output_offset, + acc, + mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], + ) @triton.jit @@ -538,6 +783,11 @@ def unified_attention( v_scale_cache=None, # [num_blocks, block_size, num_kv_heads] float32 # Chunked attention: restrict attention to aligned blocks with lookback. chunk_lookback=-1, + # Tensor-descriptor mode: use ``tl.make_tensor_descriptor`` for Q/K/V + # loads and output stores. Enables HW 2D block reads on Intel Xe2/Xe3. + # The non-TD branch is dead-code-eliminated at Triton compile time so + # disabling this flag costs nothing. + use_td: bool = False, ): assert causal, "Only causal attention is supported" assert q_descale is None, "Q scales not supported" @@ -608,6 +858,52 @@ def unified_attention( head_size, sliding_window_val, q.element_size(), is_prefill=False ) + # USE_TD requires BLOCK_SIZE % TILE_SIZE == 0 (enforced by a + # ``tl.static_assert`` in the kernel). The default prefill tile + # size (32) is larger than a common ``block_size=16``, so clamp it + # down when TD is enabled. Zero overhead when disabled. + if use_td: + TILE_SIZE_PREFILL = min(TILE_SIZE_PREFILL, block_size) + TILE_SIZE_DECODE = min(TILE_SIZE_DECODE, block_size) + + # Tensor descriptors for Q load / output store require every element + # of ``block_shape`` to be a power of 2. ``num_queries_per_kv`` is + # not always pow2 (e.g. Qwen2-7B: 28 / 4 = 7), so gate the Q/O paths + # separately from the KV tile loads (whose ``block_shape`` does not + # include ``num_queries_per_kv``). + # + # The Q/O descriptors also encode ``HEAD_SIZE_PADDED`` on the inner + # axis while the backing buffers (both flat output and per-segment + # output) are laid out with ``HEAD_SIZE``. When they differ (e.g. + # Phi-3's head_size=96 → HEAD_SIZE_PADDED=128) the store would spill + # padded lanes into neighbouring heads because tensor-descriptor + # stores don't mask the padded tail. Fall back to the pointer path + # for Q/O in that case — KV tile loads are unaffected because their + # ``shape`` already matches ``block_shape`` on the inner axis. + head_size_padded = triton.next_power_of_2(head_size) + _is_pow2_nq = (num_queries_per_kv & (num_queries_per_kv - 1)) == 0 + _is_pow2_hs = head_size == head_size_padded + use_td_qo = use_td and _is_pow2_nq and _is_pow2_hs + + # ``_load_q_td`` / ``_store_output_td`` flatten ``(num_queries_per_kv, + # HEAD_SIZE)`` into a single contiguous inner axis. That's only + # equivalent to the pointer path when the ``num_queries_per_kv`` heads + # for this KV group start at ``kv_head_idx * num_queries_per_kv`` and + # lie exactly HEAD_SIZE apart — i.e. ``query_stride_1 == HEAD_SIZE`` + # and ``output_stride_1 == head_size``. This is the default vLLM + # query/output layout; assert it explicitly so we fail fast if a + # future caller passes a non-contiguous query tensor. + if use_td_qo: + assert q.stride(1) == head_size, ( + f"USE_TD_QO requires contiguous query heads " + f"(q.stride(1) = {q.stride(1)} != head_size = {head_size}); " + f"set VLLM_TRITON_ATTN_USE_TD=0 or pad the query layout." + ) + assert out.stride(1) == head_size, ( + f"USE_TD_QO requires contiguous output heads " + f"(out.stride(1) = {out.stride(1)} != head_size = {head_size})." + ) + # Launch the 2D kernel if # 1. No intermediate tiled softmax buffers for the 3D kernel have been allocated, or # 2. The batch includes at least one prefill request, or @@ -689,7 +985,7 @@ def unified_attention( BLOCK_SIZE=block_size, TILE_SIZE=tile_size, HEAD_SIZE=head_size, - HEAD_SIZE_PADDED=triton.next_power_of_2(head_size), + HEAD_SIZE_PADDED=head_size_padded, USE_ALIBI_SLOPES=use_alibi_slopes, USE_ALIBI_SQRT=use_alibi_sqrt, USE_QQ_BIAS=use_qq_bias, @@ -723,6 +1019,8 @@ def unified_attention( KV_QUANT_MODE=kv_quant_mode, CHUNK_LOOKBACK=chunk_lookback, CHUNK_SIZE=chunk_size, + USE_TD=use_td, + USE_TD_QO=use_td_qo, ) if use_3d: @@ -740,7 +1038,7 @@ def unified_attention( block_table_stride=block_table.stride(0), TILE_SIZE=TILE_SIZE_DECODE, HEAD_SIZE=head_size, - HEAD_SIZE_PADDED=triton.next_power_of_2(head_size), + HEAD_SIZE_PADDED=head_size_padded, query_start_len_ptr=cu_seqlens_q, BLOCK_Q=BLOCK_Q, NUM_SEGMENTS_PER_SEQ=num_par_softmax_segments,