# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """kpool (key-pooling) Triton kernels for the sparse-attention indexer. The cache stores POOLS (1 entry per ``pool_size`` consecutive tokens) rather than individual tokens. ``compress_ratio == pool_size`` on the kv_cache_spec makes the metadata builder emit pool-granular slot_mapping / seq_lens / cu_seq_lens / page_table for free; this file supplies the compress-write kernel (replacing ``indexer_k_quant_and_cache``) and the pool-level topk helpers (select pools -> expand to tokens -> append tail). """ from __future__ import annotations import torch from vllm.platforms import current_platform from vllm.triton_utils import tl, triton # The GLM-5.3-Flash indexer head dimension is fixed at 128. INDEX_HEAD_DIM = 128 FP8_DTYPE = current_platform.fp8_dtype() FP8_MAX = torch.finfo(FP8_DTYPE).max @triton.jit def _cache_k_offset( token_offset, dim_offset, head_dim: tl.constexpr, preshuffle: tl.constexpr, ): if preshuffle: return ( (token_offset // 16) * 16 * head_dim + (dim_offset // 16) * 16 * 16 + (token_offset % 16) * 16 + dim_offset % 16 ) return token_offset * head_dim + dim_offset # Hadamard-128 rotation @triton.jit def _hadamard128_stage(x, GROUPS: tl.constexpr, STRIDE: tl.constexpr): x3 = tl.reshape(x, (GROUPS, 2, STRIDE)) x3 = tl.trans(x3, 0, 2, 1) a, b = tl.split(x3) x3 = tl.join(a + b, a - b) x3 = tl.trans(x3, 0, 2, 1) return tl.reshape(x3, (128,)) @triton.jit def _hadamard128(x): x = _hadamard128_stage(x, 64, 1) x = _hadamard128_stage(x, 32, 2) x = _hadamard128_stage(x, 16, 4) x = _hadamard128_stage(x, 8, 8) x = _hadamard128_stage(x, 4, 16) x = _hadamard128_stage(x, 2, 32) x = _hadamard128_stage(x, 1, 64) return x * 0.08838834764831845 # 1/sqrt(128) # Fused pool compression and cache write. @triton.jit def _kpool_softmax_rotate_write_cache_kernel( buf_fp8_ptr, buf_fp32_ptr, slot_k_ptr, slot_score_ptr, ape_ptr, loc_ptr, write_mask_ptr, compressed_k_ptr, compressed_scale_ptr, slot_k_stride_0, slot_k_stride_1, slot_score_stride_0, slot_score_stride_1, ape_stride_0, PAGE_SIZE: tl.constexpr, BUF_NUMEL_PER_PAGE: tl.constexpr, POOL_SIZE: tl.constexpr, HEAD_DIM: tl.constexpr, S_OFFSET_NBYTES_IN_PAGE: tl.constexpr, FP8_MAX: tl.constexpr, PRESHUFFLE: tl.constexpr, ROUND_SCALE: tl.constexpr, HAS_WRITE_MASK: tl.constexpr, RETURN_COMPRESSED: tl.constexpr, WRITE_CACHE: tl.constexpr, BLOCK_D: tl.constexpr, ): """One program per pool. softmax(slot_score+ape)-weighted sum of slot_k -> Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``.""" row = tl.program_id(0) do_write = True if HAS_WRITE_MASK: do_write = tl.load(write_mask_ptr + row) offs = tl.arange(0, BLOCK_D) mask = (offs < HEAD_DIM) & do_write # --- Pass 1: per-dim max over the pool (softmax numerical stability) --- max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32) for slot in tl.static_range(0, POOL_SIZE): score = tl.load( slot_score_ptr + row * slot_score_stride_0 + slot * slot_score_stride_1 + offs, mask=mask, other=0.0, ).to(tl.float32) score += tl.load(ape_ptr + slot * ape_stride_0 + offs, mask=mask, other=0.0).to( tl.float32 ) max_score = tl.maximum(max_score, score) # --- Pass 2: softmax-weighted sum of K --- acc = tl.full((BLOCK_D,), 0.0, tl.float32) denom = tl.full((BLOCK_D,), 0.0, tl.float32) for slot in tl.static_range(0, POOL_SIZE): score = tl.load( slot_score_ptr + row * slot_score_stride_0 + slot * slot_score_stride_1 + offs, mask=mask, other=0.0, ).to(tl.float32) score += tl.load(ape_ptr + slot * ape_stride_0 + offs, mask=mask, other=0.0).to( tl.float32 ) prob = tl.exp(score - max_score) denom += prob k = tl.load( slot_k_ptr + row * slot_k_stride_0 + slot * slot_k_stride_1 + offs, mask=mask, other=0.0, ).to(tl.float32) acc += k * prob x = acc / denom x = tl.where(do_write, x, 0.0).to(tl.bfloat16).to(tl.float32) # Match the unfused pooled-K path's bf16 precision before quantization. x = _hadamard128(x).to(tl.bfloat16).to(tl.float32) # --- per-vector absmax fp8 quant --- fp8_max_inv = 1.0 / FP8_MAX absmax = tl.max(tl.abs(x), axis=0) absmax = tl.maximum(absmax, 1e-4) if ROUND_SCALE: scale = tl.exp2(tl.ceil(tl.log2(absmax * fp8_max_inv))) else: scale = absmax * fp8_max_inv quantized = x / scale quantized = tl.minimum(tl.maximum(quantized, -FP8_MAX), FP8_MAX) if WRITE_CACHE: loc = tl.load(loc_ptr + row, mask=do_write, other=0) loc_page_index = loc // PAGE_SIZE loc_token_offset_in_page = loc % PAGE_SIZE out_k_offsets = loc_page_index * BUF_NUMEL_PER_PAGE + _cache_k_offset( loc_token_offset_in_page, offs, HEAD_DIM, PRESHUFFLE, ) out_s_offset = ( loc_page_index * BUF_NUMEL_PER_PAGE // 4 + S_OFFSET_NBYTES_IN_PAGE // 4 + loc_token_offset_in_page ) tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=mask) tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write) if RETURN_COMPRESSED: tl.store( compressed_k_ptr + row * HEAD_DIM + offs, quantized, mask=offs < HEAD_DIM, ) tl.store(compressed_scale_ptr + row, scale) def kpool_compress_and_write_cache( kv_cache: torch.Tensor, slot_k: torch.Tensor, slot_score: torch.Tensor, ape: torch.Tensor, loc: torch.Tensor, pool_size: int, head_dim: int = INDEX_HEAD_DIM, write_mask: torch.Tensor | None = None, round_scale: bool = True, return_compressed: bool = False, write_cache: bool = True, ): """Compress ``pool_size`` tokens into one fp8 K and write at ``loc``. Args: kv_cache: indexer K cache ``[num_blocks, block_size, head_dim+4]`` uint8. slot_k: ``[n_pools, pool_size, head_dim]`` bf16 — raw per-token K. slot_score: ``[n_pools, pool_size, head_dim]`` — per-token gate score. ape: ``[pool_size, head_dim]`` fp32 — per-slot position bias. loc: ``[n_pools]`` int64 — flat physical slot per pool. pool_size: Number of tokens compressed into one cache entry. head_dim: Indexer head dimension. write_mask: ``[n_pools]`` bool — pools to write, or None for all. round_scale: Round each fp8 scale down to a power of two. return_compressed: Also return the compressed K and scales. write_cache: Write the compressed result into ``kv_cache``. """ assert slot_k.ndim == 3 assert slot_score.shape == slot_k.shape assert ape.shape == slot_k.shape[1:] assert slot_k.shape[2] == head_dim assert slot_k.dtype == torch.bfloat16 assert ape.dtype == torch.float32 assert kv_cache.dtype == torch.uint8 assert loc.dtype == torch.int64 assert write_cache or return_compressed page_size = kv_cache.shape[1] buf = kv_cache slot_k = slot_k.contiguous() slot_score = slot_score.contiguous() ape = ape.contiguous() loc = loc.contiguous() if write_mask is None: write_mask = torch.empty((1,), dtype=torch.bool, device=slot_k.device) has_write_mask = False else: assert write_mask.shape == (slot_k.shape[0],) write_mask = write_mask.contiguous() has_write_mask = True assert not return_compressed if slot_k.shape[0] == 0: if return_compressed: return ( torch.empty( (0, head_dim), dtype=FP8_DTYPE, device=slot_k.device, ), torch.empty((0,), dtype=torch.float32, device=slot_k.device), ) return None buf_fp8 = buf.view(FP8_DTYPE) buf_fp32 = buf.view(torch.float32) # bytes per page (last dim of kv_cache) viewed as uint8 buf_numel_per_page = buf.stride(0) s_offset_nbytes_in_page = page_size * head_dim if return_compressed: compressed_k = torch.empty( (slot_k.shape[0], head_dim), dtype=FP8_DTYPE, device=slot_k.device, ) compressed_scale = torch.empty( (slot_k.shape[0],), dtype=torch.float32, device=slot_k.device ) else: compressed_k = buf_fp8 compressed_scale = buf_fp32 if page_size > 1: assert page_size % 16 == 0, "ROCm preshuffle requires 16-token tiles" _kpool_softmax_rotate_write_cache_kernel[(slot_k.shape[0],)]( buf_fp8, buf_fp32, slot_k, slot_score, ape, loc, write_mask, compressed_k, compressed_scale, slot_k.stride(0), slot_k.stride(1), slot_score.stride(0), slot_score.stride(1), ape.stride(0), PAGE_SIZE=page_size, BUF_NUMEL_PER_PAGE=buf_numel_per_page, POOL_SIZE=slot_k.shape[1], HEAD_DIM=head_dim, S_OFFSET_NBYTES_IN_PAGE=s_offset_nbytes_in_page, FP8_MAX=FP8_MAX, PRESHUFFLE=page_size > 1, ROUND_SCALE=round_scale, HAS_WRITE_MASK=has_write_mask, RETURN_COMPRESSED=return_compressed, WRITE_CACHE=write_cache, BLOCK_D=triton.next_power_of_2(head_dim), ) if return_compressed: return compressed_k, compressed_scale return None # Seed each request's incomplete pool into its paged tail during prefill. @triton.jit def _kpool_tail_seed_kernel( key_ptr, score_ptr, tslot_ptr, tail_ptr, n_tokens, TAIL_BLOCK_ELEMS: tl.constexpr, KPOOL_HEAD: tl.constexpr, HEAD_DIM: tl.constexpr, KPOOL: tl.constexpr, RING: tl.constexpr, BLOCK_D: tl.constexpr, ): """Copy token ``i``'s raw K + gate into its request's tail ring. Token ``i`` is among its request's last KPOOL tokens iff the token KPOOL ahead belongs to a different tail block (or is past the batch / padding, slot < 0). ``tslot = block * RING + pos % RING``; the destination is ``tail[block, {0:K, 1:score}, pos % RING, :]``. """ i = tl.program_id(0) t = tl.load(tslot_ptr + i).to(tl.int64) if t < 0: return blk = t // RING # t >= 0 here, so trunc == floor ahead = tl.load(tslot_ptr + i + KPOOL, mask=i + KPOOL < n_tokens, other=-1).to( tl.int64 ) # Match the torch semantics exactly: a negative ahead slot floors to a # block id that differs from every real block -> token is in the tail. # Only divide non-negative slots (Triton int div truncates, torch floors). if ahead >= 0 and ahead // RING == blk: return offs = tl.arange(0, BLOCK_D) m = offs < HEAD_DIM block_base = blk * TAIL_BLOCK_ELEMS base = block_base + (t % RING) * HEAD_DIM k = tl.load(key_ptr + i * HEAD_DIM + offs, mask=m) s = tl.load(score_ptr + i * HEAD_DIM + offs, mask=m) tl.store(tail_ptr + base + offs, k, mask=m) tl.store( tail_ptr + block_base + KPOOL_HEAD + (t % RING) * HEAD_DIM + offs, s, mask=m ) def kpool_seed_tail_cache( tail_kv_cache: torch.Tensor, key: torch.Tensor, gate_score: torch.Tensor, tslot: torch.Tensor, kpool: int, head_dim: int = INDEX_HEAD_DIM, ) -> None: """Seed the paged tail cache from a prefill batch (see the kernel).""" assert tail_kv_cache.dtype == torch.bfloat16 assert key.dtype == torch.bfloat16 n = tslot.shape[0] if n == 0: return _kpool_tail_seed_kernel[(n,)]( key, gate_score, tslot, tail_kv_cache, n, TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0), KPOOL_HEAD=tail_kv_cache.stride(1), HEAD_DIM=head_dim, KPOOL=kpool, RING=tail_kv_cache.shape[2], BLOCK_D=triton.next_power_of_2(head_dim), ) # Update each request's tail during decode and write completed pools. @triton.jit def _kpool_decode_update_batched_kernel( buf_fp8_ptr, buf_fp32_ptr, tail_kv_ptr, tail_slot_mapping_ptr, # [B, NEXT_N] int32 key_ptr, # [B, NEXT_N, HEAD_DIM] bf16 key_stride_b, key_stride_t, slot_score_ptr, # [B, NEXT_N, HEAD_DIM] bf16 ss_stride_b, ss_stride_t, ape_ptr, ape_stride_0, slot_mapping_ptr, # [B, NEXT_N] int32 positions_ptr, # [B, NEXT_N] int32 NEXT_N, # runtime token count per request (no .item() needed) PAGE_SIZE: tl.constexpr, BUF_NUMEL_PER_PAGE: tl.constexpr, POOL_SIZE: tl.constexpr, RING: tl.constexpr, TAIL_BLOCK_ELEMS: tl.constexpr, KPOOL_HEAD: tl.constexpr, HEAD_DIM: tl.constexpr, S_OFFSET_NBYTES_IN_PAGE: tl.constexpr, FP8_MAX: tl.constexpr, PRESHUFFLE: tl.constexpr, ROUND_SCALE: tl.constexpr, BLOCK_D: tl.constexpr, ): """One program per request; iterates its NEXT_N verify tokens in order. Replaces the caller's per-token sequential launch loop. The intra-request iteration MUST stay in position order: a pool-completion at token t* reads the tail-ring slots that tokens t < t* (same request) just stashed in this same invocation. ``tl.range`` iterates sequentially within the program, so those stashes are visible to the later completion read. Cross-request programs are independent (distinct tail blocks). RING >= POOL_SIZE. """ req = tl.program_id(0) offs = tl.arange(0, BLOCK_D) dim_mask = offs < HEAD_DIM for t in tl.range(0, NEXT_N): idx = req * NEXT_N + t cache_loc = tl.load(slot_mapping_ptr + idx) pos = tl.load(positions_ptr + idx) safe_pos = tl.maximum(pos, 0) pos_valid = (cache_loc >= 0) & (pos >= 0) slot = safe_pos % POOL_SIZE phys_slot = safe_pos % RING # Derive the tail block from THIS token's tail_slot (the request's block # is constant across a pool, but a padded / invalid entry carries a # negative sentinel -- reading it from token 0 would poison every # token's base address). Clamp so an invalid entry can never form an # out-of-bounds base; the accesses below are gated on pos_valid anyway. tail_slot = tl.load(tail_slot_mapping_ptr + idx) block = tl.maximum(tail_slot, 0).to(tl.int64) // RING block_base = block * TAIL_BLOCK_ELEMS # The tail-ring stash must run for EVERY real token, so it is gated on # the token-granular tail slot -- not on `pos_valid`, which keys off the # POOL-granular `slot_mapping` and is therefore only true on the pool's # last token. Gating the stash on pos_valid dropped every intra-pool # token, so a decode-built pool compressed 3 stale ring entries (the # prefill-seeded prompt tail, frozen forever) plus the current token. stash_valid = (pos >= 0) & (tail_slot >= 0) key = tl.load( key_ptr + req * key_stride_b + t * key_stride_t + offs, mask=dim_mask, other=0.0, ).to(tl.float32) score_current = tl.load( slot_score_ptr + req * ss_stride_b + t * ss_stride_t + offs, mask=dim_mask, other=0.0, ).to(tl.float32) if pos_valid & (slot == POOL_SIZE - 1): pool_logical_start = safe_pos - slot max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32) for pool_slot in tl.static_range(0, POOL_SIZE): is_current = pool_slot == slot phys = (pool_logical_start + pool_slot) % RING score_buf = tl.load( tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs, mask=dim_mask, other=0.0, ).to(tl.float32) score = tl.where(is_current, score_current, score_buf) score += tl.load( ape_ptr + pool_slot * ape_stride_0 + offs, mask=dim_mask, other=0.0, ).to(tl.float32) max_score = tl.maximum(max_score, score) acc = tl.full((BLOCK_D,), 0.0, tl.float32) denom = tl.full((BLOCK_D,), 0.0, tl.float32) for pool_slot in tl.static_range(0, POOL_SIZE): is_current = pool_slot == slot phys = (pool_logical_start + pool_slot) % RING score_buf = tl.load( tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs, mask=dim_mask, other=0.0, ).to(tl.float32) score = tl.where(is_current, score_current, score_buf) score += tl.load( ape_ptr + pool_slot * ape_stride_0 + offs, mask=dim_mask, other=0.0, ).to(tl.float32) prob = tl.exp(score - max_score) denom += prob k_buf = tl.load( tail_kv_ptr + block_base + phys * HEAD_DIM + offs, mask=dim_mask, other=0.0, ).to(tl.float32) k = tl.where(is_current, key, k_buf) acc += k * prob x = (acc / denom).to(tl.bfloat16).to(tl.float32) x = _hadamard128(x).to(tl.bfloat16).to(tl.float32) fp8_max_inv = 1.0 / FP8_MAX absmax = tl.maximum(tl.max(tl.abs(x), axis=0), 1e-4) if ROUND_SCALE: scale = tl.exp2(tl.ceil(tl.log2(absmax * fp8_max_inv))) else: scale = absmax * fp8_max_inv quantized = tl.minimum(tl.maximum(x / scale, -FP8_MAX), FP8_MAX) loc = cache_loc.to(tl.int64) loc_page_index = loc // PAGE_SIZE loc_token_offset_in_page = loc % PAGE_SIZE out_k_offsets = loc_page_index * BUF_NUMEL_PER_PAGE + _cache_k_offset( loc_token_offset_in_page, offs, HEAD_DIM, PRESHUFFLE, ) out_s_offset = ( loc_page_index * BUF_NUMEL_PER_PAGE // 4 + S_OFFSET_NBYTES_IN_PAGE // 4 + loc_token_offset_in_page ) tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=dim_mask) tl.store(buf_fp32_ptr + out_s_offset, scale) # Stash the current token AFTER any completion read so the completion # uses prior stashes (and the current token's own key/score via # is_current), then leaves this token for future pools. Order matches # the per-token kernel: completion read first, stash second. update_mask = dim_mask & stash_valid tl.store( tail_kv_ptr + block_base + phys_slot * HEAD_DIM + offs, key, mask=update_mask, ) tl.store( tail_kv_ptr + block_base + KPOOL_HEAD + phys_slot * HEAD_DIM + offs, score_current, mask=update_mask, ) def kpool_decode_update_and_maybe_write_cache_batched( kv_cache: torch.Tensor, tail_kv_cache: torch.Tensor, tail_slot_mapping: torch.Tensor, key: torch.Tensor, slot_score: torch.Tensor, ape: torch.Tensor, slot_mapping: torch.Tensor, positions: torch.Tensor, pool_size: int, head_dim: int = INDEX_HEAD_DIM, round_scale: bool = True, ) -> None: """Batched decode-step kpool update for spec verify (``next_n > 1``). One launch replaces the caller's per-token loop. Inputs are grouped per request: ``[num_requests, next_n, ...]``. Each program handles one request's ``next_n`` tokens in position order (see the kernel docstring for why ordering is required for pool-completion correctness). Plain decode (``next_n == 1``) is handled here too — the kernel collapses to a single-iteration loop. Args: kv_cache: indexer K cache ``[num_blocks, block_size, head_dim+4]`` uint8. tail_kv_cache: paged tail cache ``[num_blocks, 2, pool_size, head_dim]`` bf16 (K at half 0, gate score at half 1). tail_slot_mapping: ``[num_requests, next_n]`` int32. key: ``[num_requests, next_n, head_dim]`` bf16. slot_score: ``[num_requests, next_n, head_dim]`` bf16. ape: ``[pool_size, head_dim]`` fp32. slot_mapping: ``[num_requests, next_n]`` int32. positions: ``[num_requests, next_n]`` int32. pool_size: Number of tokens compressed into one cache entry. head_dim: Indexer head dimension. round_scale: Round each fp8 scale down to a power of two. """ num_requests, next_n = key.shape[0], key.shape[1] if num_requests == 0 or next_n == 0: return assert tail_kv_cache.ndim == 4 assert tail_kv_cache.shape[1] == 2 ring = tail_kv_cache.shape[2] assert ring >= pool_size and ring % pool_size == 0, (ring, pool_size) assert tail_kv_cache.shape[3] == head_dim assert tail_kv_cache.dtype == torch.bfloat16 assert key.ndim == 3 and key.shape[2] == head_dim assert slot_score.shape == key.shape assert ape.shape == (pool_size, head_dim) assert tail_slot_mapping.shape == (num_requests, next_n) assert slot_mapping.shape == (num_requests, next_n) assert positions.shape == (num_requests, next_n) assert key.dtype == torch.bfloat16 assert slot_score.dtype == torch.bfloat16 assert ape.dtype == torch.float32 assert kv_cache.dtype == torch.uint8 page_size = kv_cache.shape[1] buf = kv_cache buf_fp8 = buf.view(FP8_DTYPE) buf_fp32 = buf.view(torch.float32) # The kernel indexes the int tensors as ``req * next_n + t`` (row-major), # so they must be contiguous. Callers pass either a view of a contiguous # slice or a freshly scattered tensor, making these no-ops; the calls guard # against a future caller handing over a strided view. tail_slot_mapping = tail_slot_mapping.contiguous() slot_mapping = slot_mapping.contiguous() positions = positions.contiguous() if page_size > 1: assert page_size % 16 == 0, "ROCm preshuffle requires 16-token tiles" _kpool_decode_update_batched_kernel[(num_requests,)]( buf_fp8, buf_fp32, tail_kv_cache, tail_slot_mapping, key, key.stride(0), key.stride(1), slot_score, slot_score.stride(0), slot_score.stride(1), ape, ape.stride(0), slot_mapping, positions, next_n, PAGE_SIZE=page_size, BUF_NUMEL_PER_PAGE=buf.stride(0), POOL_SIZE=pool_size, RING=ring, TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0), KPOOL_HEAD=tail_kv_cache.stride(1), HEAD_DIM=head_dim, S_OFFSET_NBYTES_IN_PAGE=page_size * head_dim, FP8_MAX=FP8_MAX, PRESHUFFLE=page_size > 1, ROUND_SCALE=round_scale, BLOCK_D=triton.next_power_of_2(head_dim), ) # Pool-level top-k helpers. def history_group_budget_for_topk(topk: int, pool_size: int) -> int: """Number of pools to select so that expanding yields ``topk`` tokens.""" assert topk % pool_size == 0 return topk // pool_size def expand_pools_to_tokens( group_ids: torch.Tensor, group_valid: torch.Tensor, topk: int, pool_size: int, page_table: torch.Tensor | None = None, topk_offsets: torch.Tensor | None = None, ) -> torch.Tensor: """Expand selected full-pool ids to a strict-width token topk tensor.""" assert group_ids.ndim == 2 assert group_valid.shape == group_ids.shape assert topk % pool_size == 0 assert group_ids.shape[1] == history_group_budget_for_topk(topk, pool_size) assert page_table is None or topk_offsets is None device = group_ids.device offsets = torch.arange(pool_size, device=device, dtype=torch.int64) token_ids = group_ids.to(torch.int64).unsqueeze(-1) * pool_size + offsets token_ids = token_ids.reshape(group_ids.shape[0], topk) valid = ( group_valid.unsqueeze(-1) .expand(-1, -1, pool_size) .reshape(group_ids.shape[0], topk) ) if page_table is not None: assert page_table.ndim == 2 safe_ids = token_ids.clamp(min=0, max=page_table.shape[1] - 1) output = torch.gather(page_table, dim=1, index=safe_ids).to(torch.int32) elif topk_offsets is not None: if topk_offsets.ndim == 2: assert topk_offsets.shape[1] == 1 topk_offsets = topk_offsets.squeeze(1) output = (token_ids + topk_offsets.to(torch.int64).unsqueeze(1)).to(torch.int32) else: output = token_ids.to(torch.int32) return torch.where(valid, output, torch.full_like(output, -1)) def append_tail_to_topk( topk_result: torch.Tensor, seq_lens: torch.Tensor, pool_lens: torch.Tensor, pool_size: int, page_table: torch.Tensor | None = None, topk_offsets: torch.Tensor | None = None, ) -> torch.Tensor: """Append non-pooled tail tokens after expanded history tokens. ``index_kpool_always_select_tail`` keeps the (incomplete) trailing pool so the most recent tokens are always attended to. """ assert topk_result.dtype == torch.int32 assert seq_lens.ndim == 1 assert pool_lens.ndim == 1 tail_pool = pool_size - 1 if tail_pool == 0: return topk_result rows, n_cols = topk_result.shape out_cols = n_cols + tail_pool out = torch.empty( (rows, out_cols), dtype=topk_result.dtype, device=topk_result.device ) # tail tokens: [pool_len*pool_size, seq_len) for each row. pool_len = pool_lens.to(torch.int32) tail_start = pool_len * pool_size seq_len = seq_lens.to(torch.int32) tail_count = seq_len - tail_start # in [0, pool_size) cols = torch.arange(out_cols, device=topk_result.device)[None, :] history_len = n_cols is_history = cols < history_len tail_off = cols - history_len is_tail = (tail_off >= 0) & (tail_off < tail_count[:, None]) # safe_hist must be per-row [rows, out_cols] so the gather reads each row's # OWN history. cols is [1, out_cols]; if used directly, gather (which does # NOT broadcast the index) would read only row 0 of topk_result, making every # query inherit row 0's history (empty for the first token) and lose all its # selected tokens — only the per-row tail would survive. This only manifests # for multi-row sparse PREFILL (decode has 1 row, so it reads its own row 0). safe_hist = torch.minimum(cols, torch.full_like(cols, n_cols - 1)).expand( rows, out_cols ) history_val = torch.gather(topk_result, 1, safe_hist) tail_raw = tail_start[:, None] + tail_off tail_val = tail_raw.to(torch.int32) if page_table is not None: safe_tail = tail_raw.clamp(min=0, max=page_table.shape[1] - 1) tail_val = torch.gather(page_table, 1, safe_tail).to(torch.int32) elif topk_offsets is not None: tail_val = (tail_raw + topk_offsets.to(torch.int64).unsqueeze(1)).to( torch.int32 ) out = torch.where(is_history, history_val, -1) out = torch.where(is_tail, tail_val, out) return out @triton.jit def _expand_pools_and_append_tail_kernel( pool_ids_ptr, # [rows, n_groups], int (any int dtype) seq_lens_ptr, # [rows], int32 (token-granular seq_len) out_ptr, # [rows, out_cols], int32 topk, # n_groups * pool_size out_cols, # topk + pool_size - 1 POOL_SIZE: tl.constexpr, BLOCK_COLS: tl.constexpr, pid_s0, out_s0, ): # Fuses expand_pools_to_tokens + append_tail_to_topk (identity path) into a # single kernel. Each program writes one (row, column-tile) of the output. row = tl.program_id(0) tile = tl.program_id(1) cols = tile * BLOCK_COLS + tl.arange(0, BLOCK_COLS) mask = cols < out_cols seq_len = tl.load(seq_lens_ptr + row) pool_len = seq_len // POOL_SIZE tail_start = pool_len * POOL_SIZE tail_count = seq_len - tail_start # in [0, POOL_SIZE) # History region [0, topk): expand selected pool g = cols // POOL_SIZE. is_history = cols < topk g = cols // POOL_SIZE o = cols % POOL_SIZE pid = tl.load(pool_ids_ptr + row * pid_s0 + g, mask=mask & is_history, other=-1) hist_val = (pid * POOL_SIZE + o).to(tl.int32) hist_out = tl.where(pid >= 0, hist_val, -1) # Tail region [topk, out_cols): the request's trailing incomplete pool. tail_off = cols - topk is_tail = (tail_off >= 0) & (tail_off < tail_count) tail_val = (tail_start + tail_off).to(tl.int32) tail_out = tl.where(is_tail, tail_val, -1) result = tl.where(is_history, hist_out, tail_out) tl.store(out_ptr + row * out_s0 + cols, result, mask=mask) def expand_pools_and_append_tail( pool_ids: torch.Tensor, seq_lens: torch.Tensor, pool_size: int, ) -> torch.Tensor: """Fuse ``expand_pools_to_tokens`` + ``append_tail_to_topk`` (identity path). Produces the same ``[rows, topk + pool_size - 1]`` int32 output as calling the two functions in sequence when neither ``page_table`` nor ``topk_offsets`` is passed — the only path used by the GLM-5.3-Flash indexer. The kernel derives ``pool_len = seq_len // pool_size`` internally, so the caller no longer needs to precompute it. Replaces ~25 elementwise kernels with one Triton launch. """ rows, n_groups = pool_ids.shape topk = n_groups * pool_size out_cols = topk + pool_size - 1 out = torch.empty((rows, out_cols), dtype=torch.int32, device=pool_ids.device) BLOCK_COLS = 128 n_tiles = triton.cdiv(out_cols, BLOCK_COLS) _expand_pools_and_append_tail_kernel[(rows, n_tiles)]( pool_ids, seq_lens, out, topk, out_cols, POOL_SIZE=pool_size, BLOCK_COLS=BLOCK_COLS, pid_s0=pool_ids.stride(0), out_s0=out.stride(0), ) return out