[Refactor][GLM-5.3-Flash] Move sparse_attn_indexer_kpool into the model folder and split AMD/NVIDIA (#55358)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -23,7 +23,6 @@ from vllm.model_executor.layers.linear import (
|
||||
from vllm.model_executor.layers.mla import MLAModules, MultiHeadLatentAttentionWrapper
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope
|
||||
from vllm.model_executor.layers.sparse_attn_indexer_kpool import SparseAttnIndexerKpool
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepSeekV2FusedQkvAProjLinear,
|
||||
DeepseekV32IndexerCache,
|
||||
@@ -31,6 +30,7 @@ from vllm.model_executor.models.deepseek_v2 import (
|
||||
)
|
||||
from vllm.model_executor.utils import maybe_disable_graph_partition
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
|
||||
from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
|
||||
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared helpers for the glm5next sparse attention indexer (kpool) layers."""
|
||||
|
||||
import torch
|
||||
|
||||
RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
|
||||
|
||||
# MXFP4 layout: 2 values packed per byte, ue8m0 (1-byte) scale per block of 32.
|
||||
MXFP4_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
def _build_decode_scatter_indices(
|
||||
decode_lens: torch.Tensor,
|
||||
num_requests: int,
|
||||
n: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-token (request id, intra-request index) for a non-uniform decode
|
||||
batch, with ``n == decode_lens.sum()`` as a host int (avoids a
|
||||
device sync and keeps both repeat_interleaves sync-free).
|
||||
|
||||
Shared by every ``_scatter_decode_tokens_by_request`` call in a step:
|
||||
building it per call would repeat the same repeat_interleave/cumsum chain
|
||||
up to 5x per layer on the eager decode break.
|
||||
"""
|
||||
device = decode_lens.device
|
||||
dl = decode_lens.to(torch.int64)
|
||||
req_id = torch.repeat_interleave(
|
||||
torch.arange(num_requests, device=device, dtype=torch.int64),
|
||||
dl,
|
||||
output_size=n,
|
||||
)
|
||||
req_starts = torch.cumsum(
|
||||
torch.cat([torch.zeros(1, device=device, dtype=torch.int64), dl[:-1]]),
|
||||
dim=0,
|
||||
)
|
||||
# Broadcast the per-request start offsets to per-token (length n ==
|
||||
# dl.sum()) so each token's intra-request index subtracts its own
|
||||
# request's start.
|
||||
starts = torch.repeat_interleave(req_starts, dl, output_size=n)
|
||||
intra = torch.arange(n, device=device, dtype=torch.int64) - starts
|
||||
return req_id, intra
|
||||
|
||||
|
||||
def _scatter_decode_tokens_by_request(
|
||||
tokens: torch.Tensor,
|
||||
pad_value,
|
||||
num_requests: int,
|
||||
lmax: int,
|
||||
scatter_indices: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
"""Group ``[N, ...]`` decode tokens into a padded ``[num_requests, lmax, ...]``
|
||||
layout: request ``r``'s tokens at row ``r`` in order; short requests padded.
|
||||
|
||||
Unlike ``pack_seq_triton`` this is dtype-agnostic (needed for the int32
|
||||
slot/pos tensors) — it scatters with the shared per-step indices from
|
||||
``_build_decode_scatter_indices``. Used only for the non-uniform
|
||||
(``requires_padding``) decode batch; uniform batches use a zero-copy
|
||||
reshape.
|
||||
"""
|
||||
req_id, intra = scatter_indices
|
||||
out = torch.full(
|
||||
(num_requests, lmax, *tokens.shape[1:]),
|
||||
pad_value,
|
||||
dtype=tokens.dtype,
|
||||
device=tokens.device,
|
||||
)
|
||||
out[req_id, intra] = tokens
|
||||
return out
|
||||
|
||||
|
||||
def _decode_topk_seq_lens(
|
||||
positions: torch.Tensor,
|
||||
decode_lens: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
batch_size: int,
|
||||
next_n: int,
|
||||
requires_padding: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Token-granular seq_len (pos + 1) per pool-topk row, layout-aware.
|
||||
|
||||
``pool_topk`` (and the logits it comes from) follow the padded
|
||||
``[batch_size, next_n]`` grid whenever ``requires_padding`` is set, so row
|
||||
``(b, t)`` corresponds to flat decode token ``offset_b + t`` -- NOT
|
||||
``b * next_n + t``. Slicing flat ``positions[: batch_size * next_n]``
|
||||
(the uniform-layout shortcut) misaligns every row after the first
|
||||
non-uniform request and, past the decode region, reads prefill tokens'
|
||||
positions; ``expand_pools_and_append_tail`` then anchors the tail at
|
||||
another request's length, dropping the row's real tail tokens or emitting
|
||||
indices past its sequence (out-of-bounds block-table reads). Padded rows
|
||||
get 0 (empty tail); they are dropped by ``unpack_seq_triton`` anyway.
|
||||
"""
|
||||
n = batch_size * next_n
|
||||
if not requires_padding:
|
||||
return positions[:n].to(torch.int32) + 1
|
||||
scatter_idx = _build_decode_scatter_indices(
|
||||
decode_lens, batch_size, num_decode_tokens
|
||||
)
|
||||
padded = _scatter_decode_tokens_by_request(
|
||||
positions[:num_decode_tokens].to(torch.int32),
|
||||
-1,
|
||||
batch_size,
|
||||
next_n,
|
||||
scatter_idx,
|
||||
)
|
||||
return padded.reshape(n) + 1 # pad rows: -1 + 1 = 0 -> empty tail
|
||||
|
||||
|
||||
def _fill_causal_indices(rows: torch.Tensor, positions: torch.Tensor) -> None:
|
||||
causal_range = torch.arange(rows.shape[1], device=rows.device, dtype=torch.int32)
|
||||
positions = positions.to(torch.int32)
|
||||
rows[:] = causal_range[None, :]
|
||||
rows[causal_range[None, :] > positions[:, None]] = -1
|
||||
|
||||
|
||||
def _fill_short_decode_causal_indices(
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
positions: torch.Tensor | None,
|
||||
num_decode_tokens: int,
|
||||
max_seq_len: int,
|
||||
topk_tokens: int,
|
||||
) -> bool:
|
||||
"""Fill exact causal rows when sparse decode would select every token."""
|
||||
if positions is None or positions.numel() == 0 or max_seq_len > topk_tokens:
|
||||
return False
|
||||
_fill_causal_indices(
|
||||
topk_indices_buffer[:num_decode_tokens], positions[:num_decode_tokens]
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _gather_workspace_shapes(
|
||||
total_seq_lens: int,
|
||||
head_dim: int,
|
||||
fp8_dtype: torch.dtype,
|
||||
use_fp4_cache: bool,
|
||||
) -> tuple[tuple[tuple[int, int], torch.dtype], tuple[tuple[int, int], torch.dtype]]:
|
||||
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
|
||||
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
|
||||
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
|
||||
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
|
||||
if use_fp4_cache:
|
||||
return (
|
||||
((total_seq_lens, head_dim // 2), torch.uint8),
|
||||
((total_seq_lens, head_dim // MXFP4_BLOCK_SIZE), torch.uint8),
|
||||
)
|
||||
return (
|
||||
((total_seq_lens, head_dim), fp8_dtype),
|
||||
((total_seq_lens, 4), torch.uint8),
|
||||
)
|
||||
|
||||
|
||||
def kv_cache_as_quant_view(
|
||||
kv_cache: torch.Tensor,
|
||||
head_dim: int,
|
||||
use_fp4_cache: bool,
|
||||
) -> torch.Tensor:
|
||||
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
|
||||
DeepGEMM, from the 3D indexer kv-cache allocation."""
|
||||
if use_fp4_cache:
|
||||
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
|
||||
num_blocks, block_size, _ = kv_cache.shape
|
||||
page_bytes = int(kv_cache.stride(0))
|
||||
fp4_bytes = head_dim // 2 + head_dim // MXFP4_BLOCK_SIZE
|
||||
return torch.as_strided(
|
||||
kv_cache,
|
||||
size=(num_blocks, block_size, 1, fp4_bytes),
|
||||
stride=(page_bytes, fp4_bytes, fp4_bytes, 1),
|
||||
)
|
||||
return kv_cache.unsqueeze(-2)
|
||||
Reference in New Issue
Block a user