[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
+24
-19
@@ -3468,7 +3468,7 @@ class rocm_aiter_ops:
|
||||
# AITER's Python wrapper allocates intermediate/output tensors without
|
||||
# explicit device arguments, so run it under the residual tensor's device.
|
||||
with torch.device(residual_flat.device):
|
||||
post_mix, comb_mix, layer_input = mhc_pre(
|
||||
args = (
|
||||
residual_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
@@ -3478,9 +3478,11 @@ class rocm_aiter_ops:
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
if norm_weight is None:
|
||||
post_mix, comb_mix, layer_input = mhc_pre(*args)
|
||||
else:
|
||||
post_mix, comb_mix, layer_input = mhc_pre(*args, norm_weight, norm_eps)
|
||||
return (
|
||||
post_mix.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix.view(*outer_shape, hc_mult, hc_mult),
|
||||
@@ -3630,23 +3632,26 @@ class rocm_aiter_ops:
|
||||
),
|
||||
)
|
||||
|
||||
args = (
|
||||
x_flat,
|
||||
residual_flat,
|
||||
post_flat,
|
||||
comb_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
with torch.device(residual_flat.device):
|
||||
post_mix, comb_mix, layer_input, next_residual = mhc_fused_post_pre(
|
||||
x_flat,
|
||||
residual_flat,
|
||||
post_flat,
|
||||
comb_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
if norm_weight is None:
|
||||
result = mhc_fused_post_pre(*args)
|
||||
else:
|
||||
result = mhc_fused_post_pre(*args, norm_weight, norm_eps)
|
||||
post_mix, comb_mix, layer_input, next_residual = result
|
||||
|
||||
return (
|
||||
next_residual.view_as(residual),
|
||||
|
||||
@@ -61,6 +61,7 @@ MTPModelTypes = Literal[
|
||||
"hy_v4_mtp",
|
||||
"gemma4_mtp",
|
||||
"inkling_mtp",
|
||||
"glm5_next_mtp",
|
||||
]
|
||||
NgramGPUTypes = Literal["ngram_gpu"]
|
||||
DFlashModelTypes = Literal["dflash"]
|
||||
@@ -1016,6 +1017,12 @@ class SpeculativeConfig:
|
||||
hf_config.update(
|
||||
{"n_predict": n_predict, "architectures": ["MiniMaxM3MTP"]}
|
||||
)
|
||||
if hf_config.model_type == "glm5_next":
|
||||
hf_config.model_type = "glm5_next_mtp"
|
||||
n_predict = hf_config.num_nextn_predict_layers
|
||||
hf_config.update(
|
||||
{"n_predict": n_predict, "architectures": ["Glm5NextMTPModel"]}
|
||||
)
|
||||
|
||||
return hf_config
|
||||
|
||||
|
||||
@@ -81,6 +81,9 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
|
||||
"DeepSeekV4MTPModel",
|
||||
"Dots3NoteForCausalLM",
|
||||
"Dots3NoteMTPModel",
|
||||
"Glm5NextForCausalLM",
|
||||
"Glm5NextForConditionalGeneration",
|
||||
"Glm5NextMTPModel",
|
||||
"GlmMoeDsaForCausalLM",
|
||||
"HYV4ForCausalLM",
|
||||
"HYV4MTPModel",
|
||||
|
||||
@@ -56,6 +56,7 @@ from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
@@ -1675,7 +1676,9 @@ class MooncakeConnectorWorker:
|
||||
block_len = region_cache.stride(0) * region_cache.element_size()
|
||||
region_base_addresses.append(base_addr)
|
||||
|
||||
if isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
|
||||
if isinstance(layer_spec, KpoolTailSpec):
|
||||
kv_block_len = layer_spec.unpadded_page_size_bytes // 2
|
||||
elif isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
|
||||
assert (
|
||||
layer_spec.page_size_bytes
|
||||
% self._physical_blocks_per_logical_kv_block
|
||||
|
||||
@@ -75,8 +75,14 @@ class MooncakeStoreCoordinator:
|
||||
retention_interval: int | None = None,
|
||||
dcp_world_size: int = 1,
|
||||
) -> None:
|
||||
# Mirrors core's resolve_kv_cache_block_sizes: the hash unit only has
|
||||
# to divide groups that participate in prefix caching. Non-shareable
|
||||
# scratch groups (e.g. GLM-5.3-Flash's kpool tail) are skipped by
|
||||
# _verify_and_split_kv_cache_groups and never probed for hits.
|
||||
assert all(
|
||||
g.kv_cache_spec.block_size % hash_block_size == 0 for g in kv_cache_groups
|
||||
g.kv_cache_spec.block_size % hash_block_size == 0
|
||||
for g in kv_cache_groups
|
||||
if g.kv_cache_spec.prefix_cacheable
|
||||
), "block_size must be divisible by hash_block_size"
|
||||
assert scheduler_block_size % hash_block_size == 0, (
|
||||
f"scheduler_block_size ({scheduler_block_size}) must be a multiple of "
|
||||
@@ -116,6 +122,12 @@ class MooncakeStoreCoordinator:
|
||||
"""
|
||||
attention_groups: list[SpecGroup] = []
|
||||
for i, g in enumerate(self.kv_cache_groups):
|
||||
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
|
||||
# kpool tail): per-request scratch, never shareable, so they must
|
||||
# not participate in hit lookup. Mirrors core's
|
||||
# KVCacheCoordinator.verify_and_split_kv_cache_groups.
|
||||
if not g.kv_cache_spec.prefix_cacheable:
|
||||
continue
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
|
||||
assert manager_cls is not None, (
|
||||
|
||||
@@ -523,6 +523,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
enable_group_semantics: bool = False,
|
||||
supports_group_ids: bool = False,
|
||||
record_operation: Callable[..., None] | None = None,
|
||||
group_participates: Sequence[bool] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
@@ -537,6 +538,11 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self.group_put_steps = group_put_steps
|
||||
self.coord = coord
|
||||
self.kv_role = kv_role
|
||||
self.group_participates = (
|
||||
list(group_participates)
|
||||
if group_participates is not None
|
||||
else [True] * len(token_databases)
|
||||
)
|
||||
# req_id -> ids of its store jobs that are still queued or running.
|
||||
# Keying by store_job_id, which never repeats for the engine's lifetime,
|
||||
# rather than counting jobs per request id makes the ledger immune to id
|
||||
@@ -744,6 +750,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
saved = self._saved_offset.get(req_meta.req_id, 0)
|
||||
puts: list[tuple[str, list[int], list[int], KeyMetadata]] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
if not self.group_participates[g_idx]:
|
||||
continue
|
||||
group_blocks = req_meta.block_ids[g_idx]
|
||||
# Distribute across ranks by the same rule as normal chunks.
|
||||
put_step = self.group_put_steps[g_idx]
|
||||
@@ -983,6 +991,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
group_indices: list[int] = []
|
||||
store_shard_ids: list[StoreShardId] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
if not self.group_participates[g_idx]:
|
||||
continue
|
||||
# Rotate the stride phase per group to balance load across ranks.
|
||||
put_step = self.group_put_steps[g_idx]
|
||||
put_step_rank = (self.tp_rank + g_idx) % put_step
|
||||
@@ -1253,6 +1263,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
disk_offload_buffer_budget_bytes: int | None = None,
|
||||
record_operation: Callable[..., None] | None = None,
|
||||
request_queue: queue.Queue[Any] | None = None,
|
||||
group_participates: Sequence[bool] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
@@ -1264,6 +1275,11 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
record_operation=record_operation,
|
||||
request_queue=request_queue,
|
||||
)
|
||||
self.group_participates = (
|
||||
list(group_participates)
|
||||
if group_participates is not None
|
||||
else [True] * len(token_databases)
|
||||
)
|
||||
# _invalid_block_ids can be access by both the Worker and RecvingThread
|
||||
self._invalid_block_ids_lock = threading.Lock()
|
||||
self._invalid_block_ids: set[int] = set()
|
||||
@@ -1311,6 +1327,8 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
key_list: list[str] = []
|
||||
block_id_list: list[int] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
if not self.group_participates[g_idx]:
|
||||
continue
|
||||
mask = load_mask_per_group[g_idx]
|
||||
chunks: list[tuple[int, int]] = []
|
||||
store_shard_ids: list[StoreShardId] = []
|
||||
@@ -1791,6 +1809,11 @@ class MooncakeStoreWorker:
|
||||
"""Construct token databases and their Store layouts."""
|
||||
token_dbs: list[ChunkedTokenDatabase] = []
|
||||
for group_idx, group in enumerate(self._kv_cache_groups):
|
||||
hash_block_size = (
|
||||
self.hash_block_size
|
||||
if group.kv_cache_spec.prefix_cacheable
|
||||
else group.kv_cache_spec.block_size
|
||||
)
|
||||
group_tp_rank = self.tp_rank
|
||||
if layout_cls is None:
|
||||
group_tp_rank //= self._group_tp_replication_factors[group_idx]
|
||||
@@ -1805,7 +1828,7 @@ class MooncakeStoreWorker:
|
||||
store_layout = layout_cls(
|
||||
group_metadata,
|
||||
group.kv_cache_spec.block_size,
|
||||
self.hash_block_size,
|
||||
hash_block_size,
|
||||
local_tp_size=self.tp_size,
|
||||
store_tp_size=self.store_tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
@@ -1815,7 +1838,7 @@ class MooncakeStoreWorker:
|
||||
ChunkedTokenDatabase(
|
||||
group_metadata,
|
||||
group.kv_cache_spec.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
store_layout=store_layout,
|
||||
)
|
||||
)
|
||||
@@ -1937,6 +1960,10 @@ class MooncakeStoreWorker:
|
||||
enable_group_semantics=self.enable_group_semantics,
|
||||
supports_group_ids=self._supports_group_ids,
|
||||
record_operation=self._record_kv_connector_operation,
|
||||
group_participates=[
|
||||
group.kv_cache_spec.prefix_cacheable
|
||||
for group in self._kv_cache_groups
|
||||
],
|
||||
)
|
||||
self.kv_send_thread.start()
|
||||
|
||||
@@ -1954,6 +1981,10 @@ class MooncakeStoreWorker:
|
||||
disk_offload_buffer_budget_bytes=self.disk_offload_buffer_budget_bytes,
|
||||
record_operation=self._record_kv_connector_operation,
|
||||
request_queue=self.recv_request_queue,
|
||||
group_participates=[
|
||||
group.kv_cache_spec.prefix_cacheable
|
||||
for group in self._kv_cache_groups
|
||||
],
|
||||
)
|
||||
recv_thread.name = f"KVCacheStoreRecvingThread-{i}"
|
||||
recv_thread.start()
|
||||
@@ -2126,6 +2157,8 @@ class MooncakeStoreWorker:
|
||||
fine_grained = self.coord.enable_partial_hash_hits
|
||||
lookup_masks = None if fine_grained else self.coord.lookup_mask(token_len)
|
||||
for g_idx, db in enumerate(self.token_dbs):
|
||||
if not self._kv_cache_groups[g_idx].kv_cache_spec.prefix_cacheable:
|
||||
continue
|
||||
spec_block_size = db.block_size
|
||||
key_prefixes = self._lookup_key_prefixes[g_idx]
|
||||
if fine_grained:
|
||||
@@ -2236,6 +2269,9 @@ class MooncakeStoreWorker:
|
||||
boundaries = []
|
||||
hit_boundary_hash_idx = hit_length // self.hash_block_size - 1
|
||||
for group_id, db in enumerate(self.token_dbs):
|
||||
if not self._kv_cache_groups[group_id].kv_cache_spec.prefix_cacheable:
|
||||
# Scratch groups are never stored, so they have no tail key.
|
||||
continue
|
||||
chunk_id = cdiv(hit_length, db.block_size) - 1
|
||||
boundary_tokens = hit_length
|
||||
contains_hit_boundary = cached_block_pool.contains(
|
||||
|
||||
@@ -74,6 +74,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
@@ -99,6 +100,46 @@ def _share_storage_and_block_stride(caches: list[torch.Tensor]) -> bool:
|
||||
return len(block_strides) == len(storage_ptrs) == 1
|
||||
|
||||
|
||||
def _tensor_byte_span_end(cache: torch.Tensor) -> int:
|
||||
"""Return the exclusive end address touched by a nonnegative-stride view."""
|
||||
if cache.numel() == 0:
|
||||
return cache.data_ptr()
|
||||
if any(stride < 0 for stride in cache.stride()):
|
||||
raise ValueError("NIXL cache views must have nonnegative strides")
|
||||
max_element_offset = sum(
|
||||
(size - 1) * stride for size, stride in zip(cache.shape, cache.stride())
|
||||
)
|
||||
return cache.data_ptr() + (max_element_offset + 1) * cache.element_size()
|
||||
|
||||
|
||||
def _uses_dense_virtual_transfer_pages(
|
||||
layer_spec: KVCacheSpec,
|
||||
cache: torch.Tensor,
|
||||
physical_page_size: int,
|
||||
num_blocks: int,
|
||||
) -> bool:
|
||||
"""Return whether a compressed kernel view can be split into NIXL pages."""
|
||||
if not (
|
||||
isinstance(layer_spec, MLAAttentionSpec)
|
||||
and layer_spec.tokens_per_state > 1
|
||||
and cache.ndim == 4
|
||||
and cache.shape[1] == 1
|
||||
and cache.is_contiguous()
|
||||
and physical_page_size > 0
|
||||
and layer_spec.state_content_size_bytes > 0
|
||||
):
|
||||
return False
|
||||
|
||||
block_stride = cache.stride(0) * cache.element_size()
|
||||
return (
|
||||
block_stride > physical_page_size
|
||||
and block_stride % physical_page_size == 0
|
||||
and physical_page_size % layer_spec.state_content_size_bytes == 0
|
||||
and cache.shape[0] * (block_stride // physical_page_size) == num_blocks
|
||||
and cache.nbytes == num_blocks * physical_page_size
|
||||
)
|
||||
|
||||
|
||||
class NixlBaseConnectorWorker:
|
||||
"""Base implementation of Worker side methods shared by pull and push."""
|
||||
|
||||
@@ -1150,6 +1191,31 @@ class NixlBaseConnectorWorker:
|
||||
# aliases even though every view shares one block-major allocation.
|
||||
packed_storage = packed_storage and not self._is_csa_linear
|
||||
|
||||
layer_specs: dict[str, KVCacheSpec] = {}
|
||||
compressed_region_owners: dict[int, torch.Tensor] = {}
|
||||
for layer_name, cache in xfer_buffers.items():
|
||||
layer_spec = self._layer_specs.get(layer_name)
|
||||
if isinstance(layer_spec, UniformTypeKVCacheSpecs):
|
||||
layer_spec = layer_spec.kv_cache_specs[layer_name]
|
||||
if layer_spec is None:
|
||||
continue
|
||||
layer_specs[layer_name] = layer_spec
|
||||
physical_page_size = (
|
||||
layer_spec.page_size_bytes
|
||||
if isinstance(layer_spec, MambaSpec)
|
||||
else layer_spec.page_size_bytes
|
||||
// self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
num_blocks = (
|
||||
self._logical_num_blocks
|
||||
if isinstance(layer_spec, MambaSpec)
|
||||
else self.num_blocks
|
||||
)
|
||||
if _uses_dense_virtual_transfer_pages(
|
||||
layer_spec, cache, physical_page_size, num_blocks
|
||||
):
|
||||
compressed_region_owners.setdefault(cache.data_ptr(), cache)
|
||||
|
||||
# K and V are packed into the content dim, so each attention layer is a
|
||||
# single NIXL region whose block transfers as one unit. Mamba layers instead
|
||||
# register separate conv/ssm sub-regions (see `_build_mamba_local`).
|
||||
@@ -1158,7 +1224,7 @@ class NixlBaseConnectorWorker:
|
||||
# kernel requires a specific block size. This leads to SSM and FA layers
|
||||
# having different num_blocks.
|
||||
# `_physical_blocks_per_logical_kv_block` ratio is used to adjust for this.
|
||||
layer_spec = self._layer_specs.get(layer_name)
|
||||
layer_spec = layer_specs.get(layer_name)
|
||||
if layer_spec is None:
|
||||
logger.debug(
|
||||
"Skipping layer %s as no KVCache spec is present. "
|
||||
@@ -1166,9 +1232,6 @@ class NixlBaseConnectorWorker:
|
||||
layer_name,
|
||||
)
|
||||
continue
|
||||
if isinstance(layer_spec, UniformTypeKVCacheSpecs):
|
||||
# DSA Indexer case: UniformTypeKVCacheSpecs merges kv_cache_specs
|
||||
layer_spec = layer_spec.kv_cache_specs[layer_name]
|
||||
# `layer_spec.page_size_bytes` only accounts for logical page_size, that is
|
||||
# the page_size assuming constant `self._logical_num_blocks`.
|
||||
physical_page_size = (
|
||||
@@ -1187,6 +1250,33 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
storage = cache.untyped_storage()
|
||||
storage_addr = storage.data_ptr()
|
||||
|
||||
if isinstance(layer_spec, KpoolTailSpec):
|
||||
compressed_owner = compressed_region_owners.get(cache.data_ptr())
|
||||
if compressed_owner is not None:
|
||||
owner_storage = compressed_owner.untyped_storage()
|
||||
owner_end = compressed_owner.data_ptr() + compressed_owner.nbytes
|
||||
tail_is_covered = (
|
||||
compressed_owner.is_contiguous()
|
||||
and owner_storage.data_ptr() == storage_addr
|
||||
and _tensor_byte_span_end(cache) <= owner_end
|
||||
)
|
||||
if not tail_is_covered:
|
||||
raise AssertionError(
|
||||
"Kpool tail cache is not fully covered by its compressed "
|
||||
f"indexer region: layer={layer_name}, "
|
||||
f"tail_shape={tuple(cache.shape)}, "
|
||||
f"tail_stride={tuple(cache.stride())}, "
|
||||
f"owner_shape={tuple(compressed_owner.shape)}, "
|
||||
f"owner_stride={tuple(compressed_owner.stride())}"
|
||||
)
|
||||
logger.debug(
|
||||
"Skipping layer %s because its compressed indexer region "
|
||||
"covers the same storage",
|
||||
layer_name,
|
||||
)
|
||||
continue
|
||||
|
||||
# Memory registration follows allocations, while transfer regions follow
|
||||
# logical layers (or contiguous head segments). This keeps strided
|
||||
# cross-layer views inside their registered allocation.
|
||||
@@ -1228,7 +1318,15 @@ class NixlBaseConnectorWorker:
|
||||
and cache.stride(2) == cache.shape[3]
|
||||
and cache.stride(1) == cache.shape[2] * cache.shape[3]
|
||||
)
|
||||
if storage_is_block_major and (
|
||||
virtual_transfer_pages = _uses_dense_virtual_transfer_pages(
|
||||
layer_spec, cache, physical_page_size, num_blocks
|
||||
)
|
||||
if virtual_transfer_pages:
|
||||
# A compressed kernel row can contain multiple NIXL transfer pages.
|
||||
region_specs = [
|
||||
(cache.data_ptr(), physical_page_size, physical_page_size)
|
||||
]
|
||||
elif storage_is_block_major and (
|
||||
(packed_storage and is_mla_region)
|
||||
or (not hnc_contiguous and not self._is_csa_linear)
|
||||
):
|
||||
@@ -1243,7 +1341,15 @@ class NixlBaseConnectorWorker:
|
||||
]
|
||||
else:
|
||||
segment_bytes = num_blocks * block_stride
|
||||
assert cache.nbytes % segment_bytes == 0
|
||||
if cache.nbytes % segment_bytes != 0:
|
||||
raise AssertionError(
|
||||
"KV cache view cannot be partitioned into NIXL regions: "
|
||||
f"layer={layer_name}, cache_nbytes={cache.nbytes}, "
|
||||
f"num_blocks={num_blocks}, block_stride={block_stride}, "
|
||||
f"physical_page_size={physical_page_size}, "
|
||||
f"cache_shape={tuple(cache.shape)}, "
|
||||
f"cache_stride={tuple(cache.stride())}"
|
||||
)
|
||||
num_segments = cache.nbytes // segment_bytes
|
||||
region_block_len = (
|
||||
block_stride if num_segments > 1 else physical_page_size
|
||||
|
||||
@@ -45,8 +45,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
|
||||
# 7: Include NIXL transfer mode (push vs pull) in the compatibility hash
|
||||
# 8: Add dcp_size and pcp_size to NixlAgentMetadata
|
||||
# 9: Add block_strides
|
||||
# 10: Add dense virtual transfer pages for compressed MLA caches
|
||||
#
|
||||
NIXL_CONNECTOR_VERSION: int = 9
|
||||
NIXL_CONNECTOR_VERSION: int = 10
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -1534,7 +1534,7 @@ class MLACommonBackend(AttentionBackend):
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return [320, 576]
|
||||
return [320, 512, 576]
|
||||
|
||||
@classmethod
|
||||
def is_mla(cls) -> bool:
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _scatter_states_kernel(
|
||||
state_ptr,
|
||||
src_ptr,
|
||||
indices_ptr,
|
||||
stride_state_batch,
|
||||
stride_src_batch,
|
||||
stride_indices,
|
||||
row_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
block_idx = tl.program_id(0)
|
||||
batch_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < row_size
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
|
||||
values = tl.load(src_ptr + batch_idx * stride_src_batch + offsets, mask=mask)
|
||||
tl.store(state_ptr + state_idx * stride_state_batch + offsets, values, mask=mask)
|
||||
|
||||
|
||||
def scatter_states(
|
||||
state: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Scatter ``src`` rows into ``state`` at ``indices`` (in place).
|
||||
|
||||
Equivalent to ``state[indices] = src`` but non-atomic and bandwidth-bound,
|
||||
since mamba cache slots are unique per sequence. ``gather_initial_states``
|
||||
is the read-side counterpart.
|
||||
"""
|
||||
assert state.ndim >= 2
|
||||
assert state.is_cuda
|
||||
assert src.ndim == state.ndim
|
||||
assert indices.ndim == 1
|
||||
assert indices.device == state.device
|
||||
assert src.shape[1:] == state.shape[1:]
|
||||
assert src.shape[0] == indices.shape[0]
|
||||
assert indices.dtype in (torch.int32, torch.int64)
|
||||
|
||||
row_size = state[0].numel()
|
||||
assert state[0].is_contiguous()
|
||||
assert src[0].is_contiguous()
|
||||
block_size = min(triton.next_power_of_2(row_size), 1024)
|
||||
grid = (triton.cdiv(row_size, block_size), indices.numel())
|
||||
_scatter_states_kernel[grid](
|
||||
state,
|
||||
src,
|
||||
indices,
|
||||
state.stride(0),
|
||||
src.stride(0),
|
||||
indices.stride(0),
|
||||
row_size=row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import inspect
|
||||
|
||||
import torch
|
||||
|
||||
# this import will also register the custom ops
|
||||
@@ -29,6 +31,67 @@ HAS_TILELANG_MHC = _has_tilelang_mhc()
|
||||
HAS_AITER_MHC = is_aiter_found_and_supported()
|
||||
|
||||
|
||||
def _has_aiter_mhc_fused() -> bool:
|
||||
if not HAS_AITER_MHC:
|
||||
return False
|
||||
try:
|
||||
from aiter.ops.mhc import mhc_fused_post_pre
|
||||
except Exception:
|
||||
return False
|
||||
return callable(mhc_fused_post_pre)
|
||||
|
||||
|
||||
def _aiter_mhc_op_accepts_norm(op_name: str) -> bool:
|
||||
if not HAS_AITER_MHC:
|
||||
return False
|
||||
try:
|
||||
from aiter.ops import mhc as aiter_mhc
|
||||
|
||||
for candidate_name in (op_name, f"{op_name}_fake"):
|
||||
op = getattr(aiter_mhc, candidate_name, None)
|
||||
if op is None:
|
||||
continue
|
||||
parameters = inspect.signature(op).parameters
|
||||
if "norm_weight" in parameters and "norm_eps" in parameters:
|
||||
return True
|
||||
except (AttributeError, ImportError, TypeError, ValueError):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
HAS_AITER_MHC_FUSED = _has_aiter_mhc_fused()
|
||||
HAS_AITER_MHC_PRE_NORM = _aiter_mhc_op_accepts_norm("mhc_pre")
|
||||
HAS_AITER_MHC_FUSED_NORM = _aiter_mhc_op_accepts_norm("mhc_fused_post_pre")
|
||||
|
||||
|
||||
def _aiter_mhc_supported(
|
||||
residual: torch.Tensor,
|
||||
norm_weight: torch.Tensor | None,
|
||||
*,
|
||||
supports_norm: bool,
|
||||
) -> bool:
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult = residual.shape[-2]
|
||||
return (
|
||||
HAS_AITER_MHC
|
||||
and hidden_size % 256 == 0
|
||||
and hc_mult == 4
|
||||
and (norm_weight is None or supports_norm)
|
||||
)
|
||||
|
||||
|
||||
def _apply_mhc_norm(
|
||||
layer_input: torch.Tensor,
|
||||
norm_weight: torch.Tensor | None,
|
||||
norm_eps: float,
|
||||
) -> torch.Tensor:
|
||||
if norm_weight is None:
|
||||
return layer_input
|
||||
from vllm import ir
|
||||
|
||||
return ir.ops.rms_norm(layer_input, norm_weight, norm_eps)
|
||||
|
||||
|
||||
# --8<-- [start:mhc_pre]
|
||||
@CustomOp.register("mhc_pre")
|
||||
class MHCPreOp(CustomOp):
|
||||
@@ -89,9 +152,11 @@ class MHCPreOp(CustomOp):
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult = residual.shape[-2]
|
||||
if HAS_AITER_MHC and hidden_size % 256 == 0 and hc_mult == 4:
|
||||
if _aiter_mhc_supported(
|
||||
residual,
|
||||
norm_weight,
|
||||
supports_norm=HAS_AITER_MHC_PRE_NORM,
|
||||
):
|
||||
return torch.ops.vllm.mhc_pre_aiter(
|
||||
residual,
|
||||
fn,
|
||||
@@ -122,7 +187,7 @@ class MHCPreOp(CustomOp):
|
||||
norm_eps,
|
||||
)
|
||||
else:
|
||||
return self.forward_native(
|
||||
post_mix, comb_mix, layer_input = self.forward_native(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
@@ -136,6 +201,11 @@ class MHCPreOp(CustomOp):
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
return (
|
||||
post_mix,
|
||||
comb_mix,
|
||||
_apply_mhc_norm(layer_input, norm_weight, norm_eps),
|
||||
)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
@@ -225,9 +295,7 @@ class MHCPostOp(CustomOp):
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult = residual.shape[-2]
|
||||
if HAS_AITER_MHC and hidden_size % 256 == 0 and hc_mult == 4:
|
||||
if _aiter_mhc_supported(residual, None, supports_norm=True):
|
||||
return torch.ops.vllm.mhc_post_aiter(
|
||||
x,
|
||||
residual,
|
||||
@@ -449,9 +517,11 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult = residual.shape[-2]
|
||||
if HAS_AITER_MHC and hidden_size % 256 == 0 and hc_mult == 4:
|
||||
if HAS_AITER_MHC_FUSED and _aiter_mhc_supported(
|
||||
residual,
|
||||
norm_weight,
|
||||
supports_norm=HAS_AITER_MHC_FUSED_NORM,
|
||||
):
|
||||
return torch.ops.vllm.mhc_fused_post_pre_aiter(
|
||||
x,
|
||||
residual,
|
||||
@@ -489,7 +559,7 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
return self.forward_native(
|
||||
residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur = self.forward_native(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
@@ -507,6 +577,12 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
return (
|
||||
residual_cur,
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
_apply_mhc_norm(layer_input_cur, norm_weight, norm_eps),
|
||||
)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
@@ -577,3 +653,13 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
|
||||
def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
"""[s, hidden_size] -> [s, n * hidden_size] by replication."""
|
||||
return x.unsqueeze(1).expand(-1, n, -1).contiguous()
|
||||
|
||||
|
||||
def hc_contract(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
"""[s, n * hidden_size] -> [s, hidden_size] by averaging."""
|
||||
return x.mean(dim=1)
|
||||
|
||||
@@ -8,6 +8,7 @@ from vllm.config import CacheConfig
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -17,7 +18,7 @@ class MLAModules:
|
||||
|
||||
kv_a_layernorm: torch.nn.Module
|
||||
kv_b_proj: torch.nn.Module
|
||||
rotary_emb: torch.nn.Module
|
||||
rotary_emb: torch.nn.Module | None
|
||||
o_proj: torch.nn.Module
|
||||
fused_qkv_a_proj: torch.nn.Module | None
|
||||
kv_a_proj_with_mqa: torch.nn.Module | None
|
||||
@@ -69,6 +70,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
skip_topk: bool = False,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
allow_short_prefill_indexer_scoring_skip: bool = False,
|
||||
fuse_qkv_rmsnorm: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -98,6 +100,9 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
# the topk_tokens buffer written by a previous layer in the same pass.
|
||||
# Refer: https://arxiv.org/abs/2603.12201 for more details.
|
||||
self.skip_topk = skip_topk
|
||||
# When True, fuse the q_a and kv_a RMSNorms into a single kernel launch
|
||||
# (MLA layers with q-LoRA). Opt-in; default False preserves other models.
|
||||
self.fuse_qkv_rmsnorm = fuse_qkv_rmsnorm
|
||||
# qrep is active when the query projection is a DCP-group-sharded layer
|
||||
# that materializes the full group head set locally.
|
||||
q_proj_layer = self.q_b_proj if self.q_lora_rank is not None else self.q_proj
|
||||
@@ -172,8 +177,9 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
|
||||
dim=-1,
|
||||
)
|
||||
q_c = self.q_a_layernorm(q_c)
|
||||
q_proj_layer = self.q_b_proj
|
||||
if not self.fuse_qkv_rmsnorm:
|
||||
q_c = self.q_a_layernorm(q_c)
|
||||
q_proj_input = q_c
|
||||
else:
|
||||
assert self.kv_a_proj_with_mqa is not None, (
|
||||
@@ -187,7 +193,20 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
q_proj_input = hidden_states
|
||||
|
||||
kv_c, k_pe = kv_lora.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
|
||||
kv_c_normed = self.kv_a_layernorm(kv_c)
|
||||
if self.fuse_qkv_rmsnorm and q_c is not None:
|
||||
# Fuse q_a + kv_a RMSNorm into one kernel launch (q_c is still the
|
||||
# pre-norm projection output here).
|
||||
assert self.q_a_layernorm is not None
|
||||
q_proj_input, kv_c_normed = fused_q_kv_rmsnorm(
|
||||
q_c,
|
||||
kv_c,
|
||||
self.q_a_layernorm.weight.data,
|
||||
self.kv_a_layernorm.weight.data,
|
||||
self.q_a_layernorm.variance_epsilon,
|
||||
)
|
||||
q_c = q_proj_input
|
||||
else:
|
||||
kv_c_normed = self.kv_a_layernorm(kv_c)
|
||||
# Add head dim of 1 to k_pe
|
||||
k_pe = k_pe.unsqueeze(1)
|
||||
|
||||
|
||||
@@ -361,30 +361,27 @@ def _shuffle_deepseek_fp8_moe_weights(
|
||||
Returns 4D weight tensors in BlockMajorK layout
|
||||
(E, K/block_k, Mn, block_k)
|
||||
"""
|
||||
from flashinfer import shuffle_matrix_a
|
||||
from flashinfer.fused_moe import convert_to_block_layout
|
||||
from flashinfer.utils import get_shuffle_matrix_a_row_indices
|
||||
|
||||
epilogue_tile_m = 64
|
||||
block_k = 128
|
||||
num_experts = w13.shape[0]
|
||||
|
||||
M13, K13 = w13.shape[1], w13.shape[2]
|
||||
M2, K2 = w2.shape[1], w2.shape[2]
|
||||
w13_out = torch.empty(
|
||||
num_experts, K13 // block_k, M13, block_k, dtype=torch.uint8, device=w13.device
|
||||
)
|
||||
w2_out = torch.empty(
|
||||
num_experts, K2 // block_k, M2, block_k, dtype=torch.uint8, device=w2.device
|
||||
)
|
||||
def shuffle_to_block_major_k(w: torch.Tensor) -> torch.Tensor:
|
||||
# shuffle_matrix_a's row permutation depends only on (M,
|
||||
# epilogue_tile_m), so it is computed once and applied to every expert
|
||||
# in a single gather instead of once per expert. Gathering through the
|
||||
# BlockMajorK-permuted view also folds convert_to_block_layout into
|
||||
# that same kernel. Per-expert loops here cost minutes for a MoE this
|
||||
# wide (~24k tiny launches plus a host round-trip each).
|
||||
num_experts, m, k = w.shape
|
||||
rows = get_shuffle_matrix_a_row_indices(
|
||||
w[0].view(torch.uint8), epilogue_tile_m
|
||||
).to(w.device)
|
||||
blocked = w.view(torch.uint8).view(num_experts, m, k // block_k, block_k)
|
||||
out = blocked.permute(0, 2, 1, 3)[:, :, rows, :].contiguous()
|
||||
return out.view(torch.float8_e4m3fn)
|
||||
|
||||
for i in range(num_experts):
|
||||
t13 = shuffle_matrix_a(w13[i].view(torch.uint8), epilogue_tile_m)
|
||||
w13_out[i] = convert_to_block_layout(t13, block_k)
|
||||
|
||||
t2 = shuffle_matrix_a(w2[i].view(torch.uint8), epilogue_tile_m)
|
||||
w2_out[i] = convert_to_block_layout(t2, block_k)
|
||||
|
||||
return w13_out.view(torch.float8_e4m3fn), w2_out.view(torch.float8_e4m3fn)
|
||||
return shuffle_to_block_major_k(w13), shuffle_to_block_major_k(w2)
|
||||
|
||||
|
||||
def _shuffle_mxfp8_moe_weights(
|
||||
@@ -396,70 +393,54 @@ def _shuffle_mxfp8_moe_weights(
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Preprocess MXFP8 weights and scales for the FlashInfer TRT-LLM kernel.
|
||||
|
||||
Following flashinfer/tests/moe/test_trtllm_gen_fused_moe.py:
|
||||
1. reorder_rows_for_gated_act_gemm (interleave gate/up rows)
|
||||
2. shuffle_matrix_a (weight data layout shuffle)
|
||||
3. shuffle_matrix_sf_a (scale factor layout shuffle)
|
||||
All three transforms (gate/up row reorder, ``shuffle_matrix_a`` weight
|
||||
shuffle, ``shuffle_matrix_sf_a`` scale shuffle) are fixed row/index
|
||||
permutations that depend only on the per-expert matrix shape, so the
|
||||
permutation is computed once and applied to every expert in a single
|
||||
gather instead of once per expert. ``block_scale_interleave`` accepts a
|
||||
batched ``(E, M, K)`` scale tensor directly. Output is bit-identical to
|
||||
the per-expert loop but ~20x faster (a 288-expert MoE otherwise costs
|
||||
seconds per layer at load).
|
||||
"""
|
||||
from flashinfer import (
|
||||
reorder_rows_for_gated_act_gemm,
|
||||
shuffle_matrix_a,
|
||||
shuffle_matrix_sf_a,
|
||||
from flashinfer.fused_moe.core import (
|
||||
get_reorder_rows_for_gated_act_gemm_row_indices,
|
||||
)
|
||||
from flashinfer.quantization.fp4_quantization import block_scale_interleave
|
||||
from flashinfer.utils import get_shuffle_matrix_a_row_indices
|
||||
|
||||
epilogue_tile_m = 128
|
||||
num_experts = w13.shape[0]
|
||||
intermediate_size = w13.shape[1] // 2
|
||||
hidden_size = w13.shape[2]
|
||||
|
||||
w13_interleaved: list[torch.Tensor] = []
|
||||
w13_scale_interleaved: list[torch.Tensor] = []
|
||||
for i in range(num_experts):
|
||||
if is_gated:
|
||||
w13_interleaved.append(
|
||||
reorder_rows_for_gated_act_gemm(
|
||||
w13[i].reshape(2 * intermediate_size, -1)
|
||||
)
|
||||
)
|
||||
w13_scale_interleaved.append(
|
||||
reorder_rows_for_gated_act_gemm(
|
||||
w13_scale[i].reshape(2 * intermediate_size, -1)
|
||||
)
|
||||
)
|
||||
else:
|
||||
w13_interleaved.append(w13[i])
|
||||
w13_scale_interleaved.append(w13_scale[i])
|
||||
w13_u = w13.view(torch.uint8)
|
||||
w2_u = w2.view(torch.uint8)
|
||||
|
||||
w13_shuffled: list[torch.Tensor] = []
|
||||
w2_shuffled: list[torch.Tensor] = []
|
||||
w13_scale_shuffled: list[torch.Tensor] = []
|
||||
w2_scale_shuffled: list[torch.Tensor] = []
|
||||
for i in range(num_experts):
|
||||
w13_shuffled.append(
|
||||
shuffle_matrix_a(w13_interleaved[i].view(torch.uint8), epilogue_tile_m)
|
||||
)
|
||||
w2_shuffled.append(shuffle_matrix_a(w2[i].view(torch.uint8), epilogue_tile_m))
|
||||
w13_scale_shuffled.append(
|
||||
shuffle_matrix_sf_a(
|
||||
w13_scale_interleaved[i]
|
||||
.view(torch.uint8)
|
||||
.reshape(2 * intermediate_size, -1),
|
||||
epilogue_tile_m,
|
||||
)
|
||||
)
|
||||
w2_scale_shuffled.append(
|
||||
shuffle_matrix_sf_a(
|
||||
w2_scale[i].view(torch.uint8).reshape(hidden_size, -1),
|
||||
epilogue_tile_m,
|
||||
)
|
||||
# 1. Interleave gate/up rows (gated activation GEMM layout).
|
||||
if is_gated:
|
||||
gate_idx = get_reorder_rows_for_gated_act_gemm_row_indices(
|
||||
w13_u[0].reshape(w13_u.shape[1], -1)
|
||||
)
|
||||
w13_u = w13_u[:, gate_idx]
|
||||
w13_scale = w13_scale[:, gate_idx]
|
||||
|
||||
w13_out = torch.stack(w13_shuffled).view(torch.float8_e4m3fn)
|
||||
w2_out = torch.stack(w2_shuffled).view(torch.float8_e4m3fn)
|
||||
w13_scale_out = torch.stack(w13_scale_shuffled).reshape(w13_scale.shape)
|
||||
w2_scale_out = torch.stack(w2_scale_shuffled).reshape(w2_scale.shape)
|
||||
def shuffle_weights(t: torch.Tensor) -> torch.Tensor:
|
||||
# Row permutation depends only on (M, epilogue_tile_m).
|
||||
idx = get_shuffle_matrix_a_row_indices(t[0], epilogue_tile_m).to(t.device)
|
||||
return t[:, idx].view(torch.float8_e4m3fn)
|
||||
|
||||
return w13_out, w2_out, w13_scale_out, w2_scale_out
|
||||
def shuffle_scales(s: torch.Tensor) -> torch.Tensor:
|
||||
# shuffle_matrix_sf_a == row-gather (same indices as the weight shuffle)
|
||||
# followed by the 128x4 block-scale interleave, which is batch-capable.
|
||||
idx = get_shuffle_matrix_a_row_indices(
|
||||
s[0].view(torch.uint8).reshape(s.shape[1], -1), epilogue_tile_m
|
||||
).to(s.device)
|
||||
interleaved = block_scale_interleave(s[:, idx])
|
||||
return interleaved.reshape(s.shape)
|
||||
|
||||
return (
|
||||
shuffle_weights(w13_u),
|
||||
shuffle_weights(w2_u),
|
||||
shuffle_scales(w13_scale),
|
||||
shuffle_scales(w2_scale),
|
||||
)
|
||||
|
||||
|
||||
def prepare_fp8_moe_layer_for_fi(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -120,6 +120,7 @@ _TEXT_GENERATION_MODELS = {
|
||||
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
|
||||
"Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
|
||||
"GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", "GlmMoeDsaForCausalLM"),
|
||||
"Glm5NextForCausalLM": ("vllm.models.glm5next", "Glm5NextForCausalLM"),
|
||||
"GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
|
||||
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
|
||||
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
|
||||
@@ -413,6 +414,10 @@ _MULTIMODAL_MODELS = {
|
||||
"Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),
|
||||
"Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),
|
||||
"GlmOcrForConditionalGeneration": ("glm_ocr", "GlmOcrForConditionalGeneration"),
|
||||
"Glm5NextForConditionalGeneration": (
|
||||
"vllm.models.glm5next",
|
||||
"Glm5NextForConditionalGeneration",
|
||||
),
|
||||
"GraniteSpeechForConditionalGeneration": (
|
||||
"granite_speech",
|
||||
"GraniteSpeechForConditionalGeneration",
|
||||
@@ -666,6 +671,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
|
||||
"Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
|
||||
"GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
|
||||
"Glm5NextMTPModel": ("vllm.models.glm5next", "Glm5NextMTP"),
|
||||
"MedusaModel": ("medusa", "Medusa"),
|
||||
"OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
|
||||
"Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
|
||||
|
||||
@@ -41,6 +41,9 @@ from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup
|
||||
from vllm.model_executor.warmup.replayssm_warmup import (
|
||||
replayssm_autotune_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.spec_decode_rejection_warmup import (
|
||||
spec_decode_rejection_warmup,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
@@ -163,6 +166,7 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
if worker.vllm_config.kernel_config.enable_jit_warmup:
|
||||
kimi_k3_triton_warmup(worker)
|
||||
fa4_cutedsl_warmup(worker)
|
||||
spec_decode_rejection_warmup(worker)
|
||||
qwen4_exp_qsa_triton_warmup(worker)
|
||||
|
||||
if current_platform.has_device_capability(90):
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm up spec-decode rejection-sampler Triton kernels.
|
||||
|
||||
The rejection sampler kernels (``_compute_local_logits_stats_kernel``,
|
||||
``_rejection_kernel``, ``_resample_kernel``) are JIT-compiled by Triton on
|
||||
first use. Without warmup, the first spec-decode request pays a multi-second
|
||||
compilation cost. This pre-compiles them with dummy data matching the
|
||||
server's vocab size and speculative config.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def spec_decode_rejection_warmup(worker: Worker) -> None:
|
||||
spec_config = worker.vllm_config.speculative_config
|
||||
if spec_config is None:
|
||||
return
|
||||
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
|
||||
rejection_sample,
|
||||
)
|
||||
|
||||
model_config = worker.vllm_config.model_config
|
||||
vocab_size = model_config.get_vocab_size()
|
||||
num_spec = spec_config.num_speculative_tokens
|
||||
if num_spec <= 0 or vocab_size <= 0:
|
||||
return
|
||||
|
||||
# Mirror the constexpr-relevant flags the runtime uses.
|
||||
rejection_method = getattr(spec_config, "rejection_sample_method", None)
|
||||
use_block_verification = rejection_method == "block"
|
||||
use_synthetic = rejection_method == "synthetic"
|
||||
|
||||
device = torch.device("cuda")
|
||||
num_reqs = 1
|
||||
tokens_per_req = num_spec + 1
|
||||
num_logits = num_reqs * tokens_per_req
|
||||
|
||||
# Triton JIT-specializes on tensor dtypes. The target logits may be fp32
|
||||
# (apply_sampling_params copies to fp32 when processing is needed) or the
|
||||
# model dtype (pass-through otherwise), while draft logits are always the
|
||||
# model dtype. Warm every (target, draft) combination the runtime can hit.
|
||||
model_dtype = model_config.dtype
|
||||
warmup_dtype_pairs = {
|
||||
(model_dtype, model_dtype),
|
||||
(torch.float32, torch.float32),
|
||||
(torch.float32, model_dtype),
|
||||
(model_dtype, torch.float32),
|
||||
}
|
||||
|
||||
logger.info(
|
||||
"Warming up spec-decode rejection sampler kernels "
|
||||
"(vocab=%d, num_spec=%d, dtype_pairs=%s, block_verify=%s).",
|
||||
vocab_size,
|
||||
num_spec,
|
||||
[(str(t), str(d)) for t, d in warmup_dtype_pairs],
|
||||
use_block_verification,
|
||||
)
|
||||
for tgt_dtype, draft_dtype in warmup_dtype_pairs:
|
||||
target_logits = torch.zeros(
|
||||
(num_logits, vocab_size), dtype=tgt_dtype, device=device
|
||||
)
|
||||
draft_logits = torch.zeros(
|
||||
(num_reqs, num_spec, vocab_size), dtype=draft_dtype, device=device
|
||||
)
|
||||
synthetic_rates = (
|
||||
torch.full((num_spec,), 0.5, dtype=torch.float32, device=device)
|
||||
if use_synthetic
|
||||
else None
|
||||
)
|
||||
try:
|
||||
rejection_sample(
|
||||
target_logits=target_logits,
|
||||
draft_logits=draft_logits,
|
||||
draft_sampled=torch.zeros(num_logits, dtype=torch.int64, device=device),
|
||||
cu_num_logits=torch.tensor(
|
||||
[0, num_logits], dtype=torch.int32, device=device
|
||||
),
|
||||
pos=torch.zeros(num_logits, dtype=torch.int64, device=device),
|
||||
idx_mapping=torch.zeros(num_reqs, dtype=torch.int32, device=device),
|
||||
expanded_idx_mapping=torch.zeros(
|
||||
num_logits, dtype=torch.int32, device=device
|
||||
),
|
||||
expanded_local_pos=torch.arange(
|
||||
num_logits, dtype=torch.int32, device=device
|
||||
),
|
||||
temperature=torch.zeros(num_reqs, dtype=torch.float32, device=device),
|
||||
seed=torch.full((num_reqs,), 42, dtype=torch.int64, device=device),
|
||||
num_speculative_steps=num_spec,
|
||||
synthetic_conditional_rates=synthetic_rates,
|
||||
use_fp64=False,
|
||||
use_block_verification=use_block_verification,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Skipping spec-decode rejection sampler warmup.", exc_info=True
|
||||
)
|
||||
return
|
||||
@@ -0,0 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_xpu():
|
||||
raise NotImplementedError("GLM-5.3-Flash does not currently support XPU.")
|
||||
|
||||
from .nvidia.model import Glm5NextForCausalLM, Glm5NextForConditionalGeneration
|
||||
from .nvidia.mtp import Glm5NextMTP
|
||||
|
||||
__all__ = [
|
||||
"Glm5NextForCausalLM",
|
||||
"Glm5NextForConditionalGeneration",
|
||||
"Glm5NextMTP",
|
||||
]
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,890 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""kpool (key-pooling) Triton kernels for the sparse-attention indexer.
|
||||
|
||||
The cache stores POOLS (1 entry per ``pool_size`` consecutive tokens) rather
|
||||
than individual tokens. ``compress_ratio == pool_size`` on the kv_cache_spec
|
||||
makes the metadata builder emit pool-granular slot_mapping / seq_lens /
|
||||
cu_seq_lens / page_table for free; this file supplies the compress-write
|
||||
kernel (replacing ``indexer_k_quant_and_cache``) and the pool-level topk
|
||||
helpers (select pools -> expand to tokens -> append tail).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
# The GLM-5.3-Flash indexer head dimension is fixed at 128.
|
||||
INDEX_HEAD_DIM = 128
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
FP8_MAX = torch.finfo(FP8_DTYPE).max
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _cache_k_offset(
|
||||
token_offset,
|
||||
dim_offset,
|
||||
head_dim: tl.constexpr,
|
||||
preshuffle: tl.constexpr,
|
||||
):
|
||||
if preshuffle:
|
||||
return (
|
||||
(token_offset // 16) * 16 * head_dim
|
||||
+ (dim_offset // 16) * 16 * 16
|
||||
+ (token_offset % 16) * 16
|
||||
+ dim_offset % 16
|
||||
)
|
||||
return token_offset * head_dim + dim_offset
|
||||
|
||||
|
||||
# Hadamard-128 rotation
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hadamard128_stage(x, GROUPS: tl.constexpr, STRIDE: tl.constexpr):
|
||||
x3 = tl.reshape(x, (GROUPS, 2, STRIDE))
|
||||
x3 = tl.trans(x3, 0, 2, 1)
|
||||
a, b = tl.split(x3)
|
||||
x3 = tl.join(a + b, a - b)
|
||||
x3 = tl.trans(x3, 0, 2, 1)
|
||||
return tl.reshape(x3, (128,))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hadamard128(x):
|
||||
x = _hadamard128_stage(x, 64, 1)
|
||||
x = _hadamard128_stage(x, 32, 2)
|
||||
x = _hadamard128_stage(x, 16, 4)
|
||||
x = _hadamard128_stage(x, 8, 8)
|
||||
x = _hadamard128_stage(x, 4, 16)
|
||||
x = _hadamard128_stage(x, 2, 32)
|
||||
x = _hadamard128_stage(x, 1, 64)
|
||||
return x * 0.08838834764831845 # 1/sqrt(128)
|
||||
|
||||
|
||||
# Map pool ids to physical cache slots.
|
||||
|
||||
|
||||
def compute_pooled_write_locs(
|
||||
page_table_64: torch.Tensor,
|
||||
pool_ids: torch.Tensor,
|
||||
pool_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Map logical pooled-K ids to physical flat cache slots.
|
||||
|
||||
``pool_size`` consecutive tokens share one pool slot that lives at the
|
||||
*first* token page of each page-group. ``page_table_64`` maps token pages
|
||||
to physical block ids; we gather the block id of each pool's page-group
|
||||
and add the in-block pool offset.
|
||||
"""
|
||||
assert page_table_64.ndim == 1
|
||||
pool_ids = pool_ids.to(torch.int64)
|
||||
block_size = 64
|
||||
pool_page_group = torch.div(pool_ids, block_size, rounding_mode="floor")
|
||||
token_page_row = pool_page_group * pool_size
|
||||
packed_page = page_table_64.index_select(0, token_page_row.to(torch.int64))
|
||||
return packed_page.to(torch.int64) * block_size + torch.remainder(
|
||||
pool_ids, block_size
|
||||
)
|
||||
|
||||
|
||||
def build_pooled_page_table(
|
||||
page_table: torch.Tensor,
|
||||
pool_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Build a pool-granular page table by taking every ``pool_size``-th
|
||||
token-page column (one pool maps to ``pool_size`` token pages).
|
||||
|
||||
Uses gather (not strided slicing) so the result is always a fresh
|
||||
row-major tensor — some downstream kernels require stride(-1) == 1.
|
||||
"""
|
||||
block_size = page_table.shape[-1]
|
||||
assert block_size % pool_size == 0, (
|
||||
f"pool_size ({pool_size}) must divide page columns ({block_size})"
|
||||
)
|
||||
idx = torch.arange(0, block_size, pool_size, device=page_table.device)
|
||||
return page_table[..., idx].contiguous()
|
||||
|
||||
|
||||
# Fused pool compression and cache write.
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _kpool_softmax_rotate_write_cache_kernel(
|
||||
buf_fp8_ptr,
|
||||
buf_fp32_ptr,
|
||||
slot_k_ptr,
|
||||
slot_score_ptr,
|
||||
ape_ptr,
|
||||
loc_ptr,
|
||||
write_mask_ptr,
|
||||
compressed_k_ptr,
|
||||
compressed_scale_ptr,
|
||||
slot_k_stride_0,
|
||||
slot_k_stride_1,
|
||||
slot_score_stride_0,
|
||||
slot_score_stride_1,
|
||||
ape_stride_0,
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
BUF_NUMEL_PER_PAGE: tl.constexpr,
|
||||
POOL_SIZE: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
PRESHUFFLE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
HAS_WRITE_MASK: tl.constexpr,
|
||||
RETURN_COMPRESSED: tl.constexpr,
|
||||
WRITE_CACHE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""One program per pool. softmax(slot_score+ape)-weighted sum of slot_k ->
|
||||
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``."""
|
||||
row = tl.program_id(0)
|
||||
do_write = True
|
||||
if HAS_WRITE_MASK:
|
||||
do_write = tl.load(write_mask_ptr + row)
|
||||
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
mask = (offs < HEAD_DIM) & do_write
|
||||
|
||||
# --- Pass 1: per-dim max over the pool (softmax numerical stability) ---
|
||||
max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32)
|
||||
for slot in tl.static_range(0, POOL_SIZE):
|
||||
score = tl.load(
|
||||
slot_score_ptr
|
||||
+ row * slot_score_stride_0
|
||||
+ slot * slot_score_stride_1
|
||||
+ offs,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
score += tl.load(ape_ptr + slot * ape_stride_0 + offs, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
max_score = tl.maximum(max_score, score)
|
||||
|
||||
# --- Pass 2: softmax-weighted sum of K ---
|
||||
acc = tl.full((BLOCK_D,), 0.0, tl.float32)
|
||||
denom = tl.full((BLOCK_D,), 0.0, tl.float32)
|
||||
for slot in tl.static_range(0, POOL_SIZE):
|
||||
score = tl.load(
|
||||
slot_score_ptr
|
||||
+ row * slot_score_stride_0
|
||||
+ slot * slot_score_stride_1
|
||||
+ offs,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
score += tl.load(ape_ptr + slot * ape_stride_0 + offs, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
prob = tl.exp(score - max_score)
|
||||
denom += prob
|
||||
k = tl.load(
|
||||
slot_k_ptr + row * slot_k_stride_0 + slot * slot_k_stride_1 + offs,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
acc += k * prob
|
||||
|
||||
x = acc / denom
|
||||
x = tl.where(do_write, x, 0.0).to(tl.bfloat16).to(tl.float32)
|
||||
|
||||
# Match the unfused pooled-K path's bf16 precision before quantization.
|
||||
x = _hadamard128(x).to(tl.bfloat16).to(tl.float32)
|
||||
|
||||
# --- per-vector absmax fp8 quant ---
|
||||
fp8_max_inv = 1.0 / FP8_MAX
|
||||
absmax = tl.max(tl.abs(x), axis=0)
|
||||
absmax = tl.maximum(absmax, 1e-4)
|
||||
if ROUND_SCALE:
|
||||
scale = tl.exp2(tl.ceil(tl.log2(absmax * fp8_max_inv)))
|
||||
else:
|
||||
scale = absmax * fp8_max_inv
|
||||
quantized = x / scale
|
||||
quantized = tl.minimum(tl.maximum(quantized, -FP8_MAX), FP8_MAX)
|
||||
|
||||
if WRITE_CACHE:
|
||||
loc = tl.load(loc_ptr + row, mask=do_write, other=0)
|
||||
loc_page_index = loc // PAGE_SIZE
|
||||
loc_token_offset_in_page = loc % PAGE_SIZE
|
||||
out_k_offsets = loc_page_index * BUF_NUMEL_PER_PAGE + _cache_k_offset(
|
||||
loc_token_offset_in_page,
|
||||
offs,
|
||||
HEAD_DIM,
|
||||
PRESHUFFLE,
|
||||
)
|
||||
out_s_offset = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE // 4
|
||||
+ S_OFFSET_NBYTES_IN_PAGE // 4
|
||||
+ loc_token_offset_in_page
|
||||
)
|
||||
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=mask)
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
|
||||
|
||||
if RETURN_COMPRESSED:
|
||||
tl.store(
|
||||
compressed_k_ptr + row * HEAD_DIM + offs,
|
||||
quantized,
|
||||
mask=offs < HEAD_DIM,
|
||||
)
|
||||
tl.store(compressed_scale_ptr + row, scale)
|
||||
|
||||
|
||||
def kpool_compress_and_write_cache(
|
||||
kv_cache: torch.Tensor,
|
||||
slot_k: torch.Tensor,
|
||||
slot_score: torch.Tensor,
|
||||
ape: torch.Tensor,
|
||||
loc: torch.Tensor,
|
||||
pool_size: int,
|
||||
head_dim: int = INDEX_HEAD_DIM,
|
||||
write_mask: torch.Tensor | None = None,
|
||||
round_scale: bool = True,
|
||||
return_compressed: bool = False,
|
||||
write_cache: bool = True,
|
||||
):
|
||||
"""Compress ``pool_size`` tokens into one fp8 K and write at ``loc``.
|
||||
|
||||
Args:
|
||||
kv_cache: indexer K cache ``[num_blocks, block_size, head_dim+4]`` uint8.
|
||||
slot_k: ``[n_pools, pool_size, head_dim]`` bf16 — raw per-token K.
|
||||
slot_score: ``[n_pools, pool_size, head_dim]`` — per-token gate score.
|
||||
ape: ``[pool_size, head_dim]`` fp32 — per-slot position bias.
|
||||
loc: ``[n_pools]`` int64 — flat physical slot per pool.
|
||||
"""
|
||||
assert slot_k.ndim == 3
|
||||
assert slot_score.shape == slot_k.shape
|
||||
assert ape.shape == slot_k.shape[1:]
|
||||
assert slot_k.shape[2] == head_dim
|
||||
assert slot_k.dtype == torch.bfloat16
|
||||
assert ape.dtype == torch.float32
|
||||
assert kv_cache.dtype == torch.uint8
|
||||
assert loc.dtype == torch.int64
|
||||
assert write_cache or return_compressed
|
||||
|
||||
page_size = kv_cache.shape[1]
|
||||
buf = kv_cache
|
||||
slot_k = slot_k.contiguous()
|
||||
slot_score = slot_score.contiguous()
|
||||
ape = ape.contiguous()
|
||||
loc = loc.contiguous()
|
||||
if write_mask is None:
|
||||
write_mask = torch.empty((1,), dtype=torch.bool, device=slot_k.device)
|
||||
has_write_mask = False
|
||||
else:
|
||||
assert write_mask.shape == (slot_k.shape[0],)
|
||||
write_mask = write_mask.contiguous()
|
||||
has_write_mask = True
|
||||
assert not return_compressed
|
||||
|
||||
if slot_k.shape[0] == 0:
|
||||
if return_compressed:
|
||||
return (
|
||||
torch.empty(
|
||||
(0, head_dim),
|
||||
dtype=FP8_DTYPE,
|
||||
device=slot_k.device,
|
||||
),
|
||||
torch.empty((0,), dtype=torch.float32, device=slot_k.device),
|
||||
)
|
||||
return None
|
||||
|
||||
buf_fp8 = buf.view(FP8_DTYPE)
|
||||
buf_fp32 = buf.view(torch.float32)
|
||||
# bytes per page (last dim of kv_cache) viewed as uint8
|
||||
buf_numel_per_page = buf.stride(0)
|
||||
s_offset_nbytes_in_page = page_size * head_dim
|
||||
|
||||
if return_compressed:
|
||||
compressed_k = torch.empty(
|
||||
(slot_k.shape[0], head_dim),
|
||||
dtype=FP8_DTYPE,
|
||||
device=slot_k.device,
|
||||
)
|
||||
compressed_scale = torch.empty(
|
||||
(slot_k.shape[0],), dtype=torch.float32, device=slot_k.device
|
||||
)
|
||||
else:
|
||||
compressed_k = buf_fp8
|
||||
compressed_scale = buf_fp32
|
||||
|
||||
if page_size > 1:
|
||||
assert page_size % 16 == 0, "ROCm preshuffle requires 16-token tiles"
|
||||
|
||||
_kpool_softmax_rotate_write_cache_kernel[(slot_k.shape[0],)](
|
||||
buf_fp8,
|
||||
buf_fp32,
|
||||
slot_k,
|
||||
slot_score,
|
||||
ape,
|
||||
loc,
|
||||
write_mask,
|
||||
compressed_k,
|
||||
compressed_scale,
|
||||
slot_k.stride(0),
|
||||
slot_k.stride(1),
|
||||
slot_score.stride(0),
|
||||
slot_score.stride(1),
|
||||
ape.stride(0),
|
||||
PAGE_SIZE=page_size,
|
||||
BUF_NUMEL_PER_PAGE=buf_numel_per_page,
|
||||
POOL_SIZE=slot_k.shape[1],
|
||||
HEAD_DIM=head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=s_offset_nbytes_in_page,
|
||||
FP8_MAX=FP8_MAX,
|
||||
PRESHUFFLE=page_size > 1,
|
||||
ROUND_SCALE=round_scale,
|
||||
HAS_WRITE_MASK=has_write_mask,
|
||||
RETURN_COMPRESSED=return_compressed,
|
||||
WRITE_CACHE=write_cache,
|
||||
BLOCK_D=triton.next_power_of_2(head_dim),
|
||||
)
|
||||
|
||||
if return_compressed:
|
||||
return compressed_k, compressed_scale
|
||||
return None
|
||||
|
||||
|
||||
# Seed each request's incomplete pool into its paged tail during prefill.
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _kpool_tail_seed_kernel(
|
||||
key_ptr,
|
||||
score_ptr,
|
||||
tslot_ptr,
|
||||
tail_ptr,
|
||||
n_tokens,
|
||||
TAIL_BLOCK_ELEMS: tl.constexpr,
|
||||
KPOOL_HEAD: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
KPOOL: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""Copy token ``i``'s raw K + gate into its request's tail block.
|
||||
|
||||
Token ``i`` is among its request's last KPOOL tokens iff the token KPOOL
|
||||
ahead belongs to a different tail block (or is past the batch / padding,
|
||||
slot < 0). ``tslot = block * KPOOL + pos % KPOOL``; the destination is
|
||||
``tail[block, {0:K, 1:score}, pos % KPOOL, :]``.
|
||||
"""
|
||||
i = tl.program_id(0)
|
||||
t = tl.load(tslot_ptr + i).to(tl.int64)
|
||||
if t < 0:
|
||||
return
|
||||
blk = t // KPOOL # t >= 0 here, so trunc == floor
|
||||
ahead = tl.load(tslot_ptr + i + KPOOL, mask=i + KPOOL < n_tokens, other=-1).to(
|
||||
tl.int64
|
||||
)
|
||||
# Match the torch semantics exactly: a negative ahead slot floors to a
|
||||
# block id that differs from every real block -> token is in the tail.
|
||||
# Only divide non-negative slots (Triton int div truncates, torch floors).
|
||||
if ahead >= 0 and ahead // KPOOL == blk:
|
||||
return
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
m = offs < HEAD_DIM
|
||||
block_base = blk * TAIL_BLOCK_ELEMS
|
||||
base = block_base + (t % KPOOL) * HEAD_DIM
|
||||
k = tl.load(key_ptr + i * HEAD_DIM + offs, mask=m)
|
||||
s = tl.load(score_ptr + i * HEAD_DIM + offs, mask=m)
|
||||
tl.store(tail_ptr + base + offs, k, mask=m)
|
||||
tl.store(
|
||||
tail_ptr + block_base + KPOOL_HEAD + (t % KPOOL) * HEAD_DIM + offs, s, mask=m
|
||||
)
|
||||
|
||||
|
||||
def kpool_seed_tail_cache(
|
||||
tail_kv_cache: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
gate_score: torch.Tensor,
|
||||
tslot: torch.Tensor,
|
||||
kpool: int,
|
||||
head_dim: int = INDEX_HEAD_DIM,
|
||||
) -> None:
|
||||
"""Seed the paged tail cache from a prefill batch (see the kernel)."""
|
||||
assert tail_kv_cache.dtype == torch.bfloat16
|
||||
assert key.dtype == torch.bfloat16
|
||||
n = tslot.shape[0]
|
||||
if n == 0:
|
||||
return
|
||||
_kpool_tail_seed_kernel[(n,)](
|
||||
key,
|
||||
gate_score,
|
||||
tslot,
|
||||
tail_kv_cache,
|
||||
n,
|
||||
TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0),
|
||||
KPOOL_HEAD=tail_kv_cache.stride(1),
|
||||
HEAD_DIM=head_dim,
|
||||
KPOOL=kpool,
|
||||
BLOCK_D=triton.next_power_of_2(head_dim),
|
||||
)
|
||||
|
||||
|
||||
# Update each request's tail during decode and write completed pools.
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _kpool_decode_update_batched_kernel(
|
||||
buf_fp8_ptr,
|
||||
buf_fp32_ptr,
|
||||
tail_kv_ptr,
|
||||
tail_slot_mapping_ptr, # [B, NEXT_N] int32
|
||||
key_ptr, # [B, NEXT_N, HEAD_DIM] bf16
|
||||
key_stride_b,
|
||||
key_stride_t,
|
||||
slot_score_ptr, # [B, NEXT_N, HEAD_DIM] bf16
|
||||
ss_stride_b,
|
||||
ss_stride_t,
|
||||
ape_ptr,
|
||||
ape_stride_0,
|
||||
slot_mapping_ptr, # [B, NEXT_N] int32
|
||||
positions_ptr, # [B, NEXT_N] int32
|
||||
NEXT_N, # runtime token count per request (no .item() needed)
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
BUF_NUMEL_PER_PAGE: tl.constexpr,
|
||||
POOL_SIZE: tl.constexpr,
|
||||
TAIL_BLOCK_ELEMS: tl.constexpr,
|
||||
KPOOL_HEAD: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
PRESHUFFLE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""One program per request; iterates its NEXT_N verify tokens in order.
|
||||
|
||||
Replaces the caller's per-token sequential launch loop. The intra-request
|
||||
iteration MUST stay in position order: a pool-completion at token t* reads
|
||||
the tail-ring slots that tokens t < t* (same request) just stashed in this
|
||||
same invocation. ``tl.range`` iterates sequentially within the program, so
|
||||
those stashes are visible to the later completion read. Cross-request
|
||||
programs are independent (distinct tail blocks). With NEXT_N < POOL_SIZE
|
||||
(the spec-verify case: NEXT_N ~= num_spec+1, POOL_SIZE=16) at most one
|
||||
completion can occur per request per call, but the ordered loop is correct
|
||||
for any NEXT_N.
|
||||
"""
|
||||
req = tl.program_id(0)
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
dim_mask = offs < HEAD_DIM
|
||||
|
||||
for t in tl.range(0, NEXT_N):
|
||||
idx = req * NEXT_N + t
|
||||
cache_loc = tl.load(slot_mapping_ptr + idx)
|
||||
pos = tl.load(positions_ptr + idx)
|
||||
safe_pos = tl.maximum(pos, 0)
|
||||
pos_valid = (cache_loc >= 0) & (pos >= 0)
|
||||
|
||||
slot = safe_pos % POOL_SIZE
|
||||
phys_slot = safe_pos % POOL_SIZE
|
||||
|
||||
# Derive the tail block from THIS token's tail_slot (the request's block
|
||||
# is constant across a pool, but a padded / invalid entry carries a
|
||||
# negative sentinel -- reading it from token 0 would poison every
|
||||
# token's base address). Clamp so an invalid entry can never form an
|
||||
# out-of-bounds base; the accesses below are gated on pos_valid anyway.
|
||||
tail_slot = tl.load(tail_slot_mapping_ptr + idx)
|
||||
block = tl.maximum(tail_slot, 0).to(tl.int64) // POOL_SIZE
|
||||
block_base = block * TAIL_BLOCK_ELEMS
|
||||
|
||||
# The tail-ring stash must run for EVERY real token, so it is gated on
|
||||
# the token-granular tail slot -- not on `pos_valid`, which keys off the
|
||||
# POOL-granular `slot_mapping` and is therefore only true on the pool's
|
||||
# last token. Gating the stash on pos_valid dropped every intra-pool
|
||||
# token, so a decode-built pool compressed 3 stale ring entries (the
|
||||
# prefill-seeded prompt tail, frozen forever) plus the current token.
|
||||
stash_valid = (pos >= 0) & (tail_slot >= 0)
|
||||
|
||||
key = tl.load(
|
||||
key_ptr + req * key_stride_b + t * key_stride_t + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
score_current = tl.load(
|
||||
slot_score_ptr + req * ss_stride_b + t * ss_stride_t + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
|
||||
if pos_valid & (slot == POOL_SIZE - 1):
|
||||
pool_logical_start = safe_pos - slot
|
||||
|
||||
max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32)
|
||||
for pool_slot in tl.static_range(0, POOL_SIZE):
|
||||
is_current = pool_slot == slot
|
||||
phys = (pool_logical_start + pool_slot) % POOL_SIZE
|
||||
score_buf = tl.load(
|
||||
tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
score = tl.where(is_current, score_current, score_buf)
|
||||
score += tl.load(
|
||||
ape_ptr + pool_slot * ape_stride_0 + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
max_score = tl.maximum(max_score, score)
|
||||
|
||||
acc = tl.full((BLOCK_D,), 0.0, tl.float32)
|
||||
denom = tl.full((BLOCK_D,), 0.0, tl.float32)
|
||||
for pool_slot in tl.static_range(0, POOL_SIZE):
|
||||
is_current = pool_slot == slot
|
||||
phys = (pool_logical_start + pool_slot) % POOL_SIZE
|
||||
score_buf = tl.load(
|
||||
tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
score = tl.where(is_current, score_current, score_buf)
|
||||
score += tl.load(
|
||||
ape_ptr + pool_slot * ape_stride_0 + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
prob = tl.exp(score - max_score)
|
||||
denom += prob
|
||||
k_buf = tl.load(
|
||||
tail_kv_ptr + block_base + phys * HEAD_DIM + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
k = tl.where(is_current, key, k_buf)
|
||||
acc += k * prob
|
||||
|
||||
x = (acc / denom).to(tl.bfloat16).to(tl.float32)
|
||||
x = _hadamard128(x).to(tl.bfloat16).to(tl.float32)
|
||||
|
||||
fp8_max_inv = 1.0 / FP8_MAX
|
||||
absmax = tl.maximum(tl.max(tl.abs(x), axis=0), 1e-4)
|
||||
if ROUND_SCALE:
|
||||
scale = tl.exp2(tl.ceil(tl.log2(absmax * fp8_max_inv)))
|
||||
else:
|
||||
scale = absmax * fp8_max_inv
|
||||
quantized = tl.minimum(tl.maximum(x / scale, -FP8_MAX), FP8_MAX)
|
||||
|
||||
loc = cache_loc.to(tl.int64)
|
||||
loc_page_index = loc // PAGE_SIZE
|
||||
loc_token_offset_in_page = loc % PAGE_SIZE
|
||||
out_k_offsets = loc_page_index * BUF_NUMEL_PER_PAGE + _cache_k_offset(
|
||||
loc_token_offset_in_page,
|
||||
offs,
|
||||
HEAD_DIM,
|
||||
PRESHUFFLE,
|
||||
)
|
||||
out_s_offset = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE // 4
|
||||
+ S_OFFSET_NBYTES_IN_PAGE // 4
|
||||
+ loc_token_offset_in_page
|
||||
)
|
||||
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=dim_mask)
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale)
|
||||
|
||||
# Stash the current token AFTER any completion read so the completion
|
||||
# uses prior stashes (and the current token's own key/score via
|
||||
# is_current), then leaves this token for future pools. Order matches
|
||||
# the per-token kernel: completion read first, stash second.
|
||||
update_mask = dim_mask & stash_valid
|
||||
tl.store(
|
||||
tail_kv_ptr + block_base + phys_slot * HEAD_DIM + offs,
|
||||
key,
|
||||
mask=update_mask,
|
||||
)
|
||||
tl.store(
|
||||
tail_kv_ptr + block_base + KPOOL_HEAD + phys_slot * HEAD_DIM + offs,
|
||||
score_current,
|
||||
mask=update_mask,
|
||||
)
|
||||
|
||||
|
||||
def kpool_decode_update_and_maybe_write_cache_batched(
|
||||
kv_cache: torch.Tensor,
|
||||
tail_kv_cache: torch.Tensor,
|
||||
tail_slot_mapping: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
slot_score: torch.Tensor,
|
||||
ape: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
pool_size: int,
|
||||
head_dim: int = INDEX_HEAD_DIM,
|
||||
round_scale: bool = True,
|
||||
) -> None:
|
||||
"""Batched decode-step kpool update for spec verify (``next_n > 1``).
|
||||
|
||||
One launch replaces the caller's per-token loop. Inputs are grouped per
|
||||
request: ``[num_requests, next_n, ...]``. Each program handles one
|
||||
request's ``next_n`` tokens in position order (see the kernel docstring for
|
||||
why ordering is required for pool-completion correctness).
|
||||
|
||||
Plain decode (``next_n == 1``) is handled here too — the kernel collapses
|
||||
to a single-iteration loop.
|
||||
|
||||
Args:
|
||||
kv_cache: indexer K cache ``[num_blocks, block_size, head_dim+4]`` uint8.
|
||||
tail_kv_cache: paged tail cache ``[num_blocks, 2, pool_size, head_dim]``
|
||||
bf16 (K at half 0, gate score at half 1).
|
||||
tail_slot_mapping: ``[num_requests, next_n]`` int32.
|
||||
key / slot_score: ``[num_requests, next_n, head_dim]`` bf16.
|
||||
ape: ``[pool_size, head_dim]`` fp32.
|
||||
slot_mapping / positions: ``[num_requests, next_n]`` int32.
|
||||
"""
|
||||
num_requests, next_n = key.shape[0], key.shape[1]
|
||||
if num_requests == 0 or next_n == 0:
|
||||
return
|
||||
assert tail_kv_cache.ndim == 4
|
||||
assert tail_kv_cache.shape[1] == 2
|
||||
assert tail_kv_cache.shape[2] == pool_size
|
||||
assert tail_kv_cache.shape[3] == head_dim
|
||||
assert tail_kv_cache.dtype == torch.bfloat16
|
||||
assert key.ndim == 3 and key.shape[2] == head_dim
|
||||
assert slot_score.shape == key.shape
|
||||
assert ape.shape == (pool_size, head_dim)
|
||||
assert tail_slot_mapping.shape == (num_requests, next_n)
|
||||
assert slot_mapping.shape == (num_requests, next_n)
|
||||
assert positions.shape == (num_requests, next_n)
|
||||
assert key.dtype == torch.bfloat16
|
||||
assert slot_score.dtype == torch.bfloat16
|
||||
assert ape.dtype == torch.float32
|
||||
assert kv_cache.dtype == torch.uint8
|
||||
|
||||
page_size = kv_cache.shape[1]
|
||||
buf = kv_cache
|
||||
buf_fp8 = buf.view(FP8_DTYPE)
|
||||
buf_fp32 = buf.view(torch.float32)
|
||||
|
||||
# The kernel indexes the int tensors as ``req * next_n + t`` (row-major),
|
||||
# so they must be contiguous. Callers pass either a view of a contiguous
|
||||
# slice or a freshly scattered tensor, making these no-ops; the calls guard
|
||||
# against a future caller handing over a strided view.
|
||||
tail_slot_mapping = tail_slot_mapping.contiguous()
|
||||
slot_mapping = slot_mapping.contiguous()
|
||||
positions = positions.contiguous()
|
||||
|
||||
if page_size > 1:
|
||||
assert page_size % 16 == 0, "ROCm preshuffle requires 16-token tiles"
|
||||
|
||||
_kpool_decode_update_batched_kernel[(num_requests,)](
|
||||
buf_fp8,
|
||||
buf_fp32,
|
||||
tail_kv_cache,
|
||||
tail_slot_mapping,
|
||||
key,
|
||||
key.stride(0),
|
||||
key.stride(1),
|
||||
slot_score,
|
||||
slot_score.stride(0),
|
||||
slot_score.stride(1),
|
||||
ape,
|
||||
ape.stride(0),
|
||||
slot_mapping,
|
||||
positions,
|
||||
next_n,
|
||||
PAGE_SIZE=page_size,
|
||||
BUF_NUMEL_PER_PAGE=buf.stride(0),
|
||||
POOL_SIZE=pool_size,
|
||||
TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0),
|
||||
KPOOL_HEAD=tail_kv_cache.stride(1),
|
||||
HEAD_DIM=head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=page_size * head_dim,
|
||||
FP8_MAX=FP8_MAX,
|
||||
PRESHUFFLE=page_size > 1,
|
||||
ROUND_SCALE=round_scale,
|
||||
BLOCK_D=triton.next_power_of_2(head_dim),
|
||||
)
|
||||
|
||||
|
||||
# Pool-level top-k helpers.
|
||||
|
||||
|
||||
def history_group_budget_for_topk(topk: int, pool_size: int) -> int:
|
||||
"""Number of pools to select so that expanding yields ``topk`` tokens."""
|
||||
assert topk % pool_size == 0
|
||||
return topk // pool_size
|
||||
|
||||
|
||||
def expand_pools_to_tokens(
|
||||
group_ids: torch.Tensor,
|
||||
group_valid: torch.Tensor,
|
||||
topk: int,
|
||||
pool_size: int,
|
||||
page_table: torch.Tensor | None = None,
|
||||
topk_offsets: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Expand selected full-pool ids to a strict-width token topk tensor."""
|
||||
assert group_ids.ndim == 2
|
||||
assert group_valid.shape == group_ids.shape
|
||||
assert topk % pool_size == 0
|
||||
assert group_ids.shape[1] == history_group_budget_for_topk(topk, pool_size)
|
||||
assert page_table is None or topk_offsets is None
|
||||
|
||||
device = group_ids.device
|
||||
offsets = torch.arange(pool_size, device=device, dtype=torch.int64)
|
||||
token_ids = group_ids.to(torch.int64).unsqueeze(-1) * pool_size + offsets
|
||||
token_ids = token_ids.reshape(group_ids.shape[0], topk)
|
||||
valid = (
|
||||
group_valid.unsqueeze(-1)
|
||||
.expand(-1, -1, pool_size)
|
||||
.reshape(group_ids.shape[0], topk)
|
||||
)
|
||||
|
||||
if page_table is not None:
|
||||
assert page_table.ndim == 2
|
||||
safe_ids = token_ids.clamp(min=0, max=page_table.shape[1] - 1)
|
||||
output = torch.gather(page_table, dim=1, index=safe_ids).to(torch.int32)
|
||||
elif topk_offsets is not None:
|
||||
if topk_offsets.ndim == 2:
|
||||
assert topk_offsets.shape[1] == 1
|
||||
topk_offsets = topk_offsets.squeeze(1)
|
||||
output = (token_ids + topk_offsets.to(torch.int64).unsqueeze(1)).to(torch.int32)
|
||||
else:
|
||||
output = token_ids.to(torch.int32)
|
||||
|
||||
return torch.where(valid, output, torch.full_like(output, -1))
|
||||
|
||||
|
||||
def append_tail_to_topk(
|
||||
topk_result: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
pool_lens: torch.Tensor,
|
||||
pool_size: int,
|
||||
page_table: torch.Tensor | None = None,
|
||||
topk_offsets: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Append non-pooled tail tokens after expanded history tokens.
|
||||
|
||||
``index_kpool_always_select_tail`` keeps the (incomplete) trailing pool so
|
||||
the most recent tokens are always attended to.
|
||||
"""
|
||||
assert topk_result.dtype == torch.int32
|
||||
assert seq_lens.ndim == 1
|
||||
assert pool_lens.ndim == 1
|
||||
|
||||
tail_pool = pool_size - 1
|
||||
if tail_pool == 0:
|
||||
return topk_result
|
||||
|
||||
rows, n_cols = topk_result.shape
|
||||
out_cols = n_cols + tail_pool
|
||||
out = torch.empty(
|
||||
(rows, out_cols), dtype=topk_result.dtype, device=topk_result.device
|
||||
)
|
||||
|
||||
# tail tokens: [pool_len*pool_size, seq_len) for each row.
|
||||
pool_len = pool_lens.to(torch.int32)
|
||||
tail_start = pool_len * pool_size
|
||||
seq_len = seq_lens.to(torch.int32)
|
||||
tail_count = seq_len - tail_start # in [0, pool_size)
|
||||
|
||||
cols = torch.arange(out_cols, device=topk_result.device)[None, :]
|
||||
history_len = n_cols
|
||||
is_history = cols < history_len
|
||||
tail_off = cols - history_len
|
||||
is_tail = (tail_off >= 0) & (tail_off < tail_count[:, None])
|
||||
|
||||
# safe_hist must be per-row [rows, out_cols] so the gather reads each row's
|
||||
# OWN history. cols is [1, out_cols]; if used directly, gather (which does
|
||||
# NOT broadcast the index) would read only row 0 of topk_result, making every
|
||||
# query inherit row 0's history (empty for the first token) and lose all its
|
||||
# selected tokens — only the per-row tail would survive. This only manifests
|
||||
# for multi-row sparse PREFILL (decode has 1 row, so it reads its own row 0).
|
||||
safe_hist = torch.minimum(cols, torch.full_like(cols, n_cols - 1)).expand(
|
||||
rows, out_cols
|
||||
)
|
||||
history_val = torch.gather(topk_result, 1, safe_hist)
|
||||
|
||||
tail_raw = tail_start[:, None] + tail_off
|
||||
tail_val = tail_raw.to(torch.int32)
|
||||
if page_table is not None:
|
||||
safe_tail = tail_raw.clamp(min=0, max=page_table.shape[1] - 1)
|
||||
tail_val = torch.gather(page_table, 1, safe_tail).to(torch.int32)
|
||||
elif topk_offsets is not None:
|
||||
tail_val = (tail_raw + topk_offsets.to(torch.int64).unsqueeze(1)).to(
|
||||
torch.int32
|
||||
)
|
||||
|
||||
out = torch.where(is_history, history_val, -1)
|
||||
out = torch.where(is_tail, tail_val, out)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _expand_pools_and_append_tail_kernel(
|
||||
pool_ids_ptr, # [rows, n_groups], int (any int dtype)
|
||||
seq_lens_ptr, # [rows], int32 (token-granular seq_len)
|
||||
out_ptr, # [rows, out_cols], int32
|
||||
topk, # n_groups * pool_size
|
||||
out_cols, # topk + pool_size - 1
|
||||
POOL_SIZE: tl.constexpr,
|
||||
BLOCK_COLS: tl.constexpr,
|
||||
pid_s0,
|
||||
out_s0,
|
||||
):
|
||||
# Fuses expand_pools_to_tokens + append_tail_to_topk (identity path) into a
|
||||
# single kernel. Each program writes one (row, column-tile) of the output.
|
||||
row = tl.program_id(0)
|
||||
tile = tl.program_id(1)
|
||||
cols = tile * BLOCK_COLS + tl.arange(0, BLOCK_COLS)
|
||||
mask = cols < out_cols
|
||||
|
||||
seq_len = tl.load(seq_lens_ptr + row)
|
||||
pool_len = seq_len // POOL_SIZE
|
||||
tail_start = pool_len * POOL_SIZE
|
||||
tail_count = seq_len - tail_start # in [0, POOL_SIZE)
|
||||
|
||||
# History region [0, topk): expand selected pool g = cols // POOL_SIZE.
|
||||
is_history = cols < topk
|
||||
g = cols // POOL_SIZE
|
||||
o = cols % POOL_SIZE
|
||||
pid = tl.load(pool_ids_ptr + row * pid_s0 + g, mask=mask & is_history, other=-1)
|
||||
hist_val = (pid * POOL_SIZE + o).to(tl.int32)
|
||||
hist_out = tl.where(pid >= 0, hist_val, -1)
|
||||
|
||||
# Tail region [topk, out_cols): the request's trailing incomplete pool.
|
||||
tail_off = cols - topk
|
||||
is_tail = (tail_off >= 0) & (tail_off < tail_count)
|
||||
tail_val = (tail_start + tail_off).to(tl.int32)
|
||||
tail_out = tl.where(is_tail, tail_val, -1)
|
||||
|
||||
result = tl.where(is_history, hist_out, tail_out)
|
||||
tl.store(out_ptr + row * out_s0 + cols, result, mask=mask)
|
||||
|
||||
|
||||
def expand_pools_and_append_tail(
|
||||
pool_ids: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
pool_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Fuse ``expand_pools_to_tokens`` + ``append_tail_to_topk`` (identity path).
|
||||
|
||||
Produces the same ``[rows, topk + pool_size - 1]`` int32 output as calling
|
||||
the two functions in sequence when neither ``page_table`` nor
|
||||
``topk_offsets`` is passed — the only path used by the GLM-5.3-Flash indexer.
|
||||
The kernel derives ``pool_len = seq_len // pool_size`` internally, so the
|
||||
caller no longer needs to precompute it. Replaces ~25 elementwise kernels
|
||||
with one Triton launch.
|
||||
"""
|
||||
rows, n_groups = pool_ids.shape
|
||||
topk = n_groups * pool_size
|
||||
out_cols = topk + pool_size - 1
|
||||
out = torch.empty((rows, out_cols), dtype=torch.int32, device=pool_ids.device)
|
||||
BLOCK_COLS = 128
|
||||
n_tiles = triton.cdiv(out_cols, BLOCK_COLS)
|
||||
_expand_pools_and_append_tail_kernel[(rows, n_tiles)](
|
||||
pool_ids,
|
||||
seq_lens,
|
||||
out,
|
||||
topk,
|
||||
out_cols,
|
||||
POOL_SIZE=pool_size,
|
||||
BLOCK_COLS=BLOCK_COLS,
|
||||
pid_s0=pool_ids.stride(0),
|
||||
out_s0=out.stride(0),
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .kernels import chunk_kda_with_fused_gate, fused_recurrent_kda
|
||||
|
||||
__all__ = ["chunk_kda_with_fused_gate", "fused_recurrent_kda"]
|
||||
@@ -0,0 +1,656 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang
|
||||
# mypy: ignore-errors
|
||||
#
|
||||
# This file contains code copied from the flash-linear-attention project.
|
||||
# The original source code was licensed under the MIT license and included
|
||||
# the following copyright notice:
|
||||
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
||||
# ruff: noqa: E501
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.op import exp
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"USE_INITIAL_STATE": lambda args: args["h0"] is not None,
|
||||
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
||||
"IS_CONTINUOUS_BATCHING": lambda args: args["ssm_state_indices"] is not None,
|
||||
"IS_SPEC_DECODING": lambda args: args["num_accepted_tokens"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.jit(do_not_specialize=["N", "T"])
|
||||
def fused_recurrent_gated_delta_rule_fwd_kernel(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
g,
|
||||
beta,
|
||||
o,
|
||||
h0,
|
||||
ht,
|
||||
cu_seqlens,
|
||||
ssm_state_indices,
|
||||
num_accepted_tokens,
|
||||
a_log,
|
||||
g_bias,
|
||||
scale,
|
||||
N: tl.int64, # num of sequences
|
||||
T: tl.int64, # num of tokens
|
||||
B: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
stride_init_state_token: tl.constexpr,
|
||||
stride_final_state_token: tl.constexpr,
|
||||
stride_indices_seq: tl.constexpr,
|
||||
stride_indices_tok: tl.constexpr,
|
||||
USE_INITIAL_STATE: tl.constexpr, # whether to use initial state
|
||||
INPLACE_FINAL_STATE: tl.constexpr, # whether to store final state inplace
|
||||
IS_BETA_HEADWISE: tl.constexpr, # whether beta is headwise vector or scalar,
|
||||
USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
IS_CONTINUOUS_BATCHING: tl.constexpr,
|
||||
IS_SPEC_DECODING: tl.constexpr,
|
||||
IS_KDA: tl.constexpr,
|
||||
SIGMOID_BETA: tl.constexpr, # beta holds raw logits; sigmoid at fp32 load
|
||||
COMPUTE_GATE: tl.constexpr, # g holds raw logits; KDA gate computed in-kernel
|
||||
SAFE_GATE: tl.constexpr, # bounded gate variant (only branch implemented)
|
||||
LOWER_BOUND: tl.constexpr,
|
||||
):
|
||||
i_k, i_v, i_nh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_n, i_hv = i_nh // HV, i_nh % HV
|
||||
i_h = i_hv // (HV // H)
|
||||
if IS_VARLEN:
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int64),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int64),
|
||||
)
|
||||
all = T
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_n * T, i_n * T + T
|
||||
all = B * T
|
||||
|
||||
if T == 0:
|
||||
# no tokens to process for this sequence
|
||||
return
|
||||
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
|
||||
p_q = q + (bos * H + i_h) * K + o_k
|
||||
p_k = k + (bos * H + i_h) * K + o_k
|
||||
p_v = v + (bos * HV + i_hv) * V + o_v
|
||||
if IS_BETA_HEADWISE:
|
||||
p_beta = beta + (bos * HV + i_hv) * V + o_v
|
||||
else:
|
||||
p_beta = beta + bos * HV + i_hv
|
||||
|
||||
if not IS_KDA:
|
||||
p_g = g + bos * HV + i_hv
|
||||
else:
|
||||
p_gk = g + (bos * HV + i_hv) * K + o_k
|
||||
|
||||
# Per-head gate amplitude, hoisted out of the token loop (COMPUTE_GATE).
|
||||
if COMPUTE_GATE:
|
||||
b_a_log = tl.exp(tl.load(a_log + i_h).to(tl.float32))
|
||||
|
||||
p_o = o + ((i_k * all + bos) * HV + i_hv) * V + o_v
|
||||
|
||||
mask_k = o_k < K
|
||||
mask_v = o_v < V
|
||||
mask_h = mask_v[:, None] & mask_k[None, :]
|
||||
|
||||
b_h = tl.zeros([BV, BK], dtype=tl.float32)
|
||||
if USE_INITIAL_STATE:
|
||||
if IS_CONTINUOUS_BATCHING:
|
||||
if IS_SPEC_DECODING:
|
||||
i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1
|
||||
else:
|
||||
i_t = 0
|
||||
# Load state index and check for invalid entries
|
||||
state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq + i_t).to(
|
||||
tl.int64
|
||||
)
|
||||
# Skip if state index is invalid (NULL_BLOCK_ID=0)
|
||||
if state_idx <= 0:
|
||||
return
|
||||
p_h0 = h0 + state_idx * stride_init_state_token
|
||||
else:
|
||||
p_h0 = h0 + bos * HV * V * K
|
||||
p_h0 = p_h0 + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
b_h += tl.load(p_h0, mask=mask_h, other=0).to(tl.float32)
|
||||
|
||||
for i_t in range(0, T):
|
||||
b_q = tl.load(p_q, mask=mask_k, other=0).to(tl.float32)
|
||||
b_k = tl.load(p_k, mask=mask_k, other=0).to(tl.float32)
|
||||
b_v = tl.load(p_v, mask=mask_v, other=0).to(tl.float32)
|
||||
|
||||
if USE_QK_L2NORM_IN_KERNEL:
|
||||
b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
|
||||
b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
|
||||
b_q = b_q * scale
|
||||
# [BV, BK]
|
||||
if not IS_KDA:
|
||||
b_g = tl.load(p_g).to(tl.float32)
|
||||
b_h *= exp(b_g)
|
||||
else:
|
||||
b_gk = tl.load(p_gk).to(tl.float32)
|
||||
if COMPUTE_GATE:
|
||||
# Replicates kda_gate_fwd_kernel's SAFE_GATE branch
|
||||
# bit-for-bit (same tl.exp, same fp32 math; the intermediate
|
||||
# gate value this replaces was stored/reloaded as fp32,
|
||||
# which is lossless): y = lb / (1 + exp(-exp(A)*(g+bias))).
|
||||
b_gk += tl.load(g_bias + i_h * K + o_k, mask=mask_k, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
b_gk = LOWER_BOUND / (1.0 + tl.exp(-(b_a_log * b_gk)))
|
||||
b_h *= exp(b_gk[None, :])
|
||||
# [BV]
|
||||
b_v -= tl.sum(b_h * b_k[None, :], 1)
|
||||
if IS_BETA_HEADWISE:
|
||||
b_beta = tl.load(p_beta, mask=mask_v, other=0).to(tl.float32)
|
||||
else:
|
||||
b_beta = tl.load(p_beta).to(tl.float32)
|
||||
# Matches torch's `x.float().sigmoid()` pre-computation bit-for-bit
|
||||
# on the input side (bf16->fp32 is exact); only the sigmoid impl itself
|
||||
# can differ by <=1 ULP.
|
||||
if SIGMOID_BETA:
|
||||
b_beta = tl.sigmoid(b_beta)
|
||||
b_v *= b_beta
|
||||
# [BV, BK]
|
||||
b_h += b_v[:, None] * b_k[None, :]
|
||||
# [BV]
|
||||
b_o = tl.sum(b_h * b_q[None, :], 1)
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v)
|
||||
|
||||
# keep the states for multi-query tokens
|
||||
if INPLACE_FINAL_STATE:
|
||||
# Load state index and check for invalid entries
|
||||
final_state_idx = tl.load(
|
||||
ssm_state_indices + i_n * stride_indices_seq + i_t
|
||||
).to(tl.int64)
|
||||
# Only store if state index is valid (not NULL_BLOCK_ID=0)
|
||||
if final_state_idx > 0:
|
||||
p_ht = ht + final_state_idx * stride_final_state_token
|
||||
p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
|
||||
else:
|
||||
p_ht = ht + (bos + i_t) * stride_final_state_token
|
||||
p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
|
||||
|
||||
p_q += H * K
|
||||
p_k += H * K
|
||||
p_o += HV * V
|
||||
p_v += HV * V
|
||||
if not IS_KDA:
|
||||
p_g += HV
|
||||
else:
|
||||
p_gk += HV * K
|
||||
p_beta += HV * (V if IS_BETA_HEADWISE else 1)
|
||||
|
||||
|
||||
def fused_recurrent_gated_delta_rule_fwd(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
inplace_final_state: bool = True,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
ssm_state_indices: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
B, T, H, K, V = *k.shape, v.shape[-1]
|
||||
HV = v.shape[2]
|
||||
N = B if cu_seqlens is None else len(cu_seqlens) - 1
|
||||
BK, BV = triton.next_power_of_2(K), min(triton.next_power_of_2(V), 32)
|
||||
NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV)
|
||||
assert NK == 1, "NK > 1 is not supported yet"
|
||||
num_stages = 3
|
||||
num_warps = 1
|
||||
|
||||
o = q.new_empty(NK, *v.shape)
|
||||
if inplace_final_state:
|
||||
final_state = initial_state
|
||||
else:
|
||||
final_state = q.new_empty(T, HV, V, K, dtype=initial_state.dtype)
|
||||
|
||||
stride_init_state_token = initial_state.stride(0)
|
||||
stride_final_state_token = final_state.stride(0)
|
||||
|
||||
if ssm_state_indices is None:
|
||||
stride_indices_seq, stride_indices_tok = 1, 1
|
||||
elif ssm_state_indices.ndim == 1:
|
||||
stride_indices_seq, stride_indices_tok = ssm_state_indices.stride(0), 1
|
||||
else:
|
||||
stride_indices_seq, stride_indices_tok = ssm_state_indices.stride()
|
||||
|
||||
grid = (NK, NV, N * HV)
|
||||
fused_recurrent_gated_delta_rule_fwd_kernel[grid](
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
o=o,
|
||||
h0=initial_state,
|
||||
ht=final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
scale=scale,
|
||||
N=N,
|
||||
T=T,
|
||||
B=B,
|
||||
H=H,
|
||||
HV=HV,
|
||||
K=K,
|
||||
V=V,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
stride_init_state_token=stride_init_state_token,
|
||||
stride_final_state_token=stride_final_state_token,
|
||||
stride_indices_seq=stride_indices_seq,
|
||||
stride_indices_tok=stride_indices_tok,
|
||||
IS_BETA_HEADWISE=beta.ndim == v.ndim,
|
||||
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
|
||||
INPLACE_FINAL_STATE=inplace_final_state,
|
||||
IS_KDA=False,
|
||||
SIGMOID_BETA=False,
|
||||
a_log=None,
|
||||
g_bias=None,
|
||||
COMPUTE_GATE=False,
|
||||
SAFE_GATE=True,
|
||||
LOWER_BOUND=-5.0,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
)
|
||||
o = o.squeeze(0)
|
||||
return o, final_state
|
||||
|
||||
|
||||
@triton.jit
|
||||
def fused_recurrent_gated_delta_rule_packed_decode_kernel(
|
||||
mixed_qkv,
|
||||
a,
|
||||
b,
|
||||
A_log,
|
||||
dt_bias,
|
||||
o,
|
||||
h0,
|
||||
ht,
|
||||
ssm_state_indices,
|
||||
scale,
|
||||
stride_mixed_qkv_tok: tl.constexpr,
|
||||
stride_a_tok: tl.constexpr,
|
||||
stride_b_tok: tl.constexpr,
|
||||
stride_init_state_token: tl.constexpr,
|
||||
stride_final_state_token: tl.constexpr,
|
||||
stride_indices_seq: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
SOFTPLUS_THRESHOLD: tl.constexpr,
|
||||
USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
|
||||
SPLIT_BATCH_HEAD_GRID: tl.constexpr,
|
||||
):
|
||||
if SPLIT_BATCH_HEAD_GRID:
|
||||
i_v, i_hv, i_n = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
else:
|
||||
i_v, i_nh = tl.program_id(0), tl.program_id(1)
|
||||
i_n, i_hv = i_nh // HV, i_nh % HV
|
||||
i_h = i_hv // (HV // H)
|
||||
|
||||
o_k = tl.arange(0, BK)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
mask_k = o_k < K
|
||||
mask_v = o_v < V
|
||||
mask_h = mask_v[:, None] & mask_k[None, :]
|
||||
|
||||
state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq).to(tl.int64)
|
||||
p_o = o + (i_n * HV + i_hv) * V + o_v
|
||||
|
||||
# Skip if state index is invalid (NULL_BLOCK_ID=0)
|
||||
if state_idx <= 0:
|
||||
zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty)
|
||||
tl.store(p_o, zero, mask=mask_v)
|
||||
return
|
||||
|
||||
p_h0 = h0 + state_idx * stride_init_state_token
|
||||
p_h0 = p_h0 + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
b_h = tl.load(p_h0, mask=mask_h, other=0).to(tl.float32)
|
||||
|
||||
p_mixed = mixed_qkv + i_n * stride_mixed_qkv_tok
|
||||
q_off = i_h * K + o_k
|
||||
k_off = (H * K) + i_h * K + o_k
|
||||
v_off = (2 * H * K) + i_hv * V + o_v
|
||||
b_q = tl.load(p_mixed + q_off, mask=mask_k, other=0).to(tl.float32)
|
||||
b_k = tl.load(p_mixed + k_off, mask=mask_k, other=0).to(tl.float32)
|
||||
b_v = tl.load(p_mixed + v_off, mask=mask_v, other=0).to(tl.float32)
|
||||
|
||||
if USE_QK_L2NORM_IN_KERNEL:
|
||||
b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
|
||||
b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
|
||||
b_q = b_q * scale
|
||||
|
||||
a_val = tl.load(a + i_n * stride_a_tok + i_hv).to(tl.float32)
|
||||
b_val = tl.load(b + i_n * stride_b_tok + i_hv).to(tl.float32)
|
||||
A_log_val = tl.load(A_log + i_hv).to(tl.float32)
|
||||
dt_bias_val = tl.load(dt_bias + i_hv).to(tl.float32)
|
||||
x = a_val + dt_bias_val
|
||||
softplus_x = tl.where(x <= SOFTPLUS_THRESHOLD, tl.log(1.0 + tl.exp(x)), x)
|
||||
g_val = -tl.exp(A_log_val) * softplus_x
|
||||
beta_val = tl.sigmoid(b_val)
|
||||
|
||||
b_h *= exp(g_val)
|
||||
b_v -= tl.sum(b_h * b_k[None, :], 1)
|
||||
b_v *= beta_val
|
||||
b_h += b_v[:, None] * b_k[None, :]
|
||||
b_o = tl.sum(b_h * b_q[None, :], 1)
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v)
|
||||
|
||||
p_ht = ht + state_idx * stride_final_state_token
|
||||
p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
|
||||
|
||||
|
||||
def fused_recurrent_gated_delta_rule_packed_decode(
|
||||
mixed_qkv: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
ssm_state_indices: torch.Tensor,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if mixed_qkv.ndim != 2:
|
||||
raise ValueError(
|
||||
f"`mixed_qkv` must be a 2D tensor (got ndim={mixed_qkv.ndim})."
|
||||
)
|
||||
if mixed_qkv.stride(-1) != 1:
|
||||
raise ValueError("`mixed_qkv` must be contiguous in the last dim.")
|
||||
if a.ndim != 2 or b.ndim != 2:
|
||||
raise ValueError(
|
||||
f"`a` and `b` must be 2D tensors (got a.ndim={a.ndim}, b.ndim={b.ndim})."
|
||||
)
|
||||
if a.stride(-1) != 1 or b.stride(-1) != 1:
|
||||
raise ValueError("`a`/`b` must be contiguous in the last dim.")
|
||||
if A_log.ndim != 1 or dt_bias.ndim != 1:
|
||||
raise ValueError("`A_log`/`dt_bias` must be 1D tensors.")
|
||||
if A_log.stride(0) != 1 or dt_bias.stride(0) != 1:
|
||||
raise ValueError("`A_log`/`dt_bias` must be contiguous.")
|
||||
if ssm_state_indices.ndim != 1:
|
||||
raise ValueError(
|
||||
f"`ssm_state_indices` must be 1D for packed decode (got ndim={ssm_state_indices.ndim})."
|
||||
)
|
||||
if not out.is_contiguous():
|
||||
raise ValueError("`out` must be contiguous.")
|
||||
|
||||
dev = mixed_qkv.device
|
||||
if (
|
||||
a.device != dev
|
||||
or b.device != dev
|
||||
or A_log.device != dev
|
||||
or dt_bias.device != dev
|
||||
or initial_state.device != dev
|
||||
or out.device != dev
|
||||
or ssm_state_indices.device != dev
|
||||
):
|
||||
raise ValueError("All inputs must be on the same device.")
|
||||
|
||||
B = mixed_qkv.shape[0]
|
||||
if a.shape[0] != B or b.shape[0] != B:
|
||||
raise ValueError(
|
||||
"Mismatched batch sizes: "
|
||||
f"mixed_qkv.shape[0]={B}, a.shape[0]={a.shape[0]}, b.shape[0]={b.shape[0]}."
|
||||
)
|
||||
if ssm_state_indices.shape[0] != B:
|
||||
raise ValueError(
|
||||
f"`ssm_state_indices` must have shape [B] (got {tuple(ssm_state_indices.shape)}; expected ({B},))."
|
||||
)
|
||||
|
||||
if initial_state.ndim != 4:
|
||||
raise ValueError(
|
||||
f"`initial_state` must be a 4D tensor (got ndim={initial_state.ndim})."
|
||||
)
|
||||
if initial_state.stride(-1) != 1:
|
||||
raise ValueError("`initial_state` must be contiguous in the last dim.")
|
||||
HV, V, K = initial_state.shape[-3:]
|
||||
if a.shape[1] != HV or b.shape[1] != HV:
|
||||
raise ValueError(
|
||||
f"`a`/`b` must have shape [B, HV] with HV={HV} (got a.shape={tuple(a.shape)}, b.shape={tuple(b.shape)})."
|
||||
)
|
||||
if A_log.numel() != HV or dt_bias.numel() != HV:
|
||||
raise ValueError(
|
||||
f"`A_log` and `dt_bias` must have {HV} elements (got A_log.numel()={A_log.numel()}, dt_bias.numel()={dt_bias.numel()})."
|
||||
)
|
||||
if out.shape != (B, 1, HV, V):
|
||||
raise ValueError(
|
||||
f"`out` must have shape {(B, 1, HV, V)} (got out.shape={tuple(out.shape)})."
|
||||
)
|
||||
|
||||
qkv_dim = mixed_qkv.shape[1]
|
||||
qk_dim = qkv_dim - HV * V
|
||||
if qk_dim <= 0 or qk_dim % 2 != 0:
|
||||
raise ValueError(
|
||||
f"Invalid packed `mixed_qkv` last dim={qkv_dim} for HV={HV}, V={V}."
|
||||
)
|
||||
q_dim = qk_dim // 2
|
||||
if q_dim % K != 0:
|
||||
raise ValueError(f"Invalid packed Q size {q_dim}: must be divisible by K={K}.")
|
||||
H = q_dim // K
|
||||
if H <= 0 or HV % H != 0:
|
||||
raise ValueError(
|
||||
f"Invalid head config inferred from mixed_qkv: H={H}, HV={HV}."
|
||||
)
|
||||
|
||||
BK = triton.next_power_of_2(K)
|
||||
if triton.cdiv(K, BK) != 1:
|
||||
raise ValueError(
|
||||
f"Packed decode kernel only supports NK=1 (got K={K}, BK={BK})."
|
||||
)
|
||||
BV = min(triton.next_power_of_2(V), 32)
|
||||
num_stages = 3
|
||||
num_warps = 1
|
||||
|
||||
stride_mixed_qkv_tok = mixed_qkv.stride(0)
|
||||
stride_a_tok = a.stride(0)
|
||||
stride_b_tok = b.stride(0)
|
||||
stride_init_state_token = initial_state.stride(0)
|
||||
stride_final_state_token = initial_state.stride(0)
|
||||
stride_indices_seq = ssm_state_indices.stride(0)
|
||||
|
||||
NV = triton.cdiv(V, BV)
|
||||
# CUDA limits grid Y/Z dimensions to 65535.
|
||||
split_batch_head_grid = B * HV > 65535
|
||||
grid = (NV, HV, B) if split_batch_head_grid else (NV, B * HV)
|
||||
fused_recurrent_gated_delta_rule_packed_decode_kernel[grid](
|
||||
mixed_qkv=mixed_qkv,
|
||||
a=a,
|
||||
b=b,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
o=out,
|
||||
h0=initial_state,
|
||||
ht=initial_state,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
scale=scale,
|
||||
stride_mixed_qkv_tok=stride_mixed_qkv_tok,
|
||||
stride_a_tok=stride_a_tok,
|
||||
stride_b_tok=stride_b_tok,
|
||||
stride_init_state_token=stride_init_state_token,
|
||||
stride_final_state_token=stride_final_state_token,
|
||||
stride_indices_seq=stride_indices_seq,
|
||||
H=H,
|
||||
HV=HV,
|
||||
K=K,
|
||||
V=V,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
SOFTPLUS_THRESHOLD=20.0,
|
||||
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
|
||||
SPLIT_BATCH_HEAD_GRID=split_batch_head_grid,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
)
|
||||
return out, initial_state
|
||||
|
||||
|
||||
class FusedRecurrentFunction(torch.autograd.Function):
|
||||
@staticmethod
|
||||
def forward(
|
||||
ctx,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
inplace_final_state: bool = True,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
ssm_state_indices: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
):
|
||||
o, final_state = fused_recurrent_gated_delta_rule_fwd(
|
||||
q=q.contiguous(),
|
||||
k=k.contiguous(),
|
||||
v=v.contiguous(),
|
||||
g=g.contiguous(),
|
||||
beta=beta.contiguous(),
|
||||
scale=scale,
|
||||
initial_state=initial_state,
|
||||
inplace_final_state=inplace_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
|
||||
)
|
||||
|
||||
return o, final_state
|
||||
|
||||
|
||||
def fused_recurrent_gated_delta_rule(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor = None,
|
||||
scale: float = None,
|
||||
initial_state: torch.Tensor = None,
|
||||
inplace_final_state: bool = True,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
ssm_state_indices: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
r"""
|
||||
Args:
|
||||
q (torch.Tensor):
|
||||
queries of shape `[B, T, H, K]`.
|
||||
k (torch.Tensor):
|
||||
keys of shape `[B, T, H, K]`.
|
||||
v (torch.Tensor):
|
||||
values of shape `[B, T, HV, V]`.
|
||||
GVA is applied if `HV > H`.
|
||||
g (torch.Tensor):
|
||||
g (decays) of shape `[B, T, HV]`.
|
||||
beta (torch.Tensor):
|
||||
betas of shape `[B, T, HV]`.
|
||||
scale (Optional[int]):
|
||||
Scale factor for the RetNet attention scores.
|
||||
If not provided, it will default to `1 / sqrt(K)`. Default: `None`.
|
||||
initial_state (Optional[torch.Tensor]):
|
||||
Initial state of shape `[N, HV, V, K]` for `N` input sequences.
|
||||
For equal-length input sequences, `N` equals the batch size `B`.
|
||||
Default: `None`.
|
||||
inplace_final_state: bool:
|
||||
Whether to store the final state in-place to save memory.
|
||||
Default: `True`.
|
||||
cu_seqlens (torch.Tensor):
|
||||
Cumulative sequence lengths of shape `[N+1]` used for variable-length training,
|
||||
consistent with the FlashAttention API.
|
||||
ssm_state_indices (Optional[torch.Tensor]):
|
||||
Indices to map the input sequences to the initial/final states.
|
||||
num_accepted_tokens (Optional[torch.Tensor]):
|
||||
Number of accepted tokens for each sequence during decoding.
|
||||
|
||||
Returns:
|
||||
o (torch.Tensor):
|
||||
Outputs of shape `[B, T, HV, V]`.
|
||||
final_state (torch.Tensor):
|
||||
Final state of shape `[N, HV, V, K]`.
|
||||
|
||||
Examples::
|
||||
>>> import torch
|
||||
>>> import torch.nn.functional as F
|
||||
>>> from einops import rearrange
|
||||
>>> from fla.ops.gated_delta_rule import fused_recurrent_gated_delta_rule
|
||||
# inputs with equal lengths
|
||||
>>> B, T, H, HV, K, V = 4, 2048, 4, 8, 512, 512
|
||||
>>> q = torch.randn(B, T, H, K, device='cuda')
|
||||
>>> k = F.normalize(torch.randn(B, T, H, K, device='cuda'), p=2, dim=-1)
|
||||
>>> v = torch.randn(B, T, HV, V, device='cuda')
|
||||
>>> g = F.logsigmoid(torch.rand(B, T, HV, device='cuda'))
|
||||
>>> beta = torch.rand(B, T, HV, device='cuda').sigmoid()
|
||||
>>> h0 = torch.randn(B, HV, V, K, device='cuda')
|
||||
>>> o, ht = fused_gated_recurrent_delta_rule(
|
||||
q, k, v, g, beta,
|
||||
initial_state=h0,
|
||||
)
|
||||
# for variable-length inputs, the batch size `B` is expected to be 1 and `cu_seqlens` is required
|
||||
>>> q, k, v, g, beta = map(lambda x: rearrange(x, 'b t ... -> 1 (b t) ...'), (q, k, v, g, beta))
|
||||
# for a batch with 4 sequences, `cu_seqlens` with 5 start/end positions are expected
|
||||
>>> cu_seqlens = q.new_tensor([0, 2048, 4096, 6144, 8192], dtype=torch.int32)
|
||||
>>> o_var, ht_var = fused_gated_recurrent_delta_rule(
|
||||
q, k, v, g, beta,
|
||||
initial_state=h0,
|
||||
cu_seqlens=cu_seqlens
|
||||
)
|
||||
"""
|
||||
if cu_seqlens is not None and q.shape[0] != 1:
|
||||
raise ValueError(
|
||||
f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`."
|
||||
f"Please flatten variable-length inputs before processing."
|
||||
)
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
else:
|
||||
assert scale > 0, "scale must be positive"
|
||||
if beta is None:
|
||||
beta = torch.ones_like(q[..., 0])
|
||||
o, final_state = FusedRecurrentFunction.apply(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
g,
|
||||
beta,
|
||||
scale,
|
||||
initial_state,
|
||||
inplace_final_state,
|
||||
cu_seqlens,
|
||||
ssm_state_indices,
|
||||
num_accepted_tokens,
|
||||
use_qk_l2norm_in_kernel,
|
||||
)
|
||||
return o, final_state
|
||||
+1362
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,590 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mla import MLAModules, MultiHeadLatentAttentionWrapper
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope
|
||||
from vllm.model_executor.layers.sparse_attn_indexer_kpool import SparseAttnIndexerKpool
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepSeekV2FusedQkvAProjLinear,
|
||||
DeepseekV32IndexerCache,
|
||||
yarn_get_mscale,
|
||||
)
|
||||
from vllm.model_executor.utils import maybe_disable_graph_partition
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
|
||||
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Shared torch.compile config for the indexer's small-kernel leaves. The MLA
|
||||
# indexer runs under breakable-CG (CompilationMode.NONE), which blocks FX-graph
|
||||
# fusion of the surrounding eager ops; carving each cluster into its own
|
||||
# @torch.compile leaf (backend==inductor) still fuses them. Matches the
|
||||
# grouped_topk / _cast_sigmoid leaf pattern.
|
||||
_INDEXER_COMPILE = dict(
|
||||
dynamic=True,
|
||||
backend=current_platform.simple_compile_backend,
|
||||
options=maybe_disable_graph_partition(current_platform.simple_compile_backend),
|
||||
)
|
||||
|
||||
|
||||
@torch.compile(**_INDEXER_COMPILE)
|
||||
def _fused_indexer_k_norm(
|
||||
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor, dim: int, eps: float
|
||||
) -> torch.Tensor:
|
||||
# Fuse fp32 cast + layer_norm + cast-back (was 3 kernels) into one.
|
||||
return F.layer_norm(x.float(), (dim,), weight, bias, eps).type_as(x)
|
||||
|
||||
|
||||
@torch.compile(**_INDEXER_COMPILE)
|
||||
def _fused_indexer_weight_scale(
|
||||
weights: torch.Tensor, q_scale: torch.Tensor, scale: float
|
||||
) -> torch.Tensor:
|
||||
# Fuse the weight-scaling muls (was 2 kernels) into one. `scale` folds
|
||||
# softmax_scale (head_dim**-0.5) and n_head**-0.5 into a single constant.
|
||||
return (weights.unsqueeze(-1) * q_scale * scale).squeeze(-1)
|
||||
|
||||
|
||||
@torch.compile(**_INDEXER_COMPILE)
|
||||
def _pad_indexer_heads(x: torch.Tensor, pad: int) -> torch.Tensor:
|
||||
# DeepGEMM MQA-logits needs num_heads in {32,64}; zero-pad the head dim.
|
||||
# Fuse new_zeros + cat (was 2 kernels) into one. Pad values are zero (exact
|
||||
# in fp8 e4m3 and zero-weight in the logits sum), so numerically a no-op.
|
||||
return torch.cat([x, x.new_zeros(x.shape[0], pad, *x.shape[2:])], dim=1)
|
||||
|
||||
|
||||
class Glm5NextIndexerCache(DeepseekV32IndexerCache):
|
||||
"""Indexer K cache that stores kpool-compressed entries.
|
||||
|
||||
Setting ``tokens_per_state = index_kpool`` on the KV cache spec makes vLLM's
|
||||
indexer metadata builder emit pool-granular ``slot_mapping`` /
|
||||
``seq_lens`` / ``cu_seq_lens`` / ``page_table`` for free, and shrinks the
|
||||
cache allocation store one state per ``index_kpool`` tokens. The pool
|
||||
*content* (softmax-weighted sum vs keep-every-Nth) is computed by the
|
||||
kpool compress kernel inside the indexer op — the cache only provides the
|
||||
addressing, which is identical for both schemes.
|
||||
|
||||
The indexer shares one block with the co-located MLA (a single
|
||||
``MLAAttentionSpec`` / block_table), so ``block_size`` is the model-wide
|
||||
``cache_config.block_size``. DeepGEMM's paged-MQA kernel
|
||||
(``csrc/apis/attention.hpp``) requires ``block_kv`` to be exactly 32 or
|
||||
64, so the storage block is virtually split into pool pages of the
|
||||
largest such size that tiles it (``storage_kernel_block_size``); this
|
||||
needs ``block_size`` to be a multiple of ``index_kpool * 32`` (512 for
|
||||
``index_kpool = 16``). A smaller block (e.g. the default 64) silently
|
||||
collapses ``storage_block_size`` (64 // 16 = 4) and only fails later at
|
||||
the opaque C++ assert; ``get_kv_cache_spec`` guards this up front
|
||||
instead.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
prefix: str,
|
||||
cache_config,
|
||||
index_kpool: int,
|
||||
):
|
||||
super().__init__(
|
||||
head_dim=head_dim, dtype=dtype, prefix=prefix, cache_config=cache_config
|
||||
)
|
||||
assert index_kpool > 1, "Glm5NextIndexerCache expects index_kpool > 1"
|
||||
# Keep chunked-prefill boundaries aligned to complete pools.
|
||||
assert cache_config.block_size % index_kpool == 0, (
|
||||
"Glm5NextIndexerCache: cache_config.block_size "
|
||||
f"({cache_config.block_size}) must be a multiple of index_kpool "
|
||||
f"({index_kpool}) so chunked-prefill boundaries stay pool-aligned."
|
||||
)
|
||||
self._index_kpool = index_kpool
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig):
|
||||
from dataclasses import replace
|
||||
|
||||
spec = super().get_kv_cache_spec(vllm_config)
|
||||
# ``tokens_per_state`` is the KV-spec representation of kpool
|
||||
# compression in the current cache-layout API.
|
||||
assert isinstance(spec, MLAAttentionSpec)
|
||||
spec = replace(spec, tokens_per_state=self._index_kpool)
|
||||
|
||||
# DeepGEMM paged-MQA takes block_kv in {32, 64}; the storage block
|
||||
# (= block_size // index_kpool) is virtually split into pool pages of
|
||||
# the largest such size that tiles it, so it must be a multiple of 32.
|
||||
storage_block_size = spec.block_size // self._index_kpool
|
||||
assert (
|
||||
spec.block_size % self._index_kpool == 0 and storage_block_size % 32 == 0
|
||||
), (
|
||||
"Glm5NextIndexerCache: kpool indexer requires cache block_size to "
|
||||
f"be a multiple of index_kpool * 32 ({self._index_kpool * 32}) so "
|
||||
"that DeepGEMM paged-MQA pool pages (32 or 64 entries) tile the "
|
||||
f"storage block, got block_size={spec.block_size} -> "
|
||||
f"storage_block_size={storage_block_size}."
|
||||
)
|
||||
max_page_size = max(PAGED_MQA_PAGE_SIZES)
|
||||
min_page_size = min(PAGED_MQA_PAGE_SIZES)
|
||||
if storage_block_size <= max_page_size:
|
||||
page_size = storage_block_size
|
||||
elif storage_block_size % max_page_size == 0:
|
||||
page_size = max_page_size
|
||||
else:
|
||||
page_size = min_page_size
|
||||
return replace(
|
||||
spec,
|
||||
storage_block_size=page_size * self._index_kpool,
|
||||
)
|
||||
|
||||
|
||||
class Glm5NextTailCache(DeepseekV32IndexerCache):
|
||||
"""Paged circular buffer for the kpool indexer's in-progress (tail) pool.
|
||||
|
||||
Holds the trailing incomplete pool's raw K + gate score: one block of
|
||||
``index_kpool`` slots per request, overwritten in place by ``pos % kpool``
|
||||
as decode/spec-decode advances. Prefill seeds it (instead of discarding the
|
||||
tail raw K+gate); the connector transfers it across PD; decode reads it to
|
||||
compress the boundary pool correctly. ``KpoolTailSpec`` /
|
||||
``KpoolTailManager`` provide the no-prune, 1-block/req allocation that lets
|
||||
the in-progress pool survive across steps and across transfer.
|
||||
|
||||
Stores raw bf16 K (``head_dim``) as the "K" half of each block and the
|
||||
bf16 gate score (``head_dim``) as the "V" half -- not the fp8-compressed
|
||||
entry, which lives in ``Glm5NextIndexerCache``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
prefix: str,
|
||||
cache_config,
|
||||
index_kpool: int,
|
||||
):
|
||||
super().__init__(
|
||||
head_dim=head_dim, dtype=dtype, prefix=prefix, cache_config=cache_config
|
||||
)
|
||||
assert index_kpool > 1, "Glm5NextTailCache expects index_kpool > 1"
|
||||
self._index_kpool = index_kpool
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig):
|
||||
# The two head slots form [K, gate score] in the generic
|
||||
# [block, head, state, content] cache view.
|
||||
return KpoolTailSpec(
|
||||
block_size=self._index_kpool,
|
||||
num_kv_heads=2,
|
||||
head_size=self.head_dim,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=self._index_kpool,
|
||||
)
|
||||
|
||||
def get_attn_backend(self):
|
||||
from vllm.v1.attention.backends.mla.indexer import KpoolTailBackend
|
||||
|
||||
return KpoolTailBackend
|
||||
|
||||
|
||||
class Indexer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: Glm5NextConfig,
|
||||
hidden_size: int,
|
||||
q_lora_rank: int,
|
||||
quant_config: QuantizationConfig | None,
|
||||
cache_config: CacheConfig | None,
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
# self.indexer_cfg = config.attn_module_list_cfg[0]["attn_index"]
|
||||
# Indexer is only constructed for v32 configs, where these sparse-indexer
|
||||
# fields are guaranteed populated; narrow away the `int | None` declared
|
||||
# on Glm5NextConfig for the optional-indexer case.
|
||||
assert config.index_topk is not None
|
||||
assert config.index_n_heads is not None
|
||||
assert config.index_head_dim is not None
|
||||
assert config.index_kpool is not None
|
||||
self.topk_tokens = config.index_topk
|
||||
self.n_head = config.index_n_heads # 64
|
||||
self.head_dim = config.index_head_dim # 128
|
||||
self.rope_dim = config.qk_rope_head_dim # 64
|
||||
self.index_kpool = config.index_kpool
|
||||
self.q_lora_rank = q_lora_rank # 1536
|
||||
|
||||
# kpool
|
||||
self.index_kpool_compress_ape = nn.Parameter(
|
||||
torch.zeros(self.index_kpool, self.head_dim, dtype=torch.float32)
|
||||
)
|
||||
# Keep the checkpoint name ``index_kpool_compress_gate`` without a
|
||||
# ``.weight`` suffix. F.linear consumes its [head_dim, hidden_size] shape.
|
||||
self.index_kpool_compress_gate = nn.Parameter(
|
||||
torch.empty(self.head_dim, hidden_size, dtype=torch.bfloat16)
|
||||
)
|
||||
|
||||
# no tensor parallel, just replicated
|
||||
self.wq_b = ReplicatedLinear(
|
||||
self.q_lora_rank,
|
||||
self.head_dim * self.n_head,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.wq_b",
|
||||
)
|
||||
# Fused wk + weights_proj: single GEMM producing [head_dim + n_head].
|
||||
# FP8 wk weights are upcasted to BF16 during loading to maintain fusion.
|
||||
self.wk_weights_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[self.head_dim, self.n_head],
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
disable_tp=True,
|
||||
prefix=f"{prefix}.wk_weights_proj",
|
||||
)
|
||||
self.k_norm = LayerNorm(self.head_dim, eps=1e-6)
|
||||
self.softmax_scale = self.head_dim**-0.5
|
||||
|
||||
# Hadamard-128 rotation of the indexer query is fused with the FP8
|
||||
# quant (see forward: fwht128_quant_fp8) -- no precomputed matrix.
|
||||
|
||||
self.scale_fmt = "ue8m0"
|
||||
self.quant_block_size = 128 # TODO: get from config
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
self._wp_fp32: torch.Tensor | None = None
|
||||
|
||||
# NOTE: (zyongye) we use fp8 naive cache,
|
||||
# where we store value in fp8 and scale in fp32
|
||||
# per self.quant_block_size element
|
||||
self.k_cache = Glm5NextIndexerCache(
|
||||
head_dim=self.head_dim + self.head_dim // self.quant_block_size * 4,
|
||||
dtype=torch.uint8,
|
||||
prefix=f"{prefix}.k_cache",
|
||||
cache_config=cache_config,
|
||||
index_kpool=self.index_kpool,
|
||||
)
|
||||
# Paged tail cache (in-progress pool's raw K + gate score). Written by
|
||||
# prefill (seeds the boundary pool) and decode (per-step stash); read by
|
||||
# the decode kernel to compress the boundary pool. Transferred across PD
|
||||
# so the decode side sees the prefill tail. See KpoolTailSpec/Manager.
|
||||
self.tail_cache = Glm5NextTailCache(
|
||||
head_dim=self.head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
prefix=f"{prefix}.tail_cache",
|
||||
cache_config=cache_config,
|
||||
index_kpool=self.index_kpool,
|
||||
)
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
self.prefix = prefix
|
||||
from vllm.v1.attention.backends.mla.indexer import get_max_prefill_buffer_size
|
||||
|
||||
self.max_total_seq_len = get_max_prefill_buffer_size(vllm_config)
|
||||
self.indexer_op = SparseAttnIndexerKpool(
|
||||
self.k_cache,
|
||||
self.quant_block_size,
|
||||
self.scale_fmt,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
tail_cache=self.tail_cache,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, qr: torch.Tensor, positions, rotary_emb
|
||||
) -> torch.Tensor:
|
||||
q, _ = self.wq_b(qr)
|
||||
q = q.view(-1, self.n_head, self.head_dim)
|
||||
|
||||
# Compute the head gate in fp32; bf16 error can change near-tie pool
|
||||
# rankings on long-context tasks. Cache it after weights are loaded.
|
||||
kw, _ = self.wk_weights_proj(hidden_states)
|
||||
k = kw[:, : self.head_dim]
|
||||
if self._wp_fp32 is None:
|
||||
self._wp_fp32 = (
|
||||
self.wk_weights_proj.weight.data[self.head_dim :, :]
|
||||
.t()
|
||||
.contiguous()
|
||||
.float()
|
||||
)
|
||||
weights = torch.mm(hidden_states.float(), self._wp_fp32)
|
||||
|
||||
k = _fused_indexer_k_norm(
|
||||
k, self.k_norm.weight, self.k_norm.bias, self.head_dim, self.k_norm.eps
|
||||
)
|
||||
|
||||
if self.rope_dim > 0:
|
||||
q_pe, q_nope = torch.split(
|
||||
q, [self.rope_dim, self.head_dim - self.rope_dim], dim=-1
|
||||
)
|
||||
k_pe, k_nope = torch.split(
|
||||
k, [self.rope_dim, self.head_dim - self.rope_dim], dim=-1
|
||||
)
|
||||
|
||||
q_pe, k_pe = rotary_emb(positions, q_pe, k_pe.unsqueeze(1))
|
||||
# Note: RoPE (NeoX) can introduce extra leading dimensions during
|
||||
# compilation so we need to reshape back to token-flattened shapes
|
||||
q_pe = q_pe.reshape(-1, self.n_head, self.rope_dim)
|
||||
k_pe = k_pe.reshape(-1, 1, self.rope_dim)
|
||||
|
||||
# `rotary_emb` is shape-preserving; `q_pe` is already
|
||||
# [num_tokens, n_head, rope_dim].
|
||||
q = torch.cat([q_pe, q_nope], dim=-1)
|
||||
# `k_pe` is [num_tokens, 1, rope_dim] (MQA).
|
||||
k = torch.cat([k_pe.squeeze(-2), k_nope], dim=-1)
|
||||
# else: qk_rope_head_dim=0 — no rope component. q is already
|
||||
# [num_tokens, n_head, head_dim] and k is [num_tokens, head_dim] (all
|
||||
# nope), so skip the rope split / rotary / cat entirely; otherwise the
|
||||
# split/reshape would build 0-element tensors (breaks dynamo tracing).
|
||||
|
||||
# Rotate Q into the cached K basis before computing fp8 MQA logits.
|
||||
# Fusing the fp32 FWHT and quantization avoids an intermediate HBM
|
||||
# round-trip and bf16 matrix-rounding bias.
|
||||
assert self.head_dim == 128 and self.quant_block_size == 128
|
||||
assert self.scale_fmt == "ue8m0"
|
||||
q = q.view(-1, self.head_dim)
|
||||
q_fp8, q_scale = fwht128_quant_fp8(q)
|
||||
q_fp8 = q_fp8.view(-1, self.n_head, self.head_dim)
|
||||
q_scale = q_scale.view(-1, self.n_head, 1)
|
||||
|
||||
weights = _fused_indexer_weight_scale(
|
||||
weights, q_scale, self.softmax_scale * self.n_head**-0.5
|
||||
)
|
||||
|
||||
# kpool: per-token gate score driving the softmax-weighted pool. Computed
|
||||
# from the same hidden_states that produced `k`, so it stays token-aligned.
|
||||
# F.linear(x, gate) = x @ gate.T with gate [head_dim, hidden_size].
|
||||
gate_score = F.linear(hidden_states, self.index_kpool_compress_gate)
|
||||
|
||||
# DeepGEMM's MQA-logits kernels (fp8_mqa_logits /
|
||||
# fp8_fp4_paged_mqa_logits) require num_heads in {32, 64}; this
|
||||
# checkpoint uses index_n_heads=16. Zero-pad q and the per-head
|
||||
# weights: logits are a weights-weighted sum over heads, so
|
||||
# zero-weight padded heads contribute exactly nothing.
|
||||
if self.n_head < 32:
|
||||
pad = 32 - self.n_head
|
||||
q_fp8 = _pad_indexer_heads(q_fp8, pad)
|
||||
weights = _pad_indexer_heads(weights, pad)
|
||||
|
||||
return self.indexer_op(
|
||||
hidden_states,
|
||||
q_fp8,
|
||||
k,
|
||||
weights,
|
||||
gate_score=gate_score,
|
||||
compress_ape=self.index_kpool_compress_ape,
|
||||
index_kpool=self.index_kpool,
|
||||
positions=positions,
|
||||
)
|
||||
|
||||
|
||||
class Glm5NextMLAAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: Glm5NextConfig,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
q_lora_rank: int | None,
|
||||
kv_lora_rank: int,
|
||||
max_position_embeddings: int = 8192,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
input_size: int | None = None,
|
||||
skip_rope: bool | None = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
|
||||
self.num_heads = num_heads
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
assert num_heads % tp_size == 0
|
||||
self.num_local_heads = num_heads // tp_size
|
||||
|
||||
self.scaling = self.qk_head_dim**-0.5
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
|
||||
# Use input_size for projection input dimensions if provided,
|
||||
# otherwise default to hidden_size (used in Eagle3 Deepseek with MLA)
|
||||
proj_input_size = input_size if input_size is not None else self.hidden_size
|
||||
|
||||
if self.q_lora_rank is not None:
|
||||
self.fused_qkv_a_proj = DeepSeekV2FusedQkvAProjLinear(
|
||||
proj_input_size,
|
||||
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.fused_qkv_a_proj",
|
||||
)
|
||||
else:
|
||||
self.kv_a_proj_with_mqa = ReplicatedLinear(
|
||||
proj_input_size,
|
||||
self.kv_lora_rank + self.qk_rope_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_a_proj_with_mqa",
|
||||
)
|
||||
|
||||
if self.q_lora_rank is not None:
|
||||
self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps)
|
||||
self.q_b_proj = ColumnParallelLinear(
|
||||
self.q_lora_rank,
|
||||
self.num_heads * self.qk_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.q_b_proj",
|
||||
)
|
||||
else:
|
||||
self.q_proj = ColumnParallelLinear(
|
||||
proj_input_size,
|
||||
self.num_heads * self.qk_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.q_proj",
|
||||
)
|
||||
self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=config.rms_norm_eps)
|
||||
self.kv_b_proj = ColumnParallelLinear(
|
||||
self.kv_lora_rank,
|
||||
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_b_proj",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.num_heads * self.v_head_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
if not skip_rope:
|
||||
assert config.rope_parameters is not None
|
||||
if config.rope_parameters["rope_type"] != "default":
|
||||
config.rope_parameters["rope_type"] = (
|
||||
"deepseek_yarn"
|
||||
if config.rope_parameters.get("apply_yarn_scaling", True)
|
||||
else "deepseek_llama_scaling"
|
||||
)
|
||||
|
||||
self.rotary_emb: RotaryEmbedding | None = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
is_neox_style=False,
|
||||
)
|
||||
|
||||
if (
|
||||
config.rope_parameters["rope_type"] != "default"
|
||||
and config.rope_parameters["rope_type"] == "deepseek_yarn"
|
||||
):
|
||||
mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
|
||||
scaling_factor = config.rope_parameters["factor"]
|
||||
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
|
||||
self.scaling = self.scaling * mscale * mscale
|
||||
else:
|
||||
self.rotary_emb = None
|
||||
|
||||
self.is_v32 = config.index_topk is not None
|
||||
|
||||
if self.is_v32:
|
||||
self.indexer_rope_emb: RotaryEmbedding | None = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
is_neox_style=not config.indexer_rope_interleave,
|
||||
)
|
||||
# The sparse indexer projects from the MLA q-lora rank, which is
|
||||
# always set for v32 MLA configs; narrow away the `int | None`.
|
||||
assert q_lora_rank is not None
|
||||
self.indexer: Indexer | None = Indexer(
|
||||
vllm_config,
|
||||
config,
|
||||
hidden_size,
|
||||
q_lora_rank,
|
||||
quant_config,
|
||||
cache_config,
|
||||
topk_indices_buffer,
|
||||
f"{prefix}.indexer",
|
||||
)
|
||||
|
||||
else:
|
||||
self.indexer_rope_emb = None
|
||||
self.indexer = None
|
||||
|
||||
mla_modules = MLAModules(
|
||||
kv_a_layernorm=self.kv_a_layernorm,
|
||||
kv_b_proj=self.kv_b_proj,
|
||||
rotary_emb=self.rotary_emb,
|
||||
o_proj=self.o_proj,
|
||||
fused_qkv_a_proj=self.fused_qkv_a_proj
|
||||
if self.q_lora_rank is not None
|
||||
else None,
|
||||
kv_a_proj_with_mqa=self.kv_a_proj_with_mqa
|
||||
if self.q_lora_rank is None
|
||||
else None,
|
||||
q_a_layernorm=self.q_a_layernorm if self.q_lora_rank is not None else None,
|
||||
q_b_proj=self.q_b_proj if self.q_lora_rank is not None else None,
|
||||
q_proj=self.q_proj if self.q_lora_rank is None else None,
|
||||
indexer=self.indexer,
|
||||
indexer_rotary_emb=self.indexer_rope_emb,
|
||||
is_sparse=self.is_v32,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
)
|
||||
|
||||
self.mla_attn = MultiHeadLatentAttentionWrapper(
|
||||
self.hidden_size,
|
||||
self.num_local_heads,
|
||||
self.scaling,
|
||||
self.qk_nope_head_dim,
|
||||
self.qk_rope_head_dim,
|
||||
self.v_head_dim,
|
||||
self.q_lora_rank,
|
||||
self.kv_lora_rank,
|
||||
mla_modules,
|
||||
cache_config,
|
||||
quant_config,
|
||||
prefix,
|
||||
skip_topk=False,
|
||||
fuse_qkv_rmsnorm=True,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, positions: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
# The wrapper also runs the sparse indexer before MLA attention.
|
||||
return self.mla_attn(positions, hidden_states)
|
||||
@@ -0,0 +1,611 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-5.3-Flash KDA layer with separate convolutions and a bounded safe gate."""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import divide
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
gather_initial_states,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.scatter_states import scatter_states
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import (
|
||||
maybe_disable_graph_partition,
|
||||
set_weight_attrs,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.glm5next.amd.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
)
|
||||
else:
|
||||
from vllm.models.glm5next.nvidia.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
)
|
||||
|
||||
|
||||
class _Glm5NextMergedColumnParallelLinear(MergedColumnParallelLinear):
|
||||
"""Merged projection with multiple replicated output shards.
|
||||
|
||||
Extends K3's ``_KimiGDNMergedColumnParallelLinear`` to support two
|
||||
replicated shards (f_a, g_a) instead of one. Pre-multiplies each
|
||||
replicated entry's output_size by tp_size so the per-rank shard
|
||||
divides back to the full size, and forces tp_rank=0 during weight
|
||||
loading for replicated shards.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_size: int,
|
||||
output_sizes: list[int],
|
||||
replicated_shard_ids: tuple[int, ...],
|
||||
tp_size: int,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.replicated_shard_ids = set(replicated_shard_ids)
|
||||
output_sizes = output_sizes.copy()
|
||||
for sid in self.replicated_shard_ids:
|
||||
output_sizes[sid] *= tp_size
|
||||
super().__init__(input_size, output_sizes, **kwargs)
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id in self.replicated_shard_ids:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
try:
|
||||
super().weight_loader(param, loaded_weight, loaded_shard_id)
|
||||
finally:
|
||||
self.tp_rank = tp_rank
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = param_tp_rank
|
||||
|
||||
def weight_loader_v2(
|
||||
self,
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id in self.replicated_shard_ids:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
try:
|
||||
super().weight_loader_v2(param, loaded_weight, loaded_shard_id)
|
||||
finally:
|
||||
self.tp_rank = tp_rank
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = param_tp_rank
|
||||
|
||||
|
||||
@torch.compile(
|
||||
dynamic=True,
|
||||
backend=current_platform.simple_compile_backend,
|
||||
options=maybe_disable_graph_partition(current_platform.simple_compile_backend),
|
||||
)
|
||||
def _cast_sigmoid(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Fuse the fp32 cast + sigmoid into one Inductor kernel."""
|
||||
return x.float().sigmoid()
|
||||
|
||||
|
||||
class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
head_dim: int
|
||||
num_heads: int
|
||||
conv_size: int
|
||||
|
||||
def get_state_dtype(
|
||||
self,
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
if self.model_config is None or self.cache_config is None:
|
||||
raise ValueError("model_config and cache_config must be set")
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
self.model_config.dtype, self.cache_config.mamba_cache_dtype
|
||||
)
|
||||
|
||||
def get_state_shape(
|
||||
self,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
# conv_state width must include num_spec so the spec-decode conv update
|
||||
# (causal_conv1d_update with num_accepted_tokens + max_query_len) can
|
||||
# slide the window across the draft-verify tokens without reading past
|
||||
# the allocated width. Matches qwen_gdn_linear_attn.get_state_shape.
|
||||
return MambaStateShapeCalculator.kda_state_shape(
|
||||
self.tp_size,
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
conv_kernel_size=self.conv_size,
|
||||
num_spec=self.num_spec,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Glm5NextConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
# KDA projections remain BF16 because fp8 checkpoints omit their scales.
|
||||
saved_quant_config = vllm_config.quant_config
|
||||
try:
|
||||
vllm_config.quant_config = None
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
finally:
|
||||
vllm_config.quant_config = saved_quant_config
|
||||
|
||||
self.head_dim = config.linear_head_dim
|
||||
self.num_heads = config.linear_num_heads
|
||||
self.conv_size = config.linear_conv_kernel_dim
|
||||
assert self.num_heads % self.tp_size == 0
|
||||
self.local_num_heads = divide(self.num_heads, self.tp_size)
|
||||
|
||||
projection_size = self.head_dim * self.num_heads
|
||||
self.local_projection_size = divide(projection_size, self.tp_size)
|
||||
|
||||
# Merge q, k, v, b, f_a, g_a projections into one GEMM (6→1 launches).
|
||||
# Order matches checkpoint's fused_qkvbfg_a_proj convention.
|
||||
# Shards 4 (f_a) and 5 (g_a) are replicated across TP ranks.
|
||||
self.in_proj_qkvbfg_a = _Glm5NextMergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
[
|
||||
projection_size, # q (shard 0)
|
||||
projection_size, # k (shard 1)
|
||||
projection_size, # v (shard 2)
|
||||
self.num_heads, # b (shard 3)
|
||||
self.head_dim, # f_a (shard 4, replicated)
|
||||
self.head_dim, # g_a (shard 5, replicated)
|
||||
],
|
||||
replicated_shard_ids=(4, 5),
|
||||
tp_size=self.tp_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.in_proj_qkvbfg_a",
|
||||
)
|
||||
|
||||
self.f_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.f_b_proj",
|
||||
)
|
||||
self.dt_bias = nn.Parameter(
|
||||
torch.empty(divide(projection_size, self.tp_size), dtype=torch.float32)
|
||||
)
|
||||
|
||||
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||
|
||||
self.q_conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.q_conv1d",
|
||||
)
|
||||
self.k_conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.k_conv1d",
|
||||
)
|
||||
self.v_conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.v_conv1d",
|
||||
)
|
||||
# unsqueeze to fit conv1d weights shape into the linear weights shape.
|
||||
# Can't do this in `weight_loader` since it already exists in
|
||||
# `ColumnParallelLinear` and `set_weight_attrs`
|
||||
# doesn't allow to override it
|
||||
self.q_conv1d.weight.data = self.q_conv1d.weight.data.unsqueeze(1)
|
||||
self.k_conv1d.weight.data = self.k_conv1d.weight.data.unsqueeze(1)
|
||||
self.v_conv1d.weight.data = self.v_conv1d.weight.data.unsqueeze(1)
|
||||
# Lazily-built merged q|k|v conv weight (built on first forward, after
|
||||
# weights are loaded). See _forward.
|
||||
self._merged_conv_weight: torch.Tensor | None = None
|
||||
|
||||
self.A_log = nn.Parameter(
|
||||
torch.empty(1, 1, self.local_num_heads, 1, dtype=torch.float32)
|
||||
)
|
||||
set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(2)})
|
||||
|
||||
self.g_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_b_proj",
|
||||
)
|
||||
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
|
||||
self.o_proj = RowParallelLinear(
|
||||
projection_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
# Checkpoints store A_log as 1-D; the model parameter is 4-D.
|
||||
def _a_log_weight_loader(param, loaded_weight):
|
||||
if loaded_weight.dim() == 1:
|
||||
loaded_weight = loaded_weight.view([1, 1, -1, 1])
|
||||
return sharded_weight_loader(2)(param, loaded_weight)
|
||||
|
||||
self.A_log.weight_loader = _a_log_weight_loader
|
||||
|
||||
# GLM-5.3-Flash uses a bounded sigmoid gate instead of the default
|
||||
# unbounded softplus gate.
|
||||
self.kda_safe_gate = True
|
||||
self.kda_lower_bound = config.linear_lower_bound
|
||||
# Process-global conv-state layout, resolved once here instead of on
|
||||
# every _forward call (it reads an env-derived flag each time).
|
||||
self._conv_state_dim_first = is_conv_state_dim_first()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = hidden_states.size(0)
|
||||
# One merged GEMM for q, k, v, b, f_a, g_a (replaces 6 separate GEMMs).
|
||||
projected = self.in_proj_qkvbfg_a(hidden_states)[0]
|
||||
qkv, beta_raw, f_a, g_a = projected.split(
|
||||
[
|
||||
3 * self.local_projection_size,
|
||||
self.local_num_heads,
|
||||
self.head_dim,
|
||||
self.head_dim,
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
# Beta stays raw (bf16) here: the recurrent kernel sigmoids it in fp32
|
||||
# at load (SIGMOID_BETA), and only the chunked prefill path needs the
|
||||
# pre-computed fp32 sigmoid — computed lazily in _forward. Pure decode
|
||||
# / spec-verify steps then skip the _cast_sigmoid kernel and its fp32
|
||||
# intermediate entirely.
|
||||
beta = beta_raw.unsqueeze(0)
|
||||
g1 = self.f_b_proj(f_a)[0]
|
||||
g1 = g1.reshape(1, -1, self.local_num_heads, self.head_dim)
|
||||
|
||||
g_proj_states = self.g_b_proj(g_a)[0]
|
||||
# Must stay 3D: rms_norm_gated reads H from g.shape[-2].
|
||||
g2 = g_proj_states.reshape(-1, self.local_num_heads, self.head_dim)
|
||||
|
||||
core_attn_out = torch.empty(
|
||||
(1, num_tokens, self.local_num_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
# Call the decorated eager break directly so host-side prefill branches
|
||||
# are not captured by PIECEWISE CUDA graphs.
|
||||
self._forward(
|
||||
qkv_proj_states=qkv,
|
||||
g1=g1,
|
||||
beta=beta,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
core_attn_out = self.o_norm(core_attn_out, g2)
|
||||
core_attn_out = core_attn_out.reshape(core_attn_out.size(1), -1)
|
||||
return self.o_proj(core_attn_out)[0]
|
||||
|
||||
@eager_break_during_capture
|
||||
def _forward(
|
||||
self,
|
||||
qkv_proj_states: torch.Tensor,
|
||||
g1: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata_raw is None:
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata_narrowed is None:
|
||||
# Profile/warmup dummy runs may omit mamba-family metadata.
|
||||
return
|
||||
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
|
||||
has_initial_state = attn_metadata_narrowed.has_initial_state
|
||||
non_spec_query_start_loc = attn_metadata_narrowed.non_spec_query_start_loc
|
||||
non_spec_state_indices_tensor = (
|
||||
attn_metadata_narrowed.non_spec_state_indices_tensor
|
||||
) # noqa: E501
|
||||
num_actual_tokens = attn_metadata_narrowed.num_actual_tokens
|
||||
# Spec-decode metadata (all None when speculative decoding is disabled).
|
||||
spec_sequence_masks = attn_metadata_narrowed.spec_sequence_masks
|
||||
spec_query_start_loc = attn_metadata_narrowed.spec_query_start_loc
|
||||
spec_state_indices_tensor = attn_metadata_narrowed.spec_state_indices_tensor
|
||||
spec_token_indx = attn_metadata_narrowed.spec_token_indx
|
||||
non_spec_token_indx = attn_metadata_narrowed.non_spec_token_indx
|
||||
num_accepted_tokens = attn_metadata_narrowed.num_accepted_tokens
|
||||
num_spec_decodes = attn_metadata_narrowed.num_spec_decodes
|
||||
use_spec = spec_sequence_masks is not None and num_spec_decodes > 0
|
||||
# Safe-gate checkpoints use the bounded sigmoid variant.
|
||||
safe_gate = self.kda_safe_gate
|
||||
lower_bound = self.kda_lower_bound
|
||||
constant_caches = self.kv_cache
|
||||
|
||||
qkv_proj_states = qkv_proj_states[:num_actual_tokens]
|
||||
g1 = g1[:, :num_actual_tokens]
|
||||
beta = beta[:, :num_actual_tokens]
|
||||
|
||||
(conv_state, recurrent_state) = constant_caches
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
# DS layout stores it that way directly; SD layout needs a transpose.
|
||||
# Layout is process-global and resolved once at init (see __init__).
|
||||
if not self._conv_state_dim_first:
|
||||
conv_state = conv_state.transpose(-1, -2)
|
||||
|
||||
# One merged short-conv over q|k|v instead of three separate calls. The
|
||||
# 1D conv is independent per channel, so concatenating q/k/v along the
|
||||
# channel dim and running a single causal_conv1d is bit-identical to
|
||||
# three calls. The merged weight is q|k|v conv weights concatenated;
|
||||
# built once and cached (params are fixed after load). conv_state is
|
||||
# already stored as the merged q|k|v state, so it is used directly.
|
||||
if self._merged_conv_weight is None:
|
||||
|
||||
def _w(m):
|
||||
return m.weight.view(m.weight.size(0), m.weight.size(2))
|
||||
|
||||
self._merged_conv_weight = torch.cat(
|
||||
[_w(self.q_conv1d), _w(self.k_conv1d), _w(self.v_conv1d)],
|
||||
dim=0,
|
||||
).contiguous()
|
||||
conv_weights = self._merged_conv_weight
|
||||
conv_bias = self.q_conv1d.bias
|
||||
|
||||
# Split projections / gating into spec (draft-verify) and non-spec token
|
||||
# groups when speculative decoding is active. Spec tokens carry
|
||||
# num_spec+1 recurrent-state columns each and are advanced with
|
||||
# num_accepted_tokens for rejection-sampling rollback; non-spec tokens
|
||||
# are one-per-request. Mirrors olmo_gdn_linear_attn.py. Projections are
|
||||
# [n, *] (token dim 0); g1/beta are [1, n, h, d] (token dim 1).
|
||||
if use_spec:
|
||||
# In a pure spec-verify step (no non-spec tokens) the metadata
|
||||
# builder sets spec_token_indx = arange(num_actual_tokens), making
|
||||
# the index_select calls below identity copies. Skip them on this
|
||||
# steady-state decode hot path. The outputs alias the inputs here;
|
||||
# the downstream conv/recurrent kernels read them without mutating
|
||||
# in place, so the aliasing is safe.
|
||||
if non_spec_token_indx is None or non_spec_token_indx.numel() == 0:
|
||||
qkv_spec = qkv_proj_states
|
||||
g1_spec = g1
|
||||
beta_spec = beta
|
||||
else:
|
||||
qkv_spec = qkv_proj_states.index_select(0, spec_token_indx)
|
||||
g1_spec = g1.index_select(1, spec_token_indx)
|
||||
beta_spec = beta.index_select(1, spec_token_indx)
|
||||
if non_spec_token_indx is not None and non_spec_token_indx.numel() > 0:
|
||||
qkv_ns = qkv_proj_states.index_select(0, non_spec_token_indx)
|
||||
g1_ns = g1.index_select(1, non_spec_token_indx)
|
||||
beta_ns = beta.index_select(1, non_spec_token_indx)
|
||||
else:
|
||||
qkv_ns = g1_ns = beta_ns = None
|
||||
else:
|
||||
qkv_spec = g1_spec = beta_spec = None
|
||||
qkv_ns, g1_ns, beta_ns = qkv_proj_states, g1, beta
|
||||
|
||||
# --- causal conv1d: spec (draft-verify) path ---
|
||||
if use_spec:
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert num_accepted_tokens is not None
|
||||
conv_idx = spec_state_indices_tensor[:, 0][:num_spec_decodes]
|
||||
conv_mql = spec_state_indices_tensor.size(-1)
|
||||
qkv_spec = causal_conv1d_update(
|
||||
qkv_spec,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
conv_bias,
|
||||
activation="silu",
|
||||
conv_state_indices=conv_idx,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
max_query_len=conv_mql,
|
||||
)
|
||||
q_spec, k_spec, v_spec = qkv_spec.split(self.local_projection_size, dim=-1)
|
||||
|
||||
# --- causal conv1d: non-spec path (prefill or plain decode) ---
|
||||
q_ns = k_ns = v_ns = None
|
||||
if attn_metadata_narrowed.num_prefills > 0:
|
||||
assert qkv_ns is not None
|
||||
qkv_ns = causal_conv1d_fn(
|
||||
qkv_ns.transpose(0, 1),
|
||||
conv_weights,
|
||||
conv_bias,
|
||||
activation="silu",
|
||||
conv_states=conv_state,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=attn_metadata_narrowed,
|
||||
).transpose(0, 1)
|
||||
q_ns, k_ns, v_ns = qkv_ns.split(self.local_projection_size, dim=-1)
|
||||
elif attn_metadata_narrowed.num_decodes > 0:
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
decode_conv_indices = non_spec_state_indices_tensor[
|
||||
: attn_metadata_narrowed.num_decodes
|
||||
]
|
||||
qkv_ns = causal_conv1d_update(
|
||||
qkv_ns,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
conv_bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
)
|
||||
q_ns, k_ns, v_ns = qkv_ns.split(self.local_projection_size, dim=-1)
|
||||
|
||||
def _rearr(x):
|
||||
return x.reshape(1, -1, self.local_num_heads, self.head_dim)
|
||||
|
||||
# --- core attention: spec (draft-verify) path ---
|
||||
core_attn_out_spec = None
|
||||
# In a pure spec-verify step (no non-spec tokens) the recurrent kernel
|
||||
# can write straight into the layer output buffer, skipping the
|
||||
# fresh allocation + copy below. Mixed steps must scatter via
|
||||
# spec_token_indx, so they keep the kernel-managed output.
|
||||
spec_out = (
|
||||
core_attn_out[0, :num_actual_tokens].unsqueeze(0)
|
||||
if non_spec_token_indx is None or non_spec_token_indx.numel() == 0
|
||||
else None
|
||||
)
|
||||
if use_spec:
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert num_accepted_tokens is not None
|
||||
assert spec_query_start_loc is not None
|
||||
# Gate computed inside the recurrent kernel (COMPUTE_GATE) from
|
||||
# raw g1 — replicates fused_kda_gate's arithmetic bit-for-bit and
|
||||
# skips its launch + fp32 [n, H, D] intermediate per layer.
|
||||
core_attn_out_spec, _ = fused_recurrent_kda(
|
||||
q=_rearr(q_spec),
|
||||
k=_rearr(k_spec),
|
||||
v=_rearr(v_spec),
|
||||
g=g1_spec,
|
||||
beta=beta_spec,
|
||||
initial_state=recurrent_state,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=spec_query_start_loc[: num_spec_decodes + 1],
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
out=spec_out,
|
||||
sigmoid_beta=True,
|
||||
a_log=self.A_log,
|
||||
g_bias=self.dt_bias,
|
||||
compute_gate=True,
|
||||
lower_bound=lower_bound,
|
||||
)
|
||||
|
||||
# --- core attention: non-spec path (prefill or plain decode) ---
|
||||
core_attn_out_non_spec = None
|
||||
# Only the plain-decode recurrent kernel can write straight into the
|
||||
# layer output buffer; the chunked prefill kernel cannot, so this
|
||||
# stays None there and the merge copy below runs as before.
|
||||
ns_out = None
|
||||
if attn_metadata_narrowed.num_prefills > 0:
|
||||
assert q_ns is not None
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
assert has_initial_state is not None
|
||||
initial_state = gather_initial_states(
|
||||
recurrent_state, non_spec_state_indices_tensor, has_initial_state
|
||||
)
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_kda_with_fused_gate(
|
||||
q=_rearr(q_ns),
|
||||
k=_rearr(k_ns),
|
||||
v=_rearr(v_ns),
|
||||
raw_g=g1_ns,
|
||||
# Chunk path wants the pre-sigmoided fp32 beta (its kernels
|
||||
# don't sigmoid); beta_ns is raw bf16 from forward.
|
||||
beta=_cast_sigmoid(beta_ns.squeeze(0)).unsqueeze(0),
|
||||
A_log=self.A_log,
|
||||
g_bias=self.dt_bias,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
safe_gate=safe_gate,
|
||||
lower_bound=lower_bound,
|
||||
)
|
||||
# Init cache
|
||||
scatter_states(
|
||||
recurrent_state,
|
||||
last_recurrent_state,
|
||||
non_spec_state_indices_tensor,
|
||||
)
|
||||
elif attn_metadata_narrowed.num_decodes > 0:
|
||||
assert non_spec_query_start_loc is not None
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
# Plain decode step (no spec tokens): token order is dense, so the
|
||||
# kernel can write straight into the layer output buffer. A mixed
|
||||
# step scatters non-spec output via non_spec_token_indx instead.
|
||||
# Gate computed in-kernel (COMPUTE_GATE), beta sigmoided in-kernel.
|
||||
if not use_spec:
|
||||
ns_out = spec_out
|
||||
core_attn_out_non_spec, _ = fused_recurrent_kda(
|
||||
q=_rearr(q_ns),
|
||||
k=_rearr(k_ns),
|
||||
v=_rearr(v_ns),
|
||||
g=g1_ns,
|
||||
beta=beta_ns,
|
||||
initial_state=recurrent_state,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=non_spec_query_start_loc[
|
||||
: attn_metadata_narrowed.num_decodes + 1
|
||||
],
|
||||
ssm_state_indices=non_spec_state_indices_tensor,
|
||||
out=ns_out,
|
||||
sigmoid_beta=True,
|
||||
a_log=self.A_log,
|
||||
g_bias=self.dt_bias,
|
||||
compute_gate=True,
|
||||
lower_bound=lower_bound,
|
||||
)
|
||||
|
||||
# --- merge spec / non-spec outputs back into token order ---
|
||||
if use_spec and core_attn_out_non_spec is not None:
|
||||
assert core_attn_out_spec is not None
|
||||
merged = torch.empty(
|
||||
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
|
||||
dtype=core_attn_out_non_spec.dtype,
|
||||
device=core_attn_out_non_spec.device,
|
||||
)
|
||||
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[0, :num_actual_tokens] = merged.squeeze(0)
|
||||
elif use_spec:
|
||||
assert core_attn_out_spec is not None
|
||||
if spec_out is None:
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_spec.squeeze(0)
|
||||
else:
|
||||
assert core_attn_out_non_spec is not None
|
||||
if ns_out is None:
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,429 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_mtp import SharedHead
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV2MixtureOfExperts
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .model import (
|
||||
Glm5NextDecoderLayer,
|
||||
Glm5NextMLAAttention,
|
||||
Glm5NextMoE,
|
||||
_try_load_fp8_attn_proj,
|
||||
_try_load_fp8_indexer_wk,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
)
|
||||
from .ops.fused_eh_norm import fused_eh_norm
|
||||
|
||||
|
||||
class Glm5NextMultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
||||
|
||||
# Reserve room for the incomplete pool tail and align the sparse MLA
|
||||
# buffer width to BLOCK_N=128.
|
||||
topk_tokens = config.index_topk
|
||||
assert topk_tokens is not None
|
||||
kpool = config.index_kpool
|
||||
assert kpool is not None
|
||||
buffer_width = topk_tokens + (kpool - 1 if kpool > 1 else 0)
|
||||
sparse_topk_block_n = 128
|
||||
buffer_width = (
|
||||
(buffer_width + sparse_topk_block_n - 1) // sparse_topk_block_n
|
||||
) * sparse_topk_block_n
|
||||
topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
buffer_width,
|
||||
dtype=torch.int32,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
# MTP layers sit past the base model's hidden layers; parse the index
|
||||
# from the prefix (e.g. "...layers.32") so the decoder builds an MLA
|
||||
# (DSA) layer rather than KDA for the MTP path.
|
||||
layer_idx = int(prefix.rsplit(".", 1)[-1])
|
||||
self.mtp_block = Glm5NextDecoderLayer(
|
||||
vllm_config=vllm_config,
|
||||
config=config,
|
||||
layer_idx=layer_idx,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
is_mtp_layer=True,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> torch.Tensor:
|
||||
assert inputs_embeds is not None
|
||||
# Fused: zero pos-0 embeds + enorm(embeds) + hnorm(prev) + cat -> [N, 2H].
|
||||
eh_input = fused_eh_norm(
|
||||
positions,
|
||||
inputs_embeds,
|
||||
previous_hidden_states,
|
||||
self.enorm.weight,
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
hidden_states = self.eh_proj(eh_input)
|
||||
# Fuse the residual add and final RMSNorm. Glm5NextMoE already performs
|
||||
# its all-reduce, so no collective is needed here. The post-norm result
|
||||
# feeds both draft logits and the next recycled hidden state.
|
||||
hidden_states, residual, _, _ = self.mtp_block(
|
||||
positions=positions, hidden_states=hidden_states, residual=None
|
||||
)
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual=residual)
|
||||
return hidden_states, hidden_states
|
||||
|
||||
|
||||
class Glm5NextMultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = config.num_nextn_predict_layers
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
{
|
||||
str(idx): Glm5NextMultiTokenPredictorLayer(
|
||||
vllm_config, f"{prefix}.layers.{idx}"
|
||||
)
|
||||
for idx in range(
|
||||
self.mtp_start_layer_idx,
|
||||
self.mtp_start_layer_idx + self.num_mtp_layers,
|
||||
)
|
||||
}
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
# Plain list for the per-propose lookup: ModuleDict[str(...)] builds a
|
||||
# string and hashes it on every draft step.
|
||||
self._mtp_layers = list(self.layers.values())
|
||||
self._mtp_mla_attns = []
|
||||
for layer in self._mtp_layers:
|
||||
self_attn = layer.mtp_block.self_attn
|
||||
assert isinstance(self_attn, Glm5NextMLAAttention)
|
||||
self._mtp_mla_attns.append(self_attn.mla_attn)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
def set_skip_topk(self, skip: bool):
|
||||
# index_share_for_mtp_iteration: step 0 computes top-k, steps 1+ reuse.
|
||||
for mla_attn in self._mtp_mla_attns:
|
||||
mla_attn.skip_topk = skip
|
||||
|
||||
def compact_topk_indices(self, slot_ids: torch.Tensor):
|
||||
"""Gather the top-k index rows at ``slot_ids`` to the front of the buffer."""
|
||||
num_slots = slot_ids.numel()
|
||||
for mla_attn in self._mtp_mla_attns:
|
||||
topk_indices_buffer = mla_attn.topk_indices_buffer
|
||||
assert topk_indices_buffer is not None
|
||||
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
return self._mtp_layers[current_step_idx](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self._mtp_layers[current_step_idx]
|
||||
# hidden_states is already post-final-norm (produced in the layer
|
||||
# forward and recycled as-is); apply the LM head only, without a
|
||||
# second RMSNorm.
|
||||
return self.logits_processor(mtp_layer.shared_head.head, hidden_states)
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self._mtp_layers[current_step_idx]
|
||||
# Vocab-parallel argmax for the greedy draft: per-rank head projection
|
||||
# + local argmax + a [batch, 2*tp] (value, index) reduce, instead of
|
||||
# materializing and all-gathering full [N, vocab] logits per draft
|
||||
# step. Tie-breaking matches the full argmax (shards are contiguous
|
||||
# and rank-ordered, so the lowest-rank winner is the lowest global
|
||||
# index), so greedy draft tokens are unchanged.
|
||||
return self.logits_processor.get_top_tokens(
|
||||
mtp_layer.shared_head.head, hidden_states
|
||||
)
|
||||
|
||||
|
||||
class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.model = Glm5NextMultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.set_moe_parameters()
|
||||
|
||||
def set_moe_parameters(self):
|
||||
self.num_moe_layers = self.config.num_nextn_predict_layers
|
||||
self.num_expert_groups = self.config.n_group
|
||||
self.moe_layers = []
|
||||
self.moe_mlp_layers = []
|
||||
example_moe = None
|
||||
for layer in self.model.layers.values():
|
||||
mlp = layer.mtp_block.mlp
|
||||
if isinstance(mlp, Glm5NextMoE):
|
||||
example_moe = mlp
|
||||
self.moe_mlp_layers.append(mlp)
|
||||
self.moe_layers.append(mlp.experts)
|
||||
self.extract_moe_parameters(example_moe)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
return self.model(
|
||||
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
# Greedy-draft path used when use_local_argmax_reduction is enabled:
|
||||
# vocab-parallel argmax, no full-vocab logits.
|
||||
return self.model.get_top_tokens(hidden_states, spec_step_idx)
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
"enorm",
|
||||
"hnorm",
|
||||
"eh_proj",
|
||||
"shared_head",
|
||||
]
|
||||
shared_weight_names = ["embed_tokens"]
|
||||
spec_layer_weight = False
|
||||
shared_weight = False
|
||||
for weight_name in spec_layer_weight_names:
|
||||
if weight_name in name:
|
||||
spec_layer_weight = True
|
||||
if weight_name in shared_weight_names:
|
||||
shared_weight = True
|
||||
break
|
||||
if not spec_layer_weight:
|
||||
name = name.replace(
|
||||
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
|
||||
)
|
||||
elif shared_weight:
|
||||
name = name.replace(f"model.layers.{spec_layer}.", "model.")
|
||||
return name
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
("fused_qkv_a_proj", "q_a_proj", 0),
|
||||
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
|
||||
("wk_weights_proj", "wk", 0),
|
||||
("wk_weights_proj", "weights_proj", 1),
|
||||
]
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
_pending_wk_fp8: dict = {}
|
||||
# GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from
|
||||
# ``kv_a_proj_with_mqa``; the FP8-to-BF16 path pads them for the model.
|
||||
kv_a_pad_size = 0
|
||||
if self.config.mla_nope and self.config.qk_rope_head_dim > 0:
|
||||
kv_a_pad_size = self.config.qk_rope_head_dim
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
# Multimodal (Glm5NextForConditionalGeneration) checkpoints prefix
|
||||
# the text-tower weights with "model.language_model."; the MTP head
|
||||
# is built as a text-only model (model.layers.*), so strip the
|
||||
# prefix to match.
|
||||
if name.startswith("model.language_model."):
|
||||
name = name.replace("model.language_model.", "model.", 1)
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
continue
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
if _try_load_fp8_indexer_wk(
|
||||
name,
|
||||
loaded_weight,
|
||||
_pending_wk_fp8,
|
||||
params_dict,
|
||||
loaded_params,
|
||||
):
|
||||
continue
|
||||
|
||||
# FP8 checkpoint: dequantize the BF16-kept MLA projections
|
||||
# (q_a_proj / kv_a_proj_with_mqa / o_proj) to BF16, mirroring the
|
||||
# target model. The model holds fused_qkv_a_proj / o_proj in BF16,
|
||||
# so the checkpoint's block-FP8 weight + weight_scale_inv for these
|
||||
# has no param home and would KeyError without this dequant.
|
||||
if _try_load_fp8_attn_proj(
|
||||
name,
|
||||
loaded_weight,
|
||||
_pending_wk_fp8,
|
||||
params_dict,
|
||||
loaded_params,
|
||||
kv_a_pad_size,
|
||||
):
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if (
|
||||
param_name == "fused_qkv_a_proj"
|
||||
) and name_mapped not in params_dict:
|
||||
continue
|
||||
else:
|
||||
name = name_mapped
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
|
||||
if weight_name not in name:
|
||||
continue
|
||||
is_expert_weight = True
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
continue
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
|
||||
if name is None:
|
||||
continue
|
||||
if (
|
||||
spec_layer != self.model.mtp_start_layer_idx
|
||||
and ".layers" not in name
|
||||
):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
loaded_layers: set[int] = set()
|
||||
for param_name in loaded_params:
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
|
||||
if spec_layer is not None:
|
||||
loaded_layers.add(spec_layer)
|
||||
for layer_idx in range(
|
||||
self.model.mtp_start_layer_idx,
|
||||
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
|
||||
):
|
||||
if layer_idx not in loaded_layers:
|
||||
raise ValueError(
|
||||
f"MTP speculative decoding layer {layer_idx} weights "
|
||||
f"missing from checkpoint."
|
||||
)
|
||||
return loaded_params
|
||||
@@ -0,0 +1,732 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-5.3-Flash vision tower and multimodal processor."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from functools import cached_property, partial
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
parallel_state,
|
||||
)
|
||||
from vllm.distributed import utils as dist_utils
|
||||
from vllm.model_executor.layers.activation import SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
|
||||
from vllm.model_executor.models.glm4_1v import (
|
||||
Glm4vMultiModalProcessor,
|
||||
Glm4vProcessingInfo,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
)
|
||||
from vllm.model_executor.models.vision import (
|
||||
get_vit_attn_backend,
|
||||
is_vit_use_data_parallel,
|
||||
)
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
class Glm5NextVisionPatchEmbed(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 14,
|
||||
temporal_patch_size: int = 1,
|
||||
in_channels: int = 3,
|
||||
hidden_size: int = 1536,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.patch_size = patch_size
|
||||
self.temporal_patch_size = temporal_patch_size
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
kernel_size = (temporal_patch_size, patch_size, patch_size)
|
||||
self.proj = Conv3dLayer(
|
||||
in_channels,
|
||||
hidden_size,
|
||||
kernel_size=kernel_size,
|
||||
stride=kernel_size,
|
||||
bias=True,
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
L, C = x.shape
|
||||
x = x.view(L, -1, self.temporal_patch_size, self.patch_size, self.patch_size)
|
||||
x = self.proj(x).view(L, self.hidden_size)
|
||||
return x
|
||||
|
||||
|
||||
class Glm5NextVisionMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_features: int,
|
||||
hidden_features: int,
|
||||
swiglu_limit: float,
|
||||
bias: bool = True,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
use_data_parallel = is_vit_use_data_parallel()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
input_size=in_features,
|
||||
output_sizes=[hidden_features] * 2,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
hidden_features,
|
||||
in_features,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
# GLM-5.3-Flash clamps the vision SwiGLU gate/up unlike GLM-OCR/GLM-4V.
|
||||
self.act_fn = SiluAndMulWithClamp(swiglu_limit=swiglu_limit)
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
x, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(x)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class Glm5NextVisionAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
embed_dim: int,
|
||||
num_heads: int,
|
||||
projection_size: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
use_data_parallel = is_vit_use_data_parallel()
|
||||
self.tp_size = (
|
||||
1 if use_data_parallel else get_tensor_model_parallel_world_size()
|
||||
)
|
||||
self.tp_rank = (
|
||||
0 if use_data_parallel else parallel_state.get_tensor_model_parallel_rank()
|
||||
)
|
||||
self.hidden_size_per_attention_head = dist_utils.divide(
|
||||
projection_size, num_heads
|
||||
)
|
||||
self.num_attention_heads_per_partition = dist_utils.divide(
|
||||
num_heads, self.tp_size
|
||||
)
|
||||
|
||||
self.head_dim = embed_dim // num_heads
|
||||
|
||||
# q/k norm eps hard-coded 1e-5 — distinct from block/post norm eps.
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=1e-5)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=1e-5)
|
||||
|
||||
self.qkv = QKVParallelLinear(
|
||||
hidden_size=embed_dim,
|
||||
head_size=self.hidden_size_per_attention_head,
|
||||
total_num_heads=num_heads,
|
||||
total_num_kv_heads=num_heads,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj" if quant_config else f"{prefix}.qkv",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.proj = RowParallelLinear(
|
||||
input_size=projection_size,
|
||||
output_size=embed_dim,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.proj",
|
||||
bias=True,
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
|
||||
self.attn = MMEncoderAttention(
|
||||
num_heads=self.num_attention_heads_per_partition,
|
||||
head_size=self.hidden_size_per_attention_head,
|
||||
scale=self.hidden_size_per_attention_head**-0.5,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
self.apply_rotary_emb = ApplyRotaryEmb(enforce_enable=True)
|
||||
|
||||
def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]:
|
||||
seq_len, bs, _ = qkv.shape
|
||||
q, k, v = qkv.chunk(3, dim=2)
|
||||
new_shape = (
|
||||
seq_len,
|
||||
bs,
|
||||
self.num_attention_heads_per_partition,
|
||||
self.hidden_size_per_attention_head,
|
||||
)
|
||||
q, k, v = (x.view(*new_shape) for x in (q, k, v))
|
||||
return q, k, v
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
rotary_pos_emb_cos: torch.Tensor,
|
||||
rotary_pos_emb_sin: torch.Tensor,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
x, _ = self.qkv(x)
|
||||
q, k, v = self.split_qkv(x)
|
||||
|
||||
# P1: fused q/k RMSNorm (two distinct weights, one launch; fp32, bit-identical).
|
||||
q_shape, k_shape = q.shape, k.shape
|
||||
q_flat = q.reshape(-1, self.head_dim)
|
||||
k_flat = k.reshape(-1, self.head_dim)
|
||||
q, k = fused_q_kv_rmsnorm(
|
||||
q_flat,
|
||||
k_flat,
|
||||
self.q_norm.weight,
|
||||
self.k_norm.weight,
|
||||
self.q_norm.variance_epsilon,
|
||||
)
|
||||
q = q.view(q_shape)
|
||||
k = k.view(k_shape)
|
||||
|
||||
q, k, v = (rearrange(t, "s b ... -> b s ...").contiguous() for t in (q, k, v))
|
||||
if rotary_pos_emb_cos is not None and rotary_pos_emb_sin is not None:
|
||||
qk_concat = torch.cat([q, k], dim=0)
|
||||
qk_rotated = self.apply_rotary_emb(
|
||||
qk_concat,
|
||||
rotary_pos_emb_cos,
|
||||
rotary_pos_emb_sin,
|
||||
)
|
||||
q, k = torch.chunk(qk_rotated, 2, dim=0)
|
||||
|
||||
context_layer = self.attn(
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
)
|
||||
context_layer = rearrange(context_layer, "b s h d -> s b (h d)").contiguous()
|
||||
|
||||
output, _ = self.proj(context_layer)
|
||||
return output
|
||||
|
||||
|
||||
class Glm5NextVisionBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
num_heads: int,
|
||||
mlp_hidden_dim: int,
|
||||
swiglu_limit: float,
|
||||
norm_layer: partial[nn.Module] | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if norm_layer is None:
|
||||
norm_layer = partial(nn.LayerNorm, eps=1e-6)
|
||||
self.norm1 = norm_layer(dim)
|
||||
self.norm2 = norm_layer(dim)
|
||||
self.attn = Glm5NextVisionAttention(
|
||||
embed_dim=dim,
|
||||
num_heads=num_heads,
|
||||
projection_size=dim,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
self.mlp = Glm5NextVisionMLP(
|
||||
dim,
|
||||
mlp_hidden_dim,
|
||||
swiglu_limit=swiglu_limit,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
rotary_pos_emb_cos: torch.Tensor,
|
||||
rotary_pos_emb_sin: torch.Tensor,
|
||||
max_seqlen: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
x_attn = self.attn(
|
||||
self.norm1(x),
|
||||
cu_seqlens=cu_seqlens,
|
||||
rotary_pos_emb_cos=rotary_pos_emb_cos,
|
||||
rotary_pos_emb_sin=rotary_pos_emb_sin,
|
||||
max_seqlen=max_seqlen,
|
||||
)
|
||||
x_fused_norm, residual = self.norm2(x, residual=x_attn)
|
||||
x = residual + self.mlp(x_fused_norm)
|
||||
return x
|
||||
|
||||
|
||||
class Glm5NextPatchMerger(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
d_model: int,
|
||||
context_dim: int,
|
||||
swiglu_limit: float,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
bias: bool = False,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
use_data_parallel = is_vit_use_data_parallel()
|
||||
self.hidden_size = d_model
|
||||
self.proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_size,
|
||||
bias=bias,
|
||||
gather_output=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.proj",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.post_projection_norm = nn.LayerNorm(self.hidden_size)
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
input_size=self.hidden_size,
|
||||
output_sizes=[context_dim] * 2,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
context_dim,
|
||||
self.hidden_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
# GLM-5.3-Flash also clamps the merger SwiGLU.
|
||||
self.act_fn = SiluAndMulWithClamp(swiglu_limit=swiglu_limit)
|
||||
self.extra_activation_func = nn.GELU()
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
x, _ = self.proj(x)
|
||||
x = self.extra_activation_func(self.post_projection_norm(x))
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class Glm5NextVisionTransformer(nn.Module):
|
||||
# Stacked-weight remap for the GLM-OCR/GLM-4V vision checkpoint layout.
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".attn.q.": (".attn.qkv.", "q"),
|
||||
".attn.k.": (".attn.qkv.", "k"),
|
||||
".attn.v.": (".attn.qkv.", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config, # noqa: ANN001
|
||||
vision_config,
|
||||
norm_eps: float = 1e-6,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
use_data_parallel = is_vit_use_data_parallel()
|
||||
self.tp_size = (
|
||||
1 if use_data_parallel else get_tensor_model_parallel_world_size()
|
||||
)
|
||||
|
||||
patch_size = vision_config.patch_size
|
||||
temporal_patch_size = vision_config.temporal_patch_size
|
||||
in_channels = vision_config.in_channels
|
||||
depth = vision_config.depth
|
||||
self.hidden_size = vision_config.hidden_size
|
||||
self.num_heads = vision_config.num_heads
|
||||
|
||||
self.patch_size = vision_config.patch_size
|
||||
self.spatial_merge_size = vision_config.spatial_merge_size
|
||||
self.out_hidden_size = vision_config.out_hidden_size
|
||||
|
||||
swiglu_limit = vision_config.swiglu_limit
|
||||
if swiglu_limit is None:
|
||||
swiglu_limit = text_config.swiglu_limit
|
||||
assert swiglu_limit is not None, (
|
||||
"GLM-5.3-Flash vision requires swiglu_limit (vision_config or text_config)"
|
||||
)
|
||||
|
||||
# Single construction pass — no abs-pos embeddings / post-conv norm (OCR delta).
|
||||
self.patch_embed = Glm5NextVisionPatchEmbed(
|
||||
patch_size=patch_size,
|
||||
temporal_patch_size=temporal_patch_size,
|
||||
in_channels=in_channels,
|
||||
hidden_size=self.hidden_size,
|
||||
)
|
||||
|
||||
norm_layer = partial(RMSNorm, eps=norm_eps)
|
||||
head_dim = self.hidden_size // self.num_heads
|
||||
self.rotary_pos_emb = get_rope(
|
||||
head_size=head_dim,
|
||||
max_position=8192,
|
||||
is_neox_style=True,
|
||||
rope_parameters={"partial_rotary_factor": 0.5},
|
||||
)
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
Glm5NextVisionBlock(
|
||||
dim=self.hidden_size,
|
||||
num_heads=self.num_heads,
|
||||
mlp_hidden_dim=vision_config.intermediate_size,
|
||||
swiglu_limit=swiglu_limit,
|
||||
norm_layer=norm_layer,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.blocks.{layer_idx}",
|
||||
)
|
||||
for layer_idx in range(depth)
|
||||
]
|
||||
)
|
||||
# GLM-5.3-Flash merger bottleneck width.
|
||||
self.merger = Glm5NextPatchMerger(
|
||||
d_model=vision_config.out_hidden_size,
|
||||
context_dim=vision_config.projection_intermediate_size,
|
||||
swiglu_limit=swiglu_limit,
|
||||
quant_config=quant_config,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.merger",
|
||||
)
|
||||
|
||||
self.downsample = Conv2dLayer(
|
||||
in_channels=vision_config.hidden_size,
|
||||
out_channels=vision_config.out_hidden_size,
|
||||
kernel_size=vision_config.spatial_merge_size,
|
||||
stride=vision_config.spatial_merge_size,
|
||||
)
|
||||
self.post_layernorm = RMSNorm(
|
||||
vision_config.hidden_size, eps=vision_config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_backend = get_vit_attn_backend(
|
||||
head_size=head_dim,
|
||||
dtype=torch.get_default_dtype(),
|
||||
)
|
||||
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return self.patch_embed.proj.weight.dtype
|
||||
|
||||
@property
|
||||
def device(self) -> torch.device:
|
||||
return self.patch_embed.proj.weight.device
|
||||
|
||||
def rot_pos_emb(
|
||||
self, grid_thw: list[list[int]]
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
pos_ids = []
|
||||
for t, h, w in grid_thw:
|
||||
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
|
||||
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
|
||||
hpos_ids = (
|
||||
hpos_ids.reshape(
|
||||
h // self.spatial_merge_size,
|
||||
self.spatial_merge_size,
|
||||
w // self.spatial_merge_size,
|
||||
self.spatial_merge_size,
|
||||
)
|
||||
.permute(0, 2, 1, 3)
|
||||
.flatten()
|
||||
)
|
||||
wpos_ids = (
|
||||
wpos_ids.reshape(
|
||||
h // self.spatial_merge_size,
|
||||
self.spatial_merge_size,
|
||||
w // self.spatial_merge_size,
|
||||
self.spatial_merge_size,
|
||||
)
|
||||
.permute(0, 2, 1, 3)
|
||||
.flatten()
|
||||
)
|
||||
pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
|
||||
pos_ids = torch.cat(pos_ids, dim=0)
|
||||
max_grid_size = max(max(h, w) for _, h, w in grid_thw)
|
||||
|
||||
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
|
||||
|
||||
pos_ids = pos_ids.to(cos.device, non_blocking=True)
|
||||
cos_combined = cos[pos_ids].flatten(1)
|
||||
sin_combined = sin[pos_ids].flatten(1)
|
||||
return cos_combined, sin_combined, pos_ids
|
||||
|
||||
def compute_attn_mask_seqlen(
|
||||
self,
|
||||
cu_seqlens: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
max_seqlen = None
|
||||
if self.attn_backend in {
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
}:
|
||||
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
|
||||
return max_seqlen
|
||||
|
||||
def prepare_encoder_metadata(
|
||||
self,
|
||||
grid_thw_list: list[list[int]],
|
||||
*,
|
||||
max_batch_size: int | None = None,
|
||||
max_frames_per_batch: int | None = None,
|
||||
max_seqlen_override: int | None = None,
|
||||
device: torch.device | None = None,
|
||||
) -> dict[str, torch.Tensor | None]:
|
||||
"""Compute encoder metadata for eager and CUDA graph execution."""
|
||||
if device is None:
|
||||
device = self.device
|
||||
|
||||
metadata: dict[str, torch.Tensor | None] = {}
|
||||
|
||||
rotary_cos, rotary_sin, _ = self.rot_pos_emb(grid_thw_list)
|
||||
metadata["rotary_pos_emb_cos"] = rotary_cos
|
||||
metadata["rotary_pos_emb_sin"] = rotary_sin
|
||||
|
||||
grid_thw_np = np.array(grid_thw_list, dtype=np.int32)
|
||||
patches_per_frame = grid_thw_np[:, 1] * grid_thw_np[:, 2]
|
||||
cu_seqlens = np.repeat(patches_per_frame, grid_thw_np[:, 0]).cumsum(
|
||||
dtype=np.int32
|
||||
)
|
||||
cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])
|
||||
|
||||
pad_to = (
|
||||
max_frames_per_batch if max_frames_per_batch is not None else max_batch_size
|
||||
)
|
||||
if pad_to is not None:
|
||||
num_seqs = len(cu_seqlens) - 1
|
||||
if num_seqs < pad_to:
|
||||
cu_seqlens = np.concatenate(
|
||||
[
|
||||
cu_seqlens,
|
||||
np.full(
|
||||
pad_to - num_seqs,
|
||||
cu_seqlens[-1],
|
||||
dtype=np.int32,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
|
||||
self.attn_backend, cu_seqlens, device
|
||||
)
|
||||
|
||||
if max_seqlen_override is not None:
|
||||
max_seqlen_val = max_seqlen_override
|
||||
else:
|
||||
max_seqlen_val = MMEncoderAttention.compute_max_seqlen(
|
||||
self.attn_backend, cu_seqlens
|
||||
)
|
||||
metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)
|
||||
|
||||
metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
|
||||
self.attn_backend,
|
||||
cu_seqlens,
|
||||
self.hidden_size,
|
||||
self.tp_size,
|
||||
device,
|
||||
)
|
||||
|
||||
return metadata
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
grid_thw: torch.Tensor | list[list[int]],
|
||||
*,
|
||||
encoder_metadata: dict[str, torch.Tensor] | None = None,
|
||||
) -> torch.Tensor:
|
||||
# patchify
|
||||
x = x.to(device=self.device, dtype=self.dtype)
|
||||
x = self.patch_embed(x)
|
||||
|
||||
if encoder_metadata is not None:
|
||||
# Encoder CUDA-graph path (PR #49852): rotary/cu_seqlens/max_seqlen are
|
||||
# precomputed by prepare_encoder_metadata (which uses rot_pos_emb exactly
|
||||
# as the eager rebuild does), so reuse them and skip the per-call CPU
|
||||
# rebuild (the low-GPU-util culprit on multimodal workloads).
|
||||
rotary_pos_emb_cos = encoder_metadata["rotary_pos_emb_cos"]
|
||||
rotary_pos_emb_sin = encoder_metadata["rotary_pos_emb_sin"]
|
||||
cu_seqlens = encoder_metadata["cu_seqlens"]
|
||||
max_seqlen = encoder_metadata["max_seqlen"]
|
||||
else:
|
||||
if isinstance(grid_thw, list):
|
||||
grid_thw = torch.tensor(grid_thw, dtype=torch.int32)
|
||||
rotary_pos_emb_cos, rotary_pos_emb_sin, _ = self.rot_pos_emb(grid_thw)
|
||||
cu_seqlens = torch.repeat_interleave(
|
||||
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
|
||||
).cumsum(dim=0, dtype=torch.int32)
|
||||
cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
|
||||
cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
|
||||
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
|
||||
|
||||
# transformers
|
||||
x = x.unsqueeze(1)
|
||||
for blk in self.blocks:
|
||||
x = blk(
|
||||
x,
|
||||
cu_seqlens=cu_seqlens,
|
||||
rotary_pos_emb_cos=rotary_pos_emb_cos,
|
||||
rotary_pos_emb_sin=rotary_pos_emb_sin,
|
||||
max_seqlen=max_seqlen,
|
||||
)
|
||||
|
||||
# adapter
|
||||
x = self.post_layernorm(x)
|
||||
x = x.view(-1, self.spatial_merge_size, self.spatial_merge_size, x.shape[-1])
|
||||
x = x.permute(0, 3, 1, 2)
|
||||
x = self.downsample(x).view(-1, self.out_hidden_size)
|
||||
x = self.merger(x)
|
||||
return x
|
||||
|
||||
def load_weights(self, weights) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
"""Wires up the vLLM-native processor for the multimodal checkpoint.
|
||||
|
||||
The checkpoint's ``processor_config.json`` declares a custom ``processor_class``
|
||||
and stores its image/video processor configs inline (no standalone
|
||||
``preprocessor_config.json``), so ``AutoProcessor`` cannot resolve the
|
||||
config. We bypass it and build our own ``Glm5NextProcessor``
|
||||
(``vllm/transformers_utils/processors/glm5next.py``), a port of the
|
||||
training-side pipeline that no longer imports transformers' GLM processor
|
||||
classes. The port applies ``patch_expand_factor`` (checkpoint ships 1)
|
||||
inside ``smart_resize``'s spatial factor.
|
||||
"""
|
||||
|
||||
@cached_property
|
||||
def _glm5_hf_processor(self):
|
||||
from vllm.transformers_utils.processors.glm5next import Glm5NextProcessor
|
||||
|
||||
return Glm5NextProcessor.from_pretrained(self.ctx.model_config.model)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
return self._glm5_hf_processor
|
||||
|
||||
def _processor_pixel_budget(self, proc) -> tuple[int, int]:
|
||||
from vllm.transformers_utils.processors.glm5next import _pixel_budget
|
||||
|
||||
return _pixel_budget(
|
||||
proc.min_image_tokens,
|
||||
proc.max_image_tokens,
|
||||
proc.patch_size,
|
||||
proc.merge_size,
|
||||
proc.temporal_patch_size,
|
||||
)
|
||||
|
||||
def _get_image_max_pixels(self) -> int:
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs({})
|
||||
if (override := mm_kwargs.get("max_pixels")) is not None:
|
||||
return int(override)
|
||||
return self._processor_pixel_budget(self.get_hf_processor().image_processor)[1]
|
||||
|
||||
def _get_video_max_pixels(self) -> int:
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs({})
|
||||
if (override := mm_kwargs.get("max_pixels")) is not None:
|
||||
return int(override)
|
||||
return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]
|
||||
|
||||
def _get_vision_info(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
num_frames: int = 16,
|
||||
do_resize: bool = True,
|
||||
max_image_pixels: int = 28 * 28 * 2 * 30000,
|
||||
) -> tuple[ImageSize, int]:
|
||||
"""GLM-5.3-Flash canvas geometry for token budgeting and dummy inputs.
|
||||
|
||||
The inherited Glm4v path resolves the pixel budget from
|
||||
``size.longest_edge`` and resizes with GLM-4V's ``smart_resize``. This
|
||||
checkpoint's ``processor_config.json`` ships the token-budget style
|
||||
(``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, and
|
||||
the alignment factor carries ``patch_expand_factor`` — resolve both
|
||||
from the vLLM-native processor so profiling matches runtime geometry.
|
||||
"""
|
||||
from vllm.transformers_utils.processors.glm5next import smart_resize
|
||||
|
||||
vision_config = self.get_hf_config().vision_config
|
||||
patch_size = vision_config.patch_size
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
temporal_patch_size = vision_config.temporal_patch_size
|
||||
|
||||
image_processor = self.get_hf_processor().image_processor
|
||||
factor = patch_size * merge_size * image_processor.patch_expand_factor
|
||||
# Keep the profiling search viable when the caller's budget is below
|
||||
# one aligned canvas of the requested duration.
|
||||
max_image_pixels = max(max_image_pixels, temporal_patch_size * factor * factor)
|
||||
|
||||
if do_resize:
|
||||
t = num_frames if num_frames > temporal_patch_size else temporal_patch_size
|
||||
resized_height, resized_width = smart_resize(
|
||||
t=t,
|
||||
h=image_height,
|
||||
w=image_width,
|
||||
t_factor=temporal_patch_size,
|
||||
h_factor=factor,
|
||||
w_factor=factor,
|
||||
min_pixels=1,
|
||||
max_pixels=max_image_pixels,
|
||||
)
|
||||
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
|
||||
else:
|
||||
preprocessed_size = ImageSize(width=image_width, height=image_height)
|
||||
|
||||
padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
|
||||
grid_t = max(padded_num_frames // temporal_patch_size, 1)
|
||||
grid_h = preprocessed_size.height // patch_size
|
||||
grid_w = preprocessed_size.width // patch_size
|
||||
|
||||
num_patches = grid_t * grid_h * grid_w
|
||||
num_vision_tokens = num_patches // (merge_size**2)
|
||||
|
||||
return preprocessed_size, num_vision_tokens
|
||||
|
||||
|
||||
class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor):
|
||||
"""The vLLM-native ``Glm5NextProcessor`` extracts image/video features
|
||||
only and passes the prompt text through unchanged, so prompt expansion
|
||||
(image token repeat, video frame/timestamp structure) is owned by vLLM's
|
||||
prompt-update machinery — the inherited ``_get_prompt_updates`` builds
|
||||
the replacement content and the placeholder scan validates against
|
||||
exactly that."""
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
@@ -0,0 +1,77 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rms_norm(x, w, eps, HIDDEN_SIZE: tl.constexpr):
|
||||
x = x.to(tl.float32)
|
||||
mean_sq = tl.sum(x * x, axis=0) / HIDDEN_SIZE
|
||||
rrms = tl.rsqrt(mean_sq + eps)
|
||||
w = w.to(tl.float32)
|
||||
return (x * rrms) * w
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_eh_norm_kernel(
|
||||
pos_ptr,
|
||||
embeds_ptr,
|
||||
embeds_stride,
|
||||
prev_ptr,
|
||||
prev_stride,
|
||||
enorm_w_ptr,
|
||||
hnorm_w_ptr,
|
||||
eps,
|
||||
out_ptr,
|
||||
out_stride,
|
||||
H: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""MTP input fusion: zero embeds at position 0, RMSNorm(embeds) with enorm
|
||||
and RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out``
|
||||
([N, 2H]) ready for the eh_proj GEMM. Replaces where + 2x RMSNorm + cat."""
|
||||
tok = tl.program_id(0)
|
||||
off = tl.arange(0, BLOCK)
|
||||
mask = off < H
|
||||
|
||||
pos = tl.load(pos_ptr + tok)
|
||||
e = tl.load(embeds_ptr + tok * embeds_stride + off, mask=mask, other=0.0)
|
||||
e = tl.where(pos == 0, 0.0, e.to(tl.float32))
|
||||
ew = tl.load(enorm_w_ptr + off, mask=mask)
|
||||
e_normed = _rms_norm(e, ew, eps, H)
|
||||
tl.store(out_ptr + tok * out_stride + off, e_normed, mask=mask)
|
||||
|
||||
p = tl.load(prev_ptr + tok * prev_stride + off, mask=mask, other=0.0)
|
||||
hw = tl.load(hnorm_w_ptr + off, mask=mask)
|
||||
p_normed = _rms_norm(p, hw, eps, H)
|
||||
tl.store(out_ptr + tok * out_stride + H + off, p_normed, mask=mask)
|
||||
|
||||
|
||||
def fused_eh_norm(
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor,
|
||||
previous_hidden: torch.Tensor,
|
||||
enorm_w: torch.Tensor,
|
||||
hnorm_w: torch.Tensor,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""Returns cat([enorm(masked embeds), hnorm(prev_hidden)]) -> [N, 2H]."""
|
||||
n, h = inputs_embeds.shape
|
||||
out = torch.empty(n, 2 * h, dtype=inputs_embeds.dtype, device=inputs_embeds.device)
|
||||
_fused_eh_norm_kernel[(n,)](
|
||||
positions,
|
||||
inputs_embeds,
|
||||
inputs_embeds.stride(0),
|
||||
previous_hidden,
|
||||
previous_hidden.stride(0),
|
||||
enorm_w,
|
||||
hnorm_w,
|
||||
eps,
|
||||
out,
|
||||
out.stride(0),
|
||||
h,
|
||||
triton.next_power_of_2(h),
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,891 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""kpool (key-pooling) Triton kernels for the sparse-attention indexer.
|
||||
|
||||
The cache stores POOLS (1 entry per ``pool_size`` consecutive tokens) rather
|
||||
than individual tokens. ``compress_ratio == pool_size`` on the kv_cache_spec
|
||||
makes the metadata builder emit pool-granular slot_mapping / seq_lens /
|
||||
cu_seq_lens / page_table for free; this file supplies the compress-write
|
||||
kernel (replacing ``indexer_k_quant_and_cache``) and the pool-level topk
|
||||
helpers (select pools -> expand to tokens -> append tail).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
# The GLM-5.3-Flash indexer head dimension is fixed at 128.
|
||||
INDEX_HEAD_DIM = 128
|
||||
|
||||
|
||||
# Hadamard-128 rotation
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hadamard128_stage(x, GROUPS: tl.constexpr, STRIDE: tl.constexpr):
|
||||
x3 = tl.reshape(x, (GROUPS, 2, STRIDE))
|
||||
x3 = tl.trans(x3, 0, 2, 1)
|
||||
a, b = tl.split(x3)
|
||||
x3 = tl.join(a + b, a - b)
|
||||
x3 = tl.trans(x3, 0, 2, 1)
|
||||
return tl.reshape(x3, (128,))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hadamard128(x):
|
||||
x = _hadamard128_stage(x, 64, 1)
|
||||
x = _hadamard128_stage(x, 32, 2)
|
||||
x = _hadamard128_stage(x, 16, 4)
|
||||
x = _hadamard128_stage(x, 8, 8)
|
||||
x = _hadamard128_stage(x, 4, 16)
|
||||
x = _hadamard128_stage(x, 2, 32)
|
||||
x = _hadamard128_stage(x, 1, 64)
|
||||
return x * 0.08838834764831845 # 1/sqrt(128)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwht_stage(x, N: tl.constexpr, GROUPS: tl.constexpr, STRIDE: tl.constexpr):
|
||||
# One FWHT butterfly stage on a flat tensor of N = GROUPS*2*STRIDE elems;
|
||||
# same construction as _hadamard128_stage but with a parametric N so it can
|
||||
# process BLOCK_R rows at once.
|
||||
x3 = tl.reshape(x, (GROUPS, 2, STRIDE))
|
||||
x3 = tl.trans(x3, 0, 2, 1)
|
||||
a, b = tl.split(x3)
|
||||
x3 = tl.join(a + b, a - b)
|
||||
x3 = tl.trans(x3, 0, 2, 1)
|
||||
return tl.reshape(x3, (N,))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwht_quant_kernel(
|
||||
q_ptr,
|
||||
qout_ptr,
|
||||
sout_ptr,
|
||||
n_rows,
|
||||
BLOCK_R: tl.constexpr,
|
||||
):
|
||||
"""Fused Hadamard-128 rotation + per-row absmax FP8 (ue8m0) quant.
|
||||
|
||||
Each row uses fp32 butterflies and scaling, rounds to bf16, then applies
|
||||
absmax quantization with a power-of-two scale.
|
||||
"""
|
||||
pid = tl.program_id(0)
|
||||
rows = pid * BLOCK_R + tl.arange(0, BLOCK_R)
|
||||
rmask = rows < n_rows
|
||||
offs = tl.arange(0, 128)
|
||||
x = tl.load(
|
||||
q_ptr + rows[:, None] * 128 + offs[None, :], mask=rmask[:, None], other=0.0
|
||||
).to(tl.float32)
|
||||
|
||||
# Flatten so each row's 128 lanes stay contiguous: every stage's
|
||||
# (GROUPS, 2, STRIDE) tiling has 2*STRIDE dividing 128, so pairs never
|
||||
# straddle a row boundary. GROUPS of each stage scales by BLOCK_R.
|
||||
N: tl.constexpr = BLOCK_R * 128
|
||||
x = tl.reshape(x, (N,))
|
||||
x = _fwht_stage(x, N, BLOCK_R * 64, 1)
|
||||
x = _fwht_stage(x, N, BLOCK_R * 32, 2)
|
||||
x = _fwht_stage(x, N, BLOCK_R * 16, 4)
|
||||
x = _fwht_stage(x, N, BLOCK_R * 8, 8)
|
||||
x = _fwht_stage(x, N, BLOCK_R * 4, 16)
|
||||
x = _fwht_stage(x, N, BLOCK_R * 2, 32)
|
||||
x = _fwht_stage(x, N, BLOCK_R, 64)
|
||||
x = x * 0.08838834764831845 # 1/sqrt(128), exact in fp32
|
||||
|
||||
# Match the unfused path's bf16 materialization before quantizing.
|
||||
x = x.to(tl.bfloat16).to(tl.float32)
|
||||
x = tl.reshape(x, (BLOCK_R, 128))
|
||||
|
||||
absmax = tl.maximum(tl.max(tl.abs(x), axis=1), 1e-4)
|
||||
scale = tl.exp2(tl.ceil(tl.log2(absmax * (1.0 / 448.0))))
|
||||
y = tl.minimum(tl.maximum(x / scale[:, None], -448.0), 448.0)
|
||||
|
||||
tl.store(qout_ptr + rows[:, None] * 128 + offs[None, :], y, mask=rmask[:, None])
|
||||
tl.store(sout_ptr + rows, scale, mask=rmask)
|
||||
|
||||
|
||||
def fwht128_quant_fp8(q: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Rotate each 128-wide row by the Hadamard-128 transform, then FP8-quant.
|
||||
|
||||
The fused kernel avoids materializing the rotated tensor and uses an exact
|
||||
fp32 ``1 / sqrt(128)`` scale before block-128 ue8m0 quantization.
|
||||
|
||||
Args:
|
||||
q: ``[rows, 128]`` bf16 — one head vector per row.
|
||||
|
||||
Returns:
|
||||
(q_fp8 ``[rows, 128]`` float8_e4m3fn, scale ``[rows, 1]`` float32).
|
||||
"""
|
||||
assert q.ndim == 2 and q.shape[1] == 128, q.shape
|
||||
assert q.dtype == torch.bfloat16
|
||||
assert q.is_contiguous()
|
||||
n_rows = q.shape[0]
|
||||
q_fp8 = torch.empty((n_rows, 128), dtype=torch.float8_e4m3fn, device=q.device)
|
||||
q_scale = torch.empty((n_rows, 1), dtype=torch.float32, device=q.device)
|
||||
if n_rows == 0:
|
||||
return q_fp8, q_scale
|
||||
BLOCK_R = 32
|
||||
grid = (triton.cdiv(n_rows, BLOCK_R),)
|
||||
_fwht_quant_kernel[grid](q, q_fp8, q_scale, n_rows, BLOCK_R=BLOCK_R, num_warps=2)
|
||||
return q_fp8, q_scale
|
||||
|
||||
|
||||
# Fused pool compression and cache write.
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _kpool_softmax_rotate_write_cache_kernel(
|
||||
buf_fp8_ptr,
|
||||
buf_fp32_ptr,
|
||||
slot_k_ptr,
|
||||
slot_score_ptr,
|
||||
ape_ptr,
|
||||
loc_ptr,
|
||||
write_mask_ptr,
|
||||
compressed_k_ptr,
|
||||
compressed_scale_ptr,
|
||||
slot_k_stride_0,
|
||||
slot_k_stride_1,
|
||||
slot_score_stride_0,
|
||||
slot_score_stride_1,
|
||||
ape_stride_0,
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
BUF_NUMEL_PER_PAGE: tl.constexpr,
|
||||
POOL_SIZE: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
HAS_WRITE_MASK: tl.constexpr,
|
||||
RETURN_COMPRESSED: tl.constexpr,
|
||||
WRITE_CACHE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""One program per pool. softmax(slot_score+ape)-weighted sum of slot_k ->
|
||||
Hadamard-128 -> per-vector fp8 absmax quant -> write to cache at ``loc``."""
|
||||
row = tl.program_id(0)
|
||||
do_write = True
|
||||
if HAS_WRITE_MASK:
|
||||
do_write = tl.load(write_mask_ptr + row)
|
||||
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
mask = (offs < HEAD_DIM) & do_write
|
||||
|
||||
# --- Pass 1: per-dim max over the pool (softmax numerical stability) ---
|
||||
max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32)
|
||||
for slot in tl.static_range(0, POOL_SIZE):
|
||||
score = tl.load(
|
||||
slot_score_ptr
|
||||
+ row * slot_score_stride_0
|
||||
+ slot * slot_score_stride_1
|
||||
+ offs,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
score += tl.load(ape_ptr + slot * ape_stride_0 + offs, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
max_score = tl.maximum(max_score, score)
|
||||
|
||||
# --- Pass 2: softmax-weighted sum of K ---
|
||||
acc = tl.full((BLOCK_D,), 0.0, tl.float32)
|
||||
denom = tl.full((BLOCK_D,), 0.0, tl.float32)
|
||||
for slot in tl.static_range(0, POOL_SIZE):
|
||||
score = tl.load(
|
||||
slot_score_ptr
|
||||
+ row * slot_score_stride_0
|
||||
+ slot * slot_score_stride_1
|
||||
+ offs,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
score += tl.load(ape_ptr + slot * ape_stride_0 + offs, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
prob = tl.exp(score - max_score)
|
||||
denom += prob
|
||||
k = tl.load(
|
||||
slot_k_ptr + row * slot_k_stride_0 + slot * slot_k_stride_1 + offs,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
acc += k * prob
|
||||
|
||||
x = acc / denom
|
||||
x = tl.where(do_write, x, 0.0).to(tl.bfloat16).to(tl.float32)
|
||||
|
||||
# Match the unfused pooled-K path's bf16 precision before quantization.
|
||||
x = _hadamard128(x).to(tl.bfloat16).to(tl.float32)
|
||||
|
||||
# --- per-vector absmax fp8 quant ---
|
||||
fp8_max = 448.0
|
||||
fp8_max_inv = 1.0 / fp8_max
|
||||
absmax = tl.max(tl.abs(x), axis=0)
|
||||
absmax = tl.maximum(absmax, 1e-4)
|
||||
if ROUND_SCALE:
|
||||
scale = tl.exp2(tl.ceil(tl.log2(absmax * fp8_max_inv)))
|
||||
else:
|
||||
scale = absmax * fp8_max_inv
|
||||
quantized = x / scale
|
||||
quantized = tl.minimum(tl.maximum(quantized, -fp8_max), fp8_max)
|
||||
|
||||
if WRITE_CACHE:
|
||||
loc = tl.load(loc_ptr + row, mask=do_write, other=0)
|
||||
loc_page_index = loc // PAGE_SIZE
|
||||
loc_token_offset_in_page = loc % PAGE_SIZE
|
||||
out_k_offsets = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE
|
||||
+ loc_token_offset_in_page * HEAD_DIM
|
||||
+ offs
|
||||
)
|
||||
out_s_offset = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE // 4
|
||||
+ S_OFFSET_NBYTES_IN_PAGE // 4
|
||||
+ loc_token_offset_in_page
|
||||
)
|
||||
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=mask)
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
|
||||
|
||||
if RETURN_COMPRESSED:
|
||||
tl.store(
|
||||
compressed_k_ptr + row * HEAD_DIM + offs,
|
||||
quantized,
|
||||
mask=offs < HEAD_DIM,
|
||||
)
|
||||
tl.store(compressed_scale_ptr + row, scale)
|
||||
|
||||
|
||||
def kpool_compress_and_write_cache(
|
||||
kv_cache: torch.Tensor,
|
||||
slot_k: torch.Tensor,
|
||||
slot_score: torch.Tensor,
|
||||
ape: torch.Tensor,
|
||||
loc: torch.Tensor,
|
||||
pool_size: int,
|
||||
head_dim: int = INDEX_HEAD_DIM,
|
||||
write_mask: torch.Tensor | None = None,
|
||||
round_scale: bool = True,
|
||||
return_compressed: bool = False,
|
||||
write_cache: bool = True,
|
||||
):
|
||||
"""Compress ``pool_size`` tokens into one fp8 K and write at ``loc``.
|
||||
|
||||
Args:
|
||||
kv_cache: indexer K cache ``[num_blocks, block_size, head_dim+4]`` uint8.
|
||||
slot_k: ``[n_pools, pool_size, head_dim]`` bf16 — raw per-token K.
|
||||
slot_score: ``[n_pools, pool_size, head_dim]`` — per-token gate score.
|
||||
ape: ``[pool_size, head_dim]`` fp32 — per-slot position bias.
|
||||
loc: ``[n_pools]`` int64 — flat physical slot per pool.
|
||||
"""
|
||||
assert slot_k.ndim == 3
|
||||
assert slot_score.shape == slot_k.shape
|
||||
assert ape.shape == slot_k.shape[1:]
|
||||
assert slot_k.shape[2] == head_dim
|
||||
assert slot_k.dtype == torch.bfloat16
|
||||
assert ape.dtype == torch.float32
|
||||
assert kv_cache.dtype == torch.uint8
|
||||
assert loc.dtype == torch.int64
|
||||
assert write_cache or return_compressed
|
||||
|
||||
page_size = kv_cache.shape[1]
|
||||
buf = kv_cache
|
||||
slot_k = slot_k.contiguous()
|
||||
slot_score = slot_score.contiguous()
|
||||
ape = ape.contiguous()
|
||||
loc = loc.contiguous()
|
||||
if write_mask is None:
|
||||
write_mask = torch.empty((1,), dtype=torch.bool, device=slot_k.device)
|
||||
has_write_mask = False
|
||||
else:
|
||||
assert write_mask.shape == (slot_k.shape[0],)
|
||||
write_mask = write_mask.contiguous()
|
||||
has_write_mask = True
|
||||
assert not return_compressed
|
||||
|
||||
if slot_k.shape[0] == 0:
|
||||
if return_compressed:
|
||||
return (
|
||||
torch.empty(
|
||||
(0, head_dim),
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=slot_k.device,
|
||||
),
|
||||
torch.empty((0,), dtype=torch.float32, device=slot_k.device),
|
||||
)
|
||||
return None
|
||||
|
||||
buf_fp8 = buf.view(torch.float8_e4m3fn)
|
||||
buf_fp32 = buf.view(torch.float32)
|
||||
# bytes per page (last dim of kv_cache) viewed as uint8
|
||||
buf_numel_per_page = buf.stride(0)
|
||||
s_offset_nbytes_in_page = page_size * head_dim
|
||||
|
||||
if return_compressed:
|
||||
compressed_k = torch.empty(
|
||||
(slot_k.shape[0], head_dim),
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=slot_k.device,
|
||||
)
|
||||
compressed_scale = torch.empty(
|
||||
(slot_k.shape[0],), dtype=torch.float32, device=slot_k.device
|
||||
)
|
||||
else:
|
||||
compressed_k = buf_fp8
|
||||
compressed_scale = buf_fp32
|
||||
|
||||
_kpool_softmax_rotate_write_cache_kernel[(slot_k.shape[0],)](
|
||||
buf_fp8,
|
||||
buf_fp32,
|
||||
slot_k,
|
||||
slot_score,
|
||||
ape,
|
||||
loc,
|
||||
write_mask,
|
||||
compressed_k,
|
||||
compressed_scale,
|
||||
slot_k.stride(0),
|
||||
slot_k.stride(1),
|
||||
slot_score.stride(0),
|
||||
slot_score.stride(1),
|
||||
ape.stride(0),
|
||||
PAGE_SIZE=page_size,
|
||||
BUF_NUMEL_PER_PAGE=buf_numel_per_page,
|
||||
POOL_SIZE=slot_k.shape[1],
|
||||
HEAD_DIM=head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=s_offset_nbytes_in_page,
|
||||
ROUND_SCALE=round_scale,
|
||||
HAS_WRITE_MASK=has_write_mask,
|
||||
RETURN_COMPRESSED=return_compressed,
|
||||
WRITE_CACHE=write_cache,
|
||||
BLOCK_D=triton.next_power_of_2(head_dim),
|
||||
)
|
||||
|
||||
if return_compressed:
|
||||
return compressed_k, compressed_scale
|
||||
return None
|
||||
|
||||
|
||||
# Seed each request's incomplete pool into its paged tail during prefill.
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _kpool_tail_seed_kernel(
|
||||
key_ptr,
|
||||
score_ptr,
|
||||
tslot_ptr,
|
||||
tail_ptr,
|
||||
n_tokens,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
KPOOL: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""Copy token ``i``'s raw K + gate into its request's tail block.
|
||||
|
||||
Token ``i`` is among its request's last KPOOL tokens iff the token KPOOL
|
||||
ahead belongs to a different tail block (or is past the batch / padding,
|
||||
slot < 0). ``tslot = block * KPOOL + pos % KPOOL``; the destination is
|
||||
``tail[block, {0:K, 1:score}, pos % KPOOL, :]``.
|
||||
"""
|
||||
i = tl.program_id(0)
|
||||
t = tl.load(tslot_ptr + i).to(tl.int64)
|
||||
if t < 0:
|
||||
return
|
||||
blk = t // KPOOL # t >= 0 here, so trunc == floor
|
||||
ahead = tl.load(tslot_ptr + i + KPOOL, mask=i + KPOOL < n_tokens, other=-1).to(
|
||||
tl.int64
|
||||
)
|
||||
# Match the torch semantics exactly: a negative ahead slot floors to a
|
||||
# block id that differs from every real block -> token is in the tail.
|
||||
# Only divide non-negative slots (Triton int div truncates, torch floors).
|
||||
if ahead >= 0 and ahead // KPOOL == blk:
|
||||
return
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
m = offs < HEAD_DIM
|
||||
base = (blk * 2 * KPOOL + t % KPOOL) * HEAD_DIM
|
||||
k = tl.load(key_ptr + i * HEAD_DIM + offs, mask=m)
|
||||
s = tl.load(score_ptr + i * HEAD_DIM + offs, mask=m)
|
||||
tl.store(tail_ptr + base + offs, k, mask=m)
|
||||
tl.store(tail_ptr + base + KPOOL * HEAD_DIM + offs, s, mask=m)
|
||||
|
||||
|
||||
def kpool_seed_tail_cache(
|
||||
tail_kv_cache: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
gate_score: torch.Tensor,
|
||||
tslot: torch.Tensor,
|
||||
kpool: int,
|
||||
head_dim: int = INDEX_HEAD_DIM,
|
||||
) -> None:
|
||||
"""Seed the paged tail cache from a prefill batch (see the kernel)."""
|
||||
assert tail_kv_cache.dtype == torch.bfloat16
|
||||
assert key.dtype == torch.bfloat16
|
||||
n = tslot.shape[0]
|
||||
if n == 0:
|
||||
return
|
||||
_kpool_tail_seed_kernel[(n,)](
|
||||
key,
|
||||
gate_score,
|
||||
tslot,
|
||||
tail_kv_cache,
|
||||
n,
|
||||
HEAD_DIM=head_dim,
|
||||
KPOOL=kpool,
|
||||
BLOCK_D=triton.next_power_of_2(head_dim),
|
||||
)
|
||||
|
||||
|
||||
# Update each request's tail during decode and write completed pools.
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _kpool_decode_update_batched_kernel(
|
||||
buf_fp8_ptr,
|
||||
buf_fp32_ptr,
|
||||
tail_kv_ptr,
|
||||
tail_slot_mapping_ptr, # [B, NEXT_N] int32
|
||||
key_ptr, # [B, NEXT_N, HEAD_DIM] bf16
|
||||
key_stride_b,
|
||||
key_stride_t,
|
||||
slot_score_ptr, # [B, NEXT_N, HEAD_DIM] bf16
|
||||
ss_stride_b,
|
||||
ss_stride_t,
|
||||
ape_ptr,
|
||||
ape_stride_0,
|
||||
slot_mapping_ptr, # [B, NEXT_N] int32
|
||||
positions_ptr, # [B, NEXT_N] int32
|
||||
NEXT_N, # runtime token count per request (no .item() needed)
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
BUF_NUMEL_PER_PAGE: tl.constexpr,
|
||||
POOL_SIZE: tl.constexpr,
|
||||
TAIL_BLOCK_ELEMS: tl.constexpr,
|
||||
KPOOL_HEAD: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""One program per request; iterates its NEXT_N verify tokens in order.
|
||||
|
||||
Replaces the caller's per-token sequential launch loop. The intra-request
|
||||
iteration MUST stay in position order: a pool-completion at token t* reads
|
||||
the tail-ring slots that tokens t < t* (same request) just stashed in this
|
||||
same invocation. ``tl.range`` iterates sequentially within the program, so
|
||||
those stashes are visible to the later completion read. Cross-request
|
||||
programs are independent (distinct tail blocks). With NEXT_N < POOL_SIZE
|
||||
(the spec-verify case: NEXT_N ~= num_spec+1, POOL_SIZE=16) at most one
|
||||
completion can occur per request per call, but the ordered loop is correct
|
||||
for any NEXT_N.
|
||||
"""
|
||||
req = tl.program_id(0)
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
dim_mask = offs < HEAD_DIM
|
||||
|
||||
for t in tl.range(0, NEXT_N):
|
||||
idx = req * NEXT_N + t
|
||||
cache_loc = tl.load(slot_mapping_ptr + idx)
|
||||
pos = tl.load(positions_ptr + idx)
|
||||
safe_pos = tl.maximum(pos, 0)
|
||||
pos_valid = (cache_loc >= 0) & (pos >= 0)
|
||||
|
||||
slot = safe_pos % POOL_SIZE
|
||||
phys_slot = safe_pos % POOL_SIZE
|
||||
|
||||
# Derive the tail block from THIS token's tail_slot (the request's block
|
||||
# is constant across a pool, but a padded / invalid entry carries a
|
||||
# negative sentinel -- reading it from token 0 would poison every
|
||||
# token's base address). Clamp so an invalid entry can never form an
|
||||
# out-of-bounds base; the accesses below are gated on pos_valid anyway.
|
||||
tail_slot = tl.load(tail_slot_mapping_ptr + idx)
|
||||
block = tl.maximum(tail_slot, 0).to(tl.int64) // POOL_SIZE
|
||||
block_base = block * TAIL_BLOCK_ELEMS
|
||||
|
||||
# The tail-ring stash must run for EVERY real token, so it is gated on
|
||||
# the token-granular tail slot -- not on `pos_valid`, which keys off the
|
||||
# POOL-granular `slot_mapping` and is therefore only true on the pool's
|
||||
# last token. Gating the stash on pos_valid dropped every intra-pool
|
||||
# token, so a decode-built pool compressed 3 stale ring entries (the
|
||||
# prefill-seeded prompt tail, frozen forever) plus the current token.
|
||||
stash_valid = (pos >= 0) & (tail_slot >= 0)
|
||||
|
||||
key = tl.load(
|
||||
key_ptr + req * key_stride_b + t * key_stride_t + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
score_current = tl.load(
|
||||
slot_score_ptr + req * ss_stride_b + t * ss_stride_t + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
|
||||
if pos_valid & (slot == POOL_SIZE - 1):
|
||||
pool_logical_start = safe_pos - slot
|
||||
|
||||
max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32)
|
||||
for pool_slot in tl.static_range(0, POOL_SIZE):
|
||||
is_current = pool_slot == slot
|
||||
phys = (pool_logical_start + pool_slot) % POOL_SIZE
|
||||
score_buf = tl.load(
|
||||
tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
score = tl.where(is_current, score_current, score_buf)
|
||||
score += tl.load(
|
||||
ape_ptr + pool_slot * ape_stride_0 + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
max_score = tl.maximum(max_score, score)
|
||||
|
||||
acc = tl.full((BLOCK_D,), 0.0, tl.float32)
|
||||
denom = tl.full((BLOCK_D,), 0.0, tl.float32)
|
||||
for pool_slot in tl.static_range(0, POOL_SIZE):
|
||||
is_current = pool_slot == slot
|
||||
phys = (pool_logical_start + pool_slot) % POOL_SIZE
|
||||
score_buf = tl.load(
|
||||
tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
score = tl.where(is_current, score_current, score_buf)
|
||||
score += tl.load(
|
||||
ape_ptr + pool_slot * ape_stride_0 + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
prob = tl.exp(score - max_score)
|
||||
denom += prob
|
||||
k_buf = tl.load(
|
||||
tail_kv_ptr + block_base + phys * HEAD_DIM + offs,
|
||||
mask=dim_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
k = tl.where(is_current, key, k_buf)
|
||||
acc += k * prob
|
||||
|
||||
x = (acc / denom).to(tl.bfloat16).to(tl.float32)
|
||||
x = _hadamard128(x).to(tl.bfloat16).to(tl.float32)
|
||||
|
||||
fp8_max = 448.0
|
||||
fp8_max_inv = 1.0 / fp8_max
|
||||
absmax = tl.maximum(tl.max(tl.abs(x), axis=0), 1e-4)
|
||||
if ROUND_SCALE:
|
||||
scale = tl.exp2(tl.ceil(tl.log2(absmax * fp8_max_inv)))
|
||||
else:
|
||||
scale = absmax * fp8_max_inv
|
||||
quantized = tl.minimum(tl.maximum(x / scale, -fp8_max), fp8_max)
|
||||
|
||||
loc = cache_loc.to(tl.int64)
|
||||
loc_page_index = loc // PAGE_SIZE
|
||||
loc_token_offset_in_page = loc % PAGE_SIZE
|
||||
out_k_offsets = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE
|
||||
+ loc_token_offset_in_page * HEAD_DIM
|
||||
+ offs
|
||||
)
|
||||
out_s_offset = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE // 4
|
||||
+ S_OFFSET_NBYTES_IN_PAGE // 4
|
||||
+ loc_token_offset_in_page
|
||||
)
|
||||
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=dim_mask)
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale)
|
||||
|
||||
# Stash the current token AFTER any completion read so the completion
|
||||
# uses prior stashes (and the current token's own key/score via
|
||||
# is_current), then leaves this token for future pools. Order matches
|
||||
# the per-token kernel: completion read first, stash second.
|
||||
update_mask = dim_mask & stash_valid
|
||||
tl.store(
|
||||
tail_kv_ptr + block_base + phys_slot * HEAD_DIM + offs,
|
||||
key,
|
||||
mask=update_mask,
|
||||
)
|
||||
tl.store(
|
||||
tail_kv_ptr + block_base + KPOOL_HEAD + phys_slot * HEAD_DIM + offs,
|
||||
score_current,
|
||||
mask=update_mask,
|
||||
)
|
||||
|
||||
|
||||
def kpool_decode_update_and_maybe_write_cache_batched(
|
||||
kv_cache: torch.Tensor,
|
||||
tail_kv_cache: torch.Tensor,
|
||||
tail_slot_mapping: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
slot_score: torch.Tensor,
|
||||
ape: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
pool_size: int,
|
||||
head_dim: int = INDEX_HEAD_DIM,
|
||||
round_scale: bool = True,
|
||||
) -> None:
|
||||
"""Batched decode-step kpool update for spec verify (``next_n > 1``).
|
||||
|
||||
One launch replaces the caller's per-token loop. Inputs are grouped per
|
||||
request: ``[num_requests, next_n, ...]``. Each program handles one
|
||||
request's ``next_n`` tokens in position order (see the kernel docstring for
|
||||
why ordering is required for pool-completion correctness).
|
||||
|
||||
Plain decode (``next_n == 1``) is handled here too — the kernel collapses
|
||||
to a single-iteration loop.
|
||||
|
||||
Args:
|
||||
kv_cache: indexer K cache ``[num_blocks, block_size, head_dim+4]`` uint8.
|
||||
tail_kv_cache: paged tail cache ``[num_blocks, 2, pool_size, head_dim]``
|
||||
bf16 (K at half 0, gate score at half 1).
|
||||
tail_slot_mapping: ``[num_requests, next_n]`` int32.
|
||||
key / slot_score: ``[num_requests, next_n, head_dim]`` bf16.
|
||||
ape: ``[pool_size, head_dim]`` fp32.
|
||||
slot_mapping / positions: ``[num_requests, next_n]`` int32.
|
||||
"""
|
||||
num_requests, next_n = key.shape[0], key.shape[1]
|
||||
if num_requests == 0 or next_n == 0:
|
||||
return
|
||||
assert tail_kv_cache.ndim == 4
|
||||
assert tail_kv_cache.shape[1] == 2
|
||||
assert tail_kv_cache.shape[2] == pool_size
|
||||
assert tail_kv_cache.shape[3] == head_dim
|
||||
assert tail_kv_cache.dtype == torch.bfloat16
|
||||
assert key.ndim == 3 and key.shape[2] == head_dim
|
||||
assert slot_score.shape == key.shape
|
||||
assert ape.shape == (pool_size, head_dim)
|
||||
assert tail_slot_mapping.shape == (num_requests, next_n)
|
||||
assert slot_mapping.shape == (num_requests, next_n)
|
||||
assert positions.shape == (num_requests, next_n)
|
||||
assert key.dtype == torch.bfloat16
|
||||
assert slot_score.dtype == torch.bfloat16
|
||||
assert ape.dtype == torch.float32
|
||||
assert kv_cache.dtype == torch.uint8
|
||||
|
||||
page_size = kv_cache.shape[1]
|
||||
buf = kv_cache
|
||||
buf_fp8 = buf.view(torch.float8_e4m3fn)
|
||||
buf_fp32 = buf.view(torch.float32)
|
||||
|
||||
# The kernel indexes the int tensors as ``req * next_n + t`` (row-major),
|
||||
# so they must be contiguous. Callers pass either a view of a contiguous
|
||||
# slice or a freshly scattered tensor, making these no-ops; the calls guard
|
||||
# against a future caller handing over a strided view.
|
||||
tail_slot_mapping = tail_slot_mapping.contiguous()
|
||||
slot_mapping = slot_mapping.contiguous()
|
||||
positions = positions.contiguous()
|
||||
|
||||
_kpool_decode_update_batched_kernel[(num_requests,)](
|
||||
buf_fp8,
|
||||
buf_fp32,
|
||||
tail_kv_cache,
|
||||
tail_slot_mapping,
|
||||
key,
|
||||
key.stride(0),
|
||||
key.stride(1),
|
||||
slot_score,
|
||||
slot_score.stride(0),
|
||||
slot_score.stride(1),
|
||||
ape,
|
||||
ape.stride(0),
|
||||
slot_mapping,
|
||||
positions,
|
||||
next_n,
|
||||
PAGE_SIZE=page_size,
|
||||
BUF_NUMEL_PER_PAGE=buf.stride(0),
|
||||
POOL_SIZE=pool_size,
|
||||
TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0),
|
||||
KPOOL_HEAD=tail_kv_cache.stride(1),
|
||||
HEAD_DIM=head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=page_size * head_dim,
|
||||
ROUND_SCALE=round_scale,
|
||||
BLOCK_D=triton.next_power_of_2(head_dim),
|
||||
)
|
||||
|
||||
|
||||
# Pool-level top-k helpers.
|
||||
|
||||
|
||||
def history_group_budget_for_topk(topk: int, pool_size: int) -> int:
|
||||
"""Number of pools to select so that expanding yields ``topk`` tokens."""
|
||||
assert topk % pool_size == 0
|
||||
return topk // pool_size
|
||||
|
||||
|
||||
def expand_pools_to_tokens(
|
||||
group_ids: torch.Tensor,
|
||||
group_valid: torch.Tensor,
|
||||
topk: int,
|
||||
pool_size: int,
|
||||
page_table: torch.Tensor | None = None,
|
||||
topk_offsets: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Expand selected full-pool ids to a strict-width token topk tensor."""
|
||||
assert group_ids.ndim == 2
|
||||
assert group_valid.shape == group_ids.shape
|
||||
assert topk % pool_size == 0
|
||||
assert group_ids.shape[1] == history_group_budget_for_topk(topk, pool_size)
|
||||
assert page_table is None or topk_offsets is None
|
||||
|
||||
device = group_ids.device
|
||||
offsets = torch.arange(pool_size, device=device, dtype=torch.int64)
|
||||
token_ids = group_ids.to(torch.int64).unsqueeze(-1) * pool_size + offsets
|
||||
token_ids = token_ids.reshape(group_ids.shape[0], topk)
|
||||
valid = (
|
||||
group_valid.unsqueeze(-1)
|
||||
.expand(-1, -1, pool_size)
|
||||
.reshape(group_ids.shape[0], topk)
|
||||
)
|
||||
|
||||
if page_table is not None:
|
||||
assert page_table.ndim == 2
|
||||
safe_ids = token_ids.clamp(min=0, max=page_table.shape[1] - 1)
|
||||
output = torch.gather(page_table, dim=1, index=safe_ids).to(torch.int32)
|
||||
elif topk_offsets is not None:
|
||||
if topk_offsets.ndim == 2:
|
||||
assert topk_offsets.shape[1] == 1
|
||||
topk_offsets = topk_offsets.squeeze(1)
|
||||
output = (token_ids + topk_offsets.to(torch.int64).unsqueeze(1)).to(torch.int32)
|
||||
else:
|
||||
output = token_ids.to(torch.int32)
|
||||
|
||||
return torch.where(valid, output, torch.full_like(output, -1))
|
||||
|
||||
|
||||
def append_tail_to_topk(
|
||||
topk_result: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
pool_lens: torch.Tensor,
|
||||
pool_size: int,
|
||||
page_table: torch.Tensor | None = None,
|
||||
topk_offsets: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Append non-pooled tail tokens after expanded history tokens.
|
||||
|
||||
``index_kpool_always_select_tail`` keeps the (incomplete) trailing pool so
|
||||
the most recent tokens are always attended to.
|
||||
"""
|
||||
assert topk_result.dtype == torch.int32
|
||||
assert seq_lens.ndim == 1
|
||||
assert pool_lens.ndim == 1
|
||||
|
||||
tail_pool = pool_size - 1
|
||||
if tail_pool == 0:
|
||||
return topk_result
|
||||
|
||||
rows, n_cols = topk_result.shape
|
||||
out_cols = n_cols + tail_pool
|
||||
out = torch.empty(
|
||||
(rows, out_cols), dtype=topk_result.dtype, device=topk_result.device
|
||||
)
|
||||
|
||||
# tail tokens: [pool_len*pool_size, seq_len) for each row.
|
||||
pool_len = pool_lens.to(torch.int32)
|
||||
tail_start = pool_len * pool_size
|
||||
seq_len = seq_lens.to(torch.int32)
|
||||
tail_count = seq_len - tail_start # in [0, pool_size)
|
||||
|
||||
cols = torch.arange(out_cols, device=topk_result.device)[None, :]
|
||||
history_len = n_cols
|
||||
is_history = cols < history_len
|
||||
tail_off = cols - history_len
|
||||
is_tail = (tail_off >= 0) & (tail_off < tail_count[:, None])
|
||||
|
||||
# safe_hist must be per-row [rows, out_cols] so the gather reads each row's
|
||||
# OWN history. cols is [1, out_cols]; if used directly, gather (which does
|
||||
# NOT broadcast the index) would read only row 0 of topk_result, making every
|
||||
# query inherit row 0's history (empty for the first token) and lose all its
|
||||
# selected tokens — only the per-row tail would survive. This only manifests
|
||||
# for multi-row sparse PREFILL (decode has 1 row, so it reads its own row 0).
|
||||
safe_hist = torch.minimum(cols, torch.full_like(cols, n_cols - 1)).expand(
|
||||
rows, out_cols
|
||||
)
|
||||
history_val = torch.gather(topk_result, 1, safe_hist)
|
||||
|
||||
tail_raw = tail_start[:, None] + tail_off
|
||||
tail_val = tail_raw.to(torch.int32)
|
||||
if page_table is not None:
|
||||
safe_tail = tail_raw.clamp(min=0, max=page_table.shape[1] - 1)
|
||||
tail_val = torch.gather(page_table, 1, safe_tail).to(torch.int32)
|
||||
elif topk_offsets is not None:
|
||||
tail_val = (tail_raw + topk_offsets.to(torch.int64).unsqueeze(1)).to(
|
||||
torch.int32
|
||||
)
|
||||
|
||||
out = torch.where(is_history, history_val, -1)
|
||||
out = torch.where(is_tail, tail_val, out)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _expand_pools_and_append_tail_kernel(
|
||||
pool_ids_ptr, # [rows, n_groups], int (any int dtype)
|
||||
seq_lens_ptr, # [rows], int32 (token-granular seq_len)
|
||||
out_ptr, # [rows, out_cols], int32
|
||||
topk, # n_groups * pool_size
|
||||
out_cols, # topk + pool_size - 1
|
||||
POOL_SIZE: tl.constexpr,
|
||||
BLOCK_COLS: tl.constexpr,
|
||||
pid_s0,
|
||||
out_s0,
|
||||
):
|
||||
# Fuses expand_pools_to_tokens + append_tail_to_topk (identity path) into a
|
||||
# single kernel. Each program writes one (row, column-tile) of the output.
|
||||
row = tl.program_id(0)
|
||||
tile = tl.program_id(1)
|
||||
cols = tile * BLOCK_COLS + tl.arange(0, BLOCK_COLS)
|
||||
mask = cols < out_cols
|
||||
|
||||
seq_len = tl.load(seq_lens_ptr + row)
|
||||
pool_len = seq_len // POOL_SIZE
|
||||
tail_start = pool_len * POOL_SIZE
|
||||
tail_count = seq_len - tail_start # in [0, POOL_SIZE)
|
||||
|
||||
# History region [0, topk): expand selected pool g = cols // POOL_SIZE.
|
||||
is_history = cols < topk
|
||||
g = cols // POOL_SIZE
|
||||
o = cols % POOL_SIZE
|
||||
pid = tl.load(pool_ids_ptr + row * pid_s0 + g, mask=mask & is_history, other=-1)
|
||||
hist_val = (pid * POOL_SIZE + o).to(tl.int32)
|
||||
hist_out = tl.where(pid >= 0, hist_val, -1)
|
||||
|
||||
# Tail region [topk, out_cols): the request's trailing incomplete pool.
|
||||
tail_off = cols - topk
|
||||
is_tail = (tail_off >= 0) & (tail_off < tail_count)
|
||||
tail_val = (tail_start + tail_off).to(tl.int32)
|
||||
tail_out = tl.where(is_tail, tail_val, -1)
|
||||
|
||||
result = tl.where(is_history, hist_out, tail_out)
|
||||
tl.store(out_ptr + row * out_s0 + cols, result, mask=mask)
|
||||
|
||||
|
||||
def expand_pools_and_append_tail(
|
||||
pool_ids: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
pool_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Fuse ``expand_pools_to_tokens`` + ``append_tail_to_topk`` (identity path).
|
||||
|
||||
Produces the same ``[rows, topk + pool_size - 1]`` int32 output as calling
|
||||
the two functions in sequence when neither ``page_table`` nor
|
||||
``topk_offsets`` is passed — the only path used by the GLM-5.3-Flash indexer.
|
||||
The kernel derives ``pool_len = seq_len // pool_size`` internally, so the
|
||||
caller no longer needs to precompute it. Replaces ~25 elementwise kernels
|
||||
with one Triton launch.
|
||||
"""
|
||||
rows, n_groups = pool_ids.shape
|
||||
topk = n_groups * pool_size
|
||||
out_cols = topk + pool_size - 1
|
||||
out = torch.empty((rows, out_cols), dtype=torch.int32, device=pool_ids.device)
|
||||
BLOCK_COLS = 128
|
||||
n_tiles = triton.cdiv(out_cols, BLOCK_COLS)
|
||||
_expand_pools_and_append_tail_kernel[(rows, n_tiles)](
|
||||
pool_ids,
|
||||
seq_lens,
|
||||
out,
|
||||
topk,
|
||||
out_cols,
|
||||
POOL_SIZE=pool_size,
|
||||
BLOCK_COLS=BLOCK_COLS,
|
||||
pid_s0=pool_ids.stride(0),
|
||||
out_s0=out.stride(0),
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .kernels import chunk_kda_with_fused_gate, fused_recurrent_kda
|
||||
|
||||
__all__ = ["chunk_kda_with_fused_gate", "fused_recurrent_kda"]
|
||||
@@ -0,0 +1,656 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang
|
||||
# mypy: ignore-errors
|
||||
#
|
||||
# This file contains code copied from the flash-linear-attention project.
|
||||
# The original source code was licensed under the MIT license and included
|
||||
# the following copyright notice:
|
||||
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
||||
# ruff: noqa: E501
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.op import exp
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"USE_INITIAL_STATE": lambda args: args["h0"] is not None,
|
||||
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
||||
"IS_CONTINUOUS_BATCHING": lambda args: args["ssm_state_indices"] is not None,
|
||||
"IS_SPEC_DECODING": lambda args: args["num_accepted_tokens"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.jit(do_not_specialize=["N", "T"])
|
||||
def fused_recurrent_gated_delta_rule_fwd_kernel(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
g,
|
||||
beta,
|
||||
o,
|
||||
h0,
|
||||
ht,
|
||||
cu_seqlens,
|
||||
ssm_state_indices,
|
||||
num_accepted_tokens,
|
||||
a_log,
|
||||
g_bias,
|
||||
scale,
|
||||
N: tl.int64, # num of sequences
|
||||
T: tl.int64, # num of tokens
|
||||
B: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
stride_init_state_token: tl.constexpr,
|
||||
stride_final_state_token: tl.constexpr,
|
||||
stride_indices_seq: tl.constexpr,
|
||||
stride_indices_tok: tl.constexpr,
|
||||
USE_INITIAL_STATE: tl.constexpr, # whether to use initial state
|
||||
INPLACE_FINAL_STATE: tl.constexpr, # whether to store final state inplace
|
||||
IS_BETA_HEADWISE: tl.constexpr, # whether beta is headwise vector or scalar,
|
||||
USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
IS_CONTINUOUS_BATCHING: tl.constexpr,
|
||||
IS_SPEC_DECODING: tl.constexpr,
|
||||
IS_KDA: tl.constexpr,
|
||||
SIGMOID_BETA: tl.constexpr, # beta holds raw logits; sigmoid at fp32 load
|
||||
COMPUTE_GATE: tl.constexpr, # g holds raw logits; KDA gate computed in-kernel
|
||||
SAFE_GATE: tl.constexpr, # bounded gate variant (only branch implemented)
|
||||
LOWER_BOUND: tl.constexpr,
|
||||
):
|
||||
i_k, i_v, i_nh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_n, i_hv = i_nh // HV, i_nh % HV
|
||||
i_h = i_hv // (HV // H)
|
||||
if IS_VARLEN:
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int64),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int64),
|
||||
)
|
||||
all = T
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_n * T, i_n * T + T
|
||||
all = B * T
|
||||
|
||||
if T == 0:
|
||||
# no tokens to process for this sequence
|
||||
return
|
||||
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
|
||||
p_q = q + (bos * H + i_h) * K + o_k
|
||||
p_k = k + (bos * H + i_h) * K + o_k
|
||||
p_v = v + (bos * HV + i_hv) * V + o_v
|
||||
if IS_BETA_HEADWISE:
|
||||
p_beta = beta + (bos * HV + i_hv) * V + o_v
|
||||
else:
|
||||
p_beta = beta + bos * HV + i_hv
|
||||
|
||||
if not IS_KDA:
|
||||
p_g = g + bos * HV + i_hv
|
||||
else:
|
||||
p_gk = g + (bos * HV + i_hv) * K + o_k
|
||||
|
||||
# Per-head gate amplitude, hoisted out of the token loop (COMPUTE_GATE).
|
||||
if COMPUTE_GATE:
|
||||
b_a_log = tl.exp(tl.load(a_log + i_h).to(tl.float32))
|
||||
|
||||
p_o = o + ((i_k * all + bos) * HV + i_hv) * V + o_v
|
||||
|
||||
mask_k = o_k < K
|
||||
mask_v = o_v < V
|
||||
mask_h = mask_v[:, None] & mask_k[None, :]
|
||||
|
||||
b_h = tl.zeros([BV, BK], dtype=tl.float32)
|
||||
if USE_INITIAL_STATE:
|
||||
if IS_CONTINUOUS_BATCHING:
|
||||
if IS_SPEC_DECODING:
|
||||
i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1
|
||||
else:
|
||||
i_t = 0
|
||||
# Load state index and check for invalid entries
|
||||
state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq + i_t).to(
|
||||
tl.int64
|
||||
)
|
||||
# Skip if state index is invalid (NULL_BLOCK_ID=0)
|
||||
if state_idx <= 0:
|
||||
return
|
||||
p_h0 = h0 + state_idx * stride_init_state_token
|
||||
else:
|
||||
p_h0 = h0 + bos * HV * V * K
|
||||
p_h0 = p_h0 + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
b_h += tl.load(p_h0, mask=mask_h, other=0).to(tl.float32)
|
||||
|
||||
for i_t in range(0, T):
|
||||
b_q = tl.load(p_q, mask=mask_k, other=0).to(tl.float32)
|
||||
b_k = tl.load(p_k, mask=mask_k, other=0).to(tl.float32)
|
||||
b_v = tl.load(p_v, mask=mask_v, other=0).to(tl.float32)
|
||||
|
||||
if USE_QK_L2NORM_IN_KERNEL:
|
||||
b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
|
||||
b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
|
||||
b_q = b_q * scale
|
||||
# [BV, BK]
|
||||
if not IS_KDA:
|
||||
b_g = tl.load(p_g).to(tl.float32)
|
||||
b_h *= exp(b_g)
|
||||
else:
|
||||
b_gk = tl.load(p_gk).to(tl.float32)
|
||||
if COMPUTE_GATE:
|
||||
# Replicates kda_gate_fwd_kernel's SAFE_GATE branch
|
||||
# bit-for-bit (same tl.exp, same fp32 math; the intermediate
|
||||
# gate value this replaces was stored/reloaded as fp32,
|
||||
# which is lossless): y = lb / (1 + exp(-exp(A)*(g+bias))).
|
||||
b_gk += tl.load(g_bias + i_h * K + o_k, mask=mask_k, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
b_gk = LOWER_BOUND / (1.0 + tl.exp(-(b_a_log * b_gk)))
|
||||
b_h *= exp(b_gk[None, :])
|
||||
# [BV]
|
||||
b_v -= tl.sum(b_h * b_k[None, :], 1)
|
||||
if IS_BETA_HEADWISE:
|
||||
b_beta = tl.load(p_beta, mask=mask_v, other=0).to(tl.float32)
|
||||
else:
|
||||
b_beta = tl.load(p_beta).to(tl.float32)
|
||||
# Matches torch's `x.float().sigmoid()` pre-computation bit-for-bit
|
||||
# on the input side (bf16->fp32 is exact); only the sigmoid impl itself
|
||||
# can differ by <=1 ULP.
|
||||
if SIGMOID_BETA:
|
||||
b_beta = tl.sigmoid(b_beta)
|
||||
b_v *= b_beta
|
||||
# [BV, BK]
|
||||
b_h += b_v[:, None] * b_k[None, :]
|
||||
# [BV]
|
||||
b_o = tl.sum(b_h * b_q[None, :], 1)
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v)
|
||||
|
||||
# keep the states for multi-query tokens
|
||||
if INPLACE_FINAL_STATE:
|
||||
# Load state index and check for invalid entries
|
||||
final_state_idx = tl.load(
|
||||
ssm_state_indices + i_n * stride_indices_seq + i_t
|
||||
).to(tl.int64)
|
||||
# Only store if state index is valid (not NULL_BLOCK_ID=0)
|
||||
if final_state_idx > 0:
|
||||
p_ht = ht + final_state_idx * stride_final_state_token
|
||||
p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
|
||||
else:
|
||||
p_ht = ht + (bos + i_t) * stride_final_state_token
|
||||
p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
|
||||
|
||||
p_q += H * K
|
||||
p_k += H * K
|
||||
p_o += HV * V
|
||||
p_v += HV * V
|
||||
if not IS_KDA:
|
||||
p_g += HV
|
||||
else:
|
||||
p_gk += HV * K
|
||||
p_beta += HV * (V if IS_BETA_HEADWISE else 1)
|
||||
|
||||
|
||||
def fused_recurrent_gated_delta_rule_fwd(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
inplace_final_state: bool = True,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
ssm_state_indices: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
B, T, H, K, V = *k.shape, v.shape[-1]
|
||||
HV = v.shape[2]
|
||||
N = B if cu_seqlens is None else len(cu_seqlens) - 1
|
||||
BK, BV = triton.next_power_of_2(K), min(triton.next_power_of_2(V), 32)
|
||||
NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV)
|
||||
assert NK == 1, "NK > 1 is not supported yet"
|
||||
num_stages = 3
|
||||
num_warps = 1
|
||||
|
||||
o = q.new_empty(NK, *v.shape)
|
||||
if inplace_final_state:
|
||||
final_state = initial_state
|
||||
else:
|
||||
final_state = q.new_empty(T, HV, V, K, dtype=initial_state.dtype)
|
||||
|
||||
stride_init_state_token = initial_state.stride(0)
|
||||
stride_final_state_token = final_state.stride(0)
|
||||
|
||||
if ssm_state_indices is None:
|
||||
stride_indices_seq, stride_indices_tok = 1, 1
|
||||
elif ssm_state_indices.ndim == 1:
|
||||
stride_indices_seq, stride_indices_tok = ssm_state_indices.stride(0), 1
|
||||
else:
|
||||
stride_indices_seq, stride_indices_tok = ssm_state_indices.stride()
|
||||
|
||||
grid = (NK, NV, N * HV)
|
||||
fused_recurrent_gated_delta_rule_fwd_kernel[grid](
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
o=o,
|
||||
h0=initial_state,
|
||||
ht=final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
scale=scale,
|
||||
N=N,
|
||||
T=T,
|
||||
B=B,
|
||||
H=H,
|
||||
HV=HV,
|
||||
K=K,
|
||||
V=V,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
stride_init_state_token=stride_init_state_token,
|
||||
stride_final_state_token=stride_final_state_token,
|
||||
stride_indices_seq=stride_indices_seq,
|
||||
stride_indices_tok=stride_indices_tok,
|
||||
IS_BETA_HEADWISE=beta.ndim == v.ndim,
|
||||
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
|
||||
INPLACE_FINAL_STATE=inplace_final_state,
|
||||
IS_KDA=False,
|
||||
SIGMOID_BETA=False,
|
||||
a_log=None,
|
||||
g_bias=None,
|
||||
COMPUTE_GATE=False,
|
||||
SAFE_GATE=True,
|
||||
LOWER_BOUND=-5.0,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
)
|
||||
o = o.squeeze(0)
|
||||
return o, final_state
|
||||
|
||||
|
||||
@triton.jit
|
||||
def fused_recurrent_gated_delta_rule_packed_decode_kernel(
|
||||
mixed_qkv,
|
||||
a,
|
||||
b,
|
||||
A_log,
|
||||
dt_bias,
|
||||
o,
|
||||
h0,
|
||||
ht,
|
||||
ssm_state_indices,
|
||||
scale,
|
||||
stride_mixed_qkv_tok: tl.constexpr,
|
||||
stride_a_tok: tl.constexpr,
|
||||
stride_b_tok: tl.constexpr,
|
||||
stride_init_state_token: tl.constexpr,
|
||||
stride_final_state_token: tl.constexpr,
|
||||
stride_indices_seq: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
SOFTPLUS_THRESHOLD: tl.constexpr,
|
||||
USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
|
||||
SPLIT_BATCH_HEAD_GRID: tl.constexpr,
|
||||
):
|
||||
if SPLIT_BATCH_HEAD_GRID:
|
||||
i_v, i_hv, i_n = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
else:
|
||||
i_v, i_nh = tl.program_id(0), tl.program_id(1)
|
||||
i_n, i_hv = i_nh // HV, i_nh % HV
|
||||
i_h = i_hv // (HV // H)
|
||||
|
||||
o_k = tl.arange(0, BK)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
mask_k = o_k < K
|
||||
mask_v = o_v < V
|
||||
mask_h = mask_v[:, None] & mask_k[None, :]
|
||||
|
||||
state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq).to(tl.int64)
|
||||
p_o = o + (i_n * HV + i_hv) * V + o_v
|
||||
|
||||
# Skip if state index is invalid (NULL_BLOCK_ID=0)
|
||||
if state_idx <= 0:
|
||||
zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty)
|
||||
tl.store(p_o, zero, mask=mask_v)
|
||||
return
|
||||
|
||||
p_h0 = h0 + state_idx * stride_init_state_token
|
||||
p_h0 = p_h0 + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
b_h = tl.load(p_h0, mask=mask_h, other=0).to(tl.float32)
|
||||
|
||||
p_mixed = mixed_qkv + i_n * stride_mixed_qkv_tok
|
||||
q_off = i_h * K + o_k
|
||||
k_off = (H * K) + i_h * K + o_k
|
||||
v_off = (2 * H * K) + i_hv * V + o_v
|
||||
b_q = tl.load(p_mixed + q_off, mask=mask_k, other=0).to(tl.float32)
|
||||
b_k = tl.load(p_mixed + k_off, mask=mask_k, other=0).to(tl.float32)
|
||||
b_v = tl.load(p_mixed + v_off, mask=mask_v, other=0).to(tl.float32)
|
||||
|
||||
if USE_QK_L2NORM_IN_KERNEL:
|
||||
b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
|
||||
b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
|
||||
b_q = b_q * scale
|
||||
|
||||
a_val = tl.load(a + i_n * stride_a_tok + i_hv).to(tl.float32)
|
||||
b_val = tl.load(b + i_n * stride_b_tok + i_hv).to(tl.float32)
|
||||
A_log_val = tl.load(A_log + i_hv).to(tl.float32)
|
||||
dt_bias_val = tl.load(dt_bias + i_hv).to(tl.float32)
|
||||
x = a_val + dt_bias_val
|
||||
softplus_x = tl.where(x <= SOFTPLUS_THRESHOLD, tl.log(1.0 + tl.exp(x)), x)
|
||||
g_val = -tl.exp(A_log_val) * softplus_x
|
||||
beta_val = tl.sigmoid(b_val)
|
||||
|
||||
b_h *= exp(g_val)
|
||||
b_v -= tl.sum(b_h * b_k[None, :], 1)
|
||||
b_v *= beta_val
|
||||
b_h += b_v[:, None] * b_k[None, :]
|
||||
b_o = tl.sum(b_h * b_q[None, :], 1)
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v)
|
||||
|
||||
p_ht = ht + state_idx * stride_final_state_token
|
||||
p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
|
||||
|
||||
|
||||
def fused_recurrent_gated_delta_rule_packed_decode(
|
||||
mixed_qkv: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
ssm_state_indices: torch.Tensor,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if mixed_qkv.ndim != 2:
|
||||
raise ValueError(
|
||||
f"`mixed_qkv` must be a 2D tensor (got ndim={mixed_qkv.ndim})."
|
||||
)
|
||||
if mixed_qkv.stride(-1) != 1:
|
||||
raise ValueError("`mixed_qkv` must be contiguous in the last dim.")
|
||||
if a.ndim != 2 or b.ndim != 2:
|
||||
raise ValueError(
|
||||
f"`a` and `b` must be 2D tensors (got a.ndim={a.ndim}, b.ndim={b.ndim})."
|
||||
)
|
||||
if a.stride(-1) != 1 or b.stride(-1) != 1:
|
||||
raise ValueError("`a`/`b` must be contiguous in the last dim.")
|
||||
if A_log.ndim != 1 or dt_bias.ndim != 1:
|
||||
raise ValueError("`A_log`/`dt_bias` must be 1D tensors.")
|
||||
if A_log.stride(0) != 1 or dt_bias.stride(0) != 1:
|
||||
raise ValueError("`A_log`/`dt_bias` must be contiguous.")
|
||||
if ssm_state_indices.ndim != 1:
|
||||
raise ValueError(
|
||||
f"`ssm_state_indices` must be 1D for packed decode (got ndim={ssm_state_indices.ndim})."
|
||||
)
|
||||
if not out.is_contiguous():
|
||||
raise ValueError("`out` must be contiguous.")
|
||||
|
||||
dev = mixed_qkv.device
|
||||
if (
|
||||
a.device != dev
|
||||
or b.device != dev
|
||||
or A_log.device != dev
|
||||
or dt_bias.device != dev
|
||||
or initial_state.device != dev
|
||||
or out.device != dev
|
||||
or ssm_state_indices.device != dev
|
||||
):
|
||||
raise ValueError("All inputs must be on the same device.")
|
||||
|
||||
B = mixed_qkv.shape[0]
|
||||
if a.shape[0] != B or b.shape[0] != B:
|
||||
raise ValueError(
|
||||
"Mismatched batch sizes: "
|
||||
f"mixed_qkv.shape[0]={B}, a.shape[0]={a.shape[0]}, b.shape[0]={b.shape[0]}."
|
||||
)
|
||||
if ssm_state_indices.shape[0] != B:
|
||||
raise ValueError(
|
||||
f"`ssm_state_indices` must have shape [B] (got {tuple(ssm_state_indices.shape)}; expected ({B},))."
|
||||
)
|
||||
|
||||
if initial_state.ndim != 4:
|
||||
raise ValueError(
|
||||
f"`initial_state` must be a 4D tensor (got ndim={initial_state.ndim})."
|
||||
)
|
||||
if initial_state.stride(-1) != 1:
|
||||
raise ValueError("`initial_state` must be contiguous in the last dim.")
|
||||
HV, V, K = initial_state.shape[-3:]
|
||||
if a.shape[1] != HV or b.shape[1] != HV:
|
||||
raise ValueError(
|
||||
f"`a`/`b` must have shape [B, HV] with HV={HV} (got a.shape={tuple(a.shape)}, b.shape={tuple(b.shape)})."
|
||||
)
|
||||
if A_log.numel() != HV or dt_bias.numel() != HV:
|
||||
raise ValueError(
|
||||
f"`A_log` and `dt_bias` must have {HV} elements (got A_log.numel()={A_log.numel()}, dt_bias.numel()={dt_bias.numel()})."
|
||||
)
|
||||
if out.shape != (B, 1, HV, V):
|
||||
raise ValueError(
|
||||
f"`out` must have shape {(B, 1, HV, V)} (got out.shape={tuple(out.shape)})."
|
||||
)
|
||||
|
||||
qkv_dim = mixed_qkv.shape[1]
|
||||
qk_dim = qkv_dim - HV * V
|
||||
if qk_dim <= 0 or qk_dim % 2 != 0:
|
||||
raise ValueError(
|
||||
f"Invalid packed `mixed_qkv` last dim={qkv_dim} for HV={HV}, V={V}."
|
||||
)
|
||||
q_dim = qk_dim // 2
|
||||
if q_dim % K != 0:
|
||||
raise ValueError(f"Invalid packed Q size {q_dim}: must be divisible by K={K}.")
|
||||
H = q_dim // K
|
||||
if H <= 0 or HV % H != 0:
|
||||
raise ValueError(
|
||||
f"Invalid head config inferred from mixed_qkv: H={H}, HV={HV}."
|
||||
)
|
||||
|
||||
BK = triton.next_power_of_2(K)
|
||||
if triton.cdiv(K, BK) != 1:
|
||||
raise ValueError(
|
||||
f"Packed decode kernel only supports NK=1 (got K={K}, BK={BK})."
|
||||
)
|
||||
BV = min(triton.next_power_of_2(V), 32)
|
||||
num_stages = 3
|
||||
num_warps = 1
|
||||
|
||||
stride_mixed_qkv_tok = mixed_qkv.stride(0)
|
||||
stride_a_tok = a.stride(0)
|
||||
stride_b_tok = b.stride(0)
|
||||
stride_init_state_token = initial_state.stride(0)
|
||||
stride_final_state_token = initial_state.stride(0)
|
||||
stride_indices_seq = ssm_state_indices.stride(0)
|
||||
|
||||
NV = triton.cdiv(V, BV)
|
||||
# CUDA limits grid Y/Z dimensions to 65535.
|
||||
split_batch_head_grid = B * HV > 65535
|
||||
grid = (NV, HV, B) if split_batch_head_grid else (NV, B * HV)
|
||||
fused_recurrent_gated_delta_rule_packed_decode_kernel[grid](
|
||||
mixed_qkv=mixed_qkv,
|
||||
a=a,
|
||||
b=b,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
o=out,
|
||||
h0=initial_state,
|
||||
ht=initial_state,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
scale=scale,
|
||||
stride_mixed_qkv_tok=stride_mixed_qkv_tok,
|
||||
stride_a_tok=stride_a_tok,
|
||||
stride_b_tok=stride_b_tok,
|
||||
stride_init_state_token=stride_init_state_token,
|
||||
stride_final_state_token=stride_final_state_token,
|
||||
stride_indices_seq=stride_indices_seq,
|
||||
H=H,
|
||||
HV=HV,
|
||||
K=K,
|
||||
V=V,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
SOFTPLUS_THRESHOLD=20.0,
|
||||
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
|
||||
SPLIT_BATCH_HEAD_GRID=split_batch_head_grid,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
)
|
||||
return out, initial_state
|
||||
|
||||
|
||||
class FusedRecurrentFunction(torch.autograd.Function):
|
||||
@staticmethod
|
||||
def forward(
|
||||
ctx,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
inplace_final_state: bool = True,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
ssm_state_indices: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
):
|
||||
o, final_state = fused_recurrent_gated_delta_rule_fwd(
|
||||
q=q.contiguous(),
|
||||
k=k.contiguous(),
|
||||
v=v.contiguous(),
|
||||
g=g.contiguous(),
|
||||
beta=beta.contiguous(),
|
||||
scale=scale,
|
||||
initial_state=initial_state,
|
||||
inplace_final_state=inplace_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
|
||||
)
|
||||
|
||||
return o, final_state
|
||||
|
||||
|
||||
def fused_recurrent_gated_delta_rule(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor = None,
|
||||
scale: float = None,
|
||||
initial_state: torch.Tensor = None,
|
||||
inplace_final_state: bool = True,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
ssm_state_indices: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
r"""
|
||||
Args:
|
||||
q (torch.Tensor):
|
||||
queries of shape `[B, T, H, K]`.
|
||||
k (torch.Tensor):
|
||||
keys of shape `[B, T, H, K]`.
|
||||
v (torch.Tensor):
|
||||
values of shape `[B, T, HV, V]`.
|
||||
GVA is applied if `HV > H`.
|
||||
g (torch.Tensor):
|
||||
g (decays) of shape `[B, T, HV]`.
|
||||
beta (torch.Tensor):
|
||||
betas of shape `[B, T, HV]`.
|
||||
scale (Optional[int]):
|
||||
Scale factor for the RetNet attention scores.
|
||||
If not provided, it will default to `1 / sqrt(K)`. Default: `None`.
|
||||
initial_state (Optional[torch.Tensor]):
|
||||
Initial state of shape `[N, HV, V, K]` for `N` input sequences.
|
||||
For equal-length input sequences, `N` equals the batch size `B`.
|
||||
Default: `None`.
|
||||
inplace_final_state: bool:
|
||||
Whether to store the final state in-place to save memory.
|
||||
Default: `True`.
|
||||
cu_seqlens (torch.Tensor):
|
||||
Cumulative sequence lengths of shape `[N+1]` used for variable-length training,
|
||||
consistent with the FlashAttention API.
|
||||
ssm_state_indices (Optional[torch.Tensor]):
|
||||
Indices to map the input sequences to the initial/final states.
|
||||
num_accepted_tokens (Optional[torch.Tensor]):
|
||||
Number of accepted tokens for each sequence during decoding.
|
||||
|
||||
Returns:
|
||||
o (torch.Tensor):
|
||||
Outputs of shape `[B, T, HV, V]`.
|
||||
final_state (torch.Tensor):
|
||||
Final state of shape `[N, HV, V, K]`.
|
||||
|
||||
Examples::
|
||||
>>> import torch
|
||||
>>> import torch.nn.functional as F
|
||||
>>> from einops import rearrange
|
||||
>>> from fla.ops.gated_delta_rule import fused_recurrent_gated_delta_rule
|
||||
# inputs with equal lengths
|
||||
>>> B, T, H, HV, K, V = 4, 2048, 4, 8, 512, 512
|
||||
>>> q = torch.randn(B, T, H, K, device='cuda')
|
||||
>>> k = F.normalize(torch.randn(B, T, H, K, device='cuda'), p=2, dim=-1)
|
||||
>>> v = torch.randn(B, T, HV, V, device='cuda')
|
||||
>>> g = F.logsigmoid(torch.rand(B, T, HV, device='cuda'))
|
||||
>>> beta = torch.rand(B, T, HV, device='cuda').sigmoid()
|
||||
>>> h0 = torch.randn(B, HV, V, K, device='cuda')
|
||||
>>> o, ht = fused_gated_recurrent_delta_rule(
|
||||
q, k, v, g, beta,
|
||||
initial_state=h0,
|
||||
)
|
||||
# for variable-length inputs, the batch size `B` is expected to be 1 and `cu_seqlens` is required
|
||||
>>> q, k, v, g, beta = map(lambda x: rearrange(x, 'b t ... -> 1 (b t) ...'), (q, k, v, g, beta))
|
||||
# for a batch with 4 sequences, `cu_seqlens` with 5 start/end positions are expected
|
||||
>>> cu_seqlens = q.new_tensor([0, 2048, 4096, 6144, 8192], dtype=torch.int32)
|
||||
>>> o_var, ht_var = fused_gated_recurrent_delta_rule(
|
||||
q, k, v, g, beta,
|
||||
initial_state=h0,
|
||||
cu_seqlens=cu_seqlens
|
||||
)
|
||||
"""
|
||||
if cu_seqlens is not None and q.shape[0] != 1:
|
||||
raise ValueError(
|
||||
f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`."
|
||||
f"Please flatten variable-length inputs before processing."
|
||||
)
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
else:
|
||||
assert scale > 0, "scale must be positive"
|
||||
if beta is None:
|
||||
beta = torch.ones_like(q[..., 0])
|
||||
o, final_state = FusedRecurrentFunction.apply(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
g,
|
||||
beta,
|
||||
scale,
|
||||
initial_state,
|
||||
inplace_final_state,
|
||||
cu_seqlens,
|
||||
ssm_state_indices,
|
||||
num_accepted_tokens,
|
||||
use_qk_l2norm_in_kernel,
|
||||
)
|
||||
return o, final_state
|
||||
File diff suppressed because it is too large
Load Diff
@@ -140,6 +140,24 @@ class VideoLoader:
|
||||
"""Sampling-algorithm-specific metadata adjustment hook."""
|
||||
return source
|
||||
|
||||
@classmethod
|
||||
def read_frames(
|
||||
cls,
|
||||
cap: "cv2.VideoCapture",
|
||||
frame_idx: list[int],
|
||||
total_frames_num: int,
|
||||
*,
|
||||
frame_recovery: bool = False,
|
||||
) -> tuple[npt.NDArray, list[int]]:
|
||||
from vllm.multimodal.video_decoders.opencv import OpenCVVideoBackendMixin
|
||||
|
||||
return OpenCVVideoBackendMixin.read_frames(
|
||||
cap,
|
||||
frame_idx,
|
||||
total_frames_num,
|
||||
frame_recovery=frame_recovery,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def load_bytes(
|
||||
@@ -664,6 +682,100 @@ class GLM46VVideoBackend(VideoBackend):
|
||||
)
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register(
|
||||
"glm5next",
|
||||
# Both spellings: the borrowed-config type string (``Glm5Next...``) and
|
||||
# the dedicated transformers classes landing with the new checkpoint
|
||||
# (``Glm5nextVideoProcessor``, matching ``Glm5nextImageProcessor``).
|
||||
video_processor=("Glm5NextVideoProcessor", "Glm5nextVideoProcessor"),
|
||||
)
|
||||
class Glm5NextVideoBackend(VideoBackend):
|
||||
"""GLM-5.3-Flash fps-interval video backend.
|
||||
|
||||
Selects frames with the same ``glm_sample_frame_indices`` sampler the
|
||||
processor falls back to, so only the sampled frames are
|
||||
materialized. ``fps_interval`` semantics (default 2.0) with a
|
||||
temporal-patch-scaled greedy walk, frame count capped at 2048, temporal
|
||||
pairs kept even. Request overrides: ``fps`` -> fps interval,
|
||||
``max_frames`` -> frame cap, ``temporal_patch_size`` (default 2).
|
||||
"""
|
||||
|
||||
_SEEK_GAP_THRESHOLD: ClassVar[int] = 64
|
||||
|
||||
@classmethod
|
||||
def compute_frames_index_to_sample(
|
||||
cls,
|
||||
source: VideoSourceMetadata,
|
||||
target: VideoTargetMetadata,
|
||||
**kwargs,
|
||||
) -> list[int]:
|
||||
# Lazy import: the processor module sits behind the
|
||||
# transformers_utils package init, which multimodal must not pull in.
|
||||
from vllm.transformers_utils.processors.glm5next import (
|
||||
glm_sample_frame_indices,
|
||||
)
|
||||
|
||||
return glm_sample_frame_indices(
|
||||
source.total_frames_num,
|
||||
source.original_fps,
|
||||
source.duration or 0,
|
||||
target_fps=target.fps if target.fps > 0 else None,
|
||||
max_frame_count=kwargs.get("max_frames"),
|
||||
temporal_patch_size=kwargs.get("temporal_patch_size", 2),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def read_frames(
|
||||
cls,
|
||||
cap: "cv2.VideoCapture",
|
||||
frame_idx: list[int],
|
||||
total_frames_num: int,
|
||||
*,
|
||||
frame_recovery: bool = False,
|
||||
) -> tuple[npt.NDArray, list[int]]:
|
||||
if frame_recovery:
|
||||
return super().read_frames(
|
||||
cap, frame_idx, total_frames_num, frame_recovery=frame_recovery
|
||||
)
|
||||
|
||||
wanted = sorted(set(frame_idx))
|
||||
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
||||
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
frames = np.empty((len(wanted), height, width, 3), dtype=np.uint8)
|
||||
valid_frame_indices: list[int] = []
|
||||
current: int | None = None
|
||||
for target in wanted:
|
||||
gap = target - current if current is not None else None
|
||||
if gap is not None and 2 <= gap <= cls._SEEK_GAP_THRESHOLD:
|
||||
for _ in range(gap - 1):
|
||||
if not cap.grab():
|
||||
current = None
|
||||
break
|
||||
else:
|
||||
current = target - 1
|
||||
if current != target - 1:
|
||||
cap.set(cv2.CAP_PROP_POS_FRAMES, target)
|
||||
current = target - 1
|
||||
ok, frame = cap.read()
|
||||
if ok:
|
||||
frames[len(valid_frame_indices)] = cv2.cvtColor(
|
||||
frame, cv2.COLOR_BGR2RGB
|
||||
)
|
||||
valid_frame_indices.append(target)
|
||||
current = target
|
||||
else:
|
||||
current = None
|
||||
|
||||
valid_num_frames = len(valid_frame_indices)
|
||||
if valid_num_frames < len(wanted):
|
||||
logger.warning(
|
||||
"GLM video loading expected %d sampled frames but only loaded %d.",
|
||||
len(wanted),
|
||||
valid_num_frames,
|
||||
)
|
||||
return frames[:valid_num_frames], valid_frame_indices
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register(
|
||||
"glmga",
|
||||
video_processor="GlmgaVideoProcessor",
|
||||
|
||||
@@ -42,7 +42,7 @@ def decode_opencv(
|
||||
frame_idx = loader_cls.compute_frames_index_to_sample(
|
||||
source=source, target=target, **sampling_kwargs
|
||||
)
|
||||
frames, valid = OpenCVVideoBackendMixin.read_frames(
|
||||
frames, valid = loader_cls.read_frames(
|
||||
cap,
|
||||
frame_idx,
|
||||
total_frames_num=source.total_frames_num,
|
||||
|
||||
+37
-7
@@ -86,6 +86,7 @@ def _get_backend_priorities(
|
||||
num_heads: int | None = None,
|
||||
kv_cache_dtype: CacheDType | None = None,
|
||||
use_non_causal: bool = False,
|
||||
head_size: int | None = None,
|
||||
) -> list[AttentionBackendEnum]:
|
||||
"""Get backend priorities with lazy import to avoid circular dependency."""
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
@@ -133,13 +134,21 @@ def _get_backend_priorities(
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM120,
|
||||
]
|
||||
else:
|
||||
sparse_tail = [
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
]
|
||||
flashinfer_sparse = AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90
|
||||
if head_size == 512:
|
||||
sparse_tail.insert(0, flashinfer_sparse)
|
||||
else:
|
||||
sparse_tail.append(flashinfer_sparse)
|
||||
return [
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA,
|
||||
AttentionBackendEnum.FLASHMLA,
|
||||
AttentionBackendEnum.FLASHINFER_MLA,
|
||||
AttentionBackendEnum.TRITON_MLA,
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
*sparse_tail,
|
||||
]
|
||||
else:
|
||||
# SM100f defaults to FlashInfer for TRTLLM causal attention, but its non-causal
|
||||
@@ -373,11 +382,12 @@ class CudaPlatformBase(Platform):
|
||||
invalid_reasons: dict[AttentionBackendEnum, tuple[int, list[str]]] = {}
|
||||
|
||||
backend_priorities = _get_backend_priorities(
|
||||
attn_selector_config.use_mla,
|
||||
device_capability,
|
||||
num_heads,
|
||||
attn_selector_config.kv_cache_dtype,
|
||||
attn_selector_config.use_non_causal,
|
||||
use_mla=attn_selector_config.use_mla,
|
||||
device_capability=device_capability,
|
||||
num_heads=num_heads,
|
||||
kv_cache_dtype=attn_selector_config.kv_cache_dtype,
|
||||
use_non_causal=attn_selector_config.use_non_causal,
|
||||
head_size=attn_selector_config.head_size,
|
||||
)
|
||||
for priority, backend in enumerate(backend_priorities):
|
||||
try:
|
||||
@@ -400,6 +410,26 @@ class CudaPlatformBase(Platform):
|
||||
|
||||
return valid_backends_priorities, invalid_reasons
|
||||
|
||||
@classmethod
|
||||
def _get_indexer_block_alignment(cls, vllm_config: VllmConfig) -> int | None:
|
||||
index_kpool = getattr(
|
||||
vllm_config.model_config.hf_text_config, "index_kpool", None
|
||||
)
|
||||
if not index_kpool or index_kpool <= 1:
|
||||
return None
|
||||
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
|
||||
|
||||
# kpool paged-MQA indexer: the storage block (block_size /
|
||||
# index_kpool) is virtually split into pool pages, so block_size
|
||||
# must be a multiple of index_kpool times a legal pool page.
|
||||
page = min(PAGED_MQA_PAGE_SIZES)
|
||||
if cls.is_device_capability_family(120):
|
||||
# On sm120 the DeepGEMM paged-MQA kernel only accepts block_kv
|
||||
# 64 for the fp8 indexer cache, so align to the largest pool
|
||||
# page here to make the page split land on 64 not the min 32.
|
||||
page = max(PAGED_MQA_PAGE_SIZES)
|
||||
return index_kpool * page
|
||||
|
||||
@classmethod
|
||||
def get_attn_backend_cls(
|
||||
cls,
|
||||
|
||||
@@ -763,6 +763,17 @@ class Platform:
|
||||
if cache_config.mamba_page_size_padded is not None:
|
||||
cache_config.mamba_page_size_padded = shared_page
|
||||
|
||||
@classmethod
|
||||
def _get_indexer_block_alignment(cls, vllm_config: "VllmConfig") -> int | None:
|
||||
"""Extra ``block_size`` multiple a sparse indexer needs, else ``None``.
|
||||
|
||||
The CUDA kpool paged-MQA indexer virtually splits each storage block
|
||||
into pool pages, so ``block_size`` must be a multiple of
|
||||
``index_kpool * min(PAGED_MQA_PAGE_SIZES)`` — implemented in the CUDA
|
||||
platform override. Other platforms impose no extra constraint.
|
||||
"""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _align_hybrid_block_size(
|
||||
cls,
|
||||
@@ -805,6 +816,7 @@ class Platform:
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=kv_cache_dtype,
|
||||
cache_dtype_str=cache_config.cache_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
).page_size_bytes
|
||||
elif cache_config.cache_dtype.startswith("turboquant_"):
|
||||
@@ -912,6 +924,9 @@ class Platform:
|
||||
mamba_page_size,
|
||||
kernel_block_alignment_size * attn_page_size_1_token,
|
||||
)
|
||||
indexer_align = cls._get_indexer_block_alignment(vllm_config)
|
||||
if indexer_align:
|
||||
attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align)
|
||||
|
||||
if cache_config.block_size < attn_block_size:
|
||||
cache_config.block_size = attn_block_size
|
||||
|
||||
@@ -104,6 +104,9 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
k3_dspark="K3DSparkConfig",
|
||||
funaudiochat="FunAudioChatConfig",
|
||||
granite4_vision="Granite4VisionConfig",
|
||||
glm5_next="Glm5NextConfig",
|
||||
glm5_next_text="Glm5NextTextConfig",
|
||||
glm5_next_vision="Glm5NextVisionConfig",
|
||||
hyperclovax="HyperCLOVAXConfig",
|
||||
hy_v3="HYV3Config",
|
||||
hy_v4="HYV4Config",
|
||||
|
||||
@@ -40,6 +40,9 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
|
||||
"FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
|
||||
"Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision",
|
||||
"Glm5NextConfig": "vllm.transformers_utils.configs.glm5_next",
|
||||
"Glm5NextTextConfig": "vllm.transformers_utils.configs.glm5_next",
|
||||
"Glm5NextVisionConfig": "vllm.transformers_utils.configs.glm5_next",
|
||||
"HYV3Config": "vllm.transformers_utils.configs.hy_v3",
|
||||
"HYV4Config": "vllm.transformers_utils.configs.hy_v4",
|
||||
"HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
|
||||
@@ -133,6 +136,9 @@ __all__ = [
|
||||
"FunAudioChatConfig",
|
||||
"FunAudioChatAudioEncoderConfig",
|
||||
"Granite4VisionConfig",
|
||||
"Glm5NextConfig",
|
||||
"Glm5NextTextConfig",
|
||||
"Glm5NextVisionConfig",
|
||||
"HYV3Config",
|
||||
"HYV4Config",
|
||||
"HyperCLOVAXConfig",
|
||||
|
||||
@@ -0,0 +1,420 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Glm5NextTextConfig(PretrainedConfig):
|
||||
model_type = "glm5_next_text"
|
||||
base_config_key = "text_config"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_type="glm5_next_text",
|
||||
vocab_size: int = 154880,
|
||||
hidden_size: int = 4096,
|
||||
head_dim: int | None = None,
|
||||
intermediate_size: int = 12288,
|
||||
num_hidden_layers: int = 45,
|
||||
num_attention_heads: int = 64,
|
||||
num_key_value_heads: int | None = None,
|
||||
hidden_act: str = "silu",
|
||||
rms_norm_eps: float = 1e-5,
|
||||
pad_token_id: int | None = 151329,
|
||||
bos_token_id: int | None = None,
|
||||
eos_token_id: int | list[int] | None = None,
|
||||
rope_parameters: dict | None = None,
|
||||
max_position_embeddings: int = 1048576,
|
||||
tie_word_embeddings: bool = False,
|
||||
moe_intermediate_size: int = 2048,
|
||||
moe_renormalize: bool = True,
|
||||
scoring_func: str = "sigmoid",
|
||||
n_routed_experts: int | None = 288,
|
||||
num_experts_per_token: int = 7,
|
||||
n_shared_experts: int = 1,
|
||||
routed_scaling_factor: float = 2.5,
|
||||
topk_method: str | None = None,
|
||||
first_k_dense_replace: int = 0,
|
||||
moe_layer_freq: int = 1,
|
||||
use_grouped_topk: bool = True,
|
||||
n_group: int = 1,
|
||||
topk_group: int = 1,
|
||||
mla: bool = True,
|
||||
q_lora_rank: int | None = 1536,
|
||||
kv_lora_rank: int | None = 512,
|
||||
qk_nope_head_dim: int = 256,
|
||||
qk_rope_head_dim: int = 0,
|
||||
v_head_dim: int | None = 256,
|
||||
mla_nope: bool | None = True,
|
||||
num_nextn_predict_layers: int = 1,
|
||||
# Per-layer layout: "linear_attention" | "deepseek_sparse_attention"
|
||||
layer_types: list[str] | None = None,
|
||||
# Per-layer MLP: "dense" | "sparse"
|
||||
mlp_layer_types: list[str] | None = None,
|
||||
# Linear-attention (KDA) head config (flattened from the old
|
||||
# linear_attn_config dict).
|
||||
linear_head_dim: int = 128,
|
||||
linear_num_heads: int = 64,
|
||||
linear_conv_kernel_dim: int = 4,
|
||||
linear_lower_bound: float = -5.0,
|
||||
index_head_dim: int | None = None,
|
||||
index_topk: int | None = None,
|
||||
index_n_heads: int | None = None,
|
||||
index_dsa_use_layernorm: bool = True,
|
||||
index_kpool_compress: bool = True,
|
||||
# Every ``index_kpool`` indexer K entries pool into one stored entry
|
||||
# (compress_ratio). The 300B checkpoint ships 4; topk runs at pool
|
||||
# granularity (select_k = index_topk // index_kpool).
|
||||
index_kpool: int | None = 4,
|
||||
index_kpool_always_select_tail: bool = True,
|
||||
indexer_rope_interleave: bool = False,
|
||||
mhc: bool | None = True,
|
||||
mhc_num_residual_streams: int = 4,
|
||||
hc_eps: float | None = 1e-06,
|
||||
mhc_tau: float = 0.05,
|
||||
hres_vwnstyle: bool | None = True,
|
||||
mhc_no_norm_weight: bool | None = False,
|
||||
mhc_sinkhorn_iterations: int | None = 20,
|
||||
mhc_post_mult_value: float | None = 2.0,
|
||||
swiglu_limit: float | None = None,
|
||||
logit_scale: float = 1.0,
|
||||
**kwargs,
|
||||
):
|
||||
# Preserve checkpoint field names and local aliases because their
|
||||
# consumers use different spellings.
|
||||
num_experts_per_token = kwargs.get("num_experts_per_tok", num_experts_per_token)
|
||||
moe_renormalize = kwargs.get("norm_topk_prob", moe_renormalize)
|
||||
mhc_num_residual_streams = kwargs.get("hc_mult", mhc_num_residual_streams)
|
||||
mhc_sinkhorn_iterations = kwargs.get(
|
||||
"hc_sinkhorn_iters", mhc_sinkhorn_iterations
|
||||
)
|
||||
# Checkpoint ships ``mla_use_nope`` (not ``mla_nope``); without this
|
||||
# alias self.mla_nope silently stays at the param default.
|
||||
mla_nope = kwargs.get("mla_use_nope", mla_nope)
|
||||
# Checkpoints ship the KDA head config as the ``linear_attn_config``
|
||||
# dict (head_dim / num_heads / short_conv_kernel_size /
|
||||
# gate_lower_bound) rather than the flattened top-level fields; fold it
|
||||
# in so the trained values are read instead of the param defaults
|
||||
# (which only match this checkpoint by coincidence).
|
||||
linear_cfg = kwargs.get("linear_attn_config") or {}
|
||||
if linear_cfg:
|
||||
linear_head_dim = linear_cfg.get("head_dim", linear_head_dim)
|
||||
linear_num_heads = linear_cfg.get("num_heads", linear_num_heads)
|
||||
linear_conv_kernel_dim = linear_cfg.get(
|
||||
"short_conv_kernel_size", linear_conv_kernel_dim
|
||||
)
|
||||
linear_lower_bound = linear_cfg.get("gate_lower_bound", linear_lower_bound)
|
||||
|
||||
if index_topk is not None:
|
||||
if index_dsa_use_layernorm is not True:
|
||||
raise NotImplementedError(
|
||||
"GLM-5.3 sparse indexer requires index_dsa_use_layernorm=True"
|
||||
)
|
||||
if index_kpool_compress is not True:
|
||||
raise NotImplementedError(
|
||||
"GLM-5.3 sparse indexer requires index_kpool_compress=True"
|
||||
)
|
||||
if index_kpool_always_select_tail is not True:
|
||||
raise NotImplementedError(
|
||||
"GLM-5.3 sparse indexer requires "
|
||||
"index_kpool_always_select_tail=True"
|
||||
)
|
||||
|
||||
if mhc:
|
||||
if hres_vwnstyle is not True:
|
||||
raise NotImplementedError("GLM-5.3 mHC requires hres_vwnstyle=True")
|
||||
if mhc_no_norm_weight not in (False, None):
|
||||
raise NotImplementedError(
|
||||
"GLM-5.3 mHC requires mhc_no_norm_weight=False"
|
||||
)
|
||||
|
||||
self.model_type = model_type
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.head_dim = (
|
||||
head_dim if head_dim is not None else hidden_size // num_attention_heads
|
||||
)
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.rope_parameters = rope_parameters
|
||||
|
||||
# mla config
|
||||
self.mla = mla
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.mla_nope = mla_nope
|
||||
# moe config
|
||||
self.n_routed_experts = n_routed_experts
|
||||
self.num_experts_per_token = num_experts_per_token
|
||||
self.moe_renormalize = moe_renormalize
|
||||
self.n_shared_experts = n_shared_experts
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.topk_method = topk_method
|
||||
self.scoring_func = scoring_func
|
||||
assert self.scoring_func in ("softmax", "sigmoid")
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.first_k_dense_replace = first_k_dense_replace
|
||||
self.moe_layer_freq = moe_layer_freq
|
||||
self.use_grouped_topk = use_grouped_topk
|
||||
self.n_group = n_group
|
||||
self.topk_group = topk_group
|
||||
self.num_nextn_predict_layers = num_nextn_predict_layers
|
||||
|
||||
# Per-layer attention / MLP layout. Normalize mlp_layer_types from
|
||||
# first_k_dense_replace when the new-schema field is absent so layer
|
||||
# construction sees a consistent layout (mirrors cohere2_moe).
|
||||
self.layer_types = layer_types
|
||||
if mlp_layer_types is None:
|
||||
n = self.num_hidden_layers
|
||||
if first_k_dense_replace is not None:
|
||||
mlp_layer_types = ["dense"] * first_k_dense_replace + ["sparse"] * (
|
||||
n - first_k_dense_replace
|
||||
)
|
||||
else:
|
||||
mlp_layer_types = ["sparse"] * n
|
||||
self.mlp_layer_types = mlp_layer_types
|
||||
|
||||
# Linear-attention (KDA) head config.
|
||||
self.linear_head_dim = linear_head_dim
|
||||
self.linear_num_heads = linear_num_heads
|
||||
self.linear_conv_kernel_dim = linear_conv_kernel_dim
|
||||
self.linear_lower_bound = linear_lower_bound
|
||||
|
||||
# dsa index config
|
||||
self.index_head_dim = index_head_dim
|
||||
self.index_topk = index_topk
|
||||
self.index_n_heads = index_n_heads
|
||||
self.index_dsa_use_layernorm = index_dsa_use_layernorm
|
||||
self.index_kpool_compress = index_kpool_compress
|
||||
self.index_kpool = index_kpool
|
||||
self.index_kpool_always_select_tail = index_kpool_always_select_tail
|
||||
self.indexer_rope_interleave = indexer_rope_interleave
|
||||
|
||||
# mhc config
|
||||
self.mhc = mhc
|
||||
self.mhc_num_residual_streams = mhc_num_residual_streams
|
||||
self.mhc_tau = mhc_tau
|
||||
self.hres_vwnstyle = hres_vwnstyle
|
||||
self.hc_eps = hc_eps
|
||||
self.mhc_no_norm_weight = mhc_no_norm_weight
|
||||
self.mhc_sinkhorn_iterations = mhc_sinkhorn_iterations
|
||||
self.mhc_post_mult_value = mhc_post_mult_value
|
||||
|
||||
self.swiglu_limit = swiglu_limit
|
||||
self.logit_scale = logit_scale
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def is_mla(self):
|
||||
return (
|
||||
self.q_lora_rank is not None
|
||||
or self.kv_lora_rank is not None
|
||||
or self.qk_nope_head_dim is not None
|
||||
or self.qk_rope_head_dim is not None
|
||||
or self.v_head_dim is not None
|
||||
or self.mla_nope is True
|
||||
)
|
||||
|
||||
@property
|
||||
def is_moe(self):
|
||||
return self.n_routed_experts is not None
|
||||
|
||||
@property
|
||||
def is_linear_attn(self) -> bool:
|
||||
return self.layer_types is not None and any(
|
||||
t == "linear_attention" for t in self.layer_types
|
||||
)
|
||||
|
||||
def is_kda_layer(self, layer_idx: int):
|
||||
return (
|
||||
self.layer_types is not None
|
||||
and layer_idx < len(self.layer_types)
|
||||
and self.layer_types[layer_idx] == "linear_attention"
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self):
|
||||
# Map the schema's per-layer types onto the block strings vLLM's hybrid
|
||||
# accounting (get_num_layers_by_block_type) recognizes: linear-attention
|
||||
# layers stay "linear_attention"; every other attention variant collapses
|
||||
# to "attention".
|
||||
if self.layer_types is None:
|
||||
return ["attention"] * self.num_hidden_layers
|
||||
return [
|
||||
"linear_attention" if t == "linear_attention" else "attention"
|
||||
for t in self.layer_types
|
||||
]
|
||||
|
||||
|
||||
class Glm5NextVisionConfig(PretrainedConfig):
|
||||
model_type = "glm5_next_vision"
|
||||
base_config_key = "vision_config"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
depth: int = 24,
|
||||
hidden_size: int = 1024,
|
||||
hidden_act: str = "silu",
|
||||
image_size: int = 448,
|
||||
intermediate_size: int = 4096,
|
||||
num_heads: int = 16,
|
||||
out_hidden_size: int = 4096,
|
||||
projection_intermediate_size: int = 10240,
|
||||
in_channels: int = 3,
|
||||
initializer_range: float = 0.02,
|
||||
patch_size: int = 14,
|
||||
rms_norm_eps: float = 1e-5,
|
||||
spatial_merge_size: int = 2,
|
||||
temporal_patch_size: int = 2,
|
||||
attention_dropout: float = 0.0,
|
||||
attention_bias: bool = True,
|
||||
swiglu_limit: float | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
self.depth = depth
|
||||
self.hidden_size = hidden_size
|
||||
self.hidden_act = hidden_act
|
||||
self.image_size = image_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_heads = num_heads
|
||||
self.out_hidden_size = out_hidden_size
|
||||
# GLM-5.3-Flash merger bottleneck width (absent from the generic
|
||||
# GLM-OCR vision config); the tower uses it as the PatchMerger
|
||||
# context_dim instead of text_config.intermediate_size.
|
||||
self.projection_intermediate_size = projection_intermediate_size
|
||||
self.in_channels = in_channels
|
||||
self.initializer_range = initializer_range
|
||||
self.patch_size = patch_size
|
||||
# GLM-5.3-Flash checkpoints ship vision_config.rms_norm_eps = 1e-5,
|
||||
# but the vision tower was trained with 1e-6. Serving with 1e-5 drifts
|
||||
# the RMSNorm and produces repetitive/degraded image descriptions, so
|
||||
# force the trained value regardless of the checkpoint field.
|
||||
self.rms_norm_eps = 1e-6
|
||||
self.spatial_merge_size = spatial_merge_size
|
||||
self.temporal_patch_size = temporal_patch_size
|
||||
self.attention_dropout = attention_dropout
|
||||
self.attention_bias = attention_bias
|
||||
self.swiglu_limit = swiglu_limit
|
||||
|
||||
|
||||
class Glm5NextConfig(PretrainedConfig):
|
||||
model_type = "glm5_next"
|
||||
sub_configs = {
|
||||
"vision_config": Glm5NextVisionConfig,
|
||||
"text_config": Glm5NextTextConfig,
|
||||
}
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config=None,
|
||||
vision_config=None,
|
||||
image_token_id: int = 154854,
|
||||
video_token_id: int = 154855,
|
||||
image_start_token_id: int = 154830,
|
||||
image_end_token_id: int = 154831,
|
||||
video_start_token_id: int = 154832,
|
||||
video_end_token_id: int = 154833,
|
||||
**kwargs,
|
||||
):
|
||||
# Init super() first so base-class defaults don't clobber text-config
|
||||
# values set below (PretrainedConfig has many text-related defaults
|
||||
# that differ from Glm5NextTextConfig).
|
||||
super().__init__(**kwargs)
|
||||
|
||||
if isinstance(vision_config, dict):
|
||||
self.vision_config = self.sub_configs["vision_config"](**vision_config)
|
||||
elif vision_config is None:
|
||||
self.vision_config = self.sub_configs["vision_config"]()
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
|
||||
if isinstance(text_config, dict):
|
||||
self.text_config = self.sub_configs["text_config"](**text_config)
|
||||
elif text_config is None:
|
||||
# Backward compatibility: a flat top-level checkpoint (no nested
|
||||
# text_config) folds its text fields into Glm5NextTextConfig.
|
||||
self.text_config = self.sub_configs["text_config"](**kwargs)
|
||||
else:
|
||||
self.text_config = text_config
|
||||
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.image_start_token_id = image_start_token_id
|
||||
self.image_end_token_id = image_end_token_id
|
||||
self.video_start_token_id = video_start_token_id
|
||||
self.video_end_token_id = video_end_token_id
|
||||
|
||||
# Mirror attention implementation recursively onto sub-configs.
|
||||
self._attn_implementation = kwargs.pop("attn_implementation", None)
|
||||
|
||||
# Config-metadata fields that belong to the top-level (multimodal) config
|
||||
# and must NOT be mirrored onto text_config: ``architectures`` /
|
||||
# ``torch_dtype`` differ between the top-level config and the text
|
||||
# sub-config, and mirroring them makes the top-level ``architectures``
|
||||
# silently read back as None (PretrainedConfig initializes both to None),
|
||||
# which then fails model-class resolution ("No model architectures are
|
||||
# specified").
|
||||
_UNMIRRORED_KEYS = [
|
||||
"_name_or_path",
|
||||
"model_type",
|
||||
"dtype",
|
||||
"torch_dtype",
|
||||
"architectures",
|
||||
"_attn_implementation_internal",
|
||||
]
|
||||
|
||||
def __setattr__(self, key, value):
|
||||
unmirrored = type(self)._UNMIRRORED_KEYS
|
||||
if (
|
||||
(text_config := super().__getattribute__("__dict__").get("text_config"))
|
||||
is not None
|
||||
and key not in unmirrored
|
||||
and key in text_config.__dict__
|
||||
):
|
||||
setattr(text_config, key, value)
|
||||
else:
|
||||
super().__setattr__(key, value)
|
||||
|
||||
def __getattribute__(self, key):
|
||||
unmirrored = type(self)._UNMIRRORED_KEYS
|
||||
if (
|
||||
"text_config" in super().__getattribute__("__dict__")
|
||||
and key not in unmirrored
|
||||
):
|
||||
text_config = super().__getattribute__("text_config")
|
||||
# Forward both instance attributes AND class-defined properties/
|
||||
# methods of the text config, so a flat text-only checkpoint
|
||||
# (model_type "glm5_next", no nested text_config) sees is_moe /
|
||||
# is_kda_layer / layers_block_type like a Glm5NextTextConfig.
|
||||
if key in text_config.__dict__ or key in type(text_config).__dict__:
|
||||
return getattr(text_config, key)
|
||||
|
||||
return super().__getattribute__(key)
|
||||
@@ -320,6 +320,8 @@ class ModelArchConfigConvertorBase:
|
||||
"dots3_note",
|
||||
"deepseek_mtp",
|
||||
"k3_dspark",
|
||||
"glm5_next",
|
||||
"glm5_next_text",
|
||||
"glm_moe_dsa",
|
||||
"glm4_moe_lite",
|
||||
"glm4_moe_lite_mtp",
|
||||
|
||||
@@ -18,6 +18,7 @@ __all__ = [
|
||||
"FireRedASR2Processor",
|
||||
"FunASRProcessor",
|
||||
"GLM4VProcessor",
|
||||
"Glm5NextProcessor",
|
||||
"Granite4VisionProcessor",
|
||||
"H2OVLProcessor",
|
||||
"Moondream3Processor",
|
||||
@@ -56,6 +57,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
|
||||
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
|
||||
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
|
||||
"Glm5NextProcessor": "vllm.transformers_utils.processors.glm5next",
|
||||
"Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision",
|
||||
"H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
|
||||
"InternVLProcessor": "vllm.transformers_utils.processors.internvl",
|
||||
|
||||
@@ -0,0 +1,953 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM-native multimodal processor for GLM-5.3-Flash."""
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torchvision.transforms.v2 import functional as tvF
|
||||
from transformers.image_processing_utils import BatchFeature
|
||||
from transformers.image_processing_utils_fast import (
|
||||
BaseImageProcessorFast,
|
||||
group_images_by_shape,
|
||||
reorder_images,
|
||||
)
|
||||
from transformers.image_utils import (
|
||||
OPENAI_CLIP_MEAN,
|
||||
OPENAI_CLIP_STD,
|
||||
ChannelDimension,
|
||||
ImageInput,
|
||||
PILImageResampling,
|
||||
SizeDict,
|
||||
get_image_size,
|
||||
)
|
||||
from transformers.models.auto.image_processing_auto import get_image_processor_config
|
||||
from transformers.processing_utils import (
|
||||
ImagesKwargs,
|
||||
MultiModalData,
|
||||
ProcessingKwargs,
|
||||
ProcessorMixin,
|
||||
Unpack,
|
||||
VideosKwargs,
|
||||
)
|
||||
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
||||
from transformers.utils import TensorType, logging
|
||||
from transformers.video_processing_utils import BaseVideoProcessor
|
||||
from transformers.video_utils import (
|
||||
VideoInput,
|
||||
VideoMetadata,
|
||||
group_videos_by_shape,
|
||||
reorder_videos,
|
||||
)
|
||||
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
# Cap video inputs at 30,000 vision tokens to keep encoder profiling from
|
||||
# starving the KV cache. Image inputs retain their checkpoint-defined budget.
|
||||
_MAX_VIDEO_TOKENS = 30000
|
||||
|
||||
# Frame-sampler fallbacks (mirror the checkpoint's fps_interval=2 /
|
||||
# max_frame_count_dynamic=2048); used when neither the request nor the
|
||||
# processor config overrides them.
|
||||
GLM_VIDEO_DEFAULT_FPS = 2.0
|
||||
GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048
|
||||
|
||||
|
||||
def glm_sample_frame_indices(
|
||||
total_frames: int,
|
||||
fps: float,
|
||||
duration: float,
|
||||
*,
|
||||
target_fps: float | None = None,
|
||||
max_frame_count: int | None = None,
|
||||
temporal_patch_size: int = 2,
|
||||
) -> list[int]:
|
||||
"""GLM video frame sampling (training-reference parity).
|
||||
|
||||
``target_fps`` is the ``fps_interval`` request knob. The greedy walk
|
||||
advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on
|
||||
frame-dense sources it collects more candidates than ``extract_t`` and
|
||||
the ``> extract_t`` fixup re-spreads the picks uniformly with
|
||||
``np.linspace`` -- that fallback is the intended reference behavior, not
|
||||
an accident. Short clips (fewer frames than ``extract_t``) are spread at
|
||||
evenly spaced timestamps (``floor`` sampling; the linspace variant
|
||||
samples frames unevenly and cost 4 points on video grounding evals).
|
||||
Request overrides: ``target_fps`` -> fps interval, ``max_frame_count``
|
||||
-> frame cap.
|
||||
"""
|
||||
max_frame_idx = total_frames - 1
|
||||
if not duration:
|
||||
duration = (round(max_frame_idx / fps) + 1) if fps else 0
|
||||
if max_frame_count is None:
|
||||
max_frame_count = GLM_VIDEO_DEFAULT_MAX_FRAMES
|
||||
if target_fps is None:
|
||||
target_fps = GLM_VIDEO_DEFAULT_FPS
|
||||
|
||||
extract_t = int(duration * target_fps)
|
||||
extract_t = min(extract_t, int(max_frame_count))
|
||||
|
||||
duration_per_frame = 1 / fps
|
||||
timestamps = [i * duration_per_frame for i in range(total_frames)]
|
||||
max_second = int(duration)
|
||||
|
||||
if total_frames < extract_t:
|
||||
frame_indices = [
|
||||
math.floor(_i * total_frames / extract_t) for _i in range(extract_t)
|
||||
]
|
||||
else:
|
||||
frame_indices = []
|
||||
current_second = 0.0
|
||||
inv_fps = 1 / (temporal_patch_size * target_fps)
|
||||
for frame_index in range(total_frames):
|
||||
if timestamps[frame_index] >= current_second:
|
||||
current_second += inv_fps
|
||||
frame_indices.append(frame_index)
|
||||
if current_second >= max_second:
|
||||
break
|
||||
|
||||
if len(frame_indices) < extract_t:
|
||||
if len(frame_indices) == 0:
|
||||
start, end = 0, max(total_frames - 1, 0)
|
||||
else:
|
||||
start, end = frame_indices[0], frame_indices[-1]
|
||||
frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
|
||||
elif len(frame_indices) > extract_t:
|
||||
frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist()
|
||||
|
||||
seen, uniq = set(), []
|
||||
for idx in frame_indices:
|
||||
if idx not in seen:
|
||||
seen.add(idx)
|
||||
uniq.append(int(idx))
|
||||
|
||||
if len(uniq) & 1:
|
||||
uniq.append(uniq[-1])
|
||||
|
||||
return uniq
|
||||
|
||||
|
||||
def _ceil_to_factor(value: int, factor: int) -> int:
|
||||
"""Round a positive integer upward to the nearest multiple of factor."""
|
||||
return math.ceil(value / factor) * factor
|
||||
|
||||
|
||||
def _fit_aligned_size_within_budget(
|
||||
t: int,
|
||||
h: int,
|
||||
w: int,
|
||||
h_factor: int,
|
||||
w_factor: int,
|
||||
max_pixels: int,
|
||||
) -> tuple[int, int]:
|
||||
"""Largest proportional size whose upward-aligned canvas fits the budget.
|
||||
|
||||
Binary search on the unaligned content height; each candidate is rounded
|
||||
upward to h_factor/w_factor, so the returned canvas always satisfies
|
||||
``t * aligned_h * aligned_w <= max_pixels``.
|
||||
"""
|
||||
minimum_pixels = t * h_factor * w_factor
|
||||
if max_pixels < minimum_pixels:
|
||||
raise ValueError(
|
||||
f"max_pixels={max_pixels} is too small. At least "
|
||||
f"{minimum_pixels} pixels are required for one aligned patch."
|
||||
)
|
||||
|
||||
low, high = 1, h
|
||||
best_h, best_w = h_factor, w_factor
|
||||
while low <= high:
|
||||
content_h = (low + high) // 2
|
||||
content_w = max(1, math.floor(w * content_h / h))
|
||||
aligned_h = _ceil_to_factor(content_h, h_factor)
|
||||
aligned_w = _ceil_to_factor(content_w, w_factor)
|
||||
if t * aligned_h * aligned_w <= max_pixels:
|
||||
best_h, best_w = aligned_h, aligned_w
|
||||
low = content_h + 1
|
||||
else:
|
||||
high = content_h - 1
|
||||
return best_h, best_w
|
||||
|
||||
|
||||
def smart_resize(
|
||||
t: int,
|
||||
h: int,
|
||||
w: int,
|
||||
t_factor: int = 1,
|
||||
h_factor: int = 28,
|
||||
w_factor: int = 28,
|
||||
min_pixels: int = 56 * 56,
|
||||
max_pixels: int = 14 * 14 * 4 * 1280,
|
||||
) -> tuple[int, int]:
|
||||
"""GLM-5.3-Flash ``smart_resize``: upward-aligned canvas under a
|
||||
``t_bar * h_bar * w_bar`` pixel budget.
|
||||
|
||||
Height/width always round UP to their factors (content is then padded,
|
||||
never cropped or distorted); an over-budget canvas is refit by binary
|
||||
search instead of one-shot square-root scaling. ``h_factor`` /
|
||||
``w_factor`` carry ``patch_expand_factor`` on top of
|
||||
``patch_size * merge_size``; ``t_factor`` is ``temporal_patch_size``. For
|
||||
a still image ``t = t_factor = temporal_patch_size`` so ``t_bar =
|
||||
temporal_patch_size``.
|
||||
"""
|
||||
if min(t, h, w, t_factor, h_factor, w_factor) <= 0:
|
||||
raise ValueError("Image dimensions and alignment factors must be positive.")
|
||||
if min_pixels <= 0 or max_pixels <= 0:
|
||||
raise ValueError("min_pixels and max_pixels must be positive.")
|
||||
if min_pixels > max_pixels:
|
||||
raise ValueError("min_pixels must be less than or equal to max_pixels.")
|
||||
|
||||
t_bar = max(t_factor, round(t / t_factor) * t_factor)
|
||||
h_bar = _ceil_to_factor(h, h_factor)
|
||||
w_bar = _ceil_to_factor(w, w_factor)
|
||||
|
||||
if t_bar * h_bar * w_bar > max_pixels:
|
||||
h_bar, w_bar = _fit_aligned_size_within_budget(
|
||||
t=t_bar,
|
||||
h=h,
|
||||
w=w,
|
||||
h_factor=h_factor,
|
||||
w_factor=w_factor,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
elif t_bar * h_bar * w_bar < min_pixels:
|
||||
beta = math.sqrt(min_pixels / (t * h * w))
|
||||
h_bar = _ceil_to_factor(max(1, math.ceil(h * beta)), h_factor)
|
||||
w_bar = _ceil_to_factor(max(1, math.ceil(w * beta)), w_factor)
|
||||
|
||||
# Alignment can push a candidate slightly over a tight max_pixels
|
||||
# budget. Refit it when that happens.
|
||||
if t_bar * h_bar * w_bar > max_pixels:
|
||||
h_bar, w_bar = _fit_aligned_size_within_budget(
|
||||
t=t_bar,
|
||||
h=h,
|
||||
w=w,
|
||||
h_factor=h_factor,
|
||||
w_factor=w_factor,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
|
||||
return h_bar, w_bar
|
||||
|
||||
|
||||
def _get_pad_content_size(
|
||||
image_height: int,
|
||||
image_width: int,
|
||||
canvas_height: int,
|
||||
canvas_width: int,
|
||||
allow_upscale: bool = False,
|
||||
) -> tuple[int, int]:
|
||||
"""Aspect-ratio-preserving content size that fits the canvas.
|
||||
|
||||
Oversized images are shrunk proportionally. Small images are enlarged
|
||||
only when ``allow_upscale``. Padding is applied after the resize.
|
||||
"""
|
||||
scale = min(canvas_height / image_height, canvas_width / image_width)
|
||||
if not allow_upscale:
|
||||
scale = min(1.0, scale)
|
||||
content_height = max(1, min(canvas_height, math.floor(image_height * scale)))
|
||||
content_width = max(1, min(canvas_width, math.floor(image_width * scale)))
|
||||
return content_height, content_width
|
||||
|
||||
|
||||
def _resize_or_pad(
|
||||
stacked_images: torch.Tensor,
|
||||
target_height: int,
|
||||
target_width: int,
|
||||
resize_mode: str,
|
||||
resample: "PILImageResampling | tvF.InterpolationMode | int | None",
|
||||
resize,
|
||||
allow_upscale: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Resize onto the aligned canvas, or keep the aspect ratio and
|
||||
zero-pad the right/bottom sides (``resize_mode="pad"``)."""
|
||||
height, width = stacked_images.shape[-2:]
|
||||
|
||||
if resize_mode == "resize":
|
||||
return resize(
|
||||
stacked_images,
|
||||
size=SizeDict(height=target_height, width=target_width),
|
||||
resample=resample,
|
||||
)
|
||||
|
||||
if resize_mode != "pad":
|
||||
raise ValueError("resize_mode must be either 'resize' or 'pad'.")
|
||||
|
||||
content_height, content_width = _get_pad_content_size(
|
||||
image_height=height,
|
||||
image_width=width,
|
||||
canvas_height=target_height,
|
||||
canvas_width=target_width,
|
||||
allow_upscale=allow_upscale,
|
||||
)
|
||||
|
||||
if (content_height, content_width) != (height, width):
|
||||
stacked_images = resize(
|
||||
stacked_images,
|
||||
size=SizeDict(height=content_height, width=content_width),
|
||||
resample=resample,
|
||||
)
|
||||
|
||||
# torchvision padding order: [left, top, right, bottom] -> pad only the
|
||||
# right and bottom sides.
|
||||
return tvF.pad(
|
||||
stacked_images,
|
||||
padding=[0, 0, target_width - content_width, target_height - content_height],
|
||||
fill=0,
|
||||
)
|
||||
|
||||
|
||||
def _pixel_budget(
|
||||
min_image_tokens: int | None,
|
||||
max_image_tokens: int | None,
|
||||
patch_size: int,
|
||||
merge_size: int,
|
||||
temporal_patch_size: int,
|
||||
) -> tuple[int, int]:
|
||||
"""(min_pixels, max_pixels) from the token bounds of
|
||||
``processor_config.json``; one vision token covers
|
||||
``temporal_patch_size * (patch_size * merge_size) ** 2`` pixels."""
|
||||
if min_image_tokens is None or max_image_tokens is None:
|
||||
raise ValueError(
|
||||
"min_image_tokens and max_image_tokens must be provided by "
|
||||
"processor_config.json (or per-call kwargs)."
|
||||
)
|
||||
factor = temporal_patch_size * (patch_size * merge_size) ** 2
|
||||
return min_image_tokens * factor, max_image_tokens * factor
|
||||
|
||||
|
||||
class Glm5NextImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[call-arg]
|
||||
patch_size: int | None
|
||||
temporal_patch_size: int | None
|
||||
merge_size: int | None
|
||||
patch_expand_factor: int | None
|
||||
resize_mode: str | None
|
||||
min_image_tokens: int | None
|
||||
max_image_tokens: int | None
|
||||
|
||||
|
||||
class Glm5NextImageProcessor(BaseImageProcessorFast):
|
||||
"""Fast torchvision image processor for GLM-5.3-Flash.
|
||||
|
||||
``patch_expand_factor`` multiplies into the ``smart_resize`` spatial
|
||||
factor ``patch_size * merge_size``. ``resize_mode`` picks the geometry:
|
||||
``"pad"`` (default) preserves the aspect ratio and zero-pads the
|
||||
right/bottom of the upward-aligned canvas, ``"resize"`` stretches onto
|
||||
it. Defaults mirror the checkpoint's ``image_processor`` config.
|
||||
"""
|
||||
|
||||
do_resize = True
|
||||
resample = PILImageResampling.BICUBIC
|
||||
size = {"longest_edge": 1} # unused: budgets come from the token bounds
|
||||
do_rescale = True
|
||||
do_normalize = True
|
||||
image_mean = OPENAI_CLIP_MEAN
|
||||
image_std = OPENAI_CLIP_STD
|
||||
do_convert_rgb = True
|
||||
patch_size = 14
|
||||
temporal_patch_size = 2
|
||||
merge_size = 2
|
||||
patch_expand_factor = 1
|
||||
resize_mode = "pad"
|
||||
min_image_tokens = 16
|
||||
max_image_tokens = 8000
|
||||
valid_kwargs = Glm5NextImageProcessorKwargs
|
||||
model_input_names = ["pixel_values", "image_grid_thw"]
|
||||
|
||||
def _preprocess(
|
||||
self,
|
||||
images: list[torch.Tensor],
|
||||
do_resize: bool,
|
||||
size: SizeDict,
|
||||
resample: "PILImageResampling | tvF.InterpolationMode | int | None",
|
||||
do_rescale: bool,
|
||||
rescale_factor: float,
|
||||
do_normalize: bool,
|
||||
image_mean: float | list[float] | None,
|
||||
image_std: float | list[float] | None,
|
||||
patch_size: int,
|
||||
temporal_patch_size: int,
|
||||
merge_size: int,
|
||||
patch_expand_factor: int,
|
||||
resize_mode: str | None,
|
||||
min_image_tokens: int | None,
|
||||
max_image_tokens: int | None,
|
||||
disable_grouping: bool | None,
|
||||
return_tensors: str | TensorType | None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
resize_mode = resize_mode if resize_mode is not None else self.resize_mode
|
||||
min_pixels, max_pixels = _pixel_budget(
|
||||
min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
|
||||
max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
|
||||
patch_size,
|
||||
merge_size,
|
||||
temporal_patch_size,
|
||||
)
|
||||
grouped_images, grouped_images_index = group_images_by_shape(
|
||||
images, disable_grouping=disable_grouping
|
||||
)
|
||||
resized_images_grouped = {}
|
||||
for shape, stacked_images in grouped_images.items():
|
||||
height, width = stacked_images.shape[-2:]
|
||||
if do_resize:
|
||||
resized_height, resized_width = smart_resize(
|
||||
t=temporal_patch_size,
|
||||
h=height,
|
||||
w=width,
|
||||
t_factor=temporal_patch_size,
|
||||
h_factor=patch_size * merge_size * patch_expand_factor,
|
||||
w_factor=patch_size * merge_size * patch_expand_factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
stacked_images = _resize_or_pad(
|
||||
stacked_images,
|
||||
target_height=resized_height,
|
||||
target_width=resized_width,
|
||||
resize_mode=resize_mode,
|
||||
resample=resample,
|
||||
resize=self.resize,
|
||||
allow_upscale=(temporal_patch_size * height * width < min_pixels),
|
||||
)
|
||||
resized_images_grouped[shape] = stacked_images
|
||||
|
||||
resized_images = reorder_images(resized_images_grouped, grouped_images_index)
|
||||
|
||||
grouped_images, grouped_images_index = group_images_by_shape(
|
||||
resized_images, disable_grouping=disable_grouping
|
||||
)
|
||||
processed_images_grouped = {}
|
||||
processed_grids = {}
|
||||
|
||||
for shape, stacked_images in grouped_images.items():
|
||||
resized_height, resized_width = stacked_images.shape[-2:]
|
||||
|
||||
patches = self.rescale_and_normalize(
|
||||
stacked_images,
|
||||
do_rescale,
|
||||
rescale_factor,
|
||||
do_normalize,
|
||||
image_mean,
|
||||
image_std,
|
||||
)
|
||||
if patches.ndim == 4: # (B, C, H, W)
|
||||
patches = patches.unsqueeze(1) # (B, T=1, C, H, W)
|
||||
|
||||
if patches.shape[1] % temporal_patch_size != 0:
|
||||
repeats = patches[:, -1:].repeat(
|
||||
1,
|
||||
temporal_patch_size - (patches.shape[1] % temporal_patch_size),
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
patches = torch.cat([patches, repeats], dim=1)
|
||||
|
||||
batch_size, t_len, channel = patches.shape[:3]
|
||||
grid_t = t_len // temporal_patch_size
|
||||
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
||||
|
||||
patches = patches.view(
|
||||
batch_size,
|
||||
grid_t,
|
||||
temporal_patch_size,
|
||||
channel,
|
||||
grid_h // merge_size,
|
||||
merge_size,
|
||||
patch_size,
|
||||
grid_w // merge_size,
|
||||
merge_size,
|
||||
patch_size,
|
||||
)
|
||||
# (B, grid_t, gh, gw, mh, mw, C, tp, ph, pw)
|
||||
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
|
||||
|
||||
flatten_patches = patches.reshape(
|
||||
batch_size,
|
||||
grid_t * grid_h * grid_w,
|
||||
channel * temporal_patch_size * patch_size * patch_size,
|
||||
)
|
||||
|
||||
processed_images_grouped[shape] = flatten_patches
|
||||
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
|
||||
|
||||
processed_images = reorder_images(
|
||||
processed_images_grouped, grouped_images_index
|
||||
)
|
||||
processed_grids = reorder_images(processed_grids, grouped_images_index)
|
||||
|
||||
pixel_values = torch.cat(processed_images, dim=0)
|
||||
image_grid_thw = torch.tensor(processed_grids)
|
||||
|
||||
return BatchFeature(
|
||||
data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
|
||||
def preprocess(
|
||||
self, images: ImageInput, **kwargs: Unpack[Glm5NextImageProcessorKwargs]
|
||||
) -> BatchFeature:
|
||||
return super().preprocess(images, **kwargs)
|
||||
|
||||
def get_number_of_image_patches(
|
||||
self, height: int, width: int, images_kwargs: dict | None = None
|
||||
) -> int:
|
||||
"""Number of image patches (pre-merge) for a given (height, width)."""
|
||||
images_kwargs = images_kwargs or {}
|
||||
patch_size = images_kwargs.get("patch_size", self.patch_size)
|
||||
merge_size = images_kwargs.get("merge_size", self.merge_size)
|
||||
patch_expand_factor = images_kwargs.get(
|
||||
"patch_expand_factor", self.patch_expand_factor
|
||||
)
|
||||
min_pixels, max_pixels = _pixel_budget(
|
||||
images_kwargs.get("min_image_tokens", self.min_image_tokens),
|
||||
images_kwargs.get("max_image_tokens", self.max_image_tokens),
|
||||
patch_size,
|
||||
merge_size,
|
||||
self.temporal_patch_size,
|
||||
)
|
||||
resized_height, resized_width = smart_resize(
|
||||
t=self.temporal_patch_size,
|
||||
h=height,
|
||||
w=width,
|
||||
t_factor=self.temporal_patch_size,
|
||||
h_factor=patch_size * merge_size * patch_expand_factor,
|
||||
w_factor=patch_size * merge_size * patch_expand_factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
||||
return grid_h * grid_w
|
||||
|
||||
|
||||
class Glm5NextVideoProcessorKwargs(VideosKwargs, total=False): # type: ignore[call-arg]
|
||||
fps: list[float] | float
|
||||
patch_size: int
|
||||
temporal_patch_size: int
|
||||
merge_size: int
|
||||
patch_expand_factor: int
|
||||
resize_mode: str | None
|
||||
target_fps: float | None
|
||||
max_frames: int | None
|
||||
fps_interval: int | None
|
||||
max_frame_count_dynamic: int | None
|
||||
min_image_tokens: int | None
|
||||
max_image_tokens: int | None
|
||||
|
||||
|
||||
class Glm5NextVideoProcessor(BaseVideoProcessor):
|
||||
"""Fast video processor for GLM-5.3-Flash.
|
||||
|
||||
Shares ``smart_resize`` / the pad-mode geometry / the patchify with the
|
||||
image processor, and adds GLM-5.3-Flash frame sampling
|
||||
(``glm_sample_frame_indices``: ``fps_interval`` semantics with a
|
||||
temporal-patch-scaled greedy walk). Defaults mirror the checkpoint's
|
||||
``video_processor`` config.
|
||||
"""
|
||||
|
||||
resample = PILImageResampling.BICUBIC
|
||||
size = {"longest_edge": 1} # unused: budgets come from the token bounds
|
||||
image_mean = OPENAI_CLIP_MEAN
|
||||
image_std = OPENAI_CLIP_STD
|
||||
do_resize = True
|
||||
do_rescale = True
|
||||
do_normalize = True
|
||||
do_convert_rgb = True
|
||||
do_sample_frames = True
|
||||
patch_size = 14
|
||||
temporal_patch_size = 2
|
||||
patch_expand_factor = 1
|
||||
merge_size = 2
|
||||
valid_kwargs = Glm5NextVideoProcessorKwargs
|
||||
num_frames = 16
|
||||
fps = 2
|
||||
fps_interval = 2.0
|
||||
max_frame_count_dynamic = 2048
|
||||
resize_mode = "pad"
|
||||
min_image_tokens = 16
|
||||
max_image_tokens = 240000
|
||||
model_input_names = ["pixel_values_videos", "video_grid_thw"]
|
||||
|
||||
def sample_frames(
|
||||
self,
|
||||
metadata: VideoMetadata,
|
||||
fps: int | float | None = None,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
"""Sample frame indices with GLM's fps-interval policy.
|
||||
|
||||
``fps`` / ``target_fps``, ``max_frames`` and ``fps_interval`` /
|
||||
``max_frame_count_dynamic`` are the overrides described in
|
||||
:func:`glm_sample_frame_indices`.
|
||||
"""
|
||||
if metadata is None or getattr(metadata, "fps", None) is None:
|
||||
raise ValueError(
|
||||
"Asked to sample frames per second but no video metadata was "
|
||||
"provided which is required when sampling in GLM-5.3-Flash. Please "
|
||||
"pass in `VideoMetadata` object or set `do_sample_frames=False`."
|
||||
)
|
||||
|
||||
target_fps = fps if fps is not None else kwargs.get("target_fps")
|
||||
if target_fps is None:
|
||||
target_fps = self.fps_interval
|
||||
indices = glm_sample_frame_indices(
|
||||
metadata.total_num_frames,
|
||||
metadata.fps,
|
||||
metadata.duration or 0,
|
||||
target_fps=target_fps,
|
||||
max_frame_count=kwargs.get("max_frames") or self.max_frame_count_dynamic,
|
||||
temporal_patch_size=self.temporal_patch_size,
|
||||
)
|
||||
return np.array(indices)
|
||||
|
||||
def _preprocess(
|
||||
self,
|
||||
videos: list[torch.Tensor],
|
||||
do_convert_rgb: bool = True,
|
||||
do_resize: bool = True,
|
||||
size: SizeDict | None = None,
|
||||
resample: "PILImageResampling | int | None" = PILImageResampling.BICUBIC,
|
||||
do_rescale: bool = True,
|
||||
rescale_factor: float = 1 / 255.0,
|
||||
do_normalize: bool = True,
|
||||
image_mean: float | list[float] | None = None,
|
||||
image_std: float | list[float] | None = None,
|
||||
patch_size: int | None = None,
|
||||
temporal_patch_size: int | None = None,
|
||||
patch_expand_factor: int | None = None,
|
||||
merge_size: int | None = None,
|
||||
resize_mode: str | None = None,
|
||||
min_image_tokens: int | None = None,
|
||||
max_image_tokens: int | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
patch_expand_factor = self.patch_expand_factor
|
||||
patch_size = patch_size if patch_size is not None else self.patch_size
|
||||
temporal_patch_size = (
|
||||
temporal_patch_size
|
||||
if temporal_patch_size is not None
|
||||
else self.temporal_patch_size
|
||||
)
|
||||
merge_size = merge_size if merge_size is not None else self.merge_size
|
||||
resize_mode = resize_mode if resize_mode is not None else self.resize_mode
|
||||
min_pixels, max_pixels = _pixel_budget(
|
||||
min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
|
||||
max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
|
||||
patch_size,
|
||||
merge_size,
|
||||
temporal_patch_size,
|
||||
)
|
||||
grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
|
||||
resized_videos_grouped = {}
|
||||
for shape, stacked_videos in grouped_videos.items():
|
||||
if do_convert_rgb:
|
||||
stacked_videos = self.convert_to_rgb(stacked_videos)
|
||||
b, t_len, c, h, w = stacked_videos.shape
|
||||
num_frames, height, width = t_len, h, w
|
||||
if do_resize:
|
||||
resized_height, resized_width = smart_resize(
|
||||
t=num_frames,
|
||||
h=height,
|
||||
w=width,
|
||||
t_factor=temporal_patch_size,
|
||||
h_factor=patch_size * merge_size * patch_expand_factor,
|
||||
w_factor=patch_size * merge_size * patch_expand_factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
stacked_videos = stacked_videos.view(b * t_len, c, h, w)
|
||||
stacked_videos = _resize_or_pad(
|
||||
stacked_videos,
|
||||
target_height=resized_height,
|
||||
target_width=resized_width,
|
||||
resize_mode=resize_mode,
|
||||
resample=resample,
|
||||
resize=self.resize,
|
||||
allow_upscale=(num_frames * height * width < min_pixels),
|
||||
)
|
||||
stacked_videos = stacked_videos.view(
|
||||
b, t_len, c, resized_height, resized_width
|
||||
)
|
||||
resized_videos_grouped[shape] = stacked_videos
|
||||
resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
|
||||
|
||||
grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
|
||||
processed_videos_grouped = {}
|
||||
processed_grids = {}
|
||||
for shape, stacked_videos in grouped_videos.items():
|
||||
resized_height, resized_width = get_image_size(
|
||||
stacked_videos[0], channel_dim=ChannelDimension.FIRST
|
||||
)
|
||||
stacked_videos = self.rescale_and_normalize(
|
||||
stacked_videos,
|
||||
do_rescale,
|
||||
rescale_factor,
|
||||
do_normalize,
|
||||
image_mean,
|
||||
image_std,
|
||||
)
|
||||
patches = stacked_videos
|
||||
|
||||
if pad := -patches.shape[1] % temporal_patch_size:
|
||||
repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1)
|
||||
patches = torch.cat((patches, repeats), dim=1)
|
||||
batch_size, grid_t, channel = patches.shape[:3]
|
||||
grid_t = grid_t // temporal_patch_size
|
||||
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
||||
|
||||
patches = patches.view(
|
||||
batch_size,
|
||||
grid_t,
|
||||
temporal_patch_size,
|
||||
channel,
|
||||
grid_h // merge_size,
|
||||
merge_size,
|
||||
patch_size,
|
||||
grid_w // merge_size,
|
||||
merge_size,
|
||||
patch_size,
|
||||
)
|
||||
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
|
||||
flatten_patches = patches.reshape(
|
||||
batch_size,
|
||||
grid_t * grid_h * grid_w,
|
||||
channel * temporal_patch_size * patch_size * patch_size,
|
||||
)
|
||||
|
||||
processed_videos_grouped[shape] = flatten_patches
|
||||
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
|
||||
|
||||
processed_videos = reorder_videos(
|
||||
processed_videos_grouped, grouped_videos_index
|
||||
)
|
||||
processed_grids = reorder_videos(processed_grids, grouped_videos_index)
|
||||
pixel_values_videos = torch.cat(processed_videos, dim=0)
|
||||
video_grid_thw = torch.tensor(processed_grids)
|
||||
return BatchFeature(
|
||||
data={
|
||||
"pixel_values_videos": pixel_values_videos,
|
||||
"video_grid_thw": video_grid_thw,
|
||||
},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
|
||||
|
||||
class Glm5NextProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
|
||||
images_kwargs: Glm5NextImageProcessorKwargs
|
||||
videos_kwargs: Glm5NextVideoProcessorKwargs
|
||||
_defaults = {
|
||||
"text_kwargs": {
|
||||
"padding": False,
|
||||
"return_token_type_ids": False,
|
||||
"return_mm_token_type_ids": False,
|
||||
},
|
||||
"videos_kwargs": {"return_metadata": True},
|
||||
}
|
||||
|
||||
|
||||
class Glm5NextProcessor(ProcessorMixin):
|
||||
"""Wraps a GLM-5.3-Flash image processor, video processor and tokenizer.
|
||||
|
||||
Token expansion per image = ``prod(image_grid_thw) // merge_size**2``; video
|
||||
frames are expanded with ``<|begin_of_image|>...<|end_of_image|>{ts} seconds``
|
||||
structure (mrope timestamps).
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer", "video_processor"]
|
||||
image_processor_class = "AutoImageProcessor"
|
||||
video_processor_class = "AutoVideoProcessor"
|
||||
tokenizer_class = ("PreTrainedTokenizer", "PreTrainedTokenizerFast")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor=None,
|
||||
tokenizer=None,
|
||||
video_processor=None,
|
||||
chat_template=None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
image_processor, tokenizer, video_processor, chat_template=chat_template
|
||||
)
|
||||
self.image_token = (
|
||||
"<|image|>"
|
||||
if not hasattr(tokenizer, "image_token")
|
||||
else tokenizer.image_token
|
||||
)
|
||||
self.video_token = (
|
||||
"<|video|>"
|
||||
if not hasattr(tokenizer, "video_token")
|
||||
else tokenizer.video_token
|
||||
)
|
||||
self.image_token_id = (
|
||||
tokenizer.image_token_id
|
||||
if getattr(tokenizer, "image_token_id", None)
|
||||
else tokenizer.convert_tokens_to_ids(self.image_token)
|
||||
)
|
||||
self.video_token_id = (
|
||||
tokenizer.video_token_id
|
||||
if getattr(tokenizer, "video_token_id", None)
|
||||
else tokenizer.convert_tokens_to_ids(self.video_token)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
|
||||
"""Build the processor directly from the checkpoint config.
|
||||
|
||||
GLM-5.3-Flash stores nested image/video configs in
|
||||
``processor_config.json`` and declares a custom processor class. This
|
||||
method reads those configs directly and caps only the video token budget.
|
||||
"""
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
model_path = pretrained_model_name_or_path
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_path, **kwargs)
|
||||
|
||||
def _cap_cfg(cfg: dict, *, is_video: bool) -> dict:
|
||||
# Video keeps a serving token cap (the checkpoint's 240k-token
|
||||
# budget would starve the KV cache at startup profiling); images
|
||||
# follow the checkpoint budget verbatim so preprocessing matches
|
||||
# the HF reference exactly.
|
||||
if is_video and cfg.get("max_image_tokens") is not None:
|
||||
cfg["max_image_tokens"] = min(
|
||||
cfg["max_image_tokens"], _MAX_VIDEO_TOKENS
|
||||
)
|
||||
return cfg
|
||||
|
||||
ip_cfg = _cap_cfg(
|
||||
dict(get_image_processor_config(model_path, **kwargs)), is_video=False
|
||||
)
|
||||
image_processor = Glm5NextImageProcessor(
|
||||
**{k: v for k, v in ip_cfg.items() if k != "image_processor_type"}
|
||||
)
|
||||
|
||||
processor_config = get_hf_file_to_dict(
|
||||
"processor_config.json",
|
||||
model_path,
|
||||
revision=kwargs.get("revision", "main"),
|
||||
)
|
||||
if processor_config is None:
|
||||
raise ValueError(f"Missing processor_config.json for {model_path}")
|
||||
vp_cfg = _cap_cfg(dict(processor_config["video_processor"]), is_video=True)
|
||||
video_processor = Glm5NextVideoProcessor(
|
||||
**{k: v for k, v in vp_cfg.items() if k != "video_processor_type"}
|
||||
)
|
||||
|
||||
return cls(
|
||||
image_processor=image_processor,
|
||||
tokenizer=tokenizer,
|
||||
video_processor=video_processor,
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: ImageInput | None = None,
|
||||
text: TextInput
|
||||
| PreTokenizedInput
|
||||
| list[TextInput]
|
||||
| list[PreTokenizedInput] = None,
|
||||
videos: VideoInput | None = None,
|
||||
**kwargs: Unpack[Glm5NextProcessorKwargs],
|
||||
) -> BatchFeature:
|
||||
output_kwargs = self._merge_kwargs(
|
||||
Glm5NextProcessorKwargs,
|
||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
if images is not None:
|
||||
image_inputs = self.image_processor(
|
||||
images=images, **output_kwargs["images_kwargs"]
|
||||
)
|
||||
else:
|
||||
image_inputs = {}
|
||||
|
||||
if videos is not None:
|
||||
videos_inputs = self.video_processor(
|
||||
videos=videos, **output_kwargs["videos_kwargs"]
|
||||
)
|
||||
if "return_metadata" not in kwargs:
|
||||
videos_inputs.pop("video_metadata")
|
||||
else:
|
||||
videos_inputs = {}
|
||||
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
|
||||
# Prompt updates expand the unchanged image/video markers after this call.
|
||||
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
|
||||
return_mm_token_type_ids = output_kwargs["text_kwargs"].pop(
|
||||
"return_mm_token_type_ids", False
|
||||
)
|
||||
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
|
||||
|
||||
if return_mm_token_type_ids:
|
||||
array_ids = np.array(text_inputs["input_ids"])
|
||||
mm_token_type_ids = np.zeros_like(text_inputs["input_ids"])
|
||||
mm_token_type_ids[array_ids == self.image_token_id] = 1
|
||||
text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist()
|
||||
return BatchFeature(
|
||||
data={**text_inputs, **image_inputs, **videos_inputs},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
|
||||
def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs):
|
||||
vision_data = {}
|
||||
if image_sizes is not None:
|
||||
images_kwargs = Glm5NextProcessorKwargs._defaults.get("images_kwargs", {})
|
||||
images_kwargs.update(kwargs)
|
||||
merge_size = (
|
||||
images_kwargs.get("merge_size", None) or self.image_processor.merge_size
|
||||
)
|
||||
|
||||
num_image_patches = [
|
||||
self.image_processor.get_number_of_image_patches(
|
||||
*image_size, images_kwargs
|
||||
)
|
||||
for image_size in image_sizes
|
||||
]
|
||||
num_image_tokens = [(n // merge_size**2) for n in num_image_patches]
|
||||
vision_data.update(
|
||||
{
|
||||
"num_image_tokens": num_image_tokens,
|
||||
"num_image_patches": num_image_patches,
|
||||
}
|
||||
)
|
||||
|
||||
if video_sizes is not None:
|
||||
videos_kwargs = Glm5NextProcessorKwargs._defaults.get("videos_kwargs", {})
|
||||
videos_kwargs.update(kwargs)
|
||||
num_video_patches = [
|
||||
self.video_processor.get_number_of_video_patches(
|
||||
*video_size, videos_kwargs
|
||||
)
|
||||
for video_size in video_sizes
|
||||
]
|
||||
num_video_tokens = [(n // merge_size**2) for n in num_video_patches]
|
||||
vision_data["num_video_tokens"] = num_video_tokens
|
||||
|
||||
return MultiModalData(**vision_data)
|
||||
|
||||
def post_process_image_text_to_text(
|
||||
self,
|
||||
generated_outputs,
|
||||
skip_special_tokens=True,
|
||||
clean_up_tokenization_spaces=False,
|
||||
**kwargs,
|
||||
):
|
||||
return self.tokenizer.batch_decode(
|
||||
generated_outputs,
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Glm5NextImageProcessor",
|
||||
"Glm5NextVideoProcessor",
|
||||
"Glm5NextProcessor",
|
||||
"smart_resize",
|
||||
]
|
||||
@@ -29,6 +29,11 @@ _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES: set[str] = {
|
||||
"qwen3_5_moe_text",
|
||||
}
|
||||
|
||||
# KV page sizes (in cache entries) supported by the paged-MQA logits kernels
|
||||
# (fp8_fp4_paged_mqa_logits / get_paged_mqa_logits_metadata). Larger storage
|
||||
# blocks must be virtually split into one of these page sizes.
|
||||
PAGED_MQA_PAGE_SIZES = (32, 64)
|
||||
|
||||
|
||||
def should_auto_disable_deep_gemm(model_type: str | None) -> bool:
|
||||
"""Check if DeepGemm should be auto-disabled for this model on Blackwell.
|
||||
@@ -648,6 +653,13 @@ def fp8_fp4_paged_mqa_logits(
|
||||
_lazy_init()
|
||||
if _fp8_fp4_paged_mqa_logits_impl is None:
|
||||
return _missing()
|
||||
# DeepGEMM asserts block_tables.stride(-1)==1. A trailing size-1 dim
|
||||
# (e.g. block_table shape [B,1] for short seqs under a large block_size)
|
||||
# can be a transposed view where .contiguous() is a no-op (torch treats the
|
||||
# size-1 dim's stride as irrelevant) yet stride(-1)!=1, failing the kernel.
|
||||
# clone to contiguous format to force stride(-1)==1.
|
||||
if block_tables.dim() >= 2 and block_tables.stride(-1) != 1:
|
||||
block_tables = block_tables.clone(memory_format=torch.contiguous_format)
|
||||
kwargs = {} if indices is None else {"indices": indices}
|
||||
return _fp8_fp4_paged_mqa_logits_impl(
|
||||
q,
|
||||
|
||||
@@ -292,6 +292,32 @@ def has_flashinfer_moe() -> bool:
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_sm90_nope_mla() -> bool:
|
||||
"""FlashInfer SM90 NoPE MLA (FP8 KV with in-kernel dequant, kpe=0).
|
||||
|
||||
Feature-detected via the ``ckv_scale_arr`` run() kwarg introduced with
|
||||
the SM90 NoPE support (FlashInfer >= 0.6.18), so dev builds carry the
|
||||
gate without a version parse.
|
||||
"""
|
||||
if not has_flashinfer():
|
||||
return False
|
||||
try:
|
||||
import inspect
|
||||
|
||||
from flashinfer.mla import BatchMLAPagedAttentionWrapper
|
||||
except ImportError:
|
||||
return False
|
||||
try:
|
||||
params = inspect.signature(BatchMLAPagedAttentionWrapper.run).parameters
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return (
|
||||
"ckv_scale_arr" in params
|
||||
and params["ckv_scale_arr"].kind is inspect.Parameter.KEYWORD_ONLY
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_sparse_mla_sm120() -> bool:
|
||||
"""Return ``True`` if FlashInfer sparse MLA decode support is available."""
|
||||
|
||||
@@ -610,6 +610,10 @@ class AttentionMetadataBuilder(ABC, Generic[M]):
|
||||
self.layer_names = layer_names
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
self.kernel_block_size: int | None = None
|
||||
|
||||
def set_kernel_block_size(self, kernel_block_size: int) -> None:
|
||||
self.kernel_block_size = kernel_block_size
|
||||
|
||||
@classmethod
|
||||
def get_cudagraph_support(
|
||||
|
||||
@@ -54,7 +54,10 @@ class _FlashInferMLASparseBackendBase(AttentionBackend):
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return [576]
|
||||
# 576 = 512 NoPE + 64 RoPE (with-rope layout); 512 = 512 NoPE only
|
||||
# (no-rope layout, qk_rope_head_dim == 0). Both share D_V = 512 and are
|
||||
# served by the TRTLLM-GEN sparse MLA kernel.
|
||||
return [512, 576]
|
||||
|
||||
@classmethod
|
||||
def is_mla(cls) -> bool:
|
||||
@@ -117,8 +120,20 @@ class FlashInferMLASparseTRTLLMBackend(_FlashInferMLASparseBackendBase):
|
||||
# FlashInfer MLA sparse SM10 kernel requires qk_nope_head_dim in [128, 192].
|
||||
if vllm_config.model_config is not None:
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
qk_nope_head_dim = getattr(hf_text_config, "qk_nope_head_dim", 1)
|
||||
if qk_nope_head_dim not in [128, 192]:
|
||||
qk_nope_head_dim = hf_text_config.qk_nope_head_dim
|
||||
qk_rope_head_dim = hf_text_config.qk_rope_head_dim
|
||||
kv_lora_rank = hf_text_config.kv_lora_rank
|
||||
if qk_rope_head_dim == 0:
|
||||
# Native no-rope MLA: FlashInfer only ships the one shape
|
||||
# (nope_mla_dimensions in flashinfer.mla._core).
|
||||
if qk_nope_head_dim != 256 or kv_lora_rank != 512:
|
||||
return (
|
||||
"FlashInfer native no-rope MLA requires "
|
||||
"qk_nope_head_dim=256 and kv_lora_rank=512, but got "
|
||||
f"qk_nope_head_dim={qk_nope_head_dim}, "
|
||||
f"kv_lora_rank={kv_lora_rank}"
|
||||
)
|
||||
elif qk_nope_head_dim not in [128, 192]:
|
||||
return (
|
||||
"FlashInfer MLA Sparse kernel requires qk_nope_head_dim "
|
||||
f"in [128, 192], but got {qk_nope_head_dim}"
|
||||
@@ -358,6 +373,10 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
|
||||
self.bmm1_scale: float | None = None
|
||||
self.bmm2_scale: float | None = None
|
||||
|
||||
# Native no-rope MLA additionally requires a per-query-token active
|
||||
# top-k length tensor.
|
||||
self.is_nope_mla = self.qk_rope_head_dim == 0
|
||||
|
||||
# fp8 query quantization is required when using fp8 kv_cache,
|
||||
# as the TRTLLM-GEN sparse MLA kernel requires matching dtypes
|
||||
# for query and kv_cache (mixed bf16+fp8 is not supported).
|
||||
@@ -429,6 +448,31 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
|
||||
block_tables = topk_indices_physical.unsqueeze(1)
|
||||
seq_lens_arg = seq_lens
|
||||
|
||||
# page_table width = topk buffer width, which kpool widens past
|
||||
# index_topk (topk_tokens) and rounds up to a multiple of 128. The
|
||||
# kernel treats sparse_mla_top_k as the page-table *capacity* and bounds
|
||||
# the active per-query length by ``seq_lens`` (the compacted valid
|
||||
# count), so the -1 padding slots past seq_lens are never attended to.
|
||||
# Use the actual buffer width instead of the fixed topk_tokens, which
|
||||
# mismatches the page_table when index_kpool > 1.
|
||||
sparse_topk_capacity = topk_indices_physical.shape[1]
|
||||
|
||||
extra_kwargs: dict[str, torch.Tensor] = {}
|
||||
empty_rows: torch.Tensor | None = None
|
||||
if self.is_nope_mla:
|
||||
# The native no-rope kernel takes the active top-k length per query
|
||||
# token (``seq_lens`` here is already the compacted per-token valid
|
||||
# count, int32) and rejects zero-length rows. Point empty rows at a
|
||||
# single valid dummy slot with length 1 and zero their output after
|
||||
# the launch. ``triton_convert_req_index_to_global_index`` packs the
|
||||
# valid indices into a contiguous prefix, which is what the kernel
|
||||
# requires of the page table.
|
||||
empty_rows = seq_lens == 0
|
||||
topk_indices_physical[:, 0] = topk_indices_physical[:, 0].masked_fill(
|
||||
empty_rows, 0
|
||||
)
|
||||
extra_kwargs["sparse_mla_top_k_lens"] = seq_lens.clamp(min=1)
|
||||
|
||||
kernel_out = trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=query,
|
||||
kv_cache=kv_c_and_k_pe_cache.unsqueeze(1),
|
||||
@@ -438,11 +482,12 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens_arg,
|
||||
max_seq_len=attn_metadata.topk_tokens,
|
||||
max_seq_len=sparse_topk_capacity,
|
||||
bmm1_scale=self.bmm1_scale,
|
||||
bmm2_scale=self.bmm2_scale,
|
||||
sparse_mla_top_k=attn_metadata.topk_tokens,
|
||||
sparse_mla_top_k=sparse_topk_capacity,
|
||||
return_lse=self.need_to_return_lse_for_decode,
|
||||
**extra_kwargs,
|
||||
)
|
||||
if self.need_to_return_lse_for_decode:
|
||||
assert isinstance(kernel_out, tuple)
|
||||
@@ -455,9 +500,12 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
|
||||
out = o.view(-1, o.shape[-2], o.shape[-1])
|
||||
if lse is not None:
|
||||
lse = self._normalize_lse(lse, out.shape[0], out.shape[1])
|
||||
if empty_rows is None and lse is not None:
|
||||
empty_rows = (topk_indices_physical == -1).all(dim=-1)
|
||||
if empty_rows is not None:
|
||||
out.masked_fill_(empty_rows.view(-1, 1, 1), 0.0)
|
||||
lse.masked_fill_(empty_rows.view(-1, 1), float("-inf"))
|
||||
if lse is not None:
|
||||
lse.masked_fill_(empty_rows.view(-1, 1), float("-inf"))
|
||||
return out, lse
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FlashInfer sparse MLA backend for SM90 (Hopper) NoPE models.
|
||||
|
||||
Wraps FlashInfer's ``BatchMLAPagedAttentionWrapper`` (FA2/FA3 paths), which
|
||||
as of FlashInfer 0.6.18 supports ``head_dim_kpe=0`` (GLM-5.3-Flash NoPE MLA)
|
||||
and FP8 E4M3 KV caches on SM90 with in-kernel dequantization: the FP8 cache
|
||||
is read directly (half the bf16 HBM traffic) and converted to BF16 in shared
|
||||
memory, while queries stay BF16 (no query quantization).
|
||||
|
||||
Sparsity rides the same trick the FA-based sparse backend uses: with
|
||||
``page_size=1`` the per-token top-k slot indices ARE the page table, so each
|
||||
query token becomes one varlen batch row whose ``kv_indices`` slice is its
|
||||
top-k row and whose ``kv_len`` is its valid count. Causality is already
|
||||
encoded by the indexer's selection, so ``causal=False``.
|
||||
|
||||
CUDA-graph handling: ``plan()`` copies its inputs to host unconditionally,
|
||||
so it must stay outside graph capture. Each metadata builder owns a wrapper,
|
||||
reserved capture-stable device buffers, and the plan parameters. The wrapper
|
||||
bakes the per-row ``kv_len`` into its int schedule
|
||||
at plan() time — ``run()`` never reads the device-side buffer — so the
|
||||
metadata builder replans every step (outside capture) with exact host-side
|
||||
lengths derived from the batch's sequence lengths; a full-width schedule
|
||||
would send the kernel past each row's valid count into the -1 tail of the
|
||||
converted index buffer (illegal address). Per-step content (top-k slots)
|
||||
is written into the reserved buffers by kernels inside the captured
|
||||
forward, and captured runs read the refreshed plan buffers on replay.
|
||||
|
||||
KV cache format: plain contiguous E4M3 ``[num_blocks, block_size, 512]``
|
||||
(uint8 storage) with a per-tensor ``k_scale``; BF16 caches also work. The
|
||||
per-token x 128-channel-group ``ckv_scale_arr`` layout is supported by the
|
||||
kernel but not wired yet (it needs a group-quantizing cache-write op).
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
SparseMLACommonImpl,
|
||||
)
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.flashinfer import has_flashinfer_sm90_nope_mla
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionLayer,
|
||||
CommonAttentionMetadata,
|
||||
MLAAttentionImpl,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseMetadata,
|
||||
FlashInferMLASparseMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
triton_convert_req_index_to_global_index,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
||||
|
||||
_FP8_KV_DTYPES = ("fp8", "fp8_e4m3")
|
||||
_WORKSPACE_BYTES = 128 * 1024 * 1024
|
||||
|
||||
|
||||
class FlashInferMLASparseSM90Backend(AttentionBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"bfloat16",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
return [MultipleOf(64)]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHINFER_MLA_SPARSE_SM90"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["FlashInferMLASparseSM90Builder"]:
|
||||
return FlashInferMLASparseSM90Builder
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type[MLAAttentionImpl]:
|
||||
return FlashInferMLASparseSM90Impl
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
# 512 = ckv 512 + kpe 0 (NoPE); 576 = ckv 512 + kpe 64.
|
||||
return [512, 576]
|
||||
|
||||
@classmethod
|
||||
def is_mla(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def is_sparse(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
||||
return capability.major == 9
|
||||
|
||||
@classmethod
|
||||
def supports_combination(
|
||||
cls,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType | None,
|
||||
block_size: int | None,
|
||||
use_mla: bool,
|
||||
has_sink: bool,
|
||||
use_sparse: bool,
|
||||
use_mm_prefix: bool,
|
||||
device_capability: DeviceCapability,
|
||||
) -> str | None:
|
||||
if not has_flashinfer_sm90_nope_mla():
|
||||
return (
|
||||
"FLASHINFER_MLA_SPARSE_SM90 requires FlashInfer with SM90 "
|
||||
"MLA support (ckv_scale_arr in "
|
||||
"BatchMLAPagedAttentionWrapper.run, FlashInfer >= 0.6.18)"
|
||||
)
|
||||
if not use_sparse:
|
||||
return "FLASHINFER_MLA_SPARSE_SM90 requires sparse MLA"
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
if vllm_config.model_config is not None:
|
||||
hf = vllm_config.model_config.hf_text_config
|
||||
# The SM90 FA2/FA3 kernel covers ckv=512 with kpe in {0, 64}
|
||||
# (NoPE models and DeepSeek-style rope MLA alike).
|
||||
if hf.kv_lora_rank != 512:
|
||||
return "FLASHINFER_MLA_SPARSE_SM90 requires kv_lora_rank=512"
|
||||
if hf.qk_rope_head_dim not in (0, 64):
|
||||
return "FLASHINFER_MLA_SPARSE_SM90 requires qk_rope_head_dim in (0, 64)"
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, block_size, head_size)
|
||||
|
||||
@classmethod
|
||||
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]:
|
||||
return (KVCacheLayout.LBHNC,)
|
||||
|
||||
|
||||
class _SM90State:
|
||||
"""Builder-owned wrapper, capture-stable buffers, and plan parameters.
|
||||
|
||||
One instance serves every MLA layer in an attention group because the plan
|
||||
depends only on the batch shape, not the layer.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device: torch.device,
|
||||
num_heads: int,
|
||||
kv_dtype: torch.dtype,
|
||||
max_tokens: int,
|
||||
topk_width: int,
|
||||
kv_lora_rank: int,
|
||||
qk_rope_head_dim: int,
|
||||
sm_scale: float,
|
||||
) -> None:
|
||||
from flashinfer.mla import BatchMLAPagedAttentionWrapper
|
||||
|
||||
self.workspace = torch.empty(_WORKSPACE_BYTES, dtype=torch.uint8, device=device)
|
||||
self.device = device
|
||||
self.num_heads = num_heads
|
||||
self.kv_dtype = kv_dtype
|
||||
self.max_tokens = max_tokens
|
||||
self.topk_width = topk_width
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.sm_scale = sm_scale
|
||||
# User-reserved buffers: with use_cuda_graph=True plan() refreshes
|
||||
# these in place, so run()'s captured kernels always read them.
|
||||
self.kv_indices = torch.zeros(
|
||||
max_tokens * topk_width, dtype=torch.int32, device=device
|
||||
)
|
||||
self.kv_len_arr = torch.full(
|
||||
(max_tokens,), topk_width, dtype=torch.int32, device=device
|
||||
)
|
||||
self.wrapper = BatchMLAPagedAttentionWrapper(
|
||||
self.workspace,
|
||||
qo_indptr=torch.zeros(max_tokens + 1, dtype=torch.int32, device=device),
|
||||
kv_indptr=torch.zeros(max_tokens + 1, dtype=torch.int32, device=device),
|
||||
kv_indices=self.kv_indices,
|
||||
kv_len_arr=self.kv_len_arr,
|
||||
use_cuda_graph=True,
|
||||
backend="fa3",
|
||||
)
|
||||
self._arange_cpu = torch.arange(self.max_tokens + 1, dtype=torch.int32)
|
||||
self._qo_cpu = torch.empty(self.max_tokens + 1, dtype=torch.int32)
|
||||
self._kv_cpu = torch.empty(self.max_tokens + 1, dtype=torch.int32)
|
||||
self._lens_cpu = torch.full(
|
||||
(self.max_tokens,), self.topk_width, dtype=torch.int32
|
||||
)
|
||||
|
||||
def plan(self, num_tokens: int, kv_lens: torch.Tensor) -> None:
|
||||
"""Replan with exact per-row KV lengths (CPU int32, ``[num_tokens]``).
|
||||
|
||||
The wrapper bakes kv_len into its int schedule from host values;
|
||||
run() never reads the device kv_len_arr buffer. Scheduling with the
|
||||
full buffer width while kv_indices rows carry a -1 tail past each
|
||||
row's valid count makes the kernel compute ``-1 * ckv_stride_page``
|
||||
(illegal address), so the lengths must be exact at plan time and
|
||||
replanned every step as contexts grow. Must run outside CUDA graph
|
||||
capture: the in-place refreshed plan_info/indptr buffers are what
|
||||
captured runs read.
|
||||
"""
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
"FlashInferMLASparseSM90 plan() called inside CUDA graph "
|
||||
"capture; lengths must be planned host-side before capture."
|
||||
)
|
||||
# CPU staging buffers are filled in place: plan() runs per step
|
||||
# (once per draft/verify metadata build), so per-call allocations and
|
||||
# device round trips are on the hot path. Passing CPU tensors lets
|
||||
# the wrapper's internal .to("cpu") no-op; its reserved-buffer
|
||||
# copy_ then performs the single H2D transfer per tensor.
|
||||
# use_cuda_graph=True makes the wrapper copy qo/kv indptr into its
|
||||
# fixed (max_tokens+1)-sized buffers with exact-size copy_, so the
|
||||
# indptr must always be full-size. Rows past num_tokens are padded
|
||||
# empty (qo_indptr flat at num_tokens) — zero-query rows read no q
|
||||
# and schedule no work. Padded rows keep the full width lens; the
|
||||
# value is never dereferenced.
|
||||
torch.clamp(self._arange_cpu, max=num_tokens, out=self._qo_cpu)
|
||||
torch.mul(self._qo_cpu, self.topk_width, out=self._kv_cpu)
|
||||
self._lens_cpu.fill_(self.topk_width)
|
||||
self._lens_cpu[:num_tokens] = kv_lens.to(torch.int32)
|
||||
self.wrapper.plan(
|
||||
self._qo_cpu,
|
||||
self._kv_cpu,
|
||||
self.kv_indices,
|
||||
self._lens_cpu,
|
||||
self.num_heads,
|
||||
self.kv_lora_rank, # head_dim_ckv
|
||||
self.qk_rope_head_dim, # 0 (NoPE) or 64 (rope MLA)
|
||||
1, # page_size: top-k slots are the page table
|
||||
False, # causal: encoded by the indexer's selection
|
||||
self.sm_scale,
|
||||
q_data_type=torch.bfloat16,
|
||||
kv_data_type=self.kv_dtype,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlashInferMLASparseSM90Metadata(FlashInferMLASparseMetadata):
|
||||
state: _SM90State | None = None
|
||||
|
||||
|
||||
class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
|
||||
"""Reuse the common sparse metadata (req ids, topk buffer access)."""
|
||||
|
||||
metadata_cls = FlashInferMLASparseSM90Metadata
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: "AttentionSpec",
|
||||
layer_names: list[str],
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
attention_layer = vllm_config.compilation_config.static_forward_context[
|
||||
layer_names[0]
|
||||
]
|
||||
impl = attention_layer.impl
|
||||
if not isinstance(impl, FlashInferMLASparseSM90Impl):
|
||||
raise TypeError(
|
||||
"FlashInferMLASparseSM90Builder requires an SM90 FlashInfer "
|
||||
f"implementation, got {type(impl).__name__}."
|
||||
)
|
||||
topk_indices_buffer = impl.topk_indices_buffer
|
||||
assert topk_indices_buffer is not None
|
||||
self.state = _SM90State(
|
||||
device,
|
||||
impl.num_heads,
|
||||
kv_cache_spec.dtype,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
topk_indices_buffer.shape[1],
|
||||
kv_lora_rank=impl.kv_lora_rank,
|
||||
qk_rope_head_dim=impl.qk_rope_head_dim,
|
||||
sm_scale=impl.scale,
|
||||
)
|
||||
# seq_lens_cpu_upper_bound is optimistic on decode rows under async
|
||||
# spec decode, so the fast sync-free path is only safe without it;
|
||||
# under async scheduling the exact (device) positions are used at
|
||||
# the cost of one D2H sync per metadata build.
|
||||
self._async_scheduling = bool(vllm_config.scheduler_config.async_scheduling)
|
||||
hf_config = vllm_config.model_config.hf_text_config
|
||||
assert hf_config.index_topk is not None
|
||||
self._index_topk = int(hf_config.index_topk)
|
||||
self._index_kpool = int(kv_cache_spec.tokens_per_state)
|
||||
|
||||
def _kv_lens_host(self, cam: CommonAttentionMetadata) -> tuple[int, torch.Tensor]:
|
||||
"""Exact per-row KV lengths, host-side (the flashinfer wrapper bakes
|
||||
them into its schedule at plan time; there is no device-side path).
|
||||
|
||||
A row for the j-th query token of request i attends
|
||||
``seq_lens[i] - q_len[i] + j + 1`` tokens. The indexer's selection
|
||||
then bounds the valid count: contexts up to ``index_topk`` select
|
||||
everything (valid == context); longer contexts keep the top
|
||||
``index_topk`` pool-expanded tokens plus the trailing incomplete
|
||||
pool (valid == ``index_topk + context % index_kpool``). Both match
|
||||
the count of non -1 entries the convert kernel produces.
|
||||
"""
|
||||
num_reqs = cam.num_reqs
|
||||
qsl = cam.query_start_loc_cpu[: num_reqs + 1]
|
||||
num_rows = int(qsl[-1])
|
||||
if num_rows == 0:
|
||||
return 0, torch.zeros(0, dtype=torch.int32)
|
||||
# Row context == position + 1. Without async scheduling the
|
||||
# maintained host upper bound is exact, giving a sync-free path;
|
||||
# under async scheduling it is optimistic on decode rows, so fall
|
||||
# back to the exact device positions (one D2H sync per build). The
|
||||
# seq_lens derivation equals position + 1 because positions are
|
||||
# contiguous per request.
|
||||
sl_host = cam.seq_lens_cpu_upper_bound
|
||||
positions = cam.positions
|
||||
if not self._async_scheduling and sl_host is not None:
|
||||
seq_lens = sl_host[:num_reqs].to(torch.int32)
|
||||
q_lens = qsl[1:] - qsl[:-1]
|
||||
first_pos = seq_lens - q_lens
|
||||
req_of_row = torch.repeat_interleave(
|
||||
torch.arange(num_reqs, dtype=torch.int64), q_lens.to(torch.int64)
|
||||
)
|
||||
rows = torch.arange(num_rows, dtype=torch.int32)
|
||||
ctx = (
|
||||
first_pos.to(torch.int64)[req_of_row]
|
||||
+ rows.to(torch.int64)
|
||||
- qsl.to(torch.int64)[req_of_row]
|
||||
+ 1
|
||||
)
|
||||
elif positions is not None and num_rows <= positions.shape[0]:
|
||||
ctx = positions[:num_rows].cpu().to(torch.int64) + 1
|
||||
else:
|
||||
seq_lens = cam.seq_lens[:num_reqs].cpu().to(torch.int32)
|
||||
q_lens = qsl[1:] - qsl[:-1]
|
||||
first_pos = seq_lens - q_lens
|
||||
req_of_row = torch.repeat_interleave(
|
||||
torch.arange(num_reqs, dtype=torch.int64), q_lens.to(torch.int64)
|
||||
)
|
||||
rows = torch.arange(num_rows, dtype=torch.int32)
|
||||
ctx = (
|
||||
first_pos.to(torch.int64)[req_of_row]
|
||||
+ rows.to(torch.int64)
|
||||
- qsl.to(torch.int64)[req_of_row]
|
||||
+ 1
|
||||
)
|
||||
topk = self._index_topk
|
||||
kpool = max(self._index_kpool, 1)
|
||||
lens = torch.where(ctx <= topk, ctx, topk + ctx % kpool)
|
||||
return num_rows, lens.to(torch.int32)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> FlashInferMLASparseSM90Metadata:
|
||||
metadata = super().build(common_prefix_len, common_attn_metadata, fast_build)
|
||||
assert isinstance(metadata, FlashInferMLASparseSM90Metadata)
|
||||
# Replan every step outside any CUDA graph capture with this step's
|
||||
# exact per-row lengths; captured runs read the refreshed buffers.
|
||||
num_rows, kv_lens = self._kv_lens_host(common_attn_metadata)
|
||||
self.state.plan(num_rows, kv_lens)
|
||||
metadata.state = self.state
|
||||
return metadata
|
||||
|
||||
|
||||
class FlashInferMLASparseSM90Impl(SparseMLACommonImpl[FlashInferMLASparseSM90Metadata]):
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
scale: float,
|
||||
num_kv_heads: int,
|
||||
alibi_slopes: list[float] | None,
|
||||
sliding_window: int | None,
|
||||
kv_cache_dtype: str,
|
||||
logits_soft_cap: float | None,
|
||||
attn_type: str,
|
||||
kv_sharing_target_layer_name: str | None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
indexer: Any | None = None,
|
||||
**mla_args: Any,
|
||||
) -> None:
|
||||
if any([alibi_slopes, sliding_window, logits_soft_cap]):
|
||||
raise NotImplementedError(
|
||||
"FlashInferMLASparseSM90Impl does not support alibi, sliding "
|
||||
"window, or logits soft cap."
|
||||
)
|
||||
super().__init__(
|
||||
num_heads,
|
||||
head_size,
|
||||
scale,
|
||||
num_kv_heads,
|
||||
alibi_slopes,
|
||||
sliding_window,
|
||||
kv_cache_dtype,
|
||||
logits_soft_cap,
|
||||
attn_type,
|
||||
kv_sharing_target_layer_name,
|
||||
indexer=indexer,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
**mla_args,
|
||||
)
|
||||
assert self.topk_indices_buffer is not None
|
||||
self.supports_quant_query_input = False
|
||||
self.use_fp8_kv_cache = self.kv_cache_dtype in _FP8_KV_DTYPES
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: FlashInferMLASparseSM90Metadata,
|
||||
layer: AttentionLayer,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if not isinstance(q, tuple):
|
||||
raise NotImplementedError(
|
||||
"FlashInferMLASparseSM90Impl expects split (q_nope, q_rope)."
|
||||
)
|
||||
q_nope, q_rope = q
|
||||
num_tokens = q_rope.shape[0]
|
||||
# NoPE models hand a zero-width rope tensor through; rope MLA hands
|
||||
# the real 64-dim part. The kernel takes both as-is.
|
||||
q_pe = q_rope.reshape(num_tokens, self.num_heads, self.qk_rope_head_dim)
|
||||
|
||||
assert self.topk_indices_buffer is not None
|
||||
topk_indices = self.topk_indices_buffer[:num_tokens]
|
||||
# return_valid_counts=True keeps the compacted-prefix layout: valid
|
||||
# entries at [0, valid_count), -1 past it — exactly the prefix the
|
||||
# planned per-row lengths address.
|
||||
topk_slots, _ = triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token[:num_tokens],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
return_valid_counts=True,
|
||||
)
|
||||
state = attn_metadata.state
|
||||
assert state is not None
|
||||
# Refresh top-k rows in graph and clamp masked tails to a valid slot;
|
||||
# per-row lengths are already baked into the host-side plan.
|
||||
width = topk_slots.shape[1]
|
||||
state.kv_indices[: num_tokens * width].copy_(
|
||||
topk_slots.reshape(-1).clamp_(min=0).to(torch.int32)
|
||||
)
|
||||
|
||||
flat = (
|
||||
kv_c_and_k_pe_cache.view(torch.float8_e4m3fn)
|
||||
if self.use_fp8_kv_cache
|
||||
else kv_c_and_k_pe_cache
|
||||
).reshape(-1, 1, self.head_size)
|
||||
ckv = flat[..., : self.kv_lora_rank]
|
||||
kpe = flat[..., self.kv_lora_rank :]
|
||||
|
||||
scale_kwargs = (
|
||||
{"ckv_scale": float(layer._k_scale_float or 1.0), "kpe_scale": 1.0}
|
||||
if self.use_fp8_kv_cache
|
||||
else {}
|
||||
)
|
||||
out = state.wrapper.run(q_nope, q_pe, ckv, kpe, **scale_kwargs)
|
||||
return out, None
|
||||
@@ -37,7 +37,12 @@ from vllm.v1.attention.backends.utils import (
|
||||
get_dcp_local_seq_lens,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheLayout, KVCacheSpec, MLAAttentionSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -252,6 +257,30 @@ class DeepseekV32IndexerBackend(AttentionBackend):
|
||||
return DeepseekV32IndexerMetadataBuilder
|
||||
|
||||
|
||||
class KpoolTailBackend(DeepseekV32IndexerBackend):
|
||||
"""Storage-only backend for the GLM-5.3-Flash kpool tail cache."""
|
||||
|
||||
@classmethod
|
||||
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]:
|
||||
return (KVCacheLayout.LBHNC,)
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "KPOOL_TAIL"
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
return [MultipleOf(1)]
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["KpoolTailMetadataBuilder"]: # type: ignore[override]
|
||||
return KpoolTailMetadataBuilder
|
||||
|
||||
|
||||
class DeepseekV4IndexerBackend(DeepseekV32IndexerBackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
@@ -419,6 +448,9 @@ class BuildPrefillChunkMetadataKernel(
|
||||
)
|
||||
)
|
||||
)
|
||||
index_kpool = getattr(hf_config, "index_kpool", None)
|
||||
if index_kpool and index_kpool > 1 and index_kpool not in compress_ratios:
|
||||
compress_ratios = compress_ratios + (index_kpool,)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
# Cover Triton's divisible, exact-one, and generic i32 classes.
|
||||
query_slice_start=(0, 1, 2),
|
||||
@@ -483,6 +515,7 @@ class BuildPrefillChunkMetadataKernel(
|
||||
@dataclass
|
||||
class DeepseekV32IndexerPrefillMetadata:
|
||||
chunks: list[DeepseekV32IndexerPrefillChunkMetadata]
|
||||
max_prefill_seq_len: int = -1
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -497,6 +530,9 @@ class DeepSeekV32IndexerDecodeMetadata:
|
||||
requires_padding: bool
|
||||
schedule_metadata: torch.Tensor
|
||||
global_seq_lens: torch.Tensor | None = None
|
||||
per_req_decode_lens: torch.Tensor | None = None
|
||||
decode_is_uniform: bool = True
|
||||
write_max_decode_len: int = 0
|
||||
indices: torch.Tensor | None = None
|
||||
|
||||
|
||||
@@ -519,6 +555,90 @@ class DeepseekV32IndexerMetadata:
|
||||
prefill: DeepseekV32IndexerPrefillMetadata | None = None
|
||||
|
||||
|
||||
def compute_kpool_tail_slot_mapping(
|
||||
slot_mapping: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_actual_tokens: int,
|
||||
num_reqs: int,
|
||||
kpool: int,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Map every token to its request's one circular tail block."""
|
||||
if out is None:
|
||||
out = slot_mapping.clone()
|
||||
else:
|
||||
assert out.shape == slot_mapping.shape
|
||||
out.copy_(slot_mapping)
|
||||
if num_actual_tokens == 0:
|
||||
return out
|
||||
tokens = torch.arange(num_actual_tokens, device=slot_mapping.device)
|
||||
req = torch.searchsorted(query_start_loc, tokens, right=True) - 1
|
||||
req = req.clamp_(min=0, max=num_reqs - 1)
|
||||
own_block = block_table[:num_reqs, 0].index_select(0, req).to(torch.int64)
|
||||
pos = positions[:num_actual_tokens].to(torch.int64)
|
||||
out[:num_actual_tokens] = own_block * kpool + torch.remainder(pos, kpool)
|
||||
return out
|
||||
|
||||
|
||||
class KpoolTailMetadataBuilder(AttentionMetadataBuilder):
|
||||
"""Build only the circular slot mapping needed by the storage-only tail."""
|
||||
|
||||
_cudagraph_support = AttentionCGSupport.ALWAYS
|
||||
supports_update_block_table = False
|
||||
reorder_batch_threshold = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
layer_names: list[str],
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
):
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self.slot_mapping_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> DeepseekV32IndexerMetadata:
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(common_attn_metadata)
|
||||
)
|
||||
slot_mapping = common_attn_metadata.slot_mapping
|
||||
positions = common_attn_metadata.positions
|
||||
if positions is not None:
|
||||
slot_mapping_buffer = self.slot_mapping_buffer[
|
||||
: slot_mapping.numel()
|
||||
].view_as(slot_mapping)
|
||||
slot_mapping = compute_kpool_tail_slot_mapping(
|
||||
slot_mapping,
|
||||
common_attn_metadata.block_table_tensor,
|
||||
common_attn_metadata.query_start_loc,
|
||||
positions,
|
||||
common_attn_metadata.num_actual_tokens,
|
||||
common_attn_metadata.num_reqs,
|
||||
self.kv_cache_spec.block_size,
|
||||
out=slot_mapping_buffer,
|
||||
)
|
||||
return DeepseekV32IndexerMetadata(
|
||||
seq_lens=common_attn_metadata.seq_lens,
|
||||
max_seq_len=common_attn_metadata.max_seq_len,
|
||||
slot_mapping=slot_mapping,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_prefills=num_prefills,
|
||||
num_prefill_tokens=num_prefill_tokens,
|
||||
)
|
||||
|
||||
|
||||
def get_max_prefill_buffer_size(vllm_config: VllmConfig):
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
# NOTE(Chen): 40 is a magic number for controlling the prefill buffer size.
|
||||
@@ -640,6 +760,11 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.per_req_decode_lens_buffer = torch.zeros(
|
||||
(scheduler_config.max_num_batched_tokens,),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
# Shared workspace for decode seq_lens. Native MTP views this as
|
||||
# (B, max_decode_len) at runtime, keeping context_lens contiguous even
|
||||
# when max_decode_len is smaller than next_n.
|
||||
@@ -707,6 +832,8 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.indexer_decode_block_table_buffer: torch.Tensor | None = None
|
||||
self._max_num_batched_tokens = scheduler_config.max_num_batched_tokens
|
||||
|
||||
def _dcp_localize_decode_seq_lens(
|
||||
self,
|
||||
@@ -921,7 +1048,16 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
|
||||
compressed_slot_mapping = slot_mapping
|
||||
compressed_seq_lens = seq_lens
|
||||
indexer_block_table = block_table
|
||||
if self.compress_ratio > 1:
|
||||
kernel_block_size = self.kernel_block_size
|
||||
if (
|
||||
kernel_block_size is not None
|
||||
and self.kv_cache_spec.block_size != kernel_block_size
|
||||
and self.kv_cache_spec.block_size % kernel_block_size == 0
|
||||
):
|
||||
factor = self.kv_cache_spec.block_size // kernel_block_size
|
||||
indexer_block_table = (block_table[:, ::factor] // factor).contiguous()
|
||||
padded_num_tokens = num_tokens
|
||||
if self.pcp_world_size > 1:
|
||||
padded_num_tokens = slot_mapping.shape[0] // self.pcp_world_size
|
||||
@@ -929,7 +1065,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
num_tokens,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
block_table,
|
||||
indexer_block_table,
|
||||
self.kv_cache_spec.num_states,
|
||||
self.compress_ratio,
|
||||
out=self.compressed_slot_mapping_buffer,
|
||||
@@ -979,7 +1115,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
seq_lens,
|
||||
compressed_seq_lens,
|
||||
compressed_seq_lens_cpu,
|
||||
common_attn_metadata.block_table_tensor,
|
||||
indexer_block_table,
|
||||
self.compress_ratio,
|
||||
query_slice=query_slice,
|
||||
skip_kv_gather=query_slice.start > 0,
|
||||
@@ -990,7 +1126,14 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
# Skip when total_seq_lens is 0 (i.e., no compressed token).
|
||||
if metadata is not None:
|
||||
chunks.append(metadata)
|
||||
prefill_metadata = DeepseekV32IndexerPrefillMetadata(chunks)
|
||||
prefill_metadata = DeepseekV32IndexerPrefillMetadata(
|
||||
chunks,
|
||||
max_prefill_seq_len=(
|
||||
int(seq_lens_cpu[num_decodes:].max().item())
|
||||
if num_prefills > 0
|
||||
else 0
|
||||
),
|
||||
)
|
||||
|
||||
decode_metadata = None
|
||||
if num_decodes > 0:
|
||||
@@ -999,6 +1142,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
out=self.decode_lens_buffer[:num_decodes],
|
||||
)
|
||||
decode_lens = self.decode_lens_buffer[:num_decodes]
|
||||
self.per_req_decode_lens_buffer[:num_decodes].copy_(decode_lens)
|
||||
decode_lens_cpu = torch.diff(
|
||||
common_attn_metadata.query_start_loc_cpu[: num_decodes + 1]
|
||||
)
|
||||
@@ -1018,6 +1162,8 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
block_table = common_attn_metadata.block_table_tensor[:num_decodes, ...]
|
||||
|
||||
max_decode_len = int(decode_lens_cpu.max().item())
|
||||
min_decode_len = int(decode_lens_cpu.min().item())
|
||||
write_is_uniform = min_decode_len == max_decode_len
|
||||
next_n = 1 + self.num_speculative_tokens
|
||||
# The kernel sees max_decode_len Q rows, not the configured next_n,
|
||||
# so legality is per-step: on SM90 a uniformly 3-deep batch has no
|
||||
@@ -1064,7 +1210,30 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
)
|
||||
)
|
||||
|
||||
seq_lens_is_buffer_view = not use_native or next_n > 1
|
||||
if self.compress_ratio > 1:
|
||||
kernel_block_size = self.kernel_block_size
|
||||
if (
|
||||
kernel_block_size is not None
|
||||
and self.kv_cache_spec.block_size != kernel_block_size
|
||||
and self.kv_cache_spec.block_size % kernel_block_size == 0
|
||||
):
|
||||
factor = self.kv_cache_spec.block_size // kernel_block_size
|
||||
compressed = block_table[:, ::factor] // factor
|
||||
rows, cols = compressed.shape
|
||||
if self.indexer_decode_block_table_buffer is None:
|
||||
self.indexer_decode_block_table_buffer = torch.zeros(
|
||||
(self._max_num_batched_tokens, cols),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.indexer_decode_block_table_buffer[:rows, :cols].copy_(
|
||||
compressed
|
||||
)
|
||||
block_table = self.indexer_decode_block_table_buffer[:rows, :cols]
|
||||
|
||||
seq_lens_is_buffer_view = (use_native and next_n > 1) or (
|
||||
not use_native and max_decode_len > 1
|
||||
)
|
||||
|
||||
# DCP: localize the now-expanded per-token global bounds to this
|
||||
# rank's owned KV. Done here (after expansion) so each token's global
|
||||
@@ -1113,6 +1282,9 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
schedule_metadata=schedule_metadata,
|
||||
indices=decode_indices,
|
||||
global_seq_lens=global_seq_lens_for_decode,
|
||||
per_req_decode_lens=self.per_req_decode_lens_buffer[:num_decodes],
|
||||
decode_is_uniform=write_is_uniform,
|
||||
write_max_decode_len=max_decode_len,
|
||||
)
|
||||
|
||||
attn_metadata = DeepseekV32IndexerMetadata(
|
||||
|
||||
@@ -881,7 +881,11 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
rows. A DCP rank holds ``1/dcp_world_size`` of the sequence, so the
|
||||
request's block table is always wider than the pages a row can reach.
|
||||
"""
|
||||
pages_per_block = self.kernel_block_size // self._segmented_page_size
|
||||
kernel_block_size = self.kernel_block_size
|
||||
page_size = self._segmented_page_size
|
||||
assert kernel_block_size is not None
|
||||
assert page_size is not None
|
||||
pages_per_block = kernel_block_size // page_size
|
||||
max_local_pages = row_block_table.shape[1]
|
||||
max_local_blocks = cdiv(max_local_pages, pages_per_block)
|
||||
assert max_local_blocks <= block_table.shape[1], (
|
||||
|
||||
@@ -32,6 +32,9 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
AiterMLAHelper,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_sparse_attn_prefill,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
@@ -40,6 +43,51 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _use_rocm_sparse_triton(
|
||||
*,
|
||||
kv_cache_dtype: str,
|
||||
head_size: int,
|
||||
kv_lora_rank: int,
|
||||
num_prefills: int,
|
||||
num_decodes: int,
|
||||
num_decode_tokens: int,
|
||||
max_query_len: int,
|
||||
) -> bool:
|
||||
"""Select the rope-free BF16 path not supported by AITER sparse MLA."""
|
||||
plain_decode = num_decode_tokens == num_decodes
|
||||
return (
|
||||
not kv_cache_dtype.startswith("fp8")
|
||||
and head_size == kv_lora_rank
|
||||
and plain_decode
|
||||
and (num_prefills > 0 or (num_decodes > 0 and max_query_len == 1))
|
||||
)
|
||||
|
||||
|
||||
def fit_kpool_indices_to_aiter(
|
||||
token_indices: torch.Tensor, topk_tokens: int
|
||||
) -> torch.Tensor:
|
||||
"""Keep the live kpool tail while fitting AITER's fixed top-k width."""
|
||||
if token_indices.shape[1] < topk_tokens:
|
||||
raise ValueError("token_indices width must be at least topk_tokens")
|
||||
if token_indices.shape[1] == topk_tokens:
|
||||
return token_indices
|
||||
|
||||
history = token_indices[:, :topk_tokens]
|
||||
tail = token_indices[:, topk_tokens:]
|
||||
valid_history = (history >= 0).sum(dim=1)
|
||||
valid_tail = (tail >= 0).sum(dim=1)
|
||||
keep_history = torch.minimum(valid_history, topk_tokens - valid_tail)
|
||||
|
||||
columns = torch.arange(topk_tokens, device=token_indices.device).unsqueeze(0)
|
||||
tail_offsets = columns - keep_history.unsqueeze(1)
|
||||
tail_values = torch.gather(
|
||||
tail, 1, tail_offsets.clamp(min=0, max=tail.shape[1] - 1)
|
||||
)
|
||||
output = torch.where(columns < keep_history.unsqueeze(1), history, tail_values)
|
||||
valid_output = columns < (keep_history + valid_tail).unsqueeze(1)
|
||||
return torch.where(valid_output, output, -1)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _convert_req_index_to_global_index_kernel(
|
||||
req_id_ptr, # int32 [num_tokens]
|
||||
@@ -359,6 +407,7 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
self.kv_cache_spec = kv_cache_spec
|
||||
self.model_config = vllm_config.model_config
|
||||
self.model_dtype = vllm_config.model_config.dtype
|
||||
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.device = device
|
||||
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
@@ -550,53 +599,74 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
# context lengths (both clamped to topk_tokens, past which per-token KV
|
||||
# length saturates) and num_heads; fingerprint those CPU-side and skip
|
||||
# the launch when nothing changed.
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
clamped_seq_lens = np.minimum(
|
||||
common_attn_metadata.seq_lens_cpu[:num_reqs].numpy(),
|
||||
self.topk_tokens,
|
||||
head_size = self.mla_dims.kv_lora_rank + self.mla_dims.qk_rope_head_dim
|
||||
use_triton_sparse = _use_rocm_sparse_triton(
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
head_size=head_size,
|
||||
kv_lora_rank=self.mla_dims.kv_lora_rank,
|
||||
num_prefills=num_prefills,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
max_query_len=common_attn_metadata.max_query_len,
|
||||
)
|
||||
clamped_context_lens = np.minimum(
|
||||
common_attn_metadata.seq_lens_cpu[:num_reqs].numpy() - seg_lengths,
|
||||
self.topk_tokens,
|
||||
)
|
||||
metadata_key = (
|
||||
num_tokens,
|
||||
int(common_attn_metadata.max_query_len),
|
||||
self._num_attention_heads,
|
||||
clamped_seq_lens.tobytes(),
|
||||
clamped_context_lens.tobytes(),
|
||||
seg_lengths.tobytes(),
|
||||
)
|
||||
if metadata_key != self._prev_metadata_key:
|
||||
from aiter import get_mla_metadata_v1
|
||||
|
||||
max_split_per_batch = self._sparse_decode_max_split(
|
||||
int(common_attn_metadata.max_seq_len)
|
||||
work_meta_data = None
|
||||
work_indptr = None
|
||||
work_info_set = None
|
||||
reduce_indptr = None
|
||||
reduce_final_map = None
|
||||
reduce_partial_map = None
|
||||
if not use_triton_sparse:
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
clamped_seq_lens = np.minimum(
|
||||
common_attn_metadata.seq_lens_cpu[:num_reqs].numpy(),
|
||||
self.topk_tokens,
|
||||
)
|
||||
get_mla_metadata_v1(
|
||||
qo_indptr,
|
||||
paged_kv_indptr,
|
||||
paged_kv_last_page_len,
|
||||
clamped_context_lens = np.minimum(
|
||||
common_attn_metadata.seq_lens_cpu[:num_reqs].numpy() - seg_lengths,
|
||||
self.topk_tokens,
|
||||
)
|
||||
metadata_key = (
|
||||
num_tokens,
|
||||
int(common_attn_metadata.max_query_len),
|
||||
self._num_attention_heads,
|
||||
1,
|
||||
True,
|
||||
self._mla_work_meta_data,
|
||||
self._mla_work_info_set,
|
||||
self._mla_work_indptr,
|
||||
self._mla_reduce_indptr,
|
||||
self._mla_reduce_final_map,
|
||||
self._mla_reduce_partial_map,
|
||||
page_size=1,
|
||||
kv_granularity=16,
|
||||
max_seqlen_qo=1,
|
||||
uni_seqlen_qo=1,
|
||||
fast_mode=True,
|
||||
max_split_per_batch=max_split_per_batch,
|
||||
clamped_seq_lens.tobytes(),
|
||||
clamped_context_lens.tobytes(),
|
||||
seg_lengths.tobytes(),
|
||||
)
|
||||
# The persistent metadata buffers are read by graph replay. Order
|
||||
# the async metadata write before the graph-captured decode kernel.
|
||||
torch.cuda.current_stream(self.device).synchronize()
|
||||
self._prev_metadata_key = metadata_key
|
||||
if metadata_key != self._prev_metadata_key:
|
||||
from aiter import get_mla_metadata_v1
|
||||
|
||||
max_split_per_batch = self._sparse_decode_max_split(
|
||||
int(common_attn_metadata.max_seq_len)
|
||||
)
|
||||
get_mla_metadata_v1(
|
||||
qo_indptr,
|
||||
paged_kv_indptr,
|
||||
paged_kv_last_page_len,
|
||||
self._num_attention_heads,
|
||||
1,
|
||||
True,
|
||||
self._mla_work_meta_data,
|
||||
self._mla_work_info_set,
|
||||
self._mla_work_indptr,
|
||||
self._mla_reduce_indptr,
|
||||
self._mla_reduce_final_map,
|
||||
self._mla_reduce_partial_map,
|
||||
page_size=1,
|
||||
kv_granularity=16,
|
||||
max_seqlen_qo=1,
|
||||
uni_seqlen_qo=1,
|
||||
fast_mode=True,
|
||||
max_split_per_batch=max_split_per_batch,
|
||||
)
|
||||
torch.cuda.current_stream(self.device).synchronize()
|
||||
self._prev_metadata_key = metadata_key
|
||||
work_meta_data = self._mla_work_meta_data
|
||||
work_indptr = self._mla_work_indptr
|
||||
work_info_set = self._mla_work_info_set
|
||||
reduce_indptr = self._mla_reduce_indptr
|
||||
reduce_final_map = self._mla_reduce_final_map
|
||||
reduce_partial_map = self._mla_reduce_partial_map
|
||||
|
||||
metadata = ROCMAiterMLASparseMetadata(
|
||||
num_reqs=common_attn_metadata.num_reqs,
|
||||
@@ -617,12 +687,12 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
paged_kv_last_page_len=paged_kv_last_page_len,
|
||||
paged_kv_indices=paged_kv_indices,
|
||||
paged_kv_indptr=paged_kv_indptr,
|
||||
work_meta_data=self._mla_work_meta_data,
|
||||
work_indptr=self._mla_work_indptr,
|
||||
work_info_set=self._mla_work_info_set,
|
||||
reduce_indptr=self._mla_reduce_indptr,
|
||||
reduce_final_map=self._mla_reduce_final_map,
|
||||
reduce_partial_map=self._mla_reduce_partial_map,
|
||||
work_meta_data=work_meta_data,
|
||||
work_indptr=work_indptr,
|
||||
work_info_set=work_info_set,
|
||||
reduce_indptr=reduce_indptr,
|
||||
reduce_final_map=reduce_final_map,
|
||||
reduce_partial_map=reduce_partial_map,
|
||||
)
|
||||
return metadata
|
||||
|
||||
@@ -715,6 +785,31 @@ class ROCMAiterMLASparseImpl(MLAAttentionImpl[ROCMAiterMLASparseMetadata]):
|
||||
device=q.device,
|
||||
)
|
||||
|
||||
if _use_rocm_sparse_triton(
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
head_size=q.shape[-1],
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
num_prefills=attn_metadata.num_prefills,
|
||||
num_decodes=attn_metadata.num_decodes,
|
||||
num_decode_tokens=attn_metadata.num_decode_tokens,
|
||||
max_query_len=attn_metadata.max_query_len,
|
||||
):
|
||||
rocm_sparse_attn_prefill(
|
||||
q=q,
|
||||
kv=kv_c_and_k_pe_cache.view(-1, 1, q.shape[-1]),
|
||||
indices=None,
|
||||
topk_length=None,
|
||||
scale=self.scale,
|
||||
head_dim=q.shape[-1],
|
||||
nope_head_dim=self.kv_lora_rank,
|
||||
rope_head_dim=q.shape[-1] - self.kv_lora_rank,
|
||||
attn_sink=None,
|
||||
output=output,
|
||||
ragged_indices=attn_metadata.paged_kv_indices,
|
||||
ragged_indptr=attn_metadata.paged_kv_indptr,
|
||||
)
|
||||
return AiterMLAHelper.get_mla_unpadded_o(self.num_heads, output)
|
||||
|
||||
# Build kwargs and forward the persistent MLA metadata when it has
|
||||
# been computed. The aiter mla_decode_fwd switches to its
|
||||
# work-stealing persistent kernel path when work_meta_data is given.
|
||||
@@ -766,13 +861,18 @@ class ROCMAiterMLASparseImpl(MLAAttentionImpl[ROCMAiterMLASparseMetadata]):
|
||||
)
|
||||
else:
|
||||
q = self.q_concat_buffer[: ql_nope.shape[0]]
|
||||
ops.concat_mla_q(ql_nope, q_pe, q)
|
||||
if q_pe.shape[-1] == 0:
|
||||
q.copy_(ql_nope)
|
||||
else:
|
||||
ops.concat_mla_q(ql_nope, q_pe, q)
|
||||
|
||||
num_actual_toks = attn_metadata.num_actual_tokens
|
||||
|
||||
# Get topk indices
|
||||
assert self.topk_indices_buffer is not None
|
||||
topk_indices = self.topk_indices_buffer[:num_actual_toks]
|
||||
topk_indices = fit_kpool_indices_to_aiter(
|
||||
self.topk_indices_buffer[:num_actual_toks], attn_metadata.topk_tokens
|
||||
)
|
||||
|
||||
triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token,
|
||||
@@ -795,5 +895,4 @@ class ROCMAiterMLASparseImpl(MLAAttentionImpl[ROCMAiterMLASparseMetadata]):
|
||||
attn_out = self._forward_mla(
|
||||
layer, mla_padded_q, kv_c_and_k_pe_cache, attn_metadata
|
||||
)
|
||||
|
||||
return attn_out, None
|
||||
|
||||
@@ -251,7 +251,7 @@ class ConvertReqIndexToGlobalIndexKernel(
|
||||
dict(
|
||||
HAS_PREFILL_WORKSPACE=False,
|
||||
COUNT_VALID=True,
|
||||
COMPACT_TO_FRONT=False,
|
||||
COMPACT_TO_FRONT=True,
|
||||
DCP_SIZE=1,
|
||||
DCP_RANK=0,
|
||||
DCP_INTERLEAVE=1,
|
||||
@@ -259,7 +259,7 @@ class ConvertReqIndexToGlobalIndexKernel(
|
||||
dict(
|
||||
HAS_PREFILL_WORKSPACE=True,
|
||||
COUNT_VALID=True,
|
||||
COMPACT_TO_FRONT=False,
|
||||
COMPACT_TO_FRONT=True,
|
||||
DCP_SIZE=1,
|
||||
DCP_RANK=0,
|
||||
DCP_INTERLEAVE=1,
|
||||
@@ -458,7 +458,14 @@ def triton_convert_req_index_to_global_index(
|
||||
req_id_c = req_id.contiguous()
|
||||
block_table_c = block_table.contiguous()
|
||||
token_indices_c = token_indices.contiguous()
|
||||
out = torch.empty_like(token_indices_c)
|
||||
# When return_valid_counts, the kernel scatters valid entries to a
|
||||
# contiguous prefix [0, valid_count) and leaves the tail unwritten, so
|
||||
# pre-fill -1 there. flash_mla_sparse_fwd then bounds attention to
|
||||
# [:topk_length] == exactly the valid set (no dropped tokens).
|
||||
if return_valid_counts:
|
||||
out = torch.full_like(token_indices_c, -1)
|
||||
else:
|
||||
out = torch.empty_like(token_indices_c)
|
||||
|
||||
valid_counts: torch.Tensor | None = None
|
||||
if return_valid_counts:
|
||||
@@ -491,7 +498,7 @@ def triton_convert_req_index_to_global_index(
|
||||
NUM_TOPK_TOKENS=NUM_TOPK_TOKENS,
|
||||
HAS_PREFILL_WORKSPACE=HAS_PREFILL_WORKSPACE,
|
||||
COUNT_VALID=return_valid_counts,
|
||||
COMPACT_TO_FRONT=False,
|
||||
COMPACT_TO_FRONT=return_valid_counts,
|
||||
# DCP disabled (no-op de-interleave)
|
||||
DCP_SIZE=1,
|
||||
DCP_RANK=0,
|
||||
|
||||
@@ -77,6 +77,10 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla_sparse."
|
||||
"FlashInferMLASparseSM120Backend"
|
||||
)
|
||||
FLASHINFER_MLA_SPARSE_SM90 = (
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm90."
|
||||
"FlashInferMLASparseSM90Backend"
|
||||
)
|
||||
TRITON_MLA = "vllm.v1.attention.backends.mla.triton_mla.TritonMLABackend"
|
||||
CUTLASS_MLA = "vllm.v1.attention.backends.mla.cutlass_mla.CutlassMLABackend"
|
||||
FLASHMLA = "vllm.v1.attention.backends.mla.flashmla.FlashMLABackend"
|
||||
|
||||
@@ -909,6 +909,7 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
has_decode = layer_attn_metadata.num_decodes > 0
|
||||
has_prefill = layer_attn_metadata.num_prefills > 0
|
||||
num_decode_tokens = layer_attn_metadata.num_decode_tokens
|
||||
topk_indices_buffer[: hidden_states.shape[0]] = -1
|
||||
|
||||
# during speculative decoding, k may be padded to the CUDA graph batch
|
||||
# size while slot_mapping only covers actual tokens.
|
||||
@@ -1251,15 +1252,31 @@ _DSV4_SPARSE_NOPE_DIM = 448
|
||||
_DSV4_SPARSE_ROPE_DIM = 64
|
||||
|
||||
|
||||
def _validate_sparse_dims(
|
||||
head_dim: int,
|
||||
nope_head_dim: int,
|
||||
rope_head_dim: int,
|
||||
op_name: str,
|
||||
) -> None:
|
||||
assert head_dim > 0, f"{op_name} expected a positive head_dim, got {head_dim}"
|
||||
assert nope_head_dim > 0, (
|
||||
f"{op_name} expected a positive NoPE dimension, got {nope_head_dim}"
|
||||
)
|
||||
assert rope_head_dim >= 0, (
|
||||
f"{op_name} expected a non-negative RoPE dimension, got {rope_head_dim}"
|
||||
)
|
||||
assert head_dim == nope_head_dim + rope_head_dim, (
|
||||
f"{op_name} expected head_dim={nope_head_dim + rope_head_dim}, got {head_dim}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_dsv4_sparse_dims(
|
||||
head_dim: int,
|
||||
nope_head_dim: int,
|
||||
rope_head_dim: int,
|
||||
op_name: str,
|
||||
) -> None:
|
||||
assert head_dim == nope_head_dim + rope_head_dim, (
|
||||
f"{op_name} expected head_dim={nope_head_dim + rope_head_dim}, got {head_dim}"
|
||||
)
|
||||
_validate_sparse_dims(head_dim, nope_head_dim, rope_head_dim, op_name)
|
||||
assert (
|
||||
nope_head_dim == _DSV4_SPARSE_NOPE_DIM
|
||||
and rope_head_dim == _DSV4_SPARSE_ROPE_DIM
|
||||
@@ -1351,6 +1368,12 @@ def _as_int32_contiguous_1d(x: torch.Tensor) -> torch.Tensor:
|
||||
return x.to(torch.int32).contiguous()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sparse_kv_row_offset(slot, stride):
|
||||
# A global token slot fits in int32, but its byte/element offset may not.
|
||||
return slot.to(tl.int64) * stride
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sparse_attn_prefill_ragged_kernel(
|
||||
q_ptr,
|
||||
@@ -1414,7 +1437,7 @@ def _sparse_attn_prefill_ragged_kernel(
|
||||
|
||||
kv = tl.load(
|
||||
kv_ptr
|
||||
+ safe_slot[:, None] * kv_stride_n
|
||||
+ _sparse_kv_row_offset(safe_slot[:, None], kv_stride_n)
|
||||
+ dim_offsets[None, :] * kv_stride_d,
|
||||
mask=valid[:, None] & dim_mask[None, :],
|
||||
other=0.0,
|
||||
@@ -2573,7 +2596,7 @@ def _rocm_sparse_attn_prefill_ragged_triton(
|
||||
assert indptr.numel() == num_queries + 1, (
|
||||
f"expected indptr shape [{num_queries + 1}], got {indptr.shape}"
|
||||
)
|
||||
_validate_dsv4_sparse_dims(
|
||||
_validate_sparse_dims(
|
||||
head_dim,
|
||||
nope_head_dim,
|
||||
rope_head_dim,
|
||||
@@ -3125,7 +3148,7 @@ def _rocm_sparse_attn_decode_triton(
|
||||
def rocm_sparse_attn_prefill(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
indices: torch.Tensor | None,
|
||||
topk_length: torch.Tensor | None,
|
||||
scale: float,
|
||||
head_dim: int,
|
||||
@@ -3139,7 +3162,7 @@ def rocm_sparse_attn_prefill(
|
||||
assert kv.ndim == 3 and kv.shape[1] == 1, (
|
||||
f"ROCm Triton sparse prefill expects kv=[skv,1,d], got {kv.shape}"
|
||||
)
|
||||
_validate_dsv4_sparse_dims(
|
||||
_validate_sparse_dims(
|
||||
head_dim,
|
||||
nope_head_dim,
|
||||
rope_head_dim,
|
||||
@@ -3157,6 +3180,7 @@ def rocm_sparse_attn_prefill(
|
||||
rope_head_dim=rope_head_dim,
|
||||
)
|
||||
else:
|
||||
assert indices is not None
|
||||
indices_2d = indices.reshape(indices.shape[0], -1)
|
||||
output_chunk = _rocm_sparse_attn_prefill_triton(
|
||||
q=q,
|
||||
|
||||
@@ -599,6 +599,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
# can be a multiple of hash_block_size.
|
||||
self.hash_block_size = hash_block_size
|
||||
self.dcp_world_size = dcp_world_size
|
||||
# Only groups that participate in prefix caching must satisfy the
|
||||
# divisibility constraint; groups that opt out (e.g. GLM-5.3-Flash kpool
|
||||
# tail, block_size=kpool) are scratch buffers and excluded.
|
||||
group_block_sizes = [
|
||||
manager.block_size
|
||||
for manager, group in zip(
|
||||
@@ -679,6 +682,11 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
"""
|
||||
self.attention_groups: list[SpecGroup] = []
|
||||
for i, g in enumerate(self.kv_cache_config.kv_cache_groups):
|
||||
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
|
||||
# kpool tail): their blocks are per-request scratch, never
|
||||
# shareable, so they must not participate in hit lookup (their
|
||||
# manager-level hooks already no-op). Their slot in the per-group
|
||||
# hit tuple stays empty.
|
||||
if not g.kv_cache_spec.prefix_cacheable:
|
||||
continue
|
||||
manager_cls = self.single_type_managers[i].__class__
|
||||
|
||||
@@ -10,7 +10,7 @@ from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from functools import partial
|
||||
from typing import Any, NamedTuple, NewType, TypeAlias, overload
|
||||
from typing import Any, NamedTuple, NewType, TypeAlias, cast, overload
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
@@ -24,6 +24,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
@@ -1132,6 +1133,205 @@ def _get_kv_cache_groups_uniform_type(
|
||||
return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)]
|
||||
|
||||
|
||||
def _pp_balanced_mamba_group_count(
|
||||
vllm_config: VllmConfig,
|
||||
mamba_layer_names: list[str],
|
||||
mla_layer_names: list[str],
|
||||
) -> int | None:
|
||||
"""Return a Mamba group count whose PP projections fit the MLA slots."""
|
||||
num_groups = cdiv(len(mamba_layer_names), len(mla_layer_names))
|
||||
pp_size = vllm_config.parallel_config.pipeline_parallel_size
|
||||
if pp_size == 1:
|
||||
return num_groups
|
||||
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
total_layers = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
mamba_indices = [extract_layer_index(name) for name in mamba_layer_names]
|
||||
mla_indices = [extract_layer_index(name) for name in mla_layer_names]
|
||||
for rank in range(pp_size):
|
||||
start, end = get_pp_indices(total_layers, rank, pp_size)
|
||||
num_mamba = sum(start <= index < end for index in mamba_indices)
|
||||
num_mla = sum(start <= index < end for index in mla_indices)
|
||||
if not num_mamba:
|
||||
continue
|
||||
if not num_mla:
|
||||
return None
|
||||
num_groups = max(num_groups, cdiv(num_mamba, num_mla))
|
||||
return num_groups
|
||||
|
||||
|
||||
def _get_kv_cache_groups_glm5_next(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec] | None:
|
||||
"""Build GLM-5.3-Flash groups with Mamba/MLA and tail/indexer aliasing."""
|
||||
mamba_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if isinstance(spec, MambaSpec)
|
||||
}
|
||||
tail_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if isinstance(spec, KpoolTailSpec)
|
||||
}
|
||||
attn_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if not isinstance(spec, (MambaSpec, KpoolTailSpec))
|
||||
}
|
||||
if not mamba_specs or not all(
|
||||
type(spec) is MLAAttentionSpec for spec in attn_specs.values()
|
||||
):
|
||||
return None
|
||||
|
||||
mla_specs = cast(dict[str, MLAAttentionSpec], attn_specs)
|
||||
idx_pages = {
|
||||
spec.page_size_bytes for spec in mla_specs.values() if spec.tokens_per_state > 1
|
||||
}
|
||||
if not idx_pages:
|
||||
return None
|
||||
|
||||
assert all(spec.page_size_padded is None for spec in mla_specs.values())
|
||||
assert len(idx_pages) == 1
|
||||
mla_names = [name for name, spec in mla_specs.items() if spec.tokens_per_state == 1]
|
||||
mla_pages = {mla_specs[name].page_size_bytes for name in mla_names}
|
||||
assert len(mla_pages) == 1
|
||||
mla_page = mla_pages.pop()
|
||||
uniform_spec = UniformTypeKVCacheSpecs.from_specs(attn_specs)
|
||||
assert uniform_spec is not None
|
||||
|
||||
tail_group: KVCacheGroupSpec | None = None
|
||||
if tail_specs:
|
||||
idx_page = next(iter(idx_pages))
|
||||
padded_tail_specs: dict[str, KVCacheSpec] = {
|
||||
name: replace(spec, page_size_padded=idx_page)
|
||||
for name, spec in tail_specs.items()
|
||||
}
|
||||
tail_uniform = UniformTypeKVCacheSpecs.from_specs(padded_tail_specs)
|
||||
assert tail_uniform is not None
|
||||
tail_group = KVCacheGroupSpec(list(padded_tail_specs), tail_uniform)
|
||||
|
||||
any_mamba = next(iter(mamba_specs.values()))
|
||||
assert all(spec == any_mamba for spec in mamba_specs.values())
|
||||
if any_mamba.real_page_size_bytes > mla_page:
|
||||
raise ValueError(
|
||||
f"the mamba state page ({any_mamba.real_page_size_bytes} bytes) "
|
||||
f"does not fit the MLA page ({mla_page} bytes); increase tensor "
|
||||
"parallelism or use a wider KV cache dtype"
|
||||
)
|
||||
padded_specs: dict[str, KVCacheSpec] = {
|
||||
name: replace(any_mamba, page_size_padded=mla_page) for name in mamba_specs
|
||||
}
|
||||
num_groups = _pp_balanced_mamba_group_count(
|
||||
vllm_config, list(mamba_specs), mla_names
|
||||
)
|
||||
if num_groups is None:
|
||||
raise ValueError(
|
||||
"a pipeline stage has mamba layers but no MLA layer to share "
|
||||
"slots with; realign the stage boundaries (VLLM_PP_LAYER_PARTITION)"
|
||||
)
|
||||
mamba_grouped_names: list[list[str]] = [[] for _ in range(num_groups)]
|
||||
for index, name in enumerate(mamba_specs):
|
||||
mamba_grouped_names[index % num_groups].append(name)
|
||||
|
||||
return (
|
||||
[KVCacheGroupSpec(list(attn_specs), uniform_spec)]
|
||||
+ ([tail_group] if tail_group is not None else [])
|
||||
+ create_kv_cache_group_specs(padded_specs, mamba_grouped_names)
|
||||
)
|
||||
|
||||
|
||||
def _glm5_next_tensor_layout(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> (
|
||||
tuple[
|
||||
KVCacheGroupSpec,
|
||||
list[KVCacheGroupSpec],
|
||||
list[str],
|
||||
list[str],
|
||||
int,
|
||||
int,
|
||||
list[str],
|
||||
int,
|
||||
]
|
||||
| None
|
||||
):
|
||||
"""Recognize the GLM-5.3-Flash grouping after optional PP projection."""
|
||||
uniform_groups = [
|
||||
group
|
||||
for group in kv_cache_groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
]
|
||||
mamba_groups = [
|
||||
group for group in kv_cache_groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
attn_group: KVCacheGroupSpec | None = None
|
||||
tail_group: KVCacheGroupSpec | None = None
|
||||
for group in uniform_groups:
|
||||
inner = cast(UniformTypeKVCacheSpecs, group.kv_cache_spec).kv_cache_specs
|
||||
if all(type(spec) is MLAAttentionSpec for spec in inner.values()):
|
||||
attn_group = group
|
||||
elif all(isinstance(spec, KpoolTailSpec) for spec in inner.values()):
|
||||
tail_group = group
|
||||
if attn_group is None or not mamba_groups:
|
||||
return None
|
||||
if len(uniform_groups) + len(mamba_groups) != len(kv_cache_groups):
|
||||
return None
|
||||
|
||||
attn_uniform = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
|
||||
mla_inner = cast(dict[str, MLAAttentionSpec], attn_uniform.kv_cache_specs)
|
||||
if not all(
|
||||
type(spec) is MLAAttentionSpec and spec.page_size_padded is None
|
||||
for spec in mla_inner.values()
|
||||
):
|
||||
return None
|
||||
mla_names = [
|
||||
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state == 1
|
||||
]
|
||||
idx_names = [
|
||||
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state > 1
|
||||
]
|
||||
mla_pages = {mla_inner[name].page_size_bytes for name in mla_names}
|
||||
idx_pages = {mla_inner[name].page_size_bytes for name in idx_names}
|
||||
if len(mla_pages) != 1 or len(idx_pages) != 1:
|
||||
return None
|
||||
mla_page = mla_pages.pop()
|
||||
idx_page = idx_pages.pop()
|
||||
if any(group.kv_cache_spec.page_size_bytes != mla_page for group in mamba_groups):
|
||||
return None
|
||||
|
||||
tail_names: list[str] = []
|
||||
tail_page = 0
|
||||
if tail_group is not None:
|
||||
tail_names = list(tail_group.layer_names)
|
||||
tail_inner = cast(
|
||||
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
|
||||
).kv_cache_specs
|
||||
tail_pages = {
|
||||
cast(KpoolTailSpec, spec).unpadded_page_size_bytes
|
||||
for spec in tail_inner.values()
|
||||
}
|
||||
if len(tail_pages) != 1 or len(tail_names) != len(idx_names):
|
||||
return None
|
||||
tail_page = tail_pages.pop()
|
||||
if tail_page > idx_page:
|
||||
return None
|
||||
|
||||
return (
|
||||
attn_group,
|
||||
mamba_groups,
|
||||
mla_names,
|
||||
idx_names,
|
||||
mla_page,
|
||||
idx_page,
|
||||
tail_names,
|
||||
tail_page,
|
||||
)
|
||||
|
||||
|
||||
def unify_kv_cache_spec_page_size(
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> dict[str, KVCacheSpec]:
|
||||
@@ -1359,6 +1559,10 @@ def _get_kv_cache_bytes_per_block(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> int:
|
||||
"""Return the largest cache group's bytes per block."""
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
_, _, mla_names, idx_names, mla_page, idx_page, _, _ = glm5_layout
|
||||
return len(mla_names) * mla_page + len(idx_names) * idx_page
|
||||
|
||||
bytes_per_block = max(
|
||||
sum(
|
||||
_get_per_layer_spec(group, layer_name).page_size_bytes
|
||||
@@ -1432,6 +1636,69 @@ def get_kv_cache_config_from_groups(
|
||||
),
|
||||
)
|
||||
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
(
|
||||
attn_group,
|
||||
mamba_groups,
|
||||
mla_names,
|
||||
idx_names,
|
||||
mla_page,
|
||||
idx_page,
|
||||
tail_names,
|
||||
_,
|
||||
) = glm5_layout
|
||||
bytes_per_block = len(mla_names) * mla_page + len(idx_names) * idx_page
|
||||
num_blocks = may_override_num_blocks(
|
||||
vllm_config, available_memory // bytes_per_block
|
||||
)
|
||||
size = bytes_per_block * num_blocks
|
||||
attn_specs = cast(
|
||||
UniformTypeKVCacheSpecs, attn_group.kv_cache_spec
|
||||
).kv_cache_specs
|
||||
|
||||
kv_cache_tensors: list[KVCacheTensor] = []
|
||||
|
||||
def add_tensor(layer_name: str, spec: KVCacheSpec, offset: int) -> None:
|
||||
kv_cache_tensors.append(
|
||||
KVCacheTensor(
|
||||
size=size,
|
||||
layers=[layer_name],
|
||||
layer_stride=spec.page_size_bytes * num_blocks,
|
||||
block_stride=spec.page_size_bytes,
|
||||
offset=offset,
|
||||
)
|
||||
)
|
||||
|
||||
for index, mla_name in enumerate(mla_names):
|
||||
offset = index * mla_page * num_blocks
|
||||
add_tensor(mla_name, attn_specs[mla_name], offset)
|
||||
for group in mamba_groups:
|
||||
if index < len(group.layer_names):
|
||||
add_tensor(group.layer_names[index], group.kv_cache_spec, offset)
|
||||
|
||||
idx_base = len(mla_names) * mla_page * num_blocks
|
||||
for index, idx_name in enumerate(idx_names):
|
||||
offset = idx_base + index * idx_page * num_blocks
|
||||
add_tensor(idx_name, attn_specs[idx_name], offset)
|
||||
if tail_names:
|
||||
tail_name = tail_names[index]
|
||||
tail_group = next(
|
||||
group for group in kv_cache_groups if tail_name in group.layer_names
|
||||
)
|
||||
tail_specs = cast(
|
||||
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
|
||||
).kv_cache_specs
|
||||
add_tensor(tail_name, tail_specs[tail_name], offset)
|
||||
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
prefix_cache_retention_interval=(
|
||||
vllm_config.cache_config.prefix_cache_retention_interval
|
||||
),
|
||||
)
|
||||
|
||||
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||
validate_kv_cache_layout(layout, kv_cache_groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(kv_cache_groups)
|
||||
@@ -1941,6 +2208,9 @@ def get_kv_cache_groups(
|
||||
# full attention, or all layers are sliding window attention with the
|
||||
# same window size). Put all layers into one group.
|
||||
return _get_kv_cache_groups_uniform_type(uniform_spec)
|
||||
elif glm5_groups := _get_kv_cache_groups_glm5_next(vllm_config, kv_cache_spec):
|
||||
return glm5_groups
|
||||
|
||||
# Hidden-state layers use their own block table and must not be absorbed
|
||||
# into a compatible attention bucket.
|
||||
hidden_specs = {
|
||||
@@ -2065,6 +2335,30 @@ def _max_memory_usage_bytes_from_groups(
|
||||
if not kv_cache_groups:
|
||||
return 0
|
||||
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
(
|
||||
attn_group,
|
||||
mamba_groups,
|
||||
mla_names,
|
||||
idx_names,
|
||||
mla_page,
|
||||
idx_page,
|
||||
tail_names,
|
||||
_,
|
||||
) = glm5_layout
|
||||
uniform_spec = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
|
||||
total_blocks = uniform_spec.max_memory_usage_pages(vllm_config)
|
||||
total_blocks += sum(
|
||||
cdiv(
|
||||
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
|
||||
group.kv_cache_spec.page_size_bytes,
|
||||
)
|
||||
for group in mamba_groups
|
||||
)
|
||||
if tail_names:
|
||||
total_blocks += 1
|
||||
return total_blocks * (len(mla_names) * mla_page + len(idx_names) * idx_page)
|
||||
|
||||
bytes_per_block = _pool_bytes_per_block(kv_cache_groups)
|
||||
total_blocks = 0
|
||||
for group in kv_cache_groups:
|
||||
|
||||
@@ -6,6 +6,7 @@ from collections import defaultdict
|
||||
from collections.abc import Sequence
|
||||
from typing import ClassVar
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
@@ -22,6 +23,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
CrossAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
@@ -33,6 +35,8 @@ from vllm.v1.kv_cache_interface import (
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SingleTypeKVCacheManager(ABC):
|
||||
"""
|
||||
@@ -1210,6 +1214,10 @@ class CircularBufferManager(FullAttentionManager):
|
||||
return 0
|
||||
|
||||
|
||||
class KpoolTailManager(CircularBufferManager):
|
||||
"""One-block circular scratch manager for ``KpoolTailSpec``."""
|
||||
|
||||
|
||||
class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
def __init__(self, kv_cache_spec: ChunkedLocalAttentionSpec, **kwargs) -> None:
|
||||
super().__init__(kv_cache_spec, **kwargs)
|
||||
@@ -2085,6 +2093,11 @@ def register_all_kvcache_specs(vllm_config):
|
||||
SlidingWindowManager,
|
||||
uniform_type_base_spec=SlidingWindowMLASpec,
|
||||
)
|
||||
KVCacheSpecRegistry.register(
|
||||
KpoolTailSpec,
|
||||
KpoolTailManager,
|
||||
uniform_type_base_spec=KpoolTailSpec,
|
||||
)
|
||||
|
||||
KVCacheSpecRegistry.register(
|
||||
MambaSpec, MambaManager, uniform_type_base_spec=MambaSpec
|
||||
|
||||
+12
-1
@@ -333,8 +333,19 @@ class EngineCore:
|
||||
vllm_config.cache_config.num_gpu_blocks = scheduler_kv_cache_config.num_blocks
|
||||
kv_cache_groups = scheduler_kv_cache_config.kv_cache_groups
|
||||
if kv_cache_groups:
|
||||
# Exclude groups that opt out of prefix caching (e.g. GLM-5.3-Flash
|
||||
# kpool tail, a 1-block/req scratch buffer with block_size=kpool):
|
||||
# their small block_size would otherwise drag the global block_size
|
||||
# below the real allocator block size and desync it from mamba.
|
||||
participating = [
|
||||
g.kv_cache_spec.block_size
|
||||
for g in kv_cache_groups
|
||||
if g.kv_cache_spec.prefix_cacheable
|
||||
]
|
||||
vllm_config.cache_config.block_size = min(
|
||||
g.kv_cache_spec.block_size for g in kv_cache_groups
|
||||
participating
|
||||
if participating
|
||||
else [g.kv_cache_spec.block_size for g in kv_cache_groups]
|
||||
)
|
||||
update_kv_cache_capacity(vllm_config, scheduler_kv_cache_config)
|
||||
|
||||
|
||||
@@ -158,6 +158,7 @@ class KVCacheSpec:
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
"""Whether this spec's group participates in prefix caching."""
|
||||
return True
|
||||
|
||||
@property
|
||||
@@ -555,6 +556,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
# DeepseekV4 only fields. Non-DeepseekV4 MLA models leave these at defaults.
|
||||
alignment: int | None = None # Default to None for no padding.
|
||||
model_version: str | None = None
|
||||
storage_block_size: int | None = None
|
||||
"""Token width used to view storage when it differs from the kernel block."""
|
||||
# Marks draft groups that flatten a non-causal query block into decode rows.
|
||||
non_causal_multi_token_decode: bool = False
|
||||
# MLA stores a single latent vector per state; there is no separate V.
|
||||
@@ -572,13 +575,16 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
cache_dtype_str_set = set(spec.cache_dtype_str for spec in specs)
|
||||
tokens_per_state_set = set(spec.tokens_per_state for spec in specs)
|
||||
model_version_set = set(spec.model_version for spec in specs)
|
||||
storage_block_size_set = set(spec.storage_block_size for spec in specs)
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(tokens_per_state_set) == 1
|
||||
and len(model_version_set) == 1
|
||||
and len(storage_block_size_set) == 1
|
||||
), (
|
||||
"All attention layers in the same KV cache group must use the same "
|
||||
"quantization method, tokens per state, and model version."
|
||||
"quantization method, tokens per state, model version, and storage "
|
||||
"block size."
|
||||
)
|
||||
non_causal_mtd_set = {spec.non_causal_multi_token_decode for spec in specs}
|
||||
assert len(non_causal_mtd_set) == 1, (
|
||||
@@ -597,6 +603,7 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
cache_dtype_str=cache_dtype_str_set.pop(),
|
||||
tokens_per_state=tokens_per_state_set.pop(),
|
||||
model_version=model_version_set.pop(),
|
||||
storage_block_size=storage_block_size_set.pop(),
|
||||
non_causal_multi_token_decode=non_causal_mtd_set.pop(),
|
||||
)
|
||||
for spec in specs:
|
||||
@@ -853,6 +860,28 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class KpoolTailSpec(SlidingWindowSpec):
|
||||
"""One-block circular scratch cache for a kpool indexer's raw tail."""
|
||||
|
||||
def max_admission_blocks_per_request(
|
||||
self, max_in_flight_tokens: int, max_model_len: int
|
||||
) -> int:
|
||||
return 1
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
return 1
|
||||
|
||||
def is_uniform_with_collection(
|
||||
self, kv_cache_specs: dict[str, KVCacheSpec]
|
||||
) -> bool:
|
||||
return all(isinstance(spec, KpoolTailSpec) for spec in kv_cache_specs.values())
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MambaSpec(KVCacheSpec):
|
||||
shapes: tuple[tuple[int, ...], ...]
|
||||
@@ -875,6 +904,10 @@ class MambaSpec(KVCacheSpec):
|
||||
for (shape, dtype) in zip(self.shapes, self.dtypes)
|
||||
)
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
return self.state_content_size_bytes
|
||||
|
||||
@property
|
||||
def page_size_bytes(self) -> int:
|
||||
page_size = sum(
|
||||
|
||||
@@ -1021,6 +1021,7 @@ class SpecDecodeBaseProposer:
|
||||
{
|
||||
"DeepSeekMTPModel",
|
||||
"DeepseekV32MTPModel",
|
||||
"Glm5NextMTPModel",
|
||||
"KimiK3MTPModel",
|
||||
}.intersection(architectures)
|
||||
)
|
||||
|
||||
@@ -696,6 +696,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
attn_groups_iter=(g for groups in self.attn_groups for g in groups),
|
||||
kernel_block_sizes=self.kernel_block_sizes,
|
||||
static_forward_context=self.compilation_config.static_forward_context,
|
||||
num_blocks=self.kv_cache_config.num_blocks,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
|
||||
@@ -322,6 +322,7 @@ class MambaHybridModelState(DefaultModelState):
|
||||
kv_cache_config=kv_cache_config,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
||||
positions=input_batch.positions,
|
||||
model_specific_attn_metadata=mamba_attn_metadata,
|
||||
for_cudagraph_capture=for_capture,
|
||||
rswa_prefix_lens=input_batch.prompt_lens,
|
||||
|
||||
@@ -1160,6 +1160,7 @@ class GPUModelRunner(
|
||||
kernel_block_sizes=self._kernel_block_sizes,
|
||||
runner_only_attn_layers=self.runner_only_attn_layers,
|
||||
static_forward_context=self.compilation_config.static_forward_context,
|
||||
num_blocks=self.kv_cache_config.num_blocks,
|
||||
)
|
||||
|
||||
def _zero_block_ids(self, block_ids: list[int]) -> None:
|
||||
|
||||
+26
-7
@@ -35,6 +35,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
create_kv_cache_views,
|
||||
)
|
||||
@@ -115,6 +116,7 @@ class KVBlockZeroer:
|
||||
attn_groups_iter: Iterable["AttentionGroup"],
|
||||
kernel_block_sizes: list[int],
|
||||
static_forward_context: dict[str, Any],
|
||||
num_blocks: int,
|
||||
runner_only_attn_layers: set[str] | None = None,
|
||||
) -> None:
|
||||
"""Precompute the absolute-address table for the Triton zeroing kernel.
|
||||
@@ -158,8 +160,6 @@ class KVBlockZeroer:
|
||||
continue
|
||||
kernel_bs = kernel_block_sizes[group.kv_cache_group_id]
|
||||
assert spec.block_size % kernel_bs == 0
|
||||
ratio = spec.block_size // kernel_bs
|
||||
|
||||
for layer_name in group.layer_names:
|
||||
if layer_name in runner_only_attn_layers:
|
||||
continue
|
||||
@@ -168,6 +168,12 @@ class KVBlockZeroer:
|
||||
continue
|
||||
dp = kv.data_ptr()
|
||||
|
||||
assert kv.shape[0] % num_blocks == 0, (
|
||||
f"{layer_name}: {kv.shape[0]} kernel blocks is not a "
|
||||
f"multiple of {num_blocks} logical blocks"
|
||||
)
|
||||
ratio = kv.shape[0] // num_blocks
|
||||
|
||||
el = kv.element_size()
|
||||
block_stride_bytes = kv.stride(0) * el
|
||||
assert block_stride_bytes % 4 == 0
|
||||
@@ -267,11 +273,19 @@ class AttentionGroup:
|
||||
kernel_block_size: int | None = None,
|
||||
num_metadata_builders: int = 1,
|
||||
):
|
||||
kv_cache_spec_builder = (
|
||||
self.kv_cache_spec.copy_with_new_block_size(kernel_block_size)
|
||||
if kernel_block_size is not None
|
||||
else self.kv_cache_spec
|
||||
)
|
||||
if kernel_block_size is None:
|
||||
kv_cache_spec_builder = self.kv_cache_spec
|
||||
elif (
|
||||
isinstance(self.kv_cache_spec, MLAAttentionSpec)
|
||||
and self.kv_cache_spec.storage_block_size is not None
|
||||
):
|
||||
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
|
||||
self.kv_cache_spec.storage_block_size
|
||||
)
|
||||
else:
|
||||
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
|
||||
kernel_block_size
|
||||
)
|
||||
builder_cls = self.backend.get_builder_cls()
|
||||
builder_kwargs = {}
|
||||
if builder_cls.requires_block_table_width:
|
||||
@@ -291,6 +305,9 @@ class AttentionGroup:
|
||||
)
|
||||
for _ in range(num_metadata_builders)
|
||||
]
|
||||
if kernel_block_size is not None:
|
||||
for builder in self.metadata_builders:
|
||||
builder.set_kernel_block_size(kernel_block_size)
|
||||
|
||||
def get_metadata_builder(self, ubatch_id: int = 0) -> AttentionMetadataBuilder:
|
||||
assert len(self.metadata_builders) > ubatch_id
|
||||
@@ -427,6 +444,8 @@ def allocate_kv_cache(
|
||||
kernel_block_size = None
|
||||
if kernel_block_sizes is not None and group_id < len(kernel_block_sizes):
|
||||
kernel_block_size = kernel_block_sizes[group_id]
|
||||
if isinstance(spec, MLAAttentionSpec) and spec.storage_block_size is not None:
|
||||
kernel_block_size = spec.storage_block_size
|
||||
|
||||
views = create_kv_cache_views(
|
||||
buf,
|
||||
|
||||
Reference in New Issue
Block a user