[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
+26
-7
@@ -35,6 +35,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
create_kv_cache_views,
|
||||
)
|
||||
@@ -115,6 +116,7 @@ class KVBlockZeroer:
|
||||
attn_groups_iter: Iterable["AttentionGroup"],
|
||||
kernel_block_sizes: list[int],
|
||||
static_forward_context: dict[str, Any],
|
||||
num_blocks: int,
|
||||
runner_only_attn_layers: set[str] | None = None,
|
||||
) -> None:
|
||||
"""Precompute the absolute-address table for the Triton zeroing kernel.
|
||||
@@ -158,8 +160,6 @@ class KVBlockZeroer:
|
||||
continue
|
||||
kernel_bs = kernel_block_sizes[group.kv_cache_group_id]
|
||||
assert spec.block_size % kernel_bs == 0
|
||||
ratio = spec.block_size // kernel_bs
|
||||
|
||||
for layer_name in group.layer_names:
|
||||
if layer_name in runner_only_attn_layers:
|
||||
continue
|
||||
@@ -168,6 +168,12 @@ class KVBlockZeroer:
|
||||
continue
|
||||
dp = kv.data_ptr()
|
||||
|
||||
assert kv.shape[0] % num_blocks == 0, (
|
||||
f"{layer_name}: {kv.shape[0]} kernel blocks is not a "
|
||||
f"multiple of {num_blocks} logical blocks"
|
||||
)
|
||||
ratio = kv.shape[0] // num_blocks
|
||||
|
||||
el = kv.element_size()
|
||||
block_stride_bytes = kv.stride(0) * el
|
||||
assert block_stride_bytes % 4 == 0
|
||||
@@ -267,11 +273,19 @@ class AttentionGroup:
|
||||
kernel_block_size: int | None = None,
|
||||
num_metadata_builders: int = 1,
|
||||
):
|
||||
kv_cache_spec_builder = (
|
||||
self.kv_cache_spec.copy_with_new_block_size(kernel_block_size)
|
||||
if kernel_block_size is not None
|
||||
else self.kv_cache_spec
|
||||
)
|
||||
if kernel_block_size is None:
|
||||
kv_cache_spec_builder = self.kv_cache_spec
|
||||
elif (
|
||||
isinstance(self.kv_cache_spec, MLAAttentionSpec)
|
||||
and self.kv_cache_spec.storage_block_size is not None
|
||||
):
|
||||
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
|
||||
self.kv_cache_spec.storage_block_size
|
||||
)
|
||||
else:
|
||||
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
|
||||
kernel_block_size
|
||||
)
|
||||
builder_cls = self.backend.get_builder_cls()
|
||||
builder_kwargs = {}
|
||||
if builder_cls.requires_block_table_width:
|
||||
@@ -291,6 +305,9 @@ class AttentionGroup:
|
||||
)
|
||||
for _ in range(num_metadata_builders)
|
||||
]
|
||||
if kernel_block_size is not None:
|
||||
for builder in self.metadata_builders:
|
||||
builder.set_kernel_block_size(kernel_block_size)
|
||||
|
||||
def get_metadata_builder(self, ubatch_id: int = 0) -> AttentionMetadataBuilder:
|
||||
assert len(self.metadata_builders) > ubatch_id
|
||||
@@ -427,6 +444,8 @@ def allocate_kv_cache(
|
||||
kernel_block_size = None
|
||||
if kernel_block_sizes is not None and group_id < len(kernel_block_sizes):
|
||||
kernel_block_size = kernel_block_sizes[group_id]
|
||||
if isinstance(spec, MLAAttentionSpec) and spec.storage_block_size is not None:
|
||||
kernel_block_size = spec.storage_block_size
|
||||
|
||||
views = create_kv_cache_views(
|
||||
buf,
|
||||
|
||||
Reference in New Issue
Block a user