[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -55,8 +55,9 @@ def resolve_hisparse_specs(
|
||||
if len(block_sizes) != 1:
|
||||
raise ValueError("HiSparse requires one scheduler block size.")
|
||||
backends = [attn_layers[name].get_attn_backend() for name in mla_specs]
|
||||
specs = list(mla_specs.values())
|
||||
try:
|
||||
block_size = select_common_block_size(block_sizes.pop(), backends)
|
||||
block_size = select_common_block_size(block_sizes.pop(), backends, specs)
|
||||
except ValueError as error:
|
||||
raise ValueError(
|
||||
"HiSparse requires a GPU block size supported by every sparse "
|
||||
@@ -104,16 +105,13 @@ def allocate_hisparse_kv_caches(
|
||||
if isinstance(host_spec, UniformTypeKVCacheSpecs)
|
||||
else host_spec
|
||||
)
|
||||
kernel_block_size = kernel_block_sizes[host_group_id]
|
||||
if isinstance(spec, MLAAttentionSpec) and spec.storage_block_size is not None:
|
||||
kernel_block_size = spec.storage_block_size
|
||||
views = create_kv_cache_views(
|
||||
backing,
|
||||
spec,
|
||||
kv_cache_config.num_blocks_of(tensor),
|
||||
layout,
|
||||
tensor,
|
||||
kernel_block_size=kernel_block_size,
|
||||
kernel_block_size=kernel_block_sizes[host_group_id],
|
||||
)
|
||||
kv_caches.update(zip(tensor.layers, views))
|
||||
return kv_caches
|
||||
|
||||
@@ -333,7 +333,7 @@ def get_hisparse_kv_cache_config(
|
||||
device_groups = hisparse_layout.device_groups
|
||||
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||
validate_kv_cache_layout(layout, device_groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(device_groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(device_groups, layout)
|
||||
num_blocks = may_override_num_blocks(
|
||||
vllm_config, available_memory // bytes_per_block
|
||||
)
|
||||
@@ -345,7 +345,7 @@ def get_hisparse_kv_cache_config(
|
||||
host_groups = [hisparse_layout.source_group]
|
||||
host_layout = KVCacheLayout.LBNHC
|
||||
validate_kv_cache_layout(host_layout, host_groups)
|
||||
host_bytes_per_block = _get_kv_cache_bytes_per_block(host_groups)
|
||||
host_bytes_per_block = _get_kv_cache_bytes_per_block(host_groups, host_layout)
|
||||
host_size = host_bytes_per_block * hisparse_layout.host_num_blocks
|
||||
kv_cache_tensors[:0] = _build_hisparse_kv_cache_tensors(
|
||||
host_groups,
|
||||
|
||||
Reference in New Issue
Block a user