[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+37
View File
@@ -443,6 +443,43 @@ def test_copy_cpu_kv_cache_logical_blocks_ignores_storage_padding():
assert (backing[9] == -1).all()
def test_bind_kv_cache_keeps_scheduler_block_views_for_runner():
"""Block copies index scheduler blocks, so the runner keeps the unmapped
views while a packed layer is bound with kernel blocks."""
class _Layer:
def bind_kv_cache(self, kv_cache):
self.kv_cache = kv_cache
num_blocks, row = 4, 64
storage = torch.zeros(num_blocks * row)
# Two layers packed into one block row: an 8x4 page, then a 4x2 page
# split into two 2-row kernel blocks.
mla = storage.as_strided((num_blocks, 8, 4), (row, 4, 1))
indexer = storage.as_strided((num_blocks, 1, 4, 2), (row, 8, 2, 1), 40)
# The indexer in 2-row kernel blocks: block b's kernel block j is b * 16 + j.
mapped = indexer.as_strided((50, 1, 2, 2), (4, 8, 2, 1))
kv_caches = {"layers.0.attn": mla, "layers.1.indexer": indexer}
ctx = {name: _Layer() for name in kv_caches}
runner_kv_caches: list[torch.Tensor] = []
bind_kv_cache(
kv_caches,
ctx,
runner_kv_caches,
layer_kv_caches={**kv_caches, "layers.1.indexer": mapped},
)
assert ctx["layers.1.indexer"].kv_cache is mapped
assert runner_kv_caches == [mla, indexer]
copies = [KVCacheBlockCopy(1, 3)]
with pytest.raises(AssertionError, match="not divisible"):
copy_kv_cache_blocks_inplace([mapped], num_blocks, copies)
storage[row : 2 * row] = 7
copy_kv_cache_blocks_inplace(runner_kv_caches, num_blocks, copies)
assert (mla[3] == 7).all() and (indexer[3] == 7).all()
@pytest.mark.parametrize("shared,rank", [(False, 0), (True, 0), (True, 1)])
@pytest.mark.parametrize("has_copies", [False, True])
def test_hisparse_host_copy_waits_for_writes_on_writer_only(