[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -443,6 +443,43 @@ def test_copy_cpu_kv_cache_logical_blocks_ignores_storage_padding():
|
||||
assert (backing[9] == -1).all()
|
||||
|
||||
|
||||
def test_bind_kv_cache_keeps_scheduler_block_views_for_runner():
|
||||
"""Block copies index scheduler blocks, so the runner keeps the unmapped
|
||||
views while a packed layer is bound with kernel blocks."""
|
||||
|
||||
class _Layer:
|
||||
def bind_kv_cache(self, kv_cache):
|
||||
self.kv_cache = kv_cache
|
||||
|
||||
num_blocks, row = 4, 64
|
||||
storage = torch.zeros(num_blocks * row)
|
||||
# Two layers packed into one block row: an 8x4 page, then a 4x2 page
|
||||
# split into two 2-row kernel blocks.
|
||||
mla = storage.as_strided((num_blocks, 8, 4), (row, 4, 1))
|
||||
indexer = storage.as_strided((num_blocks, 1, 4, 2), (row, 8, 2, 1), 40)
|
||||
# The indexer in 2-row kernel blocks: block b's kernel block j is b * 16 + j.
|
||||
mapped = indexer.as_strided((50, 1, 2, 2), (4, 8, 2, 1))
|
||||
kv_caches = {"layers.0.attn": mla, "layers.1.indexer": indexer}
|
||||
ctx = {name: _Layer() for name in kv_caches}
|
||||
runner_kv_caches: list[torch.Tensor] = []
|
||||
|
||||
bind_kv_cache(
|
||||
kv_caches,
|
||||
ctx,
|
||||
runner_kv_caches,
|
||||
layer_kv_caches={**kv_caches, "layers.1.indexer": mapped},
|
||||
)
|
||||
|
||||
assert ctx["layers.1.indexer"].kv_cache is mapped
|
||||
assert runner_kv_caches == [mla, indexer]
|
||||
copies = [KVCacheBlockCopy(1, 3)]
|
||||
with pytest.raises(AssertionError, match="not divisible"):
|
||||
copy_kv_cache_blocks_inplace([mapped], num_blocks, copies)
|
||||
storage[row : 2 * row] = 7
|
||||
copy_kv_cache_blocks_inplace(runner_kv_caches, num_blocks, copies)
|
||||
assert (mla[3] == 7).all() and (indexer[3] == 7).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shared,rank", [(False, 0), (True, 0), (True, 1)])
|
||||
@pytest.mark.parametrize("has_copies", [False, True])
|
||||
def test_hisparse_host_copy_waits_for_writes_on_writer_only(
|
||||
|
||||
Reference in New Issue
Block a user