[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -233,7 +233,6 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
|
||||
device=torch.device(device),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
reqs: list[CachedRequestState] = []
|
||||
@@ -327,7 +326,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
|
||||
device=torch.device(device),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
ref_input_batch: InputBatch = InputBatch(
|
||||
@@ -337,7 +335,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
|
||||
device=torch.device(device),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
|
||||
@@ -388,7 +385,6 @@ def test_condense_clears_stale_allowed_token_ids_mask(device: str):
|
||||
device=torch.device(device),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
|
||||
@@ -446,7 +442,6 @@ def test_swap_states_preserves_allowed_token_ids_mask(device: str):
|
||||
device=torch.device(device),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
|
||||
@@ -516,7 +511,6 @@ def test_pooling_prompt_lens_not_aliased(device: str):
|
||||
device=torch.device(device),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
is_pooling_model=True,
|
||||
)
|
||||
@@ -552,7 +546,6 @@ def test_placeholder_spec_token_ids_written_verbatim():
|
||||
device=torch.device("cpu"),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[1],
|
||||
)
|
||||
req = CachedRequestState(
|
||||
@@ -599,7 +592,6 @@ def test_pooling_metadata_token_id_buffers(
|
||||
device=torch.device("cpu"),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
is_pooling_model=True,
|
||||
)
|
||||
@@ -630,7 +622,6 @@ def _make_input_batch(is_pooling_model: bool = False) -> InputBatch:
|
||||
device=torch.device("cpu"),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
is_pooling_model=is_pooling_model,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user