[Kimi K3] Support internal prefix checkpoints with partial prefix caching and spec-decoding (#53614)
Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
1970f3ed4b
commit
144e79c810
@@ -92,6 +92,10 @@ def _make_builder(
|
||||
mamba_cache_mode: str = "none",
|
||||
use_recoverssm: bool = False,
|
||||
num_prefill_checkpoint_blocks: int = 0,
|
||||
mamba_block_size: int = BLOCK_SIZE,
|
||||
prefix_match_unit: int | None = None,
|
||||
use_eagle: bool = False,
|
||||
disable_eagle_block_drop: bool = False,
|
||||
) -> AttentionMetadataBuilder:
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
@@ -102,20 +106,29 @@ def _make_builder(
|
||||
method="ngram",
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
if use_eagle:
|
||||
vllm_config.speculative_config.method = "eagle3"
|
||||
vllm_config.speculative_config.disable_eagle_block_drop = (
|
||||
disable_eagle_block_drop
|
||||
)
|
||||
vllm_config.compilation_config.cudagraph_mode = (
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
vllm_config.cache_config.use_replayssm = use_recoverssm
|
||||
vllm_config.cache_config.use_kda_recoverssm = use_recoverssm
|
||||
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
|
||||
builder = builder_cls(
|
||||
kv_cache_spec=MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
block_size=mamba_block_size,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
num_speculative_blocks=(0 if use_recoverssm else num_speculative_tokens),
|
||||
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
|
||||
prefill_checkpoint_alignment=(
|
||||
16 if num_prefill_checkpoint_blocks > 0 else None
|
||||
),
|
||||
),
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=vllm_config,
|
||||
@@ -156,6 +169,7 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
|
||||
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
|
||||
mamba_cache_mode="align",
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
prefill_checkpoint_alignment=16,
|
||||
)
|
||||
|
||||
# This is the same compatibility check performed while initializing the
|
||||
@@ -177,11 +191,13 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
|
||||
cache_config=SimpleNamespace(
|
||||
mamba_cache_mode="align",
|
||||
use_kda_recoverssm=True,
|
||||
prefix_match_unit=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
speculative_config=SimpleNamespace(
|
||||
num_speculative_tokens=2,
|
||||
parallel_drafting=False,
|
||||
use_eagle_block_drop=Mock(return_value=False),
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.NONE,
|
||||
@@ -300,6 +316,60 @@ def test_internal_checkpoint_metadata_targets_last_aligned_boundary():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("disable_eagle_block_drop", "prefix_match_unit", "expected_offset"),
|
||||
[(False, 16, 80), (True, 16, 96), (False, 8, None)],
|
||||
)
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_spec_internal_checkpoint_metadata_targets_replay_boundary(
|
||||
disable_eagle_block_drop: bool,
|
||||
prefix_match_unit: int,
|
||||
expected_offset: int | None,
|
||||
) -> None:
|
||||
device = torch.device("cuda")
|
||||
batch = BatchSpec(seq_lens=[100], query_lens=[100])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, device, arange_block_indices=True
|
||||
)
|
||||
common_attn_metadata = common_attn_metadata.replace(
|
||||
is_prefilling=torch.tensor([True]),
|
||||
block_table_tensor=common_attn_metadata.block_table_tensor + 1,
|
||||
)
|
||||
builder = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=3,
|
||||
full_cuda_graph=False,
|
||||
mamba_cache_mode="align",
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
mamba_block_size=64,
|
||||
prefix_match_unit=prefix_match_unit,
|
||||
use_eagle=True,
|
||||
disable_eagle_block_drop=disable_eagle_block_drop,
|
||||
device=device,
|
||||
)
|
||||
assert isinstance(builder, KimiK3KDAMetadataBuilder)
|
||||
builder.mamba_aligned_state_indices = mamba_get_block_table_tensor(
|
||||
common_attn_metadata.block_table_tensor,
|
||||
common_attn_metadata.seq_lens,
|
||||
builder.kv_cache_spec,
|
||||
"align",
|
||||
)
|
||||
actual = builder.build(0, common_attn_metadata)
|
||||
|
||||
if expected_offset is None:
|
||||
assert actual.checkpoint is None
|
||||
return
|
||||
assert actual.checkpoint is not None
|
||||
torch.testing.assert_close(
|
||||
actual.checkpoint.state_indices,
|
||||
torch.tensor([1], dtype=torch.int32, device=device),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual.checkpoint.checkpoint_offsets,
|
||||
torch.tensor([expected_offset], dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_internal_checkpoint_metadata_skips_unaligned_offset():
|
||||
device = torch.device("cuda")
|
||||
|
||||
@@ -92,8 +92,23 @@ def make_full_mamba_manager(
|
||||
mamba_block_size: int = 4,
|
||||
num_blocks: int = 32,
|
||||
use_eagle: bool = False,
|
||||
num_speculative_blocks: int = 0,
|
||||
num_prefill_checkpoint_blocks: int = 0,
|
||||
):
|
||||
mamba_group = KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
|
||||
prefill_checkpoint_alignment=(
|
||||
16 if num_prefill_checkpoint_blocks > 0 else None
|
||||
),
|
||||
),
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[],
|
||||
@@ -107,16 +122,7 @@ def make_full_mamba_manager(
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
|
||||
),
|
||||
),
|
||||
mamba_group,
|
||||
],
|
||||
)
|
||||
scheduler_block_size = lcm(
|
||||
@@ -169,6 +175,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
||||
scheduler_block_size = block_size * dcp_world_size
|
||||
hash_block_size = 32
|
||||
mock = SimpleNamespace(
|
||||
block_size=scheduler_block_size,
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=8192,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
@@ -217,6 +224,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
|
||||
token_budget = 8192
|
||||
prompt_length = 30000
|
||||
mock = SimpleNamespace(
|
||||
block_size=block_size,
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=token_budget,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
@@ -254,6 +262,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
|
||||
long_prefill_threshold = 384
|
||||
prompt_length = 1300
|
||||
mock = SimpleNamespace(
|
||||
block_size=block_size,
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=token_budget,
|
||||
scheduler_config=SimpleNamespace(
|
||||
@@ -557,9 +566,9 @@ def test_partial_hit_then_internal_checkpoint_uses_distinct_mamba_blocks():
|
||||
assert mamba_blocks[3].block_id == running_block_id
|
||||
|
||||
|
||||
def test_internal_checkpoint_requires_block_aligned_start():
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 16
|
||||
def test_internal_checkpoint_uses_partial_hash_lifecycle():
|
||||
hash_block_size = 16
|
||||
mamba_block_size = 32
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
@@ -567,23 +576,90 @@ def test_internal_checkpoint_requires_block_aligned_start():
|
||||
mamba_block_size=mamba_block_size,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
request = make_request("producer", list(range(50)), hash_block_size, sha256)
|
||||
request = make_request("producer", list(range(120)), hash_block_size, sha256)
|
||||
|
||||
# Compute token 0 first, so the next query starts at token 1, which is not
|
||||
# aligned to the Mamba block size.
|
||||
assert manager.allocate_slots(request, 1) is not None
|
||||
request.num_computed_tokens = 1
|
||||
manager.new_step_starts()
|
||||
|
||||
new_blocks = manager.allocate_slots(request, 49)
|
||||
new_blocks = manager.allocate_slots(request, request.num_tokens)
|
||||
|
||||
assert new_blocks is not None
|
||||
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
|
||||
checkpoint_block_idx = 48 // mamba_block_size - 1
|
||||
assert mamba_blocks[checkpoint_block_idx].is_null
|
||||
assert not mamba_blocks[-1].is_null
|
||||
checkpoint_hash = request.block_hashes[48 // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(checkpoint_hash, [1]) is None
|
||||
checkpoint_idx = 2
|
||||
checkpoint_block = mamba_blocks[checkpoint_idx]
|
||||
running_block = mamba_blocks[checkpoint_idx + 1]
|
||||
assert not checkpoint_block.is_null
|
||||
assert not running_block.is_null
|
||||
assert checkpoint_block is not running_block
|
||||
|
||||
# The internal block is exported at state@112, replacing its temporary
|
||||
# full-block state@96 key while retaining #52789's req_to_blocks ownership.
|
||||
partial_hash = request.block_hashes[112 // hash_block_size - 1]
|
||||
partial_hit = manager.block_pool.get_cached_block(partial_hash, [1])
|
||||
assert partial_hit is not None
|
||||
assert partial_hit[0] is checkpoint_block
|
||||
assert checkpoint_block.block_hash_num_tokens == 112
|
||||
full_hash = request.block_hashes[96 // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(full_hash, [1]) is None
|
||||
assert drain_boundary_state_offloads(manager) == {
|
||||
request.request_id: [(1, checkpoint_block.block_id, 112)]
|
||||
}
|
||||
|
||||
manager.free(request)
|
||||
replay = make_request("replay", list(range(120)), hash_block_size, sha256)
|
||||
_, num_computed, _ = manager.get_computed_blocks(replay)
|
||||
assert num_computed == 112
|
||||
|
||||
|
||||
def test_eagle_block_aligned_checkpoint_replaces_newer_hash():
|
||||
hash_block_size = mamba_block_size = 32
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=mamba_block_size,
|
||||
use_eagle=True,
|
||||
num_speculative_blocks=3,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
request = make_request("producer", list(range(104)), hash_block_size, sha256)
|
||||
|
||||
assert manager.allocate_slots(request, request.num_tokens) is not None
|
||||
|
||||
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
|
||||
checkpoint_block = mamba_blocks[2]
|
||||
checkpoint_hash = request.block_hashes[64 // hash_block_size - 1]
|
||||
checkpoint_hit = manager.block_pool.get_cached_block(checkpoint_hash, [1])
|
||||
assert checkpoint_hit is not None
|
||||
assert checkpoint_hit[0] is checkpoint_block
|
||||
assert checkpoint_block.block_hash_num_tokens == 64
|
||||
|
||||
overwritten_full_hash = request.block_hashes[96 // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(overwritten_full_hash, [1]) is None
|
||||
|
||||
|
||||
def test_hash_aligned_query_end_uses_regular_partial_tail():
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 4
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=mamba_block_size,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
request = make_request("producer", list(range(14)), hash_block_size, sha256)
|
||||
|
||||
new_blocks = manager.allocate_slots(request, request.num_tokens)
|
||||
|
||||
assert new_blocks is not None
|
||||
mamba_manager = manager.coordinator.single_type_managers[1]
|
||||
assert mamba_manager._checkpoint_positions[request.request_id] == 0
|
||||
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
|
||||
assert sum(not block.is_null for block in mamba_blocks) == 1
|
||||
|
||||
partial_hash = request.block_hashes[14 // hash_block_size - 1]
|
||||
cached = manager.block_pool.get_cached_block(partial_hash, [1])
|
||||
assert cached is not None
|
||||
assert cached[0] is mamba_blocks[14 // mamba_block_size]
|
||||
assert cached[0].block_hash_num_tokens == 14
|
||||
|
||||
|
||||
def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
|
||||
@@ -699,18 +775,9 @@ def test_boundary_state_offloads_returns_cow_target():
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
# Step A offers the materialized aligned checkpoint immediately.
|
||||
((group_id, block_id, boundary_tokens),) = drain_boundary_state_offloads(manager)[
|
||||
"0"
|
||||
]
|
||||
assert group_id == 1
|
||||
assert boundary_tokens == 4
|
||||
aligned_hash = req0.block_hashes[4 // hash_block_size - 1]
|
||||
aligned_block = manager.block_pool.get_cached_block(
|
||||
aligned_hash, kv_cache_group_ids=[1]
|
||||
)
|
||||
assert aligned_block is not None
|
||||
assert block_id == aligned_block[0].block_id
|
||||
# The final hash-aligned boundary uses the regular partial-tail path rather
|
||||
# than allocating an internal checkpoint at the query end.
|
||||
assert drain_boundary_state_offloads(manager) == {}
|
||||
|
||||
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||
source_block = manager.block_pool.get_cached_block(
|
||||
|
||||
@@ -8,6 +8,7 @@ the wrong offset, and a later chunk crossing that boundary publishes it anyway.
|
||||
Requests resuming from it then restore a truncated state (#43559).
|
||||
"""
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
@@ -21,6 +22,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
is_mamba_prefill_checkpoint_valid,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
|
||||
@@ -37,6 +39,49 @@ PROMPT_LEN = 2002
|
||||
MAMBA_GROUP_ID = 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"query_start",
|
||||
"query_end",
|
||||
"checkpoint_position",
|
||||
"hash_size",
|
||||
"block_size",
|
||||
"alignment",
|
||||
"expected",
|
||||
),
|
||||
[
|
||||
(0, 100, 88, 8, 64, 16, False),
|
||||
(0, 100, 96, 8, 64, 16, True),
|
||||
(0, 100, 96, 8, 64, None, False),
|
||||
(8, 100, 96, 8, 64, 16, False),
|
||||
(15360, 16896, 16768, 128, 1536, 16, False),
|
||||
(15360, 16960, 16896, 128, 1536, 16, True),
|
||||
(15360, 16960, 16768, 128, 1536, 16, True),
|
||||
(16064, 18433, 18432, 128, 1536, 16, False),
|
||||
],
|
||||
)
|
||||
def test_mamba_prefill_checkpoint_valid(
|
||||
query_start: int,
|
||||
query_end: int,
|
||||
checkpoint_position: int,
|
||||
hash_size: int,
|
||||
block_size: int,
|
||||
alignment: int | None,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
assert (
|
||||
is_mamba_prefill_checkpoint_valid(
|
||||
query_start=query_start,
|
||||
query_end=query_end,
|
||||
checkpoint_position=checkpoint_position,
|
||||
hash_block_size=hash_size,
|
||||
mamba_block_size=block_size,
|
||||
checkpoint_alignment=alignment,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
def _make_hybrid_kv_cache_manager(
|
||||
num_prefill_checkpoint_blocks: int = 0,
|
||||
) -> KVCacheManager:
|
||||
@@ -62,6 +107,9 @@ def _make_hybrid_kv_cache_manager(
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=NUM_SPEC,
|
||||
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
|
||||
prefill_checkpoint_alignment=(
|
||||
16 if num_prefill_checkpoint_blocks > 0 else None
|
||||
),
|
||||
),
|
||||
),
|
||||
],
|
||||
@@ -83,20 +131,23 @@ def _split(
|
||||
use_eagle_block_drop: bool | None = None,
|
||||
partial_hit: bool = False,
|
||||
num_prefill_checkpoint_blocks: int = 0,
|
||||
max_num_scheduled_tokens: int = 16384,
|
||||
) -> int:
|
||||
"""Call the real `Scheduler._mamba_block_aligned_split` on a stub self."""
|
||||
if use_eagle_block_drop is None:
|
||||
use_eagle_block_drop = use_eagle
|
||||
stub = SimpleNamespace(
|
||||
block_size=MAMBA_BLOCK_SIZE,
|
||||
cache_config=SimpleNamespace(block_size=MAMBA_BLOCK_SIZE),
|
||||
use_eagle_block_drop=use_eagle_block_drop,
|
||||
max_num_scheduled_tokens=16384,
|
||||
max_num_scheduled_tokens=max_num_scheduled_tokens,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
# `prefix_match_unit` finer than the block size (#46384).
|
||||
mamba_partial_cache_hit=partial_hit,
|
||||
hash_block_size=ATTN_BLOCK_SIZE,
|
||||
mamba_has_prefill_checkpoint_blocks=(
|
||||
num_prefill_checkpoint_blocks > 0 and not use_eagle
|
||||
mamba_has_prefill_checkpoint_blocks=(num_prefill_checkpoint_blocks > 0),
|
||||
mamba_prefill_checkpoint_alignment=(
|
||||
16 if num_prefill_checkpoint_blocks > 0 else None
|
||||
),
|
||||
)
|
||||
return Scheduler._mamba_block_aligned_split(stub, request, num_new_tokens)
|
||||
@@ -106,8 +157,11 @@ def _split(
|
||||
("prompt_len", "num_new_tokens", "use_eagle", "expected"),
|
||||
[
|
||||
(2002, 2002, False, 2002),
|
||||
(3602, 2000, False, 2000),
|
||||
(3602, 3602, True, MAMBA_BLOCK_SIZE),
|
||||
(3602, 2000, False, MAMBA_BLOCK_SIZE),
|
||||
(3602, 3602, True, 3602),
|
||||
(2002, 2002, True, 2002),
|
||||
(3602, 512, True, 0),
|
||||
(3602, 1700, True, MAMBA_BLOCK_SIZE),
|
||||
],
|
||||
)
|
||||
def test_internal_checkpoint_split(
|
||||
@@ -123,12 +177,39 @@ def test_internal_checkpoint_split(
|
||||
)
|
||||
== expected
|
||||
)
|
||||
if not use_eagle:
|
||||
manager = _make_hybrid_kv_cache_manager(num_prefill_checkpoint_blocks=1)
|
||||
assert manager.allocate_slots(request, expected) is not None
|
||||
mamba_manager = manager.coordinator.single_type_managers[MAMBA_GROUP_ID]
|
||||
blocks = mamba_manager.req_to_blocks[request.request_id]
|
||||
assert all(not block.is_null for block in blocks) # checkpoint + running state
|
||||
|
||||
|
||||
def test_partial_checkpoint_resume_stops_at_mamba_block_boundary() -> None:
|
||||
prompt_len = 3602
|
||||
resume_at = 1984
|
||||
(request,) = create_requests(1, num_tokens=prompt_len, block_size=ATTN_BLOCK_SIZE)
|
||||
request.num_computed_tokens = resume_at
|
||||
|
||||
# Without an internal checkpoint export, the forward must stop at the
|
||||
# physical Mamba boundary so it does not publish a stale running state.
|
||||
assert (
|
||||
_split(
|
||||
request,
|
||||
prompt_len - resume_at,
|
||||
use_eagle=False,
|
||||
partial_hit=True,
|
||||
)
|
||||
== MAMBA_BLOCK_SIZE - resume_at % MAMBA_BLOCK_SIZE
|
||||
)
|
||||
|
||||
# A hash-aligned resume inside a physical Mamba block cannot use the fixed
|
||||
# internal-checkpoint column: it aliases the initial-state column. Enabling
|
||||
# checkpoints must therefore preserve the normal aligned split.
|
||||
assert (
|
||||
_split(
|
||||
request,
|
||||
prompt_len - resume_at,
|
||||
use_eagle=False,
|
||||
partial_hit=True,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
== MAMBA_BLOCK_SIZE - resume_at % MAMBA_BLOCK_SIZE
|
||||
)
|
||||
|
||||
|
||||
def test_disabling_eagle_block_drop_keeps_the_trailing_cache_boundary() -> None:
|
||||
@@ -252,8 +333,6 @@ def test_poisoning_is_block_size_independent(
|
||||
budgets: list[int],
|
||||
) -> None:
|
||||
"""The invariant is per-block, so large mamba blocks are not safer."""
|
||||
import sys
|
||||
|
||||
monkeypatch.setattr(sys.modules[__name__], "MAMBA_BLOCK_SIZE", block_size)
|
||||
assert _prefill(prompt_len, budgets=budgets) > 0
|
||||
|
||||
|
||||
@@ -1162,6 +1162,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
|
||||
# junction (absolute) and stops the chunk there (req_2 has num_computed 0).
|
||||
# Create minimal mock with just the needed attributes
|
||||
mock = SimpleNamespace(
|
||||
block_size=block_size,
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=3 * block_size,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
|
||||
Reference in New Issue
Block a user