[Core][Mamba] Remove leftovers of mamba_cache_mode="all" (#60043)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Jiangyun Zhu <zhujiangyun@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
bc3ddb198a
commit
2cc2e68d1c
@@ -180,7 +180,6 @@ def _run_k3_partial_prefix_reuse(
|
||||
block_size=block_size,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="align",
|
||||
) as runner:
|
||||
assert (
|
||||
runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
|
||||
|
||||
@@ -23,4 +23,4 @@ generation_config:
|
||||
extra_body:
|
||||
chat_template_kwargs:
|
||||
enable_thinking: true
|
||||
server_args: "--tensor-parallel-size 4 --mamba-cache-mode align"
|
||||
server_args: "--tensor-parallel-size 4"
|
||||
|
||||
@@ -96,15 +96,17 @@ def mixer2_gated_norm_tensor_parallel(
|
||||
gate_states = torch.randn(batch_size, seq_len, hidden_size)
|
||||
|
||||
# create gated-norm with TP
|
||||
mixer = Mixer2RMSNormGated(
|
||||
full_hidden_size=hidden_size,
|
||||
full_n_groups=n_groups,
|
||||
)
|
||||
with ensure_current_vllm_config():
|
||||
mixer = Mixer2RMSNormGated(
|
||||
full_hidden_size=hidden_size,
|
||||
full_n_groups=n_groups,
|
||||
)
|
||||
mixer.weight.weight_loader(mixer.weight, weight) # load
|
||||
|
||||
# create gated-norm without TP to compute reference
|
||||
# - utilize mock patching to disable TP when
|
||||
with (
|
||||
ensure_current_vllm_config(),
|
||||
unittest.mock.patch(
|
||||
"vllm.model_executor.layers.mamba.mamba_mixer2."
|
||||
"get_tensor_model_parallel_world_size",
|
||||
|
||||
@@ -38,9 +38,7 @@ def _manager(
|
||||
num_prefill_lookahead=0,
|
||||
):
|
||||
init_none_hash(sha256)
|
||||
config = _make_hybrid_kv_cache_config(
|
||||
block_size, num_blocks, ["full", "mamba_align"]
|
||||
)
|
||||
config = _make_hybrid_kv_cache_config(block_size, num_blocks, ["full", "mamba"])
|
||||
if eagle_group is not None:
|
||||
groups = list(config.kv_cache_groups)
|
||||
groups[eagle_group] = replace(groups[eagle_group], is_eagle_group=True)
|
||||
|
||||
@@ -278,7 +278,14 @@ class TestCSALinearGrouping:
|
||||
neither divides the hit length nor bounds the (empty) block list."""
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
|
||||
config.cache_config.mamba_cache_mode = "align"
|
||||
specs = {
|
||||
name: replace(spec, mamba_cache_mode="align")
|
||||
if isinstance(spec, MambaSpec)
|
||||
else spec
|
||||
for name, spec in _make_csa_linear_specs().items()
|
||||
}
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=BYTES_PER_BLOCK * 64
|
||||
)
|
||||
|
||||
@@ -1617,6 +1617,7 @@ def make_kv_cache_config_hybrid_model(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
|
||||
return KVCacheConfig(
|
||||
@@ -1652,6 +1653,7 @@ def make_kv_cache_config_three_types(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
elif third_spec_type == "sliding_window":
|
||||
third_spec = SlidingWindowSpec(
|
||||
@@ -1732,6 +1734,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
@@ -1757,10 +1760,13 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
|
||||
req0 = make_request("0", common_token_ids + [99] * 7, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
req0, len(req0.prompt_token_ids), 0, computed_blocks
|
||||
)
|
||||
assert blocks is not None
|
||||
# Align-mode Mamba keeps a state only where a chunk ends, so end the first
|
||||
# chunk on the shared prefix.
|
||||
for num_new_tokens in (len(common_token_ids), 7):
|
||||
blocks = manager.allocate_slots(req0, num_new_tokens, 0, computed_blocks)
|
||||
assert blocks is not None
|
||||
req0.num_computed_tokens += num_new_tokens
|
||||
computed_blocks = None
|
||||
|
||||
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
@@ -2285,7 +2291,7 @@ def _make_hybrid_kv_cache_config(
|
||||
- "full": FullAttentionSpec
|
||||
- "sliding_window": SlidingWindowSpec with window=2*block_size
|
||||
- "sliding_window_large": SlidingWindowSpec with window=4*block_size
|
||||
- "mamba": MambaSpec
|
||||
- "mamba": MambaSpec in prefix-caching ("align") mode
|
||||
|
||||
"""
|
||||
spec_map = {
|
||||
@@ -2313,11 +2319,6 @@ def _make_hybrid_kv_cache_config(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
),
|
||||
"mamba_align": lambda: MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
}
|
||||
@@ -2427,14 +2428,16 @@ def test_prefill_hybrid_model_combinations(spec_types: list[str]):
|
||||
assert not computed_blocks.blocks[0] # No cache hit initially
|
||||
assert num_computed_tokens == 0
|
||||
|
||||
blocks = manager.allocate_slots(
|
||||
req0, 55, len(computed_blocks.blocks[0]) * block_size, computed_blocks
|
||||
)
|
||||
assert blocks is not None
|
||||
# Should have blocks for all groups
|
||||
assert len(blocks.get_block_ids()) == num_groups
|
||||
|
||||
manager.new_step_starts()
|
||||
# Align-mode Mamba keeps a state only where a chunk ends, so end the first
|
||||
# chunk on the common prefix.
|
||||
for num_new_tokens in (len(common_token_ids), len(unique_token_ids)):
|
||||
blocks = manager.allocate_slots(req0, num_new_tokens, 0, computed_blocks)
|
||||
assert blocks is not None
|
||||
# Should have blocks for all groups
|
||||
assert len(blocks.get_block_ids()) == num_groups
|
||||
req0.num_computed_tokens += num_new_tokens
|
||||
computed_blocks = None
|
||||
manager.new_step_starts()
|
||||
|
||||
# Second request: should hit cached blocks for common prefix
|
||||
req1 = make_request("1", common_token_ids + [4] * 5, block_size, hash_fn)
|
||||
@@ -2549,7 +2552,7 @@ def test_prefill_hybrid_model_mamba_align():
|
||||
num_blocks = 30
|
||||
|
||||
kv_cache_config = _make_hybrid_kv_cache_config(
|
||||
block_size, num_blocks, ["full", "mamba_align"]
|
||||
block_size, num_blocks, ["full", "mamba"]
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config,
|
||||
@@ -2585,7 +2588,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
|
||||
"""
|
||||
block_size = 16
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 30, ["full", "mamba_align"]),
|
||||
_make_hybrid_kv_cache_config(block_size, 30, ["full", "mamba"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
@@ -2660,7 +2663,7 @@ def test_hybrid_model_mamba_align_with_dynamic_draft_tokens():
|
||||
num_blocks = 30
|
||||
|
||||
kv_cache_config = _make_hybrid_kv_cache_config(
|
||||
block_size, num_blocks, ["full", "mamba_align"]
|
||||
block_size, num_blocks, ["full", "mamba"]
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
kv_cache_config,
|
||||
@@ -5896,49 +5899,6 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
|
||||
assert retained(0, None) == {14}
|
||||
|
||||
|
||||
def test_mamba_shared_prefix_survives_zero_retention():
|
||||
"""Manager-level check of the full wiring: a pinned shared-prefix boundary
|
||||
(``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style
|
||||
detection) keeps its Mamba state block cached under
|
||||
``prefix_cache_retention_interval=0``, which otherwise retains only the
|
||||
end-of-prompt replay boundary. Without this, a shared prefix (junction
|
||||
before ``num_prompt``) would be recomputed by every sharing request."""
|
||||
block_size = 16
|
||||
|
||||
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
|
||||
def cached_mamba_blocks(shared_prefix_boundary):
|
||||
# Fresh manager per scenario so cached blocks don't leak between runs.
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 100, ["full", "mamba"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=0,
|
||||
)
|
||||
req = make_request("r", token_ids, block_size, sha256)
|
||||
req.shared_prefix_boundary = shared_prefix_boundary
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req, len(token_ids), num_computed, computed_blocks
|
||||
)
|
||||
assert blocks is not None
|
||||
pool = manager.block_pool
|
||||
return {
|
||||
i
|
||||
for i in range(16)
|
||||
if pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
|
||||
is not None
|
||||
}
|
||||
|
||||
# Without a pinned boundary, retention=0 keeps only the replay boundary (14).
|
||||
assert cached_mamba_blocks(0) == {14}
|
||||
# Pinning the shared prefix at token 96 (state block 5) retains it too, so a
|
||||
# later request sharing that prefix can hit the Mamba state.
|
||||
assert cached_mamba_blocks(96) == {5, 14}
|
||||
|
||||
|
||||
def test_mamba_shared_prefix_reuse_under_zero_retention():
|
||||
"""Full cross-request Marconi flow: a partial shared prefix cached by the
|
||||
detecting request must stay reusable by a later request under
|
||||
@@ -5949,7 +5909,7 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]),
|
||||
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
|
||||
@@ -898,7 +898,6 @@ def _run_mamba_prefix_cache_mrv1(
|
||||
load_format="dummy",
|
||||
enable_prefix_caching=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
mamba_cache_mode="align",
|
||||
speculative_config={
|
||||
"method": "qwen3_next_mtp",
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
@@ -1158,7 +1157,6 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
skip_tokenizer_init=True,
|
||||
enable_prefix_caching=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
mamba_cache_mode="align",
|
||||
speculative_config={
|
||||
"method": "qwen3_next_mtp",
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
|
||||
@@ -88,7 +88,6 @@ def test_mtp_speculative_mixed_batch_short_prefill(
|
||||
trust_remote_code=True,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=enable_prefix_caching,
|
||||
mamba_cache_mode="align" if enable_prefix_caching else "none",
|
||||
) as llm:
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
@@ -161,7 +160,6 @@ def test_qwen_mtp_mamba_prefix_cache_hit_is_bounded(vllm_runner, monkeypatch):
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="align",
|
||||
speculative_config={"method": "mtp", "num_speculative_tokens": 2},
|
||||
) as runner:
|
||||
llm = runner.get_llm()
|
||||
|
||||
@@ -60,7 +60,6 @@ def _check_replayssm_parity(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
mamba_backend=mamba_backend,
|
||||
)
|
||||
@@ -130,7 +129,6 @@ def test_replayssm_flashinfer_spec_decode_matches_baseline(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
speculative_config={
|
||||
"method": "ngram",
|
||||
"num_speculative_tokens": 3,
|
||||
@@ -182,7 +180,6 @@ def test_replayssm_flashinfer_mtp(vllm_runner, monkeypatch, use_v2_model_runner)
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
mamba_backend="flashinfer",
|
||||
tensor_parallel_size=2,
|
||||
disable_log_stats=False,
|
||||
@@ -250,7 +247,6 @@ def _check_replayssm_prefix_caching_parity(
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=True,
|
||||
enable_chunked_prefill=True,
|
||||
mamba_cache_mode="align",
|
||||
disable_log_stats=False, # required for llm.get_metrics()
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
)
|
||||
|
||||
@@ -60,7 +60,6 @@ vllm serve "$MODEL" \
|
||||
--block-size 128 \
|
||||
--trust-remote-code \
|
||||
--enable-prefix-caching \
|
||||
--mamba-cache-mode align \
|
||||
--kv-transfer-config "$KV_CONFIG" \
|
||||
"${EXTRA_ARGS[@]}" &
|
||||
|
||||
@@ -78,7 +77,6 @@ vllm serve "$MODEL" \
|
||||
--block-size 128 \
|
||||
--trust-remote-code \
|
||||
--enable-prefix-caching \
|
||||
--mamba-cache-mode align \
|
||||
--kv-transfer-config "$KV_CONFIG" \
|
||||
"${EXTRA_ARGS[@]}" &
|
||||
|
||||
|
||||
@@ -583,7 +583,6 @@ def test_mamba_cpu_offload_boundary(model: str, block_size: int, tp_size: int):
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
language_model_only=True,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="align",
|
||||
# Use lossless state storage so this exact-equality regression isolates
|
||||
# offload boundary selection from low-precision Mamba checkpointing.
|
||||
mamba_cache_dtype="float32",
|
||||
|
||||
@@ -632,6 +632,8 @@ def test_p_side_chunked_prefill_mamba(mock_platform):
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = False
|
||||
# mamba_cache_mode "none" only runs without prefix caching.
|
||||
vllm_config.cache_config.enable_prefix_caching = False
|
||||
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=BLOCK_SIZE,
|
||||
@@ -855,6 +857,7 @@ def test_async_load_does_not_double_reserve_mamba_spec_blocks(
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
vllm_config.cache_config.enable_prefix_caching = mamba_cache_mode == "align"
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size,
|
||||
mamba_enabled=True,
|
||||
|
||||
@@ -49,6 +49,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
KVCacheTensor,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
@@ -2355,8 +2356,6 @@ def _make_hybrid_attention_mamba_scheduler(
|
||||
enable_kv_cache_events: bool = False,
|
||||
) -> SchedulerFixture:
|
||||
"""Build a scheduler for one attention group plus one Mamba group."""
|
||||
scheduler_block_size = scheduler_block_size or block_size
|
||||
hash_block_size = hash_block_size or block_size
|
||||
attention_spec = FullAttentionSpec(
|
||||
block_size=attention_block_size,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
@@ -2369,9 +2368,38 @@ def _make_hybrid_attention_mamba_scheduler(
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
)
|
||||
return _make_two_group_scheduler(
|
||||
attention_spec,
|
||||
("mamba", mamba_spec),
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
scheduler_block_size=scheduler_block_size or block_size,
|
||||
hash_block_size=hash_block_size or block_size,
|
||||
dcp_world_size=dcp_world_size,
|
||||
lazy=lazy,
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
enable_kv_cache_events=enable_kv_cache_events,
|
||||
)
|
||||
|
||||
|
||||
def _make_two_group_scheduler(
|
||||
attention_spec: FullAttentionSpec,
|
||||
second_group: tuple[str, KVCacheSpec],
|
||||
*,
|
||||
num_cpu_blocks: int,
|
||||
num_gpu_blocks: int,
|
||||
scheduler_block_size: int,
|
||||
hash_block_size: int,
|
||||
dcp_world_size: int,
|
||||
lazy: bool = False,
|
||||
mamba_cache_mode: MambaCacheMode = "align",
|
||||
enable_kv_cache_events: bool = False,
|
||||
) -> SchedulerFixture:
|
||||
"""Build a scheduler for one attention group plus one named second group."""
|
||||
second_name, second_spec = second_group
|
||||
groups = [
|
||||
KVCacheGroupSpec(["attention"], attention_spec),
|
||||
KVCacheGroupSpec(["mamba"], mamba_spec),
|
||||
KVCacheGroupSpec([second_name], second_spec),
|
||||
]
|
||||
tensors = [
|
||||
KVCacheTensor(
|
||||
@@ -2380,7 +2408,7 @@ def _make_hybrid_attention_mamba_scheduler(
|
||||
layer_stride=spec.page_size_bytes * num_gpu_blocks,
|
||||
block_stride=spec.page_size_bytes,
|
||||
)
|
||||
for group, spec in zip(groups, (attention_spec, mamba_spec))
|
||||
for group, spec in zip(groups, (attention_spec, second_spec))
|
||||
]
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_gpu_blocks,
|
||||
@@ -2428,34 +2456,48 @@ def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
|
||||
silently offloads only a fraction of them.
|
||||
"""
|
||||
attention_block_size = BLOCK_SIZE
|
||||
mamba_block_size = 4 * BLOCK_SIZE
|
||||
replicated_block_size = 4 * BLOCK_SIZE
|
||||
dcp_world_size = 2
|
||||
fix = _make_hybrid_attention_mamba_scheduler(
|
||||
fix = _make_two_group_scheduler(
|
||||
FullAttentionSpec(
|
||||
block_size=attention_block_size,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
),
|
||||
(
|
||||
"replicated_attention",
|
||||
FullAttentionSpec(
|
||||
block_size=replicated_block_size,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
dcp_sharded=False,
|
||||
),
|
||||
),
|
||||
num_cpu_blocks=32,
|
||||
num_gpu_blocks=32,
|
||||
attention_block_size=attention_block_size,
|
||||
block_size=mamba_block_size,
|
||||
scheduler_block_size=replicated_block_size,
|
||||
# Every prefix-cacheable group's resolved block size must be a multiple
|
||||
# of the hash block, so it cannot exceed the smallest of them.
|
||||
hash_block_size=BLOCK_SIZE,
|
||||
dcp_world_size=dcp_world_size,
|
||||
mamba_cache_mode="none",
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
attention_size, mamba_size = sched.group_block_sizes
|
||||
attention_size, replicated_size = sched.group_block_sizes
|
||||
assert attention_size == attention_block_size * dcp_world_size
|
||||
assert mamba_size == mamba_block_size
|
||||
assert replicated_size == replicated_block_size
|
||||
|
||||
confirmed = 2 * sched.block_size
|
||||
req = _make_cp_request(num_blocks=8, virtual_block_size=BLOCK_SIZE)
|
||||
attention_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, confirmed // attention_size, attention_size, group_id=0
|
||||
)
|
||||
mamba_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, confirmed // mamba_size, mamba_size, group_id=1
|
||||
replicated_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, confirmed // replicated_size, replicated_size, group_id=1
|
||||
)
|
||||
kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
|
||||
kv_blocks = KVCacheBlocks(blocks=(attention_blocks, replicated_blocks))
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
@@ -2476,11 +2518,11 @@ def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
|
||||
# count.
|
||||
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [
|
||||
confirmed // attention_size,
|
||||
confirmed // mamba_size,
|
||||
confirmed // replicated_size,
|
||||
]
|
||||
assert set(meta.store_gpu_blocks) == {
|
||||
*(b.block_id for b in attention_blocks),
|
||||
*(b.block_id for b in mamba_blocks),
|
||||
*(b.block_id for b in replicated_blocks),
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user