[KV Offload] Define clean backend configuration boundary (#48150)
Signed-off-by: Change72 <changg@nvidia.com> Signed-off-by: Chang Guo <cguo51@asu.edu> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
Claude Fable 5
Or Ozeri
OpenAI Codex
parent
8c3393f373
commit
a9531edfa6
@@ -8,6 +8,9 @@ import torch
|
||||
from tests.v1.kv_connector.unit.utils import create_vllm_config
|
||||
from vllm.config import KVEventsConfig, KVTransferConfig
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
|
||||
build_offloading_config,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.events import (
|
||||
OffloadingEventGroupSpec,
|
||||
OffloadingEventsTracker,
|
||||
@@ -70,15 +73,15 @@ def _group_config(
|
||||
*,
|
||||
group_idx: int = 0,
|
||||
block_size: int = 4,
|
||||
block_size_factor: int = 1,
|
||||
sliding_window_size_in_blocks: int | None = None,
|
||||
blocks_per_chunk: int = 1,
|
||||
sliding_window_size_in_chunks: int | None = None,
|
||||
) -> GroupOffloadConfig:
|
||||
return GroupOffloadConfig(
|
||||
group_idx=group_idx,
|
||||
gpu_block_size=block_size,
|
||||
offloaded_block_size=block_size * block_size_factor,
|
||||
hash_block_size_factor=block_size_factor,
|
||||
sliding_window_size_in_blocks=sliding_window_size_in_blocks,
|
||||
tokens_per_block=block_size,
|
||||
tokens_per_chunk=block_size * blocks_per_chunk,
|
||||
hashes_per_chunk=blocks_per_chunk,
|
||||
sliding_window_size_in_chunks=sliding_window_size_in_chunks,
|
||||
kv_event_group_spec=_FULL_ATTENTION_EVENT_SPEC,
|
||||
)
|
||||
|
||||
@@ -90,7 +93,7 @@ def _record_chunks(
|
||||
num_chunks: int,
|
||||
) -> list[OffloadKey]:
|
||||
keys: list[OffloadKey] = []
|
||||
hbf = group_config.hash_block_size_factor
|
||||
hbf = group_config.hashes_per_chunk
|
||||
for chunk_idx in range(num_chunks):
|
||||
tail_hash = req.block_hashes[(chunk_idx + 1) * hbf - 1]
|
||||
assert tail_hash is not None
|
||||
@@ -149,14 +152,14 @@ def test_take_events_publishes_routable_block_stored():
|
||||
|
||||
def test_take_events_factor_gt_1_chunk_store_and_remove():
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
blocks_per_chunk = 3
|
||||
tracker = _tracker()
|
||||
group_config = _group_config(
|
||||
block_size=block_size, block_size_factor=block_size_factor
|
||||
block_size=block_size, blocks_per_chunk=blocks_per_chunk
|
||||
)
|
||||
req = _request(
|
||||
block_hashes=[_hash(i) for i in range(6)],
|
||||
token_count=block_size * block_size_factor * 2,
|
||||
token_count=block_size * blocks_per_chunk * 2,
|
||||
)
|
||||
keys = _record_chunks(tracker, req, group_config, num_chunks=2)
|
||||
|
||||
@@ -169,17 +172,17 @@ def test_take_events_factor_gt_1_chunk_store_and_remove():
|
||||
expected_chunk_hashes = [
|
||||
_wire_hash(_hash(i))
|
||||
for i in range(
|
||||
chunk_idx * block_size_factor,
|
||||
(chunk_idx + 1) * block_size_factor,
|
||||
chunk_idx * blocks_per_chunk,
|
||||
(chunk_idx + 1) * blocks_per_chunk,
|
||||
)
|
||||
]
|
||||
assert event.block_hashes == expected_chunk_hashes
|
||||
assert event.block_size == block_size
|
||||
assert len(event.token_ids) == block_size * block_size_factor
|
||||
assert len(event.token_ids) == block_size * blocks_per_chunk
|
||||
if chunk_idx == 0:
|
||||
assert event.parent_block_hash is None
|
||||
else:
|
||||
assert event.parent_block_hash == _wire_hash(_hash(block_size_factor - 1))
|
||||
assert event.parent_block_hash == _wire_hash(_hash(blocks_per_chunk - 1))
|
||||
expected_hashes.extend(expected_chunk_hashes)
|
||||
|
||||
assert len(tracker._pending_event_metadata) == 2
|
||||
@@ -194,12 +197,12 @@ def test_take_events_factor_gt_1_chunk_store_and_remove():
|
||||
|
||||
|
||||
def test_take_events_factor_gt_1_store_is_order_independent():
|
||||
block_size_factor = 3
|
||||
blocks_per_chunk = 3
|
||||
tracker = _tracker()
|
||||
group_config = _group_config(block_size_factor=block_size_factor)
|
||||
group_config = _group_config(blocks_per_chunk=blocks_per_chunk)
|
||||
req = _request(
|
||||
block_hashes=[_hash(i) for i in range(6)],
|
||||
token_count=4 * block_size_factor * 2,
|
||||
token_count=4 * blocks_per_chunk * 2,
|
||||
)
|
||||
keys = _record_chunks(tracker, req, group_config, num_chunks=2)
|
||||
unknown_key = make_offload_key(_hash(12345), 0)
|
||||
@@ -244,7 +247,7 @@ def test_take_events_opt_out_keeps_placeholders():
|
||||
|
||||
def test_record_store_skips_sliding_window_group():
|
||||
tracker = _tracker()
|
||||
group_config = _group_config(sliding_window_size_in_blocks=2)
|
||||
group_config = _group_config(sliding_window_size_in_chunks=2)
|
||||
req = _request(block_hashes=[_hash(i) for i in range(3)], token_count=12)
|
||||
keys = _record_chunks(tracker, req, group_config, num_chunks=3)
|
||||
|
||||
@@ -258,8 +261,8 @@ def test_record_store_skips_sliding_window_group():
|
||||
|
||||
def test_take_events_groups_removed_hashes_by_kv_group():
|
||||
tracker = _tracker()
|
||||
group0_config = _group_config(group_idx=0, block_size_factor=2)
|
||||
group1_config = _group_config(group_idx=1, block_size_factor=2)
|
||||
group0_config = _group_config(group_idx=0, blocks_per_chunk=2)
|
||||
group1_config = _group_config(group_idx=1, blocks_per_chunk=2)
|
||||
req0 = _request(block_hashes=[_hash(0), _hash(1)], token_count=8)
|
||||
req1 = _request(block_hashes=[_hash(10), _hash(11)], token_count=8)
|
||||
key0 = _record_chunks(tracker, req0, group0_config, num_chunks=1)[0]
|
||||
@@ -293,7 +296,7 @@ def test_take_events_supports_restore_after_eviction():
|
||||
assert not tracker._pending_event_metadata
|
||||
|
||||
req.all_token_ids = [5, 6, 7, 8]
|
||||
tracker.record_store(req, group_config, offload_block_idx=0, offload_key=key)
|
||||
tracker.record_store(req, group_config, chunk_idx=0, offload_key=key)
|
||||
|
||||
second_store = list(tracker.take_events([_stored_event([key])]))
|
||||
assert len(second_store) == 1
|
||||
@@ -351,4 +354,4 @@ def test_tiering_rejects_self_describing_kv_events():
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="TieringOffloadingSpec"):
|
||||
TieringOffloadingSpec(vllm_config, kv_cache_config)
|
||||
TieringOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
|
||||
|
||||
@@ -105,20 +105,20 @@ def test_scheduler_reports_lookup_async_delay_on_resolve(request_runner):
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
num_gpu_blocks = 100
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
# 3 blocks, store just the middle block (skip first and last)
|
||||
# blocks = [0, 1, 2], [3, 4, 5], [6, 7, 8]
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 3)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * 3)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(list(keys)[1:2])
|
||||
)
|
||||
@@ -126,7 +126,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
|
||||
# add block missing 1 token -> no offload
|
||||
runner.run(
|
||||
decoded_tokens=[0] * (offloaded_block_size - 1),
|
||||
decoded_tokens=[0] * (tokens_per_chunk - 1),
|
||||
expected_stored=(3, 4, 5),
|
||||
)
|
||||
runner.manager.touch.assert_not_called()
|
||||
@@ -141,7 +141,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
runner.run(decoded_tokens=[0] * (offloaded_block_size + 1))
|
||||
runner.run(decoded_tokens=[0] * (tokens_per_chunk + 1))
|
||||
|
||||
# 1 more block (+ token for kicking off offloading)
|
||||
# now check touch was called with all 6 blocks
|
||||
@@ -149,7 +149,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[0] * (offloaded_block_size + 1),
|
||||
decoded_tokens=[0] * (tokens_per_chunk + 1),
|
||||
expected_stored=(15, 16, 17),
|
||||
)
|
||||
runner.manager.touch.assert_called()
|
||||
@@ -160,7 +160,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
|
||||
# create a new request differing only on the last token
|
||||
runner.new_request(token_ids=[0] * (offloaded_block_size * 6 - 1) + [1])
|
||||
runner.new_request(token_ids=[0] * (tokens_per_chunk * 6 - 1) + [1])
|
||||
runner.run(decoded_tokens=[0])
|
||||
runner.manager.touch.assert_called()
|
||||
block_hashes2 = list(runner.manager.touch.call_args.args[0])
|
||||
@@ -173,12 +173,12 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
# terminate request
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=tuple(range(6 * block_size_factor)),
|
||||
expected_stored=tuple(range(6 * blocks_per_chunk)),
|
||||
)
|
||||
|
||||
# full_block_tokens - num_computed_tokens < offloaded_block_size
|
||||
# full_block_tokens - num_computed_tokens < tokens_per_chunk
|
||||
runner.new_request(
|
||||
token_ids=[0] * block_size + [1] * (offloaded_block_size - block_size)
|
||||
token_ids=[0] * block_size + [1] * (tokens_per_chunk - block_size)
|
||||
)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
@@ -187,7 +187,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
runner.manager.lookup.assert_not_called()
|
||||
|
||||
# single block lookup with no hits
|
||||
runner.new_request(token_ids=[1] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[1] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
@@ -196,7 +196,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
|
||||
# single block lookup with a hit
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
@@ -204,9 +204,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_loaded=(0, 1, 2))
|
||||
|
||||
# single block lookup with a hit in a middle block
|
||||
runner.new_request(
|
||||
token_ids=[0] * offloaded_block_size * 2 + [1] * offloaded_block_size
|
||||
)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * 2 + [1] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
@@ -217,15 +215,15 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_request_preemption(request_runner, async_scheduling: bool):
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
num_gpu_blocks = 100
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
free_block_queue = runner.scheduler.kv_cache_manager.block_pool.free_block_queue
|
||||
@@ -233,7 +231,7 @@ def test_request_preemption(request_runner, async_scheduling: bool):
|
||||
|
||||
# 2 blocks, store all, without flushing
|
||||
# blocks = [0, 1, 2], [3, 4, 5]
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 2)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * 2)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -247,7 +245,7 @@ def test_request_preemption(request_runner, async_scheduling: bool):
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[0] * (2 * offloaded_block_size - block_size),
|
||||
decoded_tokens=[0] * (2 * tokens_per_chunk - block_size),
|
||||
complete_transfers=False,
|
||||
)
|
||||
|
||||
@@ -297,14 +295,14 @@ def test_on_request_finished_is_not_deferred_until_store_completion(
|
||||
still arrive afterward for already-submitted transfer jobs.
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=async_scheduling,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
# Record the order of per-request connector calls on the (mocked) manager.
|
||||
@@ -324,10 +322,10 @@ def test_on_request_finished_is_not_deferred_until_store_completion(
|
||||
|
||||
# Decode a couple of blocks, keeping every transfer in flight
|
||||
# (complete_transfers=False) so no store completes while the request runs.
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 2)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * 2)
|
||||
runner.run(decoded_tokens=[0], complete_transfers=False)
|
||||
runner.run(
|
||||
decoded_tokens=[0] * (2 * offloaded_block_size),
|
||||
decoded_tokens=[0] * (2 * tokens_per_chunk),
|
||||
complete_transfers=False,
|
||||
)
|
||||
|
||||
@@ -347,7 +345,7 @@ def test_on_request_finished_is_not_deferred_until_store_completion(
|
||||
runner.run(
|
||||
decoded_tokens=[],
|
||||
complete_transfers=True,
|
||||
expected_stored=tuple(range(4 * block_size_factor)),
|
||||
expected_stored=tuple(range(4 * blocks_per_chunk)),
|
||||
)
|
||||
|
||||
# on_request_finished is issued exactly once.
|
||||
@@ -364,19 +362,19 @@ def test_on_request_finished_is_not_deferred_until_store_completion(
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling: bool):
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
num_gpu_blocks = 100
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
# store 1 blocks
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -387,7 +385,7 @@ def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling:
|
||||
|
||||
# start a request to load the first block, but don't complete
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.run(
|
||||
decoded_tokens=[],
|
||||
@@ -399,7 +397,7 @@ def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling:
|
||||
assert transfer_jobs
|
||||
|
||||
# start a new request to load the same first block
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.run(
|
||||
decoded_tokens=[],
|
||||
@@ -428,19 +426,19 @@ def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling:
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_abort_loading_requests(request_runner, async_scheduling: bool):
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
num_gpu_blocks = 100
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
# store 1 blocks
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -451,7 +449,7 @@ def test_abort_loading_requests(request_runner, async_scheduling: bool):
|
||||
|
||||
# start a request to load the first block, but don't complete
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.run(
|
||||
decoded_tokens=[],
|
||||
@@ -483,7 +481,7 @@ def test_abort_loading_requests(request_runner, async_scheduling: bool):
|
||||
def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bool):
|
||||
block_size = 4
|
||||
num_gpu_blocks = 100
|
||||
# sliding_window=8 -> 2 offloaded blocks (block_size_factor=1)
|
||||
# sliding_window=8 -> 2 offloaded chunks (blocks_per_chunk=1)
|
||||
sliding_window = 8
|
||||
|
||||
kv_cache_groups = [
|
||||
@@ -518,8 +516,8 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
||||
# Verify group configs: group 0 = full attention, group 1 = sliding window
|
||||
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
|
||||
assert len(kv_group_configs) == 2
|
||||
assert kv_group_configs[0].sliding_window_size_in_blocks is None
|
||||
assert kv_group_configs[1].sliding_window_size_in_blocks == 2
|
||||
assert kv_group_configs[0].sliding_window_size_in_chunks is None
|
||||
assert kv_group_configs[1].sliding_window_size_in_chunks == 2
|
||||
|
||||
# Blocks [0, 1, 2] miss
|
||||
runner.new_request(token_ids=[0] * block_size * 3)
|
||||
@@ -587,16 +585,16 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool):
|
||||
hash_block_size = 4
|
||||
tokens_per_hash = 4
|
||||
num_gpu_blocks = 100
|
||||
|
||||
# Group 0: block_size=12 (offloaded_block_size=12)
|
||||
# Group 1: block_size=16 (offloaded_block_size=16)
|
||||
# Group 0: block_size=12 (tokens_per_chunk=12)
|
||||
# Group 1: block_size=16 (tokens_per_chunk=16)
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size * 3,
|
||||
block_size=tokens_per_hash * 3,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
@@ -605,7 +603,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size * 4,
|
||||
block_size=tokens_per_hash * 4,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
@@ -614,7 +612,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
||||
]
|
||||
|
||||
runner = request_runner(
|
||||
block_size=hash_block_size,
|
||||
block_size=tokens_per_hash,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
@@ -623,10 +621,10 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
||||
# Verify group configs
|
||||
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
|
||||
assert len(kv_group_configs) == 2
|
||||
assert kv_group_configs[0].gpu_block_size == 12
|
||||
assert kv_group_configs[0].offloaded_block_size == 12
|
||||
assert kv_group_configs[1].gpu_block_size == 16
|
||||
assert kv_group_configs[1].offloaded_block_size == 16
|
||||
assert kv_group_configs[0].tokens_per_block == 12
|
||||
assert kv_group_configs[0].tokens_per_chunk == 12
|
||||
assert kv_group_configs[1].tokens_per_block == 16
|
||||
assert kv_group_configs[1].tokens_per_chunk == 16
|
||||
|
||||
# Prompt: 25 tokens, unaligned to both block sizes.
|
||||
# Group 0 blocks: [0, 1], ending_token_offset = 24
|
||||
@@ -932,20 +930,20 @@ class TestSlidingWindowLookup:
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling: bool):
|
||||
"""With REQUEST_LEVEL policy, all blocks are stored — including prefix hits."""
|
||||
gpu_block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = gpu_block_size * block_size_factor
|
||||
tokens_per_block = 4
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = tokens_per_block * blocks_per_chunk
|
||||
num_gpu_blocks = 100
|
||||
|
||||
runner = request_runner(
|
||||
block_size_factor=block_size_factor,
|
||||
block_size=gpu_block_size,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
block_size=tokens_per_block,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
)
|
||||
|
||||
# Store 1 offloaded block (3 GPU blocks) via a normal request.
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
# Store 1 offloaded chunk (3 GPU blocks) via a normal request.
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -962,14 +960,14 @@ def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling
|
||||
policy=OffloadPolicy.REQUEST_LEVEL
|
||||
)
|
||||
|
||||
# New request with 2 offloaded blocks; first matches what's in CPU.
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 2)
|
||||
# New request with 2 offloaded chunks; first matches what's in CPU.
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * 2)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
# Load the first offloaded block from CPU.
|
||||
# Load the first offloaded chunk from CPU.
|
||||
runner.run(decoded_tokens=[0], expected_loaded=(0, 1, 2))
|
||||
|
||||
# Store must include ALL 6 GPU blocks (both the loaded prefix and
|
||||
@@ -989,7 +987,7 @@ def test_loads_do_not_populate_fence_index(request_runner):
|
||||
"""Loads don't populate _block_id_to_pending_jobs (protected by
|
||||
delay_free_blocks while in flight)."""
|
||||
runner = request_runner(
|
||||
block_size_factor=3,
|
||||
blocks_per_chunk=3,
|
||||
block_size=4,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
@@ -1008,7 +1006,7 @@ def test_fence_at_update_state_after_alloc(request_runner):
|
||||
req1 just freed.
|
||||
"""
|
||||
runner = request_runner(
|
||||
block_size_factor=1,
|
||||
blocks_per_chunk=1,
|
||||
block_size=4,
|
||||
num_gpu_blocks=2,
|
||||
async_scheduling=False,
|
||||
@@ -1059,7 +1057,7 @@ def test_fence_at_build_store_jobs(request_runner):
|
||||
reusing a finished request's pending-store block is flushed by
|
||||
_build_store_jobs's fence."""
|
||||
runner = request_runner(
|
||||
block_size_factor=1,
|
||||
blocks_per_chunk=1,
|
||||
block_size=4,
|
||||
num_gpu_blocks=2,
|
||||
async_scheduling=False,
|
||||
@@ -1108,16 +1106,16 @@ def test_fence_at_build_store_jobs(request_runner):
|
||||
def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
|
||||
"""complete_store fires per-job, not deferred to request finish.
|
||||
Each call carries only that store's keys."""
|
||||
gpu_block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = gpu_block_size * block_size_factor
|
||||
tokens_per_block = 4
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = tokens_per_block * blocks_per_chunk
|
||||
runner = request_runner(
|
||||
block_size_factor=block_size_factor,
|
||||
block_size=gpu_block_size,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
block_size=tokens_per_block,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=async_scheduling,
|
||||
)
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -1131,7 +1129,7 @@ def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
|
||||
|
||||
# Second store: fires when block 1 is fully populated, with different keys.
|
||||
runner.run(
|
||||
decoded_tokens=[0] * (offloaded_block_size + 1),
|
||||
decoded_tokens=[0] * (tokens_per_chunk + 1),
|
||||
expected_stored=(3, 4, 5),
|
||||
)
|
||||
assert runner.manager.complete_store.call_count == 1
|
||||
@@ -1148,25 +1146,25 @@ def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
|
||||
def test_max_offload_tokens_validation(request_runner, async_scheduling: bool):
|
||||
"""Validates max_offload_tokens: type coercion, boundary values, and capping.
|
||||
|
||||
Setup: 3 offloaded blocks × 3 GPU blocks each = 9 GPU block offsets (0–8).
|
||||
Setup: 3 offloaded chunks × 3 GPU blocks each = 9 GPU block offsets (0–8).
|
||||
"""
|
||||
gpu_block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = gpu_block_size * block_size_factor # 12
|
||||
tokens_per_block = 4
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = tokens_per_block * blocks_per_chunk # 12
|
||||
num_gpu_blocks = 100
|
||||
all_offsets = (0, 1, 2, 3, 4, 5, 6, 7, 8)
|
||||
|
||||
def make_runner():
|
||||
return request_runner(
|
||||
block_size=gpu_block_size,
|
||||
block_size=tokens_per_block,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
def setup(r, max_offload_tokens):
|
||||
r.new_request(
|
||||
token_ids=[0] * offloaded_block_size * 3,
|
||||
token_ids=[0] * tokens_per_chunk * 3,
|
||||
kv_transfer_params={"max_offload_tokens": max_offload_tokens},
|
||||
)
|
||||
r.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
@@ -1228,9 +1226,9 @@ def test_max_offload_tokens_validation(request_runner, async_scheduling: bool):
|
||||
setup(r, 0)
|
||||
r.run(decoded_tokens=[EOS_TOKEN_ID], expected_stored=())
|
||||
|
||||
# positive int cap -> limits offload to first 2 offloaded blocks (offsets 0–5)
|
||||
# positive int cap -> limits offload to first 2 chunks (offsets 0–5)
|
||||
r = make_runner()
|
||||
setup(r, 24) # 24 tokens = 2 offloaded blocks × 12 tokens each
|
||||
setup(r, 24) # 24 tokens = 2 offloaded chunks × 12 tokens each
|
||||
r.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=(0, 1, 2, 3, 4, 5),
|
||||
@@ -1242,8 +1240,8 @@ def test_max_offload_tokens_validation(request_runner, async_scheduling: bool):
|
||||
def test_offload_prompt_only(request_runner, async_scheduling: bool):
|
||||
"""offload_prompt_only=True offloads prompt blocks but never decode blocks.
|
||||
|
||||
Setup: a 2-offloaded-block prompt followed by enough decode tokens to fill
|
||||
4 more offloaded blocks. The flag clamps the offloadable token count to the
|
||||
Setup: a 2-chunk prompt followed by enough decode tokens to fill
|
||||
4 more offloaded chunks. The flag clamps the offloadable token count to the
|
||||
prompt length, so only the prompt's blocks (GPU offsets 0-5) are ever
|
||||
eligible for store; the decode blocks (offsets >= 6) are skipped.
|
||||
|
||||
@@ -1253,18 +1251,18 @@ def test_offload_prompt_only(request_runner, async_scheduling: bool):
|
||||
subtleties. The decode steps are still enough for the prompt store to
|
||||
complete and show up in expected_stored.
|
||||
"""
|
||||
gpu_block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = gpu_block_size * block_size_factor # 12
|
||||
tokens_per_block = 4
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = tokens_per_block * blocks_per_chunk # 12
|
||||
num_prompt_blocks = 2
|
||||
num_decode_blocks = 4
|
||||
prompt_offsets = (0, 1, 2, 3, 4, 5)
|
||||
|
||||
runner = request_runner(
|
||||
block_size=gpu_block_size,
|
||||
block_size=tokens_per_block,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=async_scheduling,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
extra_config_overrides={"offload_prompt_only": True},
|
||||
)
|
||||
|
||||
@@ -1272,9 +1270,9 @@ def test_offload_prompt_only(request_runner, async_scheduling: bool):
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * num_prompt_blocks)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * num_prompt_blocks)
|
||||
runner.run(
|
||||
decoded_tokens=[0] * (offloaded_block_size * num_decode_blocks),
|
||||
decoded_tokens=[0] * (tokens_per_chunk * num_decode_blocks),
|
||||
expected_stored=prompt_offsets,
|
||||
)
|
||||
|
||||
@@ -1291,21 +1289,21 @@ def test_offload_prompt_only(request_runner, async_scheduling: bool):
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_reset_cache(request_runner, async_scheduling: bool):
|
||||
"""reset_cache flushes in-flight loads, calls manager.reset_cache(), resets
|
||||
next_stored_block_idx for active requests and clears job tracking."""
|
||||
next_stored_chunk_idx for active requests and clears job tracking."""
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
num_gpu_blocks = 100
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
# Store 1 offloaded block (3 GPU blocks) to CPU.
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
# Store 1 offloaded chunk (3 GPU blocks) to CPU.
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -1317,7 +1315,7 @@ def test_reset_cache(request_runner, async_scheduling: bool):
|
||||
# Reset GPU prefix cache then start a request that loads from CPU.
|
||||
# Leave the load in-flight so that reset_cache must flush it.
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
@@ -1335,11 +1333,11 @@ def test_reset_cache(request_runner, async_scheduling: bool):
|
||||
# Record job counter to verify the reset counter is set correctly.
|
||||
job_counter_before_reset = runner.connector_scheduler._job_counter
|
||||
|
||||
# After update_state_after_alloc, next_stored_block_idx is advanced to
|
||||
# After update_state_after_alloc, next_stored_chunk_idx is advanced to
|
||||
# skip the loaded prefix; reset_cache must bring it back to 0.
|
||||
for req_status in runner.connector_scheduler._req_status.values():
|
||||
for group_state in req_status.group_states:
|
||||
assert group_state.next_stored_block_idx > 0
|
||||
assert group_state.next_stored_chunk_idx > 0
|
||||
|
||||
# Reset the cache
|
||||
runner.connector_scheduler.reset_cache()
|
||||
@@ -1354,18 +1352,18 @@ def test_reset_cache(request_runner, async_scheduling: bool):
|
||||
# All internal job tracking must be cleared.
|
||||
assert not runner.connector_scheduler._jobs
|
||||
assert not runner.connector_scheduler._block_id_to_pending_jobs
|
||||
if runner.connector_scheduler._blocks_being_loaded is not None:
|
||||
assert not runner.connector_scheduler._blocks_being_loaded
|
||||
if runner.connector_scheduler._chunks_being_loaded is not None:
|
||||
assert not runner.connector_scheduler._chunks_being_loaded
|
||||
|
||||
# Job reset counter must equal the job counter so that completions for
|
||||
# pre-reset jobs arriving from workers are silently discarded.
|
||||
assert runner.connector_scheduler._stale_job_threshold == job_counter_before_reset
|
||||
|
||||
# next_stored_block_idx must be reset to 0 for every active request so
|
||||
# next_stored_chunk_idx must be reset to 0 for every active request so
|
||||
# that post-reset stores restart from block 0.
|
||||
for req_status in runner.connector_scheduler._req_status.values():
|
||||
for group_state in req_status.group_states:
|
||||
assert group_state.next_stored_block_idx == 0
|
||||
assert group_state.next_stored_chunk_idx == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
@@ -1376,14 +1374,14 @@ def test_reset_cache_finalizes_finished_request_with_pending_store(
|
||||
without calling on_request_finished twice.
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=async_scheduling,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
finalized: list[str] = []
|
||||
@@ -1396,10 +1394,10 @@ def test_reset_cache_finalizes_finished_request_with_pending_store(
|
||||
|
||||
# Decode a couple of blocks and keep every transfer in flight, so the
|
||||
# request has pending store jobs.
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 2)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * 2)
|
||||
runner.run(decoded_tokens=[0], complete_transfers=False)
|
||||
runner.run(
|
||||
decoded_tokens=[0] * (2 * offloaded_block_size),
|
||||
decoded_tokens=[0] * (2 * tokens_per_chunk),
|
||||
complete_transfers=False,
|
||||
)
|
||||
|
||||
@@ -1430,7 +1428,7 @@ def test_pending_transfer_defers_prefix_lookup():
|
||||
With async scheduling, a preempted request's store can be flushed by the
|
||||
worker before the scheduler consumes its completion. If the request is
|
||||
re-admitted in that window, the connector should defer it instead of
|
||||
looking up offloaded blocks and later asserting when a load is queued while
|
||||
looking up offloaded chunks and later asserting when a load is queued while
|
||||
the store job is still tracked.
|
||||
"""
|
||||
scheduler = object.__new__(OffloadingConnectorScheduler)
|
||||
@@ -1464,27 +1462,27 @@ def test_async_preempt_readmit_before_transfer_output_is_deferred(request_runner
|
||||
re-admission path must defer while the scheduler still tracks the store.
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=True,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
free_block_queue = runner.scheduler.kv_cache_manager.block_pool.free_block_queue
|
||||
num_free_blocks_empty = free_block_queue.num_free_blocks
|
||||
|
||||
req_id = "0"
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 2)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * 2)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
runner.run(decoded_tokens=[0], complete_transfers=False)
|
||||
runner.run(
|
||||
decoded_tokens=[0] * (2 * offloaded_block_size - block_size),
|
||||
decoded_tokens=[0] * (2 * tokens_per_chunk - block_size),
|
||||
complete_transfers=False,
|
||||
)
|
||||
|
||||
@@ -1530,8 +1528,8 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
- Group 0: full attention (MLA-like), block_size=16
|
||||
- Group 1: SWA, block_size=4, sliding_window=8
|
||||
|
||||
alignment_block_count = 16 / 4 = 4 SWA blocks per alignment segment.
|
||||
sliding_window_size_in_blocks = ceil(8 / 4) = 2.
|
||||
alignment_chunk_count = 16 / 4 = 4 SWA blocks per alignment segment.
|
||||
sliding_window_size_in_chunks = ceil(8 / 4) = 2.
|
||||
Within each segment of 4 SWA blocks, only the trailing 2 are stored.
|
||||
|
||||
With 32 tokens (2 full-attn blocks, 8 SWA blocks):
|
||||
@@ -1574,17 +1572,17 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
)
|
||||
|
||||
# Verify config: alignment_block_count computed correctly
|
||||
# Verify config: alignment_chunk_count computed correctly
|
||||
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
|
||||
assert len(kv_group_configs) == 2
|
||||
# Group 0: full attention -> no alignment skip
|
||||
assert kv_group_configs[0].alignment_block_count is None
|
||||
assert kv_group_configs[0].sliding_window_size_in_blocks is None
|
||||
assert kv_group_configs[0].offloaded_block_size == full_attn_block_size
|
||||
# Group 1: SWA -> alignment_block_count = 16/4 = 4, tail = 2
|
||||
assert kv_group_configs[1].alignment_block_count == 4
|
||||
assert kv_group_configs[1].sliding_window_size_in_blocks == 2
|
||||
assert kv_group_configs[1].offloaded_block_size == swa_block_size
|
||||
assert kv_group_configs[0].alignment_chunk_count is None
|
||||
assert kv_group_configs[0].sliding_window_size_in_chunks is None
|
||||
assert kv_group_configs[0].tokens_per_chunk == full_attn_block_size
|
||||
# Group 1: SWA -> alignment_chunk_count = 16/4 = 4, tail = 2
|
||||
assert kv_group_configs[1].alignment_chunk_count == 4
|
||||
assert kv_group_configs[1].sliding_window_size_in_chunks == 2
|
||||
assert kv_group_configs[1].tokens_per_chunk == swa_block_size
|
||||
|
||||
# Send 32 tokens = 2 full-attn blocks (block_size=16) = 8 SWA blocks
|
||||
# (block_size=4). Decode 1 token to kick off processing (stores are
|
||||
@@ -1602,9 +1600,9 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
# Group 0 (full attn, block_size=16): 2 offloaded blocks
|
||||
# Group 0 (full attn, block_size=16): 2 offloaded chunks
|
||||
# -> GPU blocks (0, 0) and (0, 1)
|
||||
# Group 1 (SWA, block_size=4): 8 offloaded blocks, skip first 2
|
||||
# Group 1 (SWA, block_size=4): 8 offloaded chunks, skip first 2
|
||||
# per segment of 4:
|
||||
# Segment 0 (blocks 0-3): skip 0,1 -> store (1, 2), (1, 3)
|
||||
# Segment 1 (blocks 4-7): skip 4,5 -> store (1, 6), (1, 7)
|
||||
@@ -1625,7 +1623,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 2
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
# Group 0: full prefix lookup hits 2 offloaded blocks
|
||||
# Group 0: full prefix lookup hits 2 offloaded chunks
|
||||
# -> loads GPU blocks (0, 0), (0, 1)
|
||||
# Group 1: sliding window lookup finds trailing 2 from last segment
|
||||
# (blocks 6, 7 which were stored)
|
||||
@@ -1684,7 +1682,7 @@ def test_stale_sliding_window_block_after_prepare_store_failure(
|
||||
runner.new_request(token_ids=[0] * block_size * 3)
|
||||
|
||||
# First step: prepare_store FAILS -> offloading delayed.
|
||||
# next_stored_block_idx stays at 0, block_ids[0] still holds the
|
||||
# next_stored_chunk_idx stays at 0, block_ids[0] still holds the
|
||||
# original block_id for position 0.
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: None
|
||||
runner.run(decoded_tokens=[0])
|
||||
@@ -1719,19 +1717,19 @@ def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
|
||||
"""When skip_reading_prefix_cache=True, the offloading connector must not
|
||||
load any blocks from CPU even if a matching prefix is cached there."""
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
num_gpu_blocks = 100
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
# Populate the CPU offload cache with one block.
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -1747,7 +1745,7 @@ def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
|
||||
# The offloading connector must not load anything from CPU, but must
|
||||
# still offload the freshly computed blocks (state management intact).
|
||||
runner.new_request(
|
||||
token_ids=[0] * offloaded_block_size,
|
||||
token_ids=[0] * tokens_per_chunk,
|
||||
skip_reading_prefix_cache=True,
|
||||
)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
@@ -2219,7 +2217,7 @@ class TestEagle:
|
||||
def test_full_attn_store_excludes_trailing_decode_block(
|
||||
self, request_runner, async_scheduling: bool
|
||||
):
|
||||
"""Eagle full-attention group excludes the trailing block only while
|
||||
"""Eagle full-attention group excludes the trailing chunk only while
|
||||
decoding.
|
||||
|
||||
Setup: 2 groups — group 0 is normal full-attention, group 1 is
|
||||
@@ -2229,8 +2227,8 @@ class TestEagle:
|
||||
draft-layer KV is volatile until the next block starts).
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 1
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 1
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
num_gpu_blocks = 100
|
||||
|
||||
kv_cache_groups = [
|
||||
@@ -2260,7 +2258,7 @@ class TestEagle:
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
|
||||
@@ -2268,7 +2266,7 @@ class TestEagle:
|
||||
assert not kv_group_configs[0].is_eagle_group
|
||||
assert kv_group_configs[1].is_eagle_group
|
||||
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 3)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * 3)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -2292,7 +2290,7 @@ class TestEagle:
|
||||
self, request_runner, async_scheduling: bool
|
||||
):
|
||||
"""Eagle sliding-window group stores all prompt blocks but excludes
|
||||
the trailing block while decoding."""
|
||||
the trailing chunk while decoding."""
|
||||
block_size = 4
|
||||
sliding_window = 8
|
||||
num_gpu_blocks = 100
|
||||
@@ -2321,7 +2319,7 @@ class TestEagle:
|
||||
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
|
||||
assert len(kv_group_configs) == 1
|
||||
assert kv_group_configs[0].is_eagle_group
|
||||
assert kv_group_configs[0].sliding_window_size_in_blocks == 2
|
||||
assert kv_group_configs[0].sliding_window_size_in_chunks == 2
|
||||
|
||||
runner.new_request(token_ids=[0] * block_size * 3)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
@@ -2340,8 +2338,8 @@ class TestEagle:
|
||||
"""An eagle group with a single-block prompt stores it at the end of
|
||||
prefill: prompt blocks are stable, so no tail is held back."""
|
||||
block_size = 4
|
||||
block_size_factor = 1
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 1
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
num_gpu_blocks = 100
|
||||
|
||||
kv_cache_groups = [
|
||||
@@ -2362,10 +2360,10 @@ class TestEagle:
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -2378,18 +2376,18 @@ class TestEagle:
|
||||
"""Eagle store must not drop interior blocks across prefill chunks.
|
||||
|
||||
Regression: the trailing-block exclusion (num_blocks - 1) was applied
|
||||
when collecting keys, but next_stored_block_idx advanced by the
|
||||
non-decremented count, so the trailing block of every chunked-prefill
|
||||
when collecting keys, but next_stored_chunk_idx advanced by the
|
||||
non-decremented count, so the trailing chunk of every chunked-prefill
|
||||
chunk was skipped and never re-considered. With the harness chunk budget
|
||||
(1000 tokens) and block_size 4, a prompt longer than one chunk lost the
|
||||
block at the chunk boundary, leaving a permanent gap that caps prefix
|
||||
reuse at the first hole. Only the trailing decode block may be held
|
||||
back; all other blocks must be stored exactly once (no duplicates from
|
||||
next_stored_block_idx regressing at the prefill->decode transition).
|
||||
next_stored_chunk_idx regressing at the prefill->decode transition).
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 1
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 1
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
num_gpu_blocks = 1000
|
||||
|
||||
kv_cache_groups = [
|
||||
@@ -2409,13 +2407,13 @@ class TestEagle:
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
assert runner.connector_scheduler.config.kv_group_configs[0].is_eagle_group
|
||||
|
||||
# Prompt spans more than one prefill chunk (chunk budget 1000 tokens).
|
||||
num_blocks = 256
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * num_blocks)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * num_blocks)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -2429,7 +2427,7 @@ class TestEagle:
|
||||
for b in t.gpu_blocks
|
||||
)
|
||||
# The stored blocks must be contiguous from 0: no interior block is
|
||||
# dropped at a chunk boundary. (The bug left a gap at offloaded block
|
||||
# dropped at a chunk boundary. (The bug left a gap at offloaded chunk
|
||||
# 249, the tail of the first 1000-token chunk.)
|
||||
assert offsets == list(range(len(offsets))), (
|
||||
f"interior hole in stored blocks: {offsets}"
|
||||
@@ -2439,14 +2437,14 @@ class TestEagle:
|
||||
def test_full_attn_store_then_load(self, request_runner, async_scheduling: bool):
|
||||
"""Eagle group constrains load: convergence tightens both groups.
|
||||
|
||||
Store 3 offloaded blocks per group (all prompt blocks, so the eagle
|
||||
Store 3 offloaded chunks per group (all prompt chunks, so the eagle
|
||||
group stores all 3 as well). Then a new request loads from CPU. The
|
||||
eagle group pops its trailing hit block on load, tightening the hit
|
||||
to 2 blocks for both groups.
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 1
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 1
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
num_gpu_blocks = 100
|
||||
|
||||
kv_cache_groups = [
|
||||
@@ -2476,10 +2474,10 @@ class TestEagle:
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 3)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * 3)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -2497,7 +2495,7 @@ class TestEagle:
|
||||
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 3 + [1])
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * 3 + [1])
|
||||
runner.manager.lookup.return_value = LookupResult.HIT
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
@@ -2527,8 +2525,8 @@ def test_request_finished_with_pending_stores_populates_fence(request_runner):
|
||||
GPU blocks before the store completes.
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 1
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 1
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
|
||||
# Use 2 GPU blocks so the second run reuses the same blocks,
|
||||
# triggering a fence-based flush of the in-flight job from run 1.
|
||||
@@ -2536,11 +2534,11 @@ def test_request_finished_with_pending_stores_populates_fence(request_runner):
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=2,
|
||||
async_scheduling=False,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
# 4 prompt tokens → 1 GPU block (block 0)
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -2576,7 +2574,7 @@ def test_request_finished_with_pending_stores_populates_fence(request_runner):
|
||||
|
||||
# Run 2: block reuse triggers fence-based flush → cleanup.
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -2603,26 +2601,26 @@ def test_multiple_in_flight_stores_all_flushed_by_fence(request_runner):
|
||||
- Run 3: block reuse → both jobs flushed via fence
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 1
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
blocks_per_chunk = 1
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
|
||||
# 4 GPU blocks: block 0 is null, blocks 1-3 are usable.
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=4,
|
||||
async_scheduling=False,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
# Prompt: 4 tokens → block 1
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
# Run 1: 4 decoded tokens → block 2 full → job_0 created for block 1.
|
||||
runner.run(
|
||||
decoded_tokens=[0] * offloaded_block_size,
|
||||
decoded_tokens=[0] * tokens_per_chunk,
|
||||
complete_transfers=False,
|
||||
)
|
||||
assert len(runner.connector_scheduler._jobs) >= 1
|
||||
@@ -2630,7 +2628,7 @@ def test_multiple_in_flight_stores_all_flushed_by_fence(request_runner):
|
||||
# Run 2: 4 more tokens + EOS → block 3 full → more jobs created.
|
||||
# Request finishes → all jobs registered in fence.
|
||||
runner.run(
|
||||
decoded_tokens=[0] * offloaded_block_size + [EOS_TOKEN_ID],
|
||||
decoded_tokens=[0] * tokens_per_chunk + [EOS_TOKEN_ID],
|
||||
complete_transfers=False,
|
||||
)
|
||||
num_jobs = len(runner.connector_scheduler._jobs)
|
||||
@@ -2638,7 +2636,7 @@ def test_multiple_in_flight_stores_all_flushed_by_fence(request_runner):
|
||||
|
||||
# Run 3: block reuse → fence flushes both jobs.
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 3)
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * 3)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
@@ -96,11 +96,12 @@ def _make_worker(kv_cache_config: KVCacheConfig):
|
||||
)
|
||||
|
||||
spec = MagicMock(spec=OffloadingSpec)
|
||||
spec.kv_cache_config = kv_cache_config
|
||||
spec.vllm_config = MagicMock()
|
||||
spec.get_worker.return_value = MagicMock()
|
||||
|
||||
worker = OffloadingConnectorWorker(spec=spec)
|
||||
worker = OffloadingConnectorWorker(
|
||||
spec=spec,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
worker.worker = MagicMock()
|
||||
|
||||
return worker, spec
|
||||
|
||||
@@ -17,7 +17,6 @@ from vllm import SamplingParams
|
||||
from vllm.config import (
|
||||
KVEventsConfig,
|
||||
KVTransferConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole
|
||||
@@ -57,6 +56,7 @@ from vllm.v1.kv_offload.base import (
|
||||
TransferResult,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.config import OffloadingConfig
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
@@ -123,8 +123,8 @@ class MockOffloadingWorker(OffloadingWorker):
|
||||
|
||||
|
||||
class MockOffloadingSpec(OffloadingSpec):
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig):
|
||||
super().__init__(vllm_config, kv_cache_config)
|
||||
def __init__(self, config: OffloadingConfig):
|
||||
super().__init__(config)
|
||||
|
||||
self.manager = MagicMock(spec=OffloadingManager)
|
||||
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
|
||||
@@ -175,17 +175,17 @@ class RequestRunner:
|
||||
self,
|
||||
block_size: int,
|
||||
num_gpu_blocks: int,
|
||||
block_size_factor: int = 1,
|
||||
blocks_per_chunk: int = 1,
|
||||
async_scheduling: bool = True,
|
||||
kv_cache_groups: list[KVCacheGroupSpec] | None = None,
|
||||
extra_config_overrides: dict[str, Any] | None = None,
|
||||
):
|
||||
assert block_size_factor == 1 or kv_cache_groups is None, (
|
||||
"block_size_factor > 1 requires all groups to have the same "
|
||||
assert blocks_per_chunk == 1 or kv_cache_groups is None, (
|
||||
"blocks_per_chunk > 1 requires all groups to have the same "
|
||||
"block size, so kv_cache_groups must be None (use default group)"
|
||||
)
|
||||
|
||||
self.block_size_factor: int = block_size_factor
|
||||
self.blocks_per_chunk: int = blocks_per_chunk
|
||||
self.block_size: int = block_size
|
||||
self.num_gpu_blocks: int = num_gpu_blocks
|
||||
self.async_scheduling: bool = async_scheduling
|
||||
@@ -208,8 +208,8 @@ class RequestRunner:
|
||||
# opt-out tests override this to cover the legacy placeholders.
|
||||
"self_describing_kv_events": True,
|
||||
}
|
||||
if block_size_factor > 1:
|
||||
extra_config["block_size"] = block_size * block_size_factor
|
||||
if blocks_per_chunk > 1:
|
||||
extra_config["block_size"] = block_size * blocks_per_chunk
|
||||
if extra_config_overrides:
|
||||
extra_config.update(extra_config_overrides)
|
||||
|
||||
@@ -313,11 +313,9 @@ class RequestRunner:
|
||||
self.connector_scheduler.config.kv_group_configs,
|
||||
kv_cache_config.kv_cache_groups,
|
||||
):
|
||||
gpu_block_size = kv_cache_group.kv_cache_spec.block_size
|
||||
assert group_config.gpu_block_size == gpu_block_size
|
||||
assert (
|
||||
group_config.offloaded_block_size == gpu_block_size * block_size_factor
|
||||
)
|
||||
tokens_per_block = kv_cache_group.kv_cache_spec.block_size
|
||||
assert group_config.tokens_per_block == tokens_per_block
|
||||
assert group_config.tokens_per_chunk == tokens_per_block * blocks_per_chunk
|
||||
|
||||
# extract OffloadingSpec of worker_connector
|
||||
connector_worker = self.worker_connector.connector_worker
|
||||
@@ -389,7 +387,7 @@ class RequestRunner:
|
||||
for block_id in dst_spec.block_ids:
|
||||
self.flushed_gpu_blocks.add(self.gpu_blocks[block_id.item()])
|
||||
|
||||
block_size_factor = self.block_size_factor
|
||||
blocks_per_chunk = self.blocks_per_chunk
|
||||
|
||||
for src_spec, dst_spec in self.offloading_spec.get_completed_transfers():
|
||||
if isinstance(src_spec, GPULoadStoreSpec):
|
||||
@@ -412,7 +410,7 @@ class RequestRunner:
|
||||
# list of (offload_key, sub_block_offset)
|
||||
offload_addresses: list[Any] = []
|
||||
for offload_key in offload_spec.offload_keys:
|
||||
for sub_block_idx in range(block_size_factor):
|
||||
for sub_block_idx in range(blocks_per_chunk):
|
||||
offload_addresses.append((offload_key, sub_block_idx))
|
||||
|
||||
assert gpu_spec.block_indices is not None
|
||||
@@ -426,7 +424,7 @@ class RequestRunner:
|
||||
gpu_block_end_offset = gpu_block_offset + group_size
|
||||
assert gpu_block_end_offset <= len(gpu_blocks)
|
||||
|
||||
offload_addresses_to_skip = logical_offset % block_size_factor
|
||||
offload_addresses_to_skip = logical_offset % blocks_per_chunk
|
||||
offload_addresses_end_offset = (
|
||||
offload_address_offset + offload_addresses_to_skip + group_size
|
||||
)
|
||||
@@ -651,14 +649,14 @@ def request_runner():
|
||||
block_size,
|
||||
num_gpu_blocks,
|
||||
async_scheduling,
|
||||
block_size_factor=1,
|
||||
blocks_per_chunk=1,
|
||||
kv_cache_groups=None,
|
||||
extra_config_overrides=None,
|
||||
):
|
||||
runner = RequestRunner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
extra_config_overrides=extra_config_overrides,
|
||||
|
||||
@@ -45,7 +45,7 @@ if current_platform.is_cuda():
|
||||
# Falcon-H1: parallel hybrid (every layer has both attention and SSM).
|
||||
# The mamba and attention groups end up with different GPU block sizes
|
||||
# after page-size unification, so we leave cpu_block_size=None
|
||||
# (block_size_factor stays 1).
|
||||
# (blocks_per_chunk stays 1).
|
||||
("tiiuae/Falcon-H1-0.5B-Instruct", None, None, True),
|
||||
]
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
NUM_GPU_BLOCKS = [64]
|
||||
NUM_CPU_BLOCKS = [256]
|
||||
GPU_PAGE_SIZES = [512, 1024]
|
||||
BLOCK_SIZE_FACTORS = [1, 3]
|
||||
BLOCKS_PER_CHUNK_VALUES = [1, 3]
|
||||
NUM_TENSORS = [4]
|
||||
SEEDS = [0]
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
@@ -35,7 +35,7 @@ NUM_MAPPINGS_PER_GROUP = [2]
|
||||
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
|
||||
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
|
||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||
@pytest.mark.parametrize("block_size_factor", BLOCK_SIZE_FACTORS)
|
||||
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
||||
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_tensors", NUM_TENSORS)
|
||||
@@ -48,7 +48,7 @@ def test_transfer(
|
||||
gpu_to_cpu: bool,
|
||||
num_mappings: int,
|
||||
gpu_page_size_bytes: int,
|
||||
block_size_factor: int,
|
||||
blocks_per_chunk: int,
|
||||
num_gpu_blocks: int,
|
||||
num_cpu_blocks: int,
|
||||
num_tensors: int,
|
||||
@@ -92,7 +92,7 @@ def test_transfer(
|
||||
mmap_region: SharedOffloadRegion | None = None
|
||||
if use_shared_memory:
|
||||
cpu_page_size = round_up(
|
||||
gpu_page_size_bytes * num_tensors * block_size_factor,
|
||||
gpu_page_size_bytes * num_tensors * blocks_per_chunk,
|
||||
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
|
||||
)
|
||||
mmap_region = SharedOffloadRegion(
|
||||
@@ -105,25 +105,25 @@ def test_transfer(
|
||||
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
mmap_region=mmap_region,
|
||||
)
|
||||
|
||||
# select block mappings
|
||||
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * block_size_factor)
|
||||
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk)
|
||||
cpu_blocks = random.sample(range(num_cpu_blocks), num_mappings)
|
||||
|
||||
# expand cpu blocks to gpu-page granularity for uniform comparison:
|
||||
# each cpu block maps to block_size_factor consecutive sub-blocks
|
||||
# each cpu block maps to blocks_per_chunk consecutive sub-blocks
|
||||
cpu_blocks_expanded = [
|
||||
cpu_block * block_size_factor + j
|
||||
cpu_block * blocks_per_chunk + j
|
||||
for cpu_block in cpu_blocks
|
||||
for j in range(block_size_factor)
|
||||
for j in range(blocks_per_chunk)
|
||||
]
|
||||
|
||||
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU block
|
||||
blocks_to_skip = block_size_factor - 1
|
||||
blocks_to_skip = blocks_per_chunk - 1
|
||||
if blocks_to_skip > 0:
|
||||
gpu_blocks = gpu_blocks[blocks_to_skip:]
|
||||
cpu_blocks_expanded = cpu_blocks_expanded[blocks_to_skip:]
|
||||
@@ -214,7 +214,7 @@ def test_transfer(
|
||||
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
|
||||
@pytest.mark.parametrize("num_mappings_per_group", NUM_MAPPINGS_PER_GROUP)
|
||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||
@pytest.mark.parametrize("block_size_factor", BLOCK_SIZE_FACTORS)
|
||||
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
||||
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@@ -225,7 +225,7 @@ def test_transfer_multi_group(
|
||||
gpu_to_cpu: bool,
|
||||
num_mappings_per_group: int,
|
||||
gpu_page_size_bytes: int,
|
||||
block_size_factor: int,
|
||||
blocks_per_chunk: int,
|
||||
num_gpu_blocks: int,
|
||||
num_cpu_blocks: int,
|
||||
seed: int,
|
||||
@@ -234,7 +234,7 @@ def test_transfer_multi_group(
|
||||
"""Test transfers with three KV cache groups:
|
||||
- Group 0: aligned transfer with num_mappings_per_group blocks
|
||||
- Group 1: zero blocks (empty group)
|
||||
- Group 2: unaligned CPU->GPU transfer (logical_offset=block_size_factor-1,
|
||||
- Group 2: unaligned CPU->GPU transfer (logical_offset=blocks_per_chunk-1,
|
||||
causing the implementation to skip source sub-blocks) with
|
||||
num_mappings_per_group blocks
|
||||
"""
|
||||
@@ -275,7 +275,7 @@ def test_transfer_multi_group(
|
||||
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=canonical_kv_caches,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
)
|
||||
|
||||
@@ -283,7 +283,7 @@ def test_transfer_multi_group(
|
||||
group_sizes_in_cpu_blocks = [num_mappings_per_group, 0, num_mappings_per_group]
|
||||
|
||||
total_cpu_blocks = sum(group_sizes_in_cpu_blocks)
|
||||
total_gpu_blocks_needed = total_cpu_blocks * block_size_factor
|
||||
total_gpu_blocks_needed = total_cpu_blocks * blocks_per_chunk
|
||||
gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed)
|
||||
cpu_blocks_all = random.sample(range(num_cpu_blocks), total_cpu_blocks)
|
||||
|
||||
@@ -293,7 +293,7 @@ def test_transfer_multi_group(
|
||||
gpu_offset = 0
|
||||
cpu_offset = 0
|
||||
for size in group_sizes_in_cpu_blocks:
|
||||
gpu_count = size * block_size_factor
|
||||
gpu_count = size * blocks_per_chunk
|
||||
gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count])
|
||||
cpu_blocks_per_group.append(cpu_blocks_all[cpu_offset : cpu_offset + size])
|
||||
gpu_offset += gpu_count
|
||||
@@ -302,15 +302,15 @@ def test_transfer_multi_group(
|
||||
# expand cpu blocks to gpu-page granularity
|
||||
cpu_blocks_expanded_per_group = [
|
||||
[
|
||||
cpu_block * block_size_factor + j
|
||||
cpu_block * blocks_per_chunk + j
|
||||
for cpu_block in cpu_blocks
|
||||
for j in range(block_size_factor)
|
||||
for j in range(blocks_per_chunk)
|
||||
]
|
||||
for cpu_blocks in cpu_blocks_per_group
|
||||
]
|
||||
|
||||
# skip sub-blocks from group 2 to test unaligned transfers.
|
||||
sub_blocks_to_skip = block_size_factor - 1 # e.g. 2 when block_size_factor=3
|
||||
sub_blocks_to_skip = blocks_per_chunk - 1 # e.g. 2 when blocks_per_chunk=3
|
||||
if sub_blocks_to_skip > 0:
|
||||
gpu_blocks_per_group[2] = gpu_blocks_per_group[2][
|
||||
sub_blocks_to_skip:-sub_blocks_to_skip
|
||||
@@ -347,7 +347,7 @@ def test_transfer_multi_group(
|
||||
cpu_blocks_expanded_per_group, gpu_blocks_per_group
|
||||
)
|
||||
]
|
||||
num_dst_sub_blocks = num_cpu_blocks * block_size_factor
|
||||
num_dst_sub_blocks = num_cpu_blocks * blocks_per_chunk
|
||||
else:
|
||||
handler = worker._load_handler
|
||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
|
||||
@@ -11,17 +11,33 @@ These tests verify:
|
||||
4. Error paths — unregistered specs, missing config, duplicate registration.
|
||||
"""
|
||||
|
||||
from typing import cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import KVTransferConfig
|
||||
from vllm.config import KVTransferConfig, ParallelConfig, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
|
||||
build_offloading_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import OffloadingHistogramMetadata, OffloadingSpec
|
||||
from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
OffloadingHistogramMetadata,
|
||||
OffloadingManager,
|
||||
OffloadingSpec,
|
||||
OffloadingWorker,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
||||
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
|
||||
@@ -39,6 +55,17 @@ def restore_registry():
|
||||
OffloadingSpecFactory._registry = original
|
||||
|
||||
|
||||
def _get_extra_config(config: VllmConfig) -> dict:
|
||||
assert config.kv_transfer_config is not None
|
||||
return config.kv_transfer_config.kv_connector_extra_config
|
||||
|
||||
|
||||
def _create_spec(config: VllmConfig, kv_cache_config: KVCacheConfig) -> OffloadingSpec:
|
||||
return OffloadingSpecFactory.create_spec(
|
||||
build_offloading_config(config, kv_cache_config)
|
||||
)
|
||||
|
||||
|
||||
def _make_vllm_config(
|
||||
spec_name: str | None = "CPUOffloadingSpec",
|
||||
cpu_bytes_to_use: int | None = None,
|
||||
@@ -95,6 +122,46 @@ def _make_vllm_config(
|
||||
)
|
||||
|
||||
|
||||
def _make_layout_vllm_config(
|
||||
spec_name: str = "CPUOffloadingSpec",
|
||||
cpu_bytes_to_use: int | None = None,
|
||||
extra_config: dict | None = None,
|
||||
tensor_parallel_size: int = 1,
|
||||
pipeline_parallel_size: int = 1,
|
||||
prefill_context_parallel_size: int = 1,
|
||||
decode_context_parallel_size: int = 1,
|
||||
) -> VllmConfig:
|
||||
config = MagicMock()
|
||||
config.cache_config.block_size = 16
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
config.cache_config.prefix_match_unit = None
|
||||
config.cache_config.cache_dtype = torch.float16
|
||||
config.model_config.model = "test-model"
|
||||
world_size = (
|
||||
tensor_parallel_size * pipeline_parallel_size * prefill_context_parallel_size
|
||||
)
|
||||
with patch.object(current_platform, "device_count", return_value=world_size):
|
||||
config.parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
prefill_context_parallel_size=prefill_context_parallel_size,
|
||||
decode_context_parallel_size=decode_context_parallel_size,
|
||||
)
|
||||
config.kv_events_config = None
|
||||
config.use_v2_model_runner = False
|
||||
|
||||
connector_extra_config = dict(extra_config or {})
|
||||
connector_extra_config["spec_name"] = spec_name
|
||||
if cpu_bytes_to_use is not None:
|
||||
connector_extra_config["cpu_bytes_to_use"] = cpu_bytes_to_use
|
||||
config.kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config=connector_extra_config,
|
||||
)
|
||||
return cast(VllmConfig, config)
|
||||
|
||||
|
||||
def _make_kv_cache_config():
|
||||
"""Build a minimal KVCacheConfig with one KV cache tensor."""
|
||||
num_blocks = 16
|
||||
@@ -122,6 +189,78 @@ def _make_kv_cache_config():
|
||||
)
|
||||
|
||||
|
||||
def _make_sizing_kv_cache_config(packed: bool) -> KVCacheConfig:
|
||||
num_blocks = 4
|
||||
if packed:
|
||||
kv_cache_tensors = [
|
||||
KVCacheTensor(
|
||||
size=64,
|
||||
shared_by=[layer_name],
|
||||
block_stride=16,
|
||||
)
|
||||
for layer_name in ("layer0", "layer1")
|
||||
]
|
||||
else:
|
||||
kv_cache_tensors = [
|
||||
KVCacheTensor(size=40, shared_by=["layer0"]),
|
||||
KVCacheTensor(size=24, shared_by=["layer1"]),
|
||||
]
|
||||
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer0", "layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _make_hybrid_kv_cache_config() -> KVCacheConfig:
|
||||
return KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=40, shared_by=["full_layer"]),
|
||||
KVCacheTensor(size=24, shared_by=["mla_layer"]),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full_layer"],
|
||||
FullAttentionSpec(
|
||||
block_size=12,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mla_layer"],
|
||||
MLAAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class SingleArgExternalOffloadingSpec(OffloadingSpec):
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-registration integrity (CI sentinel)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -154,7 +293,7 @@ def test_tiering_spec_registered():
|
||||
def test_get_spec_cls_returns_registered_class():
|
||||
"""Registered spec_name returns correct class."""
|
||||
config = _make_vllm_config(spec_name="CPUOffloadingSpec")
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
assert spec_cls is CPUOffloadingSpec
|
||||
|
||||
|
||||
@@ -162,7 +301,7 @@ def test_get_spec_cls_default_to_cpu():
|
||||
"""Default spec_name (absent from config) resolves to CPUOffloadingSpec."""
|
||||
config = _make_vllm_config(spec_name=None)
|
||||
config.kv_transfer_config.kv_connector_extra_config.pop("spec_name", None)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
assert spec_cls is CPUOffloadingSpec
|
||||
|
||||
|
||||
@@ -176,16 +315,193 @@ def test_create_cpu_offloading_spec_end_to_end():
|
||||
|
||||
Verifies:
|
||||
- cpu_bytes_to_use validation and num_blocks calculation
|
||||
- block_size % hash_block_size assertion
|
||||
- block_size % tokens_per_hash assertion
|
||||
- spec instance is CPUOffloadingSpec
|
||||
"""
|
||||
config = _make_vllm_config(cpu_bytes_to_use=65536)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
spec = OffloadingSpecFactory.create_spec(config, kv_cache_config)
|
||||
spec = _create_spec(config, kv_cache_config)
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.num_blocks > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("packed", [False, True])
|
||||
def test_cpu_spec_sizing_preserves_tensor_layout(packed: bool):
|
||||
cpu_bytes_to_use = 1920
|
||||
config = _make_layout_vllm_config(
|
||||
cpu_bytes_to_use=cpu_bytes_to_use,
|
||||
extra_config={"block_size": 32},
|
||||
tensor_parallel_size=3,
|
||||
pipeline_parallel_size=2,
|
||||
)
|
||||
|
||||
spec = _create_spec(config, _make_sizing_kv_cache_config(packed))
|
||||
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 32
|
||||
assert spec.kv_bytes_per_chunk == 192
|
||||
assert spec.num_blocks == cpu_bytes_to_use // 192
|
||||
|
||||
|
||||
def test_cpu_spec_rejects_partially_packed_tensor_layout():
|
||||
config = _make_layout_vllm_config(cpu_bytes_to_use=65536)
|
||||
kv_cache_config = _make_sizing_kv_cache_config(packed=False)
|
||||
kv_cache_config.kv_cache_tensors[0].block_stride = 16
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
_create_spec(config, kv_cache_config)
|
||||
|
||||
|
||||
def test_cpu_spec_zero_blocks_skips_tensor_layout_validation():
|
||||
config = _make_layout_vllm_config(cpu_bytes_to_use=65536)
|
||||
kv_cache_config = _make_sizing_kv_cache_config(packed=False)
|
||||
kv_cache_config.num_blocks = 0
|
||||
kv_cache_config.kv_cache_tensors[0].block_stride = 16
|
||||
|
||||
spec = _create_spec(config, kv_cache_config)
|
||||
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 0
|
||||
assert spec.kv_bytes_per_chunk == 0
|
||||
assert spec.num_blocks == 0
|
||||
|
||||
|
||||
def test_tiering_spec_aligns_row_size():
|
||||
alignment = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT
|
||||
cpu_bytes_to_use = alignment * 3
|
||||
config = _make_layout_vllm_config(
|
||||
spec_name="TieringOffloadingSpec",
|
||||
cpu_bytes_to_use=cpu_bytes_to_use,
|
||||
extra_config={"block_size": 32},
|
||||
tensor_parallel_size=3,
|
||||
pipeline_parallel_size=2,
|
||||
)
|
||||
|
||||
spec = _create_spec(config, _make_sizing_kv_cache_config(packed=False))
|
||||
|
||||
assert isinstance(spec, TieringOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 32
|
||||
assert spec.kv_bytes_per_chunk == alignment
|
||||
assert spec.num_blocks == cpu_bytes_to_use // alignment
|
||||
|
||||
|
||||
def test_offloading_spec_resolves_prefill_context_parallel_block_sizes():
|
||||
config = _make_layout_vllm_config(
|
||||
cpu_bytes_to_use=65536,
|
||||
extra_config={"block_size": 64},
|
||||
prefill_context_parallel_size=2,
|
||||
)
|
||||
|
||||
spec = _create_spec(config, _make_kv_cache_config())
|
||||
|
||||
assert spec.tokens_per_block == (32,)
|
||||
assert spec.tokens_per_hash == 32
|
||||
assert spec.blocks_per_chunk == 2
|
||||
|
||||
|
||||
def test_offloading_config_preserves_data_parallel_index():
|
||||
config = _make_layout_vllm_config()
|
||||
config.parallel_config.data_parallel_index = 2
|
||||
|
||||
offloading_config = build_offloading_config(config, _make_kv_cache_config())
|
||||
|
||||
assert offloading_config.parallel.data_parallel_index == 2
|
||||
|
||||
|
||||
def test_offloading_spec_resolves_heterogeneous_hybrid_block_sizes():
|
||||
config = _make_layout_vllm_config(cpu_bytes_to_use=65536)
|
||||
config.cache_config.block_size = 4
|
||||
|
||||
spec = _create_spec(config, _make_hybrid_kv_cache_config())
|
||||
|
||||
assert spec.tokens_per_block == (12, 16)
|
||||
assert spec.tokens_per_hash == 4
|
||||
assert spec.blocks_per_chunk == 1
|
||||
|
||||
|
||||
def _full_attention_spec(block_size: int = 16) -> FullAttentionSpec:
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=4, head_size=128, dtype=torch.float32
|
||||
)
|
||||
|
||||
|
||||
def _parallelism_agnostic(kv_cache_groups: list[KVCacheGroupSpec]) -> bool:
|
||||
config = _make_layout_vllm_config()
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=0, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups
|
||||
)
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
return offloading_config.parallel.is_parallelism_agnostic
|
||||
|
||||
|
||||
def test_parallelism_agnostic_for_single_full_attention_group():
|
||||
assert _parallelism_agnostic([KVCacheGroupSpec(["l0"], _full_attention_spec())])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_groups",
|
||||
[
|
||||
# MLA latent KV is replicated per rank, never head-sharded.
|
||||
[
|
||||
KVCacheGroupSpec(
|
||||
["l0"],
|
||||
MLAAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=576, dtype=torch.float32
|
||||
),
|
||||
)
|
||||
],
|
||||
# Sliding window is not full attention.
|
||||
[
|
||||
KVCacheGroupSpec(
|
||||
["l0"],
|
||||
SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=128,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
)
|
||||
],
|
||||
# Hybrid model: more than one KV cache group.
|
||||
[
|
||||
KVCacheGroupSpec(["l0"], _full_attention_spec()),
|
||||
KVCacheGroupSpec(["l1"], _full_attention_spec()),
|
||||
],
|
||||
],
|
||||
)
|
||||
def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
|
||||
assert not _parallelism_agnostic(kv_cache_groups)
|
||||
|
||||
|
||||
def test_parallelism_agnostic_disabled_on_v2_model_runner():
|
||||
config = _make_layout_vllm_config()
|
||||
config.use_v2_model_runner = True
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=0,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["l0"], _full_attention_spec())],
|
||||
)
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
assert not offloading_config.parallel.is_parallelism_agnostic
|
||||
|
||||
|
||||
def test_create_dynamic_spec_receives_translated_config():
|
||||
config = _make_layout_vllm_config(
|
||||
spec_name="SingleArgExternalOffloadingSpec",
|
||||
extra_config={
|
||||
"spec_module_path": "tests.v1.kv_offload.test_factory",
|
||||
},
|
||||
)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
|
||||
spec = OffloadingSpecFactory.create_spec(offloading_config)
|
||||
|
||||
assert isinstance(spec, SingleArgExternalOffloadingSpec)
|
||||
assert spec.config is offloading_config
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dynamic import via spec_module_path
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -205,7 +521,7 @@ def test_dynamic_load_via_spec_module_path():
|
||||
config.kv_transfer_config.kv_connector_extra_config["spec_module_path"] = (
|
||||
"vllm.v1.kv_offload.cpu.spec"
|
||||
)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
assert spec_cls is CPUOffloadingSpec
|
||||
|
||||
|
||||
@@ -218,12 +534,12 @@ def test_unregistered_spec_without_module_path_raises():
|
||||
"""spec_name not in registry + no spec_module_path → ValueError."""
|
||||
config = _make_vllm_config(spec_name="NonexistentSpec")
|
||||
with pytest.raises(ValueError, match="Unsupported spec type"):
|
||||
OffloadingSpecFactory.get_spec_cls(config)
|
||||
OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
|
||||
# create_spec should also fail (calls get_spec_cls internally)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
with pytest.raises(ValueError, match="Unsupported spec type"):
|
||||
OffloadingSpecFactory.create_spec(config, kv_cache_config)
|
||||
_create_spec(config, kv_cache_config)
|
||||
|
||||
|
||||
def test_cpu_spec_missing_cpu_bytes_to_use_raises():
|
||||
@@ -232,7 +548,7 @@ def test_cpu_spec_missing_cpu_bytes_to_use_raises():
|
||||
config.kv_transfer_config.kv_connector_extra_config.pop("cpu_bytes_to_use", None)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
with pytest.raises(Exception, match="cpu_bytes_to_use must be specified"):
|
||||
OffloadingSpecFactory.create_spec(config, kv_cache_config)
|
||||
_create_spec(config, kv_cache_config)
|
||||
|
||||
|
||||
def test_duplicate_registration_raises():
|
||||
@@ -253,7 +569,7 @@ def test_build_metric_definitions_below_threshold():
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=1)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
metrics = spec_cls.build_metric_definitions(
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
@@ -266,7 +582,7 @@ def test_build_metric_definitions_allocation_size_histogram():
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=0)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
metrics = spec_cls.build_metric_definitions(
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
@@ -291,7 +607,7 @@ def test_build_metric_definitions_returns_counter_at_threshold():
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=2)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(_get_extra_config(config))
|
||||
metrics = spec_cls.build_metric_definitions(
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
|
||||
@@ -4,80 +4,60 @@
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
OffloadingSpec,
|
||||
make_offload_key,
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec, make_offload_key
|
||||
from vllm.v1.kv_offload.config import (
|
||||
OffloadingCacheConfig,
|
||||
OffloadingConfig,
|
||||
OffloadingGroupConfig,
|
||||
OffloadingModelConfig,
|
||||
OffloadingParallelConfig,
|
||||
)
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared mocks (mirrors test_fs_tier.py pattern)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MOCK_VLLM_CONFIG = MagicMock()
|
||||
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
|
||||
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
|
||||
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
|
||||
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
|
||||
|
||||
_MOCK_KV_CACHE_CONFIG = MagicMock()
|
||||
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
|
||||
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock(spec=OffloadingSpec)
|
||||
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
|
||||
"""Helper to create FileMapper with customizable mock config."""
|
||||
# Create a copy of the mock config to avoid modifying the global one
|
||||
mock_vllm_config = MagicMock()
|
||||
mock_vllm_config.model_config.model = kwargs.get("model_name", "test-model")
|
||||
mock_vllm_config.cache_config.block_size = kwargs.get("hash_block_size", 16)
|
||||
mock_vllm_config.cache_config.cache_dtype = (
|
||||
f"torch.{kwargs.get('dtype', 'float16')}"
|
||||
"""Build a FileMapper from a mocked spec carrying a hand-built config."""
|
||||
config = OffloadingConfig(
|
||||
groups=tuple(
|
||||
OffloadingGroupConfig(
|
||||
tokens_per_block=tokens_per_block,
|
||||
layer_names=(layer_name,),
|
||||
)
|
||||
for tokens_per_block, layer_name in kwargs.get("groups", ())
|
||||
),
|
||||
worker_kv_bytes_per_block=0,
|
||||
enable_kv_cache_events=False,
|
||||
extra_config={},
|
||||
engine_id="test-engine",
|
||||
model=OffloadingModelConfig(
|
||||
name=kwargs.get("model_name", "test-model"),
|
||||
dtype=kwargs.get("dtype", "float16"),
|
||||
),
|
||||
cache=OffloadingCacheConfig(
|
||||
tokens_per_hash=kwargs.get("tokens_per_hash", 16),
|
||||
blocks_per_chunk=kwargs.get("blocks_per_chunk", 1),
|
||||
),
|
||||
parallel=OffloadingParallelConfig(
|
||||
rank=kwargs.get("rank", 0),
|
||||
world_size=kwargs.get("world_size", 1),
|
||||
tp_size=kwargs.get("tp_size", 1),
|
||||
pp_size=kwargs.get("pp_size", 1),
|
||||
pcp_size=kwargs.get("pcp_size", 1),
|
||||
dcp_size=kwargs.get("dcp_size", 1),
|
||||
data_parallel_index=0,
|
||||
is_parallelism_agnostic=kwargs.get("is_parallelism_agnostic", False),
|
||||
),
|
||||
)
|
||||
mock_vllm_config.parallel_config.tensor_parallel_size = kwargs.get("tp_size", 1)
|
||||
mock_vllm_config.parallel_config.pipeline_parallel_size = kwargs.get("pp_size", 1)
|
||||
mock_vllm_config.parallel_config.prefill_context_parallel_size = kwargs.get(
|
||||
"pcp_size", 1
|
||||
)
|
||||
mock_vllm_config.parallel_config.decode_context_parallel_size = kwargs.get(
|
||||
"dcp_size", 1
|
||||
)
|
||||
mock_vllm_config.parallel_config.rank = kwargs.get("rank", 0)
|
||||
mock_vllm_config.use_v2_model_runner = kwargs.get("use_v2_model_runner", False)
|
||||
|
||||
mock_kv_cache_config = MagicMock()
|
||||
mock_kv_cache_config.kv_cache_groups = kwargs.get("kv_cache_groups", [])
|
||||
|
||||
mock_offloading_spec = MagicMock(spec=OffloadingSpec)
|
||||
mock_offloading_spec.vllm_config = mock_vllm_config
|
||||
mock_offloading_spec.kv_cache_config = mock_kv_cache_config
|
||||
mock_offloading_spec.block_size_factor = kwargs.get("block_size_factor", 1)
|
||||
|
||||
spec = MagicMock(spec=OffloadingSpec)
|
||||
spec.config = config
|
||||
return FileMapper.from_offloading_spec(
|
||||
root_dir=kwargs.get("root_dir", "/tmp/cache"),
|
||||
offloading_spec=mock_offloading_spec,
|
||||
gpu_blocks_per_file=mock_offloading_spec.block_size_factor,
|
||||
offloading_spec=spec,
|
||||
blocks_per_file=config.cache.blocks_per_chunk,
|
||||
parallel_agnostic=kwargs.get("parallel_agnostic", False),
|
||||
)
|
||||
|
||||
@@ -92,7 +72,7 @@ def test_get_file_name_full_structure():
|
||||
Path must match: <base_path>_r<rank>/<hhh>/<hh>_g<group_idx>/<hash_hex>.bin
|
||||
|
||||
Concretely:
|
||||
- The segment immediately after base_path must end with `_r0`
|
||||
- The segment immediately after base_path must end with `_r3`
|
||||
- The next segment is the first 3 hex chars of the block hash
|
||||
- The next segment is <2 hex chars>_g<group_idx>
|
||||
- The final segment is <full hash hex>.bin
|
||||
@@ -105,7 +85,7 @@ def test_get_file_name_full_structure():
|
||||
path = fm.get_file_name(key)
|
||||
|
||||
expected_path = (
|
||||
"/tmp/cache/test-model_588656ebcc66_r3/000/10_g2/0001020304050607.bin"
|
||||
"/tmp/cache/test-model_42b94bdc9933_r3/000/10_g2/0001020304050607.bin"
|
||||
)
|
||||
assert path == expected_path
|
||||
|
||||
@@ -114,19 +94,30 @@ def test_get_run_config_fields():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
model_name="my-model",
|
||||
dtype="bfloat16",
|
||||
tp_size=2,
|
||||
tp_size=4,
|
||||
pp_size=3,
|
||||
pcp_size=2,
|
||||
dcp_size=2,
|
||||
groups=((64, "layer0"),),
|
||||
tokens_per_hash=64,
|
||||
blocks_per_chunk=3,
|
||||
)
|
||||
cfg = fm.get_run_config()
|
||||
assert cfg == {
|
||||
"model_name": "my-model",
|
||||
"hash_block_size": 16,
|
||||
"gpu_blocks_per_file": 1,
|
||||
"tp_size": 2,
|
||||
"pp_size": 1,
|
||||
"pcp_size": 1,
|
||||
"dcp_size": 1,
|
||||
"tokens_per_hash": 64,
|
||||
"blocks_per_file": 3,
|
||||
"tp_size": 4,
|
||||
"pp_size": 3,
|
||||
"pcp_size": 2,
|
||||
"dcp_size": 2,
|
||||
"dtype": "bfloat16",
|
||||
"kv_cache_groups": [],
|
||||
"kv_cache_groups": [
|
||||
{
|
||||
"tokens_per_block": 64,
|
||||
"layer_names": ["layer0"],
|
||||
}
|
||||
],
|
||||
"inference_engine": "vllm",
|
||||
}
|
||||
|
||||
@@ -137,90 +128,61 @@ def test_get_config_file_path():
|
||||
assert config_path == f"{fm.base_path}/config.json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parallel_agnostic: honored only for a single non-MLA full-attention group
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _full_attention_group() -> KVCacheGroupSpec:
|
||||
return KVCacheGroupSpec(
|
||||
layer_names=["layer0"],
|
||||
kv_cache_spec=FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=128, dtype=torch.float32
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _sliding_window_group() -> KVCacheGroupSpec:
|
||||
return KVCacheGroupSpec(
|
||||
layer_names=["layer0"],
|
||||
kv_cache_spec=SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=128,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_parallel_agnostic_enabled_for_single_full_attention():
|
||||
# tp/rank are collapsed out of the namespace so the cache is shared
|
||||
# across tensor-parallel sizes.
|
||||
def test_hybrid_file_identity_uses_resolved_tokens_per_hash():
|
||||
# For heterogeneous groups the namespace records the resolved hash
|
||||
# granularity (GCD of the group block sizes), which is the actual
|
||||
# granularity of the offload block hashes.
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
groups=((12, "full_layer"), (16, "mla_layer")),
|
||||
tokens_per_hash=4,
|
||||
)
|
||||
assert fm.fields["tokens_per_hash"] == 4
|
||||
assert fm.fields["kv_cache_groups"] == [
|
||||
{"tokens_per_block": 12, "layer_names": ["full_layer"]},
|
||||
{"tokens_per_block": 16, "layer_names": ["mla_layer"]},
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parallel_agnostic: opt-in honored only when the config marks the layout
|
||||
# parallelism-agnostic (predicate computation is covered in test_factory.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parallel_agnostic_collapses_namespace_when_config_allows():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=4,
|
||||
pp_size=3,
|
||||
pcp_size=2,
|
||||
dcp_size=2,
|
||||
rank=1,
|
||||
kv_cache_groups=[_full_attention_group()],
|
||||
is_parallelism_agnostic=True,
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 1
|
||||
assert fm.fields["pp_size"] == 1
|
||||
assert fm.fields["pcp_size"] == 1
|
||||
assert fm.fields["dcp_size"] == 1
|
||||
assert fm.rank == 0
|
||||
|
||||
|
||||
def test_parallel_agnostic_disabled_for_multiple_groups():
|
||||
# More than one KV-cache group (hybrid model) => keep per-layout namespacing.
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
kv_cache_groups=[_full_attention_group(), _full_attention_group()],
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
|
||||
|
||||
def test_parallel_agnostic_disabled_for_non_full_attention():
|
||||
# Single group but not full attention (sliding window) => keep namespacing.
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
kv_cache_groups=[_sliding_window_group()],
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
|
||||
|
||||
def test_parallel_agnostic_excludes_mla():
|
||||
# MLA latent KV is replicated per rank, so its offloaded blocks are not
|
||||
# parallelism-invariant: the opt-in must not collapse tp/rank.
|
||||
group = KVCacheGroupSpec(
|
||||
layer_names=["layer0"],
|
||||
kv_cache_spec=MLAAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=576, dtype=torch.float32
|
||||
),
|
||||
)
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2, rank=1, kv_cache_groups=[group], parallel_agnostic=True
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
assert fm.rank == 1
|
||||
|
||||
|
||||
def test_parallel_agnostic_disabled_on_v2_model_runner():
|
||||
# V2's KV layout is not known to be parallelism-invariant: don't collapse.
|
||||
def test_parallel_agnostic_ignored_when_config_disallows():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
rank=1,
|
||||
kv_cache_groups=[_full_attention_group()],
|
||||
use_v2_model_runner=True,
|
||||
is_parallelism_agnostic=False,
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
assert fm.rank == 1
|
||||
|
||||
|
||||
def test_namespace_kept_without_parallel_agnostic_opt_in():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
rank=1,
|
||||
is_parallelism_agnostic=True,
|
||||
parallel_agnostic=False,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
assert fm.rank == 1
|
||||
|
||||
@@ -47,7 +47,7 @@ class TestDataTransportBase:
|
||||
def test_config_fingerprint_deterministic(self):
|
||||
"""Same config fields → same fingerprint."""
|
||||
view = self._make_view()
|
||||
fields = {"model": "llama", "dtype": "float16", "block_size_factor": 1}
|
||||
fields = {"model": "llama", "dtype": "float16", "blocks_per_chunk": 1}
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
t1 = NixlTransport("test:1", view, config_fields=fields)
|
||||
t2 = NixlTransport("test:2", view, config_fields=fields)
|
||||
|
||||
@@ -1419,9 +1419,9 @@ class TestBindHostPortDefaults:
|
||||
or SimpleNamespace(),
|
||||
)
|
||||
spec = SimpleNamespace(
|
||||
block_size_factor=1,
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(data_parallel_index=dp_index)
|
||||
blocks_per_chunk=1,
|
||||
config=SimpleNamespace(
|
||||
parallel=SimpleNamespace(data_parallel_index=dp_index)
|
||||
),
|
||||
)
|
||||
mgr = P2PSecondaryTierManager(spec, memoryview(b""), **kwargs)
|
||||
|
||||
@@ -22,11 +22,18 @@ from vllm.distributed.kv_events import MEDIUM_FS
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadingEvent,
|
||||
OffloadingKVEventsConfig,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
ScheduleEndContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.config import (
|
||||
OffloadingCacheConfig,
|
||||
OffloadingConfig,
|
||||
OffloadingModelConfig,
|
||||
OffloadingParallelConfig,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import JobMetadata
|
||||
from vllm.v1.kv_offload.tiering.fs.manager import (
|
||||
FileSystemTierManager,
|
||||
@@ -41,35 +48,40 @@ _BLOCK_ELEMENTS = 128 * mmap.PAGESIZE # 2MB per block for pagesize 4096.
|
||||
_DTYPE: torch.dtype = torch.float32
|
||||
_CTX = ReqContext(req_id="test")
|
||||
|
||||
_MOCK_VLLM_CONFIG = MagicMock()
|
||||
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
|
||||
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
|
||||
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
|
||||
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
|
||||
|
||||
_MOCK_KV_CACHE_CONFIG = MagicMock()
|
||||
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
|
||||
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock()
|
||||
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
|
||||
|
||||
|
||||
def _make_offloading_spec(enable_kv_cache_events: bool) -> MagicMock:
|
||||
"""Mock spec with an explicit global KV events flag."""
|
||||
spec = MagicMock()
|
||||
spec.vllm_config = _MOCK_VLLM_CONFIG
|
||||
spec.kv_cache_config = _MOCK_KV_CACHE_CONFIG
|
||||
spec.block_size_factor = 1
|
||||
spec.kv_events_config.enable_kv_cache_events = enable_kv_cache_events
|
||||
spec.config = OffloadingConfig(
|
||||
groups=(),
|
||||
worker_kv_bytes_per_block=0,
|
||||
enable_kv_cache_events=enable_kv_cache_events,
|
||||
extra_config={},
|
||||
engine_id="test-engine",
|
||||
model=OffloadingModelConfig(name="test-model", dtype="float32"),
|
||||
cache=OffloadingCacheConfig(tokens_per_hash=16, blocks_per_chunk=1),
|
||||
parallel=OffloadingParallelConfig(
|
||||
rank=0,
|
||||
world_size=1,
|
||||
tp_size=1,
|
||||
pp_size=1,
|
||||
pcp_size=1,
|
||||
dcp_size=1,
|
||||
data_parallel_index=0,
|
||||
is_parallelism_agnostic=False,
|
||||
),
|
||||
)
|
||||
spec.blocks_per_chunk = 1
|
||||
spec.kv_events_config = OffloadingKVEventsConfig(
|
||||
enable_kv_cache_events=enable_kv_cache_events,
|
||||
self_describing_kv_events=False,
|
||||
)
|
||||
return spec
|
||||
|
||||
|
||||
_MOCK_OFFLOADING_SPEC = _make_offloading_spec(enable_kv_cache_events=False)
|
||||
|
||||
|
||||
def key(n: int) -> OffloadKey:
|
||||
return make_offload_key(n.to_bytes(8, "big"), 0)
|
||||
|
||||
|
||||
@@ -19,11 +19,18 @@ import torch
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadingKVEventsConfig,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
ScheduleEndContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.config import (
|
||||
OffloadingCacheConfig,
|
||||
OffloadingConfig,
|
||||
OffloadingModelConfig,
|
||||
OffloadingParallelConfig,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
|
||||
from vllm.v1.kv_offload.tiering.obj.config import ObjStoreConfig
|
||||
from vllm.v1.kv_offload.tiering.obj.manager import ObjectStoreSecondaryTierManager
|
||||
@@ -33,24 +40,30 @@ from vllm.v1.kv_offload.tiering.obj.manager import ObjectStoreSecondaryTierManag
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_vllm_config():
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(model="test/model"),
|
||||
cache_config=SimpleNamespace(block_size=16, cache_dtype="float16"),
|
||||
parallel_config=SimpleNamespace(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
prefill_context_parallel_size=1,
|
||||
decode_context_parallel_size=1,
|
||||
def _make_offloading_config(enable_kv_cache_events: bool) -> OffloadingConfig:
|
||||
return OffloadingConfig(
|
||||
groups=(),
|
||||
worker_kv_bytes_per_block=0,
|
||||
enable_kv_cache_events=enable_kv_cache_events,
|
||||
extra_config={},
|
||||
engine_id="test-engine",
|
||||
model=OffloadingModelConfig(name="test/model", dtype="float16"),
|
||||
cache=OffloadingCacheConfig(tokens_per_hash=16, blocks_per_chunk=1),
|
||||
parallel=OffloadingParallelConfig(
|
||||
rank=0,
|
||||
world_size=1,
|
||||
tp_size=1,
|
||||
pp_size=1,
|
||||
pcp_size=1,
|
||||
dcp_size=1,
|
||||
data_parallel_index=0,
|
||||
is_parallelism_agnostic=False,
|
||||
),
|
||||
use_v2_model_runner=False,
|
||||
)
|
||||
|
||||
|
||||
_OFFLOADING_SPEC = SimpleNamespace(
|
||||
vllm_config=_make_vllm_config(),
|
||||
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
|
||||
config=_make_offloading_config(enable_kv_cache_events=False),
|
||||
)
|
||||
|
||||
_STORE_CONFIG = {
|
||||
@@ -182,9 +195,11 @@ class MockNixlAgent:
|
||||
def _make_events_spec(enable_kv_cache_events: bool) -> SimpleNamespace:
|
||||
"""Offloading spec stub with an explicit global KV events flag."""
|
||||
return SimpleNamespace(
|
||||
vllm_config=_make_vllm_config(),
|
||||
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
|
||||
kv_events_config=SimpleNamespace(enable_kv_cache_events=enable_kv_cache_events),
|
||||
config=_make_offloading_config(enable_kv_cache_events),
|
||||
kv_events_config=OffloadingKVEventsConfig(
|
||||
enable_kv_cache_events=enable_kv_cache_events,
|
||||
self_describing_kv_events=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user