[Bugfix][KV Connector] Populate SimpleCPUOffload BlockStored metadata (#54325)
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for SimpleCPUOffloadConnector-level behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.v1.kv_connector.unit.utils import create_vllm_config
|
||||
from tests.v1.simple_kv_offload.test_scheduler import (
|
||||
_BYTES_PER_BLOCK,
|
||||
BLOCK_SIZE,
|
||||
DTYPE,
|
||||
HEAD_SIZE,
|
||||
NUM_KV_HEADS,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
|
||||
SimpleCPUOffloadConnector,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
|
||||
|
||||
def _make_kv_cache_config(num_blocks: int = 16) -> KVCacheConfig:
|
||||
fa_layers = ["layer0"]
|
||||
spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
groups = [KVCacheGroupSpec(fa_layers, spec)]
|
||||
size = _BYTES_PER_BLOCK * num_blocks
|
||||
tensors = [
|
||||
KVCacheTensor(
|
||||
size=size,
|
||||
layers=fa_layers,
|
||||
layer_stride=size,
|
||||
block_stride=_BYTES_PER_BLOCK,
|
||||
)
|
||||
]
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=tensors,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
|
||||
|
||||
def _make_connector(
|
||||
extra_config: dict | None = None,
|
||||
num_gpu_blocks: int = 16,
|
||||
cpu_bytes: int = _BYTES_PER_BLOCK * 8,
|
||||
) -> SimpleCPUOffloadConnector:
|
||||
kv_cache_config = _make_kv_cache_config(num_gpu_blocks)
|
||||
config_dict = {
|
||||
"cpu_bytes_to_use": cpu_bytes,
|
||||
}
|
||||
if extra_config:
|
||||
config_dict.update(extra_config)
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="SimpleCPUOffloadConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config=config_dict,
|
||||
)
|
||||
connector = SimpleCPUOffloadConnector(
|
||||
vllm_config=vllm_config,
|
||||
role=KVConnectorRole.SCHEDULER,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
return connector
|
||||
|
||||
|
||||
def test_disk_mode_rejects_non_positive_capacity() -> None:
|
||||
"""Disk mode with disk_capacity_bytes <= 0 raises ValueError.
|
||||
|
||||
Without this guard, the worker operates in disk mode but the scheduler
|
||||
classifies events as MEDIUM_CPU instead of MEDIUM_STORAGE, silently
|
||||
mislabeling events.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="disk_capacity_bytes > 0"):
|
||||
_make_connector(
|
||||
extra_config={
|
||||
"kv_offload_backend": "disk",
|
||||
"disk_path": "/tmp/fake",
|
||||
"disk_capacity_bytes": 0,
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,790 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""KV cache event emission tests for SimpleCPUOffloadScheduler.
|
||||
|
||||
Verifies that BlockStored/BlockRemoved events carry the correct storage
|
||||
medium (MEDIUM_CPU / MEDIUM_STORAGE) and per-group metadata when
|
||||
``enable_kv_cache_events`` is set, and that no events are emitted when it
|
||||
is unset.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tests.v1.simple_kv_offload.test_scheduler import (
|
||||
_BYTES_PER_BLOCK,
|
||||
BLOCK_SIZE,
|
||||
DTYPE,
|
||||
HEAD_SIZE,
|
||||
NUM_KV_HEADS,
|
||||
SchedulerFixture,
|
||||
_alloc_and_register,
|
||||
_allocate_cp_gpu_blocks,
|
||||
_make_cp_request,
|
||||
_make_kv_cache_config,
|
||||
_make_vllm_config,
|
||||
make_request,
|
||||
make_scheduler_output,
|
||||
simulate_store_completion,
|
||||
)
|
||||
from vllm import SamplingParams
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed.kv_events import (
|
||||
MEDIUM_CPU,
|
||||
MEDIUM_STORAGE,
|
||||
BlockRemoved,
|
||||
BlockStored,
|
||||
)
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
get_block_hash,
|
||||
get_request_block_hasher,
|
||||
make_block_hash_with_group_id,
|
||||
maybe_convert_block_hash,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.simple_kv_offload.manager import (
|
||||
SimpleCPUOffloadScheduler,
|
||||
StoreRequestState,
|
||||
)
|
||||
|
||||
|
||||
def _make_mixed_kv_cache_config(
|
||||
num_blocks: int,
|
||||
sliding_window: int | None = None,
|
||||
with_mamba: bool = False,
|
||||
mamba_block_size: int = BLOCK_SIZE,
|
||||
) -> KVCacheConfig:
|
||||
"""Build a KVCacheConfig with a FullAttention group plus optional
|
||||
SlidingWindow and Mamba groups for per-group metadata tests."""
|
||||
register_all_kvcache_specs(vllm_config=None)
|
||||
groups = []
|
||||
tensors = []
|
||||
|
||||
# Group 0 is always FullAttention (the scheduler requires one FA group).
|
||||
fa_layers = ["layer_fa"]
|
||||
fa_spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
groups.append(KVCacheGroupSpec(fa_layers, fa_spec))
|
||||
fa_bytes = _BYTES_PER_BLOCK * num_blocks
|
||||
tensors.append(
|
||||
KVCacheTensor(
|
||||
size=fa_bytes,
|
||||
layers=fa_layers,
|
||||
layer_stride=fa_bytes,
|
||||
block_stride=_BYTES_PER_BLOCK,
|
||||
)
|
||||
)
|
||||
|
||||
if sliding_window is not None:
|
||||
sw_layers = ["layer_sw"]
|
||||
sw_spec = SlidingWindowSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
groups.append(KVCacheGroupSpec(sw_layers, sw_spec))
|
||||
sw_bytes = _BYTES_PER_BLOCK * num_blocks
|
||||
tensors.append(
|
||||
KVCacheTensor(
|
||||
size=sw_bytes,
|
||||
layers=sw_layers,
|
||||
layer_stride=sw_bytes,
|
||||
block_stride=_BYTES_PER_BLOCK,
|
||||
)
|
||||
)
|
||||
|
||||
if with_mamba:
|
||||
m_layers = ["layer_m"]
|
||||
m_spec = MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=((NUM_KV_HEADS, HEAD_SIZE, HEAD_SIZE),),
|
||||
dtypes=(DTYPE,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
groups.append(KVCacheGroupSpec(m_layers, m_spec))
|
||||
m_bytes = _BYTES_PER_BLOCK * num_blocks
|
||||
tensors.append(
|
||||
KVCacheTensor(
|
||||
size=m_bytes,
|
||||
layers=m_layers,
|
||||
layer_stride=m_bytes,
|
||||
block_stride=_BYTES_PER_BLOCK,
|
||||
)
|
||||
)
|
||||
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=tensors,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
|
||||
|
||||
def make_events_scheduler(
|
||||
num_cpu_blocks: int = 8,
|
||||
num_gpu_blocks: int = 16,
|
||||
num_groups: int = 1,
|
||||
lazy: bool = False,
|
||||
disk_capacity_bytes: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
kv_cache_config: KVCacheConfig | None = None,
|
||||
enable_events: bool = True,
|
||||
) -> SchedulerFixture:
|
||||
"""Build a SimpleCPUOffloadScheduler with kv cache events enabled."""
|
||||
if kv_cache_config is None:
|
||||
kv_cache_config = _make_kv_cache_config(num_gpu_blocks, num_groups)
|
||||
vllm_config = _make_vllm_config()
|
||||
if enable_events:
|
||||
vllm_config.kv_events_config = KVEventsConfig(enable_kv_cache_events=True)
|
||||
vllm_config.parallel_config.decode_context_parallel_size = dcp_world_size
|
||||
|
||||
virtual_block_size = BLOCK_SIZE * dcp_world_size
|
||||
cpu_capacity_bytes = _BYTES_PER_BLOCK * num_cpu_blocks * num_groups
|
||||
|
||||
sched = SimpleCPUOffloadScheduler(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
cpu_capacity_bytes=cpu_capacity_bytes,
|
||||
scheduler_block_size=virtual_block_size,
|
||||
hash_block_size=virtual_block_size,
|
||||
lazy_offload=lazy,
|
||||
disk_capacity_bytes=disk_capacity_bytes,
|
||||
)
|
||||
|
||||
gpu_block_pool = BlockPool(
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
enable_caching=True,
|
||||
hash_block_size=virtual_block_size,
|
||||
)
|
||||
sched.bind_gpu_block_pool(gpu_block_pool)
|
||||
return SchedulerFixture(
|
||||
scheduler=sched,
|
||||
gpu_block_pool=gpu_block_pool,
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
num_groups=num_groups,
|
||||
)
|
||||
|
||||
|
||||
def _do_eager_store(
|
||||
fix: SchedulerFixture,
|
||||
num_blocks: int = 2,
|
||||
lora_request: LoRARequest | None = None,
|
||||
) -> tuple[object, Request]:
|
||||
"""Run an eager store of ``num_blocks`` and complete it.
|
||||
|
||||
Returns (meta, req) for further assertions.
|
||||
"""
|
||||
sched = fix.scheduler
|
||||
if lora_request is not None:
|
||||
# lora_request must be set in the constructor so block_hashes
|
||||
# (computed in __init__) include the lora extra_keys.
|
||||
req = Request(
|
||||
request_id="req-lora",
|
||||
prompt_token_ids=list(range(num_blocks * BLOCK_SIZE + 1)),
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
lora_request=lora_request,
|
||||
block_hasher=get_request_block_hasher(BLOCK_SIZE, sha256),
|
||||
)
|
||||
else:
|
||||
req = make_request(num_blocks=num_blocks)
|
||||
kv_blocks = _alloc_and_register(fix, req, num_blocks)
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
block_ids = kv_blocks.get_block_ids()
|
||||
sched_out = make_scheduler_output(
|
||||
{req.request_id: num_blocks * BLOCK_SIZE},
|
||||
new_reqs={req.request_id: block_ids},
|
||||
)
|
||||
meta = sched.build_connector_meta(sched_out)
|
||||
assert meta.store_event >= 0
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
return meta, req
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# events disabled -> no events
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_events_disabled_no_events() -> None:
|
||||
"""When enable_kv_cache_events is False, take_events returns nothing."""
|
||||
fix = make_events_scheduler(enable_events=False)
|
||||
sched = fix.scheduler
|
||||
assert sched.enable_kv_cache_events is False
|
||||
_do_eager_store(fix, num_blocks=2)
|
||||
events = list(sched.take_events())
|
||||
assert events == [], f"expected no events, got {events}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# eager store emits BlockStored with per-group metadata
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_block_stored_per_group_metadata_full_attention() -> None:
|
||||
"""BlockStored carries medium, group_idx, kind, sliding_window, block_size,
|
||||
and the correct block hash."""
|
||||
fix = make_events_scheduler()
|
||||
sched = fix.scheduler
|
||||
assert sched.kv_event_medium == MEDIUM_CPU
|
||||
_, req = _do_eager_store(fix, num_blocks=2)
|
||||
events = list(sched.take_events())
|
||||
stored = [e for e in events if isinstance(e, BlockStored)]
|
||||
assert len(stored) == 2
|
||||
ev = stored[0]
|
||||
assert ev.medium == MEDIUM_CPU
|
||||
assert ev.group_idx == 0
|
||||
assert ev.kv_cache_spec_kind == "full_attention"
|
||||
assert ev.kv_cache_spec_sliding_window is None
|
||||
assert ev.block_size == BLOCK_SIZE
|
||||
assert ev.locality == "LOCAL"
|
||||
expected_hash_0 = maybe_convert_block_hash(
|
||||
get_block_hash(make_block_hash_with_group_id(req.block_hashes[0], 0))
|
||||
)
|
||||
assert ev.block_hashes == [expected_hash_0]
|
||||
assert ev.parent_block_hash is None
|
||||
assert ev.token_ids == req.prompt_token_ids[0:BLOCK_SIZE]
|
||||
|
||||
# Second block's parent_block_hash equals the first block's hash.
|
||||
ev1 = stored[1]
|
||||
expected_hash_1 = maybe_convert_block_hash(
|
||||
get_block_hash(make_block_hash_with_group_id(req.block_hashes[1], 0))
|
||||
)
|
||||
assert ev1.block_hashes == [expected_hash_1]
|
||||
assert ev1.parent_block_hash == expected_hash_0
|
||||
assert ev1.token_ids == req.prompt_token_ids[BLOCK_SIZE : 2 * BLOCK_SIZE]
|
||||
|
||||
|
||||
def test_eager_store_lora_metadata() -> None:
|
||||
"""BlockStored carries lora_id and lora_name from req.lora_request."""
|
||||
fix = make_events_scheduler()
|
||||
sched = fix.scheduler
|
||||
|
||||
_do_eager_store(
|
||||
fix,
|
||||
num_blocks=1,
|
||||
lora_request=LoRARequest(
|
||||
lora_name="test-lora",
|
||||
lora_int_id=42,
|
||||
lora_path="/fake/lora/path",
|
||||
),
|
||||
)
|
||||
|
||||
events = list(sched.take_events())
|
||||
stored = [e for e in events if isinstance(e, BlockStored)]
|
||||
assert len(stored) == 1
|
||||
ev = stored[0]
|
||||
assert ev.lora_id == 42, f"expected lora_id=42, got {ev.lora_id}"
|
||||
assert ev.lora_name == "test-lora", (
|
||||
f"expected lora_name='test-lora', got {ev.lora_name!r}"
|
||||
)
|
||||
assert ev.extra_keys is not None, "expected extra_keys for lora request"
|
||||
assert any("test-lora" in keys for keys in ev.extra_keys), (
|
||||
f"expected 'test-lora' in extra_keys, got {ev.extra_keys}"
|
||||
)
|
||||
assert len(ev.token_ids) == BLOCK_SIZE
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# disk mode -> MEDIUM_STORAGE
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_disk_mode_medium_storage_on_block_stored() -> None:
|
||||
"""disk_capacity_bytes>0 sets kv_event_medium=STORAGE on BlockStored."""
|
||||
fix = make_events_scheduler(
|
||||
disk_capacity_bytes=_BYTES_PER_BLOCK * 8,
|
||||
enable_events=True,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
assert sched.kv_event_medium == MEDIUM_STORAGE
|
||||
_do_eager_store(fix, num_blocks=1)
|
||||
events = list(sched.take_events())
|
||||
stored = [e for e in events if isinstance(e, BlockStored)]
|
||||
assert len(stored) == 1
|
||||
assert stored[0].medium == MEDIUM_STORAGE
|
||||
assert stored[0].locality == "LOCAL"
|
||||
|
||||
|
||||
def test_finished_eager_store_emits_all_storage_events() -> None:
|
||||
"""Finished stores preserve and announce every registered hash."""
|
||||
fix = make_events_scheduler(
|
||||
disk_capacity_bytes=_BYTES_PER_BLOCK * 8,
|
||||
enable_events=True,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
req = make_request(num_blocks=2)
|
||||
kv_blocks = _alloc_and_register(fix, req, num_blocks=2)
|
||||
gpu_blocks = kv_blocks.blocks[0]
|
||||
primary_hashes = []
|
||||
for gpu_block in gpu_blocks:
|
||||
assert gpu_block.block_hash is not None
|
||||
primary_hashes.append(gpu_block.block_hash)
|
||||
|
||||
fine_grained_req = Request(
|
||||
request_id="req-finished-fine-grained-hash",
|
||||
prompt_token_ids=req.prompt_token_ids,
|
||||
sampling_params=req.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=get_request_block_hasher(BLOCK_SIZE // 2, sha256),
|
||||
)
|
||||
secondary_hash = make_block_hash_with_group_id(fine_grained_req.block_hashes[0], 0)
|
||||
fix.gpu_block_pool._insert_block_hash(
|
||||
secondary_hash, gpu_blocks[0], num_tokens=BLOCK_SIZE // 2
|
||||
)
|
||||
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
sched.request_finished_all_groups(req, kv_blocks.get_block_ids())
|
||||
meta = sched.build_connector_meta(make_scheduler_output({}))
|
||||
assert meta.store_event >= 0
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
|
||||
primary_cpu_block = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block(
|
||||
primary_hashes[0]
|
||||
)
|
||||
secondary_cpu_block = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block(
|
||||
secondary_hash
|
||||
)
|
||||
assert primary_cpu_block is not None
|
||||
assert secondary_cpu_block is primary_cpu_block
|
||||
|
||||
stored = [e for e in sched.take_events() if isinstance(e, BlockStored)]
|
||||
primary_event_hashes = [
|
||||
maybe_convert_block_hash(get_block_hash(block_hash))
|
||||
for block_hash in primary_hashes
|
||||
]
|
||||
secondary_event_hash = maybe_convert_block_hash(get_block_hash(secondary_hash))
|
||||
by_hash = {event.block_hashes[0]: event for event in stored}
|
||||
assert set(by_hash) == {*primary_event_hashes, secondary_event_hash}
|
||||
assert (
|
||||
by_hash[primary_event_hashes[0]].token_ids == req.prompt_token_ids[:BLOCK_SIZE]
|
||||
)
|
||||
assert (
|
||||
by_hash[primary_event_hashes[1]].token_ids
|
||||
== req.prompt_token_ids[BLOCK_SIZE : 2 * BLOCK_SIZE]
|
||||
)
|
||||
assert by_hash[primary_event_hashes[1]].parent_block_hash == primary_event_hashes[0]
|
||||
secondary_event = by_hash[secondary_event_hash]
|
||||
assert secondary_event.medium == MEDIUM_STORAGE
|
||||
assert secondary_event.block_size == 0
|
||||
assert secondary_event.token_ids == []
|
||||
assert secondary_event.parent_block_hash is None
|
||||
|
||||
sched.cpu_block_pool.evict_blocks({primary_cpu_block.block_id})
|
||||
removed = [
|
||||
event for event in sched.take_events() if isinstance(event, BlockRemoved)
|
||||
]
|
||||
assert {event.block_hashes[0] for event in removed} == {
|
||||
primary_event_hashes[0],
|
||||
secondary_event_hash,
|
||||
}
|
||||
assert all(event.medium == MEDIUM_STORAGE for event in removed)
|
||||
|
||||
|
||||
def test_secondary_hash_block_stored_metadata() -> None:
|
||||
"""Secondary hashes carry their own token range and block size."""
|
||||
fix = make_events_scheduler(
|
||||
disk_capacity_bytes=_BYTES_PER_BLOCK * 8,
|
||||
enable_events=True,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
sched.hash_block_size = BLOCK_SIZE // 2
|
||||
req = Request(
|
||||
request_id="req-secondary-hash",
|
||||
prompt_token_ids=list(range(BLOCK_SIZE + 1)),
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=get_request_block_hasher(BLOCK_SIZE // 2, sha256),
|
||||
)
|
||||
req.num_computed_tokens = BLOCK_SIZE
|
||||
gpu_block = fix.gpu_block_pool.get_new_blocks(1)[0]
|
||||
primary_hash = make_block_hash_with_group_id(req.block_hashes[1], 0)
|
||||
secondary_hash = make_block_hash_with_group_id(req.block_hashes[0], 0)
|
||||
fix.gpu_block_pool._insert_block_hash(
|
||||
primary_hash, gpu_block, num_tokens=BLOCK_SIZE
|
||||
)
|
||||
fix.gpu_block_pool._insert_block_hash(
|
||||
secondary_hash, gpu_block, num_tokens=BLOCK_SIZE // 2
|
||||
)
|
||||
state = StoreRequestState(
|
||||
request=req,
|
||||
block_ids=([gpu_block.block_id],),
|
||||
num_stored_blocks=[0],
|
||||
)
|
||||
|
||||
gpu_block_ids, advanced_per_group, block_meta = sched._select_eager_blocks_to_store(
|
||||
state,
|
||||
([gpu_block.block_id],),
|
||||
)
|
||||
assert gpu_block_ids == [gpu_block.block_id]
|
||||
assert advanced_per_group == [1]
|
||||
assert block_meta is not None
|
||||
assert set(block_meta[0]) == {primary_hash, secondary_hash}
|
||||
|
||||
cpu_block = sched.cpu_block_pool.get_new_blocks(1)[0]
|
||||
sched._process_store_completion(
|
||||
gpu_block_ids,
|
||||
[cpu_block.block_id],
|
||||
block_meta,
|
||||
)
|
||||
|
||||
primary_event_hash = maybe_convert_block_hash(get_block_hash(primary_hash))
|
||||
secondary_event_hash = maybe_convert_block_hash(get_block_hash(secondary_hash))
|
||||
stored = [event for event in sched.take_events() if isinstance(event, BlockStored)]
|
||||
by_hash = {event.block_hashes[0]: event for event in stored}
|
||||
assert set(by_hash) == {primary_event_hash, secondary_event_hash}
|
||||
assert by_hash[primary_event_hash].block_size == BLOCK_SIZE
|
||||
assert by_hash[primary_event_hash].token_ids == req.prompt_token_ids[:BLOCK_SIZE]
|
||||
assert by_hash[secondary_event_hash].block_size == BLOCK_SIZE // 2
|
||||
assert (
|
||||
by_hash[secondary_event_hash].token_ids
|
||||
== req.prompt_token_ids[: BLOCK_SIZE // 2]
|
||||
)
|
||||
|
||||
primary_cpu_block = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block(
|
||||
primary_hash
|
||||
)
|
||||
secondary_cpu_block = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block(
|
||||
secondary_hash
|
||||
)
|
||||
assert primary_cpu_block is not None
|
||||
assert secondary_cpu_block is primary_cpu_block
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# BlockRemoved relabeled to correct medium
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_block_removed_relabeled_medium_cpu() -> None:
|
||||
"""Evicted BlockRemoved (emitted as GPU by BlockPool) is relabeled CPU."""
|
||||
# 5 total = 4 usable (null_block takes 1). Fill 4, then store 2 more to
|
||||
# evict 2 LRU blocks, producing BlockRemoved events.
|
||||
fix = make_events_scheduler(num_cpu_blocks=5, num_gpu_blocks=16)
|
||||
sched = fix.scheduler
|
||||
|
||||
# Fill CPU with 4 blocks (2 + 2).
|
||||
_do_eager_store(fix, num_blocks=2)
|
||||
# Drain store events so they don't clutter the eviction check.
|
||||
list(sched.take_events())
|
||||
_do_eager_store(fix, num_blocks=2)
|
||||
list(sched.take_events())
|
||||
|
||||
# Now store 2 more -> evicts 2 LRU blocks -> BlockRemoved emitted by pool
|
||||
# with MEDIUM_GPU, then relabeled by scheduler.take_events().
|
||||
_do_eager_store(fix, num_blocks=2)
|
||||
events = list(sched.take_events())
|
||||
|
||||
removed = [e for e in events if isinstance(e, BlockRemoved)]
|
||||
assert len(removed) >= 2, (
|
||||
f"expected >=2 BlockRemoved from eviction, got {len(removed)}"
|
||||
)
|
||||
for ev in removed:
|
||||
assert ev.medium == MEDIUM_CPU, (
|
||||
f"BlockRemoved should be relabeled CPU, got {ev.medium}"
|
||||
)
|
||||
assert ev.locality == "LOCAL", (
|
||||
f"BlockRemoved should be LOCAL, got {ev.locality}"
|
||||
)
|
||||
stored = [e for e in events if isinstance(e, BlockStored)]
|
||||
assert len(stored) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# lazy store emits BlockStored
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_lazy_store_emits_block_stored() -> None:
|
||||
"""Lazy-mode store completion also emits BlockStored with MEDIUM_CPU."""
|
||||
fix = make_events_scheduler(num_cpu_blocks=8, num_gpu_blocks=8, lazy=True)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
|
||||
# Allocate, hash, free -> hashed blocks in free queue.
|
||||
req = make_request(num_blocks=2)
|
||||
gpu_blocks = []
|
||||
for blk in gpu_pool.get_new_blocks(2):
|
||||
gpu_blocks.append(blk)
|
||||
gpu_pool.cache_full_blocks(
|
||||
request=req,
|
||||
blocks=gpu_blocks,
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=2,
|
||||
block_size=BLOCK_SIZE,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
gpu_pool.free_blocks(gpu_blocks)
|
||||
|
||||
# Push hashed blocks to LRU head (8 total - 1 null = 7 usable; 2 freed
|
||||
# hashed + 5 other free -> consume 5 fillers).
|
||||
fillers = gpu_pool.get_new_blocks(5)
|
||||
|
||||
meta = sched.build_connector_meta(make_scheduler_output({}))
|
||||
assert meta.store_event >= 0
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
gpu_pool.free_blocks(fillers)
|
||||
|
||||
events = list(sched.take_events())
|
||||
stored = [e for e in events if isinstance(e, BlockStored)]
|
||||
assert len(stored) == 2, f"expected 2 BlockStored, got {len(stored)}"
|
||||
for ev in stored:
|
||||
assert ev.medium == MEDIUM_CPU
|
||||
assert ev.group_idx == 0
|
||||
assert ev.kv_cache_spec_kind == "full_attention"
|
||||
assert ev.locality == "LOCAL"
|
||||
assert ev.token_ids == []
|
||||
assert ev.parent_block_hash is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cp_world_size scales block_size for non-Mamba
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_cp_world_size_scales_block_size() -> None:
|
||||
"""block_size on BlockStored = spec.block_size * cp_world_size (FA)."""
|
||||
fix = make_events_scheduler(dcp_world_size=2)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
vbs = BLOCK_SIZE * 2
|
||||
assert sched.cp_world_size == 2
|
||||
|
||||
req = _make_cp_request(num_blocks=1, virtual_block_size=vbs)
|
||||
gpu_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 1, vbs)
|
||||
kv_blocks = KVCacheBlocks(blocks=(gpu_blocks,))
|
||||
req.num_computed_tokens = vbs
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
sched_out = make_scheduler_output(
|
||||
{req.request_id: vbs},
|
||||
new_reqs={req.request_id: kv_blocks.get_block_ids()},
|
||||
)
|
||||
meta = sched.build_connector_meta(sched_out)
|
||||
assert meta.store_event >= 0
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
|
||||
events = list(sched.take_events())
|
||||
stored = [e for e in events if isinstance(e, BlockStored)]
|
||||
assert len(stored) == 1
|
||||
# spec.block_size=BLOCK_SIZE, cp_world_size=2 -> BLOCK_SIZE * 2
|
||||
assert stored[0].block_size == BLOCK_SIZE * 2
|
||||
assert stored[0].locality == "LOCAL"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mamba block_size NOT scaled by cp_world_size
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_mamba_block_size_not_scaled() -> None:
|
||||
"""Mamba group: block_size = spec.block_size (no cp scaling).
|
||||
|
||||
With cp_world_size=2, scheduler_block_size=hash_block_size=32. The
|
||||
coordinator constraints force Mamba spec.block_size=32 (must divide
|
||||
scheduler_block_size and be a multiple of hash_block_size). The event
|
||||
block_size for Mamba must be 32 (spec.block_size * 1), NOT 64
|
||||
(spec.block_size * cp_world_size) -- that is the bug this guards.
|
||||
"""
|
||||
kv_cfg = _make_mixed_kv_cache_config(
|
||||
num_blocks=16, with_mamba=True, mamba_block_size=32
|
||||
)
|
||||
fix = make_events_scheduler(
|
||||
num_cpu_blocks=8,
|
||||
num_gpu_blocks=16,
|
||||
kv_cache_config=kv_cfg,
|
||||
dcp_world_size=2,
|
||||
num_groups=2,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
|
||||
cpu_pool = sched.cpu_block_pool
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
|
||||
cpu_blk = cpu_pool.get_new_blocks(1)[0]
|
||||
gpu_blk = gpu_pool.get_new_blocks(1)[0]
|
||||
|
||||
raw_hash = make_request(num_blocks=1).block_hashes[0]
|
||||
mamba_hash = make_block_hash_with_group_id(raw_hash, group_id=1)
|
||||
gpu_blk._block_hash = mamba_hash # type: ignore[attr-defined]
|
||||
|
||||
sched._process_store_completion(
|
||||
gpu_block_ids=[gpu_blk.block_id],
|
||||
cpu_block_ids=[cpu_blk.block_id],
|
||||
)
|
||||
|
||||
events = list(sched.take_events())
|
||||
stored = [e for e in events if isinstance(e, BlockStored)]
|
||||
assert len(stored) == 1
|
||||
ev = stored[0]
|
||||
assert ev.group_idx == 1
|
||||
assert ev.kv_cache_spec_kind == "mamba"
|
||||
assert ev.kv_cache_spec_sliding_window is None
|
||||
assert ev.locality == "LOCAL"
|
||||
assert ev.block_size == 32, (
|
||||
f"Mamba block_size should be unscaled (32), got {ev.block_size}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# multi-group FA+SW stores carry correct group_idx and kind
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_multi_group_fa_block_stored_group_idx() -> None:
|
||||
"""Two distinct groups (FA + SW): each BlockStored carries its group_idx
|
||||
and kind. HybridKVCacheCoordinator requires >=2 distinct spec groups, so
|
||||
we use FA (group 0) + SlidingWindow (group 1)."""
|
||||
kv_cfg = _make_mixed_kv_cache_config(num_blocks=16, sliding_window=32)
|
||||
fix = make_events_scheduler(
|
||||
num_cpu_blocks=8,
|
||||
num_gpu_blocks=16,
|
||||
kv_cache_config=kv_cfg,
|
||||
num_groups=2,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
|
||||
cpu_pool = sched.cpu_block_pool
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
raw_hash = make_request(num_blocks=1).block_hashes[0]
|
||||
|
||||
# Store one block for each group via direct _process_store_completion.
|
||||
for gidx in (0, 1):
|
||||
cpu_blk = cpu_pool.get_new_blocks(1)[0]
|
||||
gpu_blk = gpu_pool.get_new_blocks(1)[0]
|
||||
gpu_blk._block_hash = make_block_hash_with_group_id( # type: ignore
|
||||
raw_hash, group_id=gidx
|
||||
)
|
||||
sched._process_store_completion(
|
||||
gpu_block_ids=[gpu_blk.block_id],
|
||||
cpu_block_ids=[cpu_blk.block_id],
|
||||
)
|
||||
|
||||
events = list(sched.take_events())
|
||||
stored = [e for e in events if isinstance(e, BlockStored)]
|
||||
assert len(stored) == 2
|
||||
by_group = {ev.group_idx: ev for ev in stored}
|
||||
assert set(by_group) == {0, 1}
|
||||
assert by_group[0].kv_cache_spec_kind == "full_attention"
|
||||
assert by_group[0].kv_cache_spec_sliding_window is None
|
||||
assert by_group[1].kv_cache_spec_kind == "sliding_window"
|
||||
assert by_group[1].kv_cache_spec_sliding_window == 32
|
||||
for ev in stored:
|
||||
assert ev.locality == "LOCAL"
|
||||
|
||||
|
||||
def test_disk_mode_block_removed_relabeled_storage() -> None:
|
||||
"""Disk mode: evicted BlockRemoved relabeled to MEDIUM_STORAGE."""
|
||||
fix = make_events_scheduler(
|
||||
num_cpu_blocks=5,
|
||||
num_gpu_blocks=16,
|
||||
disk_capacity_bytes=_BYTES_PER_BLOCK * 5,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
assert sched.kv_event_medium == MEDIUM_STORAGE
|
||||
|
||||
# Fill disk with 4 blocks (2 + 2).
|
||||
_do_eager_store(fix, num_blocks=2)
|
||||
list(sched.take_events())
|
||||
_do_eager_store(fix, num_blocks=2)
|
||||
list(sched.take_events())
|
||||
|
||||
# Store 2 more -> evicts 2 LRU blocks -> BlockRemoved relabeled STORAGE.
|
||||
_do_eager_store(fix, num_blocks=2)
|
||||
events = list(sched.take_events())
|
||||
|
||||
removed = [e for e in events if isinstance(e, BlockRemoved)]
|
||||
assert len(removed) >= 2, (
|
||||
f"expected >=2 BlockRemoved from eviction, got {len(removed)}"
|
||||
)
|
||||
for ev in removed:
|
||||
assert ev.medium == MEDIUM_STORAGE, (
|
||||
f"BlockRemoved should be relabeled STORAGE, got {ev.medium}"
|
||||
)
|
||||
assert ev.locality == "LOCAL", (
|
||||
f"BlockRemoved should be LOCAL, got {ev.locality}"
|
||||
)
|
||||
stored = [e for e in events if isinstance(e, BlockStored)]
|
||||
assert len(stored) == 2
|
||||
for ev in stored:
|
||||
assert ev.medium == MEDIUM_STORAGE
|
||||
|
||||
|
||||
def test_mamba_dcp_metadata_guard() -> None:
|
||||
"""Mamba+DCP: token_ids guard emits [] when capture and emission sizes differ.
|
||||
|
||||
The metadata capture at _prepare_eager_store_specs uses
|
||||
g_block_size = spec.block_size * cp_world_size = 32 * 2 = 64 to slice
|
||||
tokens. The event emission uses block_size = spec.block_size * 1 = 32
|
||||
for Mamba (SSM state is not CP-sharded). The guard in
|
||||
_process_store_completion detects len(token_ids) != block_size and emits
|
||||
token_ids=[] instead of misleading data. When #49962 fixes the capture
|
||||
path, the guard stops triggering and correct token_ids flow through.
|
||||
"""
|
||||
vbs = BLOCK_SIZE * 2
|
||||
kv_cfg = _make_mixed_kv_cache_config(
|
||||
num_blocks=16, with_mamba=True, mamba_block_size=32
|
||||
)
|
||||
fix = make_events_scheduler(
|
||||
num_cpu_blocks=8,
|
||||
num_gpu_blocks=16,
|
||||
kv_cache_config=kv_cfg,
|
||||
dcp_world_size=2,
|
||||
num_groups=2,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
|
||||
# Mamba g_block_size = 32 * 2 = 64, so 2 Mamba blocks need 128 computed
|
||||
# tokens to be "ready"; FA g_block_size = 16 * 2 = 32 needs only 64.
|
||||
req = _make_cp_request(num_blocks=4, virtual_block_size=vbs)
|
||||
fa_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 2, vbs, group_id=0)
|
||||
mamba_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 2, vbs, group_id=1)
|
||||
kv_blocks = KVCacheBlocks(blocks=(fa_blocks, mamba_blocks))
|
||||
req.num_computed_tokens = 4 * vbs
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
|
||||
block_ids = kv_blocks.get_block_ids()
|
||||
sched_out = make_scheduler_output(
|
||||
{req.request_id: 4 * vbs},
|
||||
new_reqs={req.request_id: block_ids},
|
||||
)
|
||||
meta = sched.build_connector_meta(sched_out)
|
||||
assert meta.store_event >= 0
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
|
||||
events = list(sched.take_events())
|
||||
stored = [e for e in events if isinstance(e, BlockStored)]
|
||||
assert len(stored) == 4, (
|
||||
f"expected 4 BlockStored (2 FA + 2 Mamba), got {len(stored)}"
|
||||
)
|
||||
|
||||
by_group: dict[int, list[BlockStored]] = {}
|
||||
for ev in stored:
|
||||
by_group.setdefault(ev.group_idx, []).append(ev)
|
||||
|
||||
# FA group: capture size (64) == emission size (64) → token_ids present.
|
||||
fa_events = by_group[0]
|
||||
assert len(fa_events) == 2
|
||||
for ev in fa_events:
|
||||
assert ev.block_size == vbs
|
||||
assert len(ev.token_ids) == vbs, (
|
||||
f"FA token_ids should be {vbs}, got {len(ev.token_ids)}"
|
||||
)
|
||||
|
||||
# Mamba group: capture size (64) != emission size (32) → guard emits [].
|
||||
mamba_events = by_group[1]
|
||||
assert len(mamba_events) == 2
|
||||
for ev in mamba_events:
|
||||
assert ev.block_size == 32
|
||||
assert ev.token_ids == [], (
|
||||
f"Mamba token_ids should be [] (guard), got {ev.token_ids}"
|
||||
)
|
||||
assert ev.parent_block_hash is None, (
|
||||
"Mamba parent_block_hash should be None (guard)"
|
||||
)
|
||||
assert ev.extra_keys is None, "Mamba extra_keys should be None (guard)"
|
||||
@@ -111,6 +111,10 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
raise ValueError(
|
||||
'kv_offload_backend="disk" requires disk_path to be set.'
|
||||
)
|
||||
if disk_capacity_bytes <= 0:
|
||||
raise ValueError(
|
||||
'kv_offload_backend="disk" requires disk_capacity_bytes > 0.'
|
||||
)
|
||||
else:
|
||||
ignored = [k for k in _DISK_ONLY_KEYS if k in extra_config]
|
||||
if ignored:
|
||||
|
||||
@@ -8,7 +8,13 @@ from dataclasses import dataclass, field, replace
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_events import KVCacheEvent
|
||||
from vllm.distributed.kv_events import (
|
||||
MEDIUM_CPU,
|
||||
MEDIUM_STORAGE,
|
||||
BlockRemoved,
|
||||
BlockStored,
|
||||
KVCacheEvent,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import yield_req_data
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -17,11 +23,23 @@ from vllm.v1.core.kv_cache_coordinator import (
|
||||
KVCacheCoordinator,
|
||||
get_kv_cache_coordinator,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHashWithGroupId,
|
||||
ExternalBlockHash,
|
||||
generate_block_hash_extra_keys,
|
||||
get_block_hash,
|
||||
get_group_id,
|
||||
make_block_hash_with_group_id,
|
||||
maybe_convert_block_hash,
|
||||
resolve_block_hashes,
|
||||
)
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
get_kv_cache_spec_kind,
|
||||
get_kv_cache_spec_sliding_window,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.simple_kv_offload.metadata import (
|
||||
@@ -38,10 +56,27 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BlockStoreMeta:
|
||||
"""Per-block metadata snapshot for BlockStored event emission.
|
||||
|
||||
Captured at store-prep time (when the Request is available) and carried
|
||||
through TransferMeta to async completion, where the Request is gone.
|
||||
"""
|
||||
|
||||
token_ids: list[int]
|
||||
parent_block_hash: ExternalBlockHash | None
|
||||
lora_id: int | None
|
||||
lora_name: str | None
|
||||
extra_keys: tuple[Any, ...] | None
|
||||
block_size: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransferMeta:
|
||||
gpu_block_ids: list[int]
|
||||
cpu_block_ids: list[int]
|
||||
block_meta: list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -98,6 +133,7 @@ class SimpleCPUOffloadScheduler:
|
||||
kv_cache_config, offload_capacity
|
||||
)
|
||||
self.num_cpu_blocks = self.cpu_kv_cache_config.num_blocks
|
||||
self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU
|
||||
# Find the full attention kv group for prefix cache matching.
|
||||
self.fa_gidx = -1
|
||||
for g_idx, g in enumerate(self.cpu_kv_cache_config.kv_cache_groups):
|
||||
@@ -414,17 +450,22 @@ class SimpleCPUOffloadScheduler:
|
||||
) -> SimpleCPUOffloadMetadata:
|
||||
# --- Stores ---
|
||||
store_event = -1
|
||||
store_gpu, store_cpu, store_req_ids = self.prepare_store_specs(scheduler_output)
|
||||
store_gpu, store_cpu, store_req_ids, store_meta = self.prepare_store_specs(
|
||||
scheduler_output
|
||||
)
|
||||
for transfer in self._pending_finished_stores:
|
||||
store_gpu.extend(transfer.gpu_block_ids)
|
||||
store_cpu.extend(transfer.cpu_block_ids)
|
||||
if store_meta is not None:
|
||||
assert transfer.block_meta is not None
|
||||
store_meta.extend(transfer.block_meta)
|
||||
self._pending_finished_stores.clear()
|
||||
|
||||
if store_gpu:
|
||||
store_event = self._store_event_counter
|
||||
self._store_event_counter += 1
|
||||
self._store_event_to_blocks[store_event] = TransferMeta(
|
||||
store_gpu, store_cpu
|
||||
store_gpu, store_cpu, store_meta
|
||||
)
|
||||
if store_req_ids: # For eager mode only, track req->blocks mapping
|
||||
self._store_event_to_reqs[store_event] = store_req_ids
|
||||
@@ -469,7 +510,12 @@ class SimpleCPUOffloadScheduler:
|
||||
|
||||
def prepare_store_specs(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> tuple[list[int], list[int], list[str]]:
|
||||
) -> tuple[
|
||||
list[int],
|
||||
list[int],
|
||||
list[str],
|
||||
list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None,
|
||||
]:
|
||||
"""Prepare store specs for the store event."""
|
||||
if self._lazy_mode:
|
||||
return self._prepare_lazy_store_specs()
|
||||
@@ -478,7 +524,12 @@ class SimpleCPUOffloadScheduler:
|
||||
|
||||
def _prepare_lazy_store_specs(
|
||||
self,
|
||||
) -> tuple[list[int], list[int], list[str]]:
|
||||
) -> tuple[
|
||||
list[int],
|
||||
list[int],
|
||||
list[str],
|
||||
list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None,
|
||||
]:
|
||||
"""Single-pass cursor walk: offload cached GPU blocks near eviction.
|
||||
|
||||
Walks the GPU free queue from the cursor, counting blocks that are
|
||||
@@ -487,7 +538,7 @@ class SimpleCPUOffloadScheduler:
|
||||
"""
|
||||
gpu_pool = self._gpu_block_pool
|
||||
if gpu_pool is None or self._target_free <= 0:
|
||||
return [], [], []
|
||||
return [], [], [], None
|
||||
|
||||
free_queue = gpu_pool.free_block_queue
|
||||
cpu_pool = self.cpu_block_pool
|
||||
@@ -525,11 +576,16 @@ class SimpleCPUOffloadScheduler:
|
||||
else:
|
||||
cpu_ids = []
|
||||
|
||||
return gpu_ids, cpu_ids, []
|
||||
return gpu_ids, cpu_ids, [], None
|
||||
|
||||
def _prepare_eager_store_specs(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> tuple[list[int], list[int], list[str]]:
|
||||
) -> tuple[
|
||||
list[int],
|
||||
list[int],
|
||||
list[str],
|
||||
list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None,
|
||||
]:
|
||||
"""Identify newly computed blocks to offload from scheduler requests.
|
||||
|
||||
Only considers blocks whose KV data has been **confirmed computed** by
|
||||
@@ -537,19 +593,22 @@ class SimpleCPUOffloadScheduler:
|
||||
the finish-time flush if the request completes first.
|
||||
|
||||
Returns:
|
||||
(gpu_block_ids, cpu_block_ids, req_ids) for the store event.
|
||||
(gpu_block_ids, cpu_block_ids, req_ids, block_meta) for the store
|
||||
event. ``block_meta`` is None when kv cache events are disabled.
|
||||
"""
|
||||
|
||||
merged_gpu_block_ids: list[int] = []
|
||||
merged_cpu_block_ids: list[int] = []
|
||||
req_ids: list[str] = []
|
||||
merged_block_meta: list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None = (
|
||||
[] if self.enable_kv_cache_events else None
|
||||
)
|
||||
|
||||
gpu_block_pool = self._gpu_block_pool
|
||||
if gpu_block_pool is None:
|
||||
return [], [], []
|
||||
return [], [], [], merged_block_meta
|
||||
cpu_block_pool = self.cpu_block_pool
|
||||
kv_cache_groups = self.cpu_kv_cache_config.kv_cache_groups
|
||||
num_groups = len(kv_cache_groups)
|
||||
num_groups = len(self.cpu_kv_cache_config.kv_cache_groups)
|
||||
# Dedup against blocks already scheduled.
|
||||
in_flight = self._in_flight_store_gpu_blocks
|
||||
|
||||
@@ -558,13 +617,13 @@ class SimpleCPUOffloadScheduler:
|
||||
if state is None or state.finished:
|
||||
continue
|
||||
|
||||
# Accumulate new block IDs.
|
||||
if preempted:
|
||||
state.block_ids = tuple([] for _ in range(num_groups))
|
||||
state.num_stored_blocks = [0] * num_groups
|
||||
if new_block_id_groups:
|
||||
for g in range(min(num_groups, len(new_block_id_groups))):
|
||||
if new_block_id_groups[g] is not None:
|
||||
# Accumulate new block IDs.
|
||||
state.block_ids[g].extend(new_block_id_groups[g])
|
||||
|
||||
num_new_tokens = scheduler_output.num_scheduled_tokens.get(req_id, 0)
|
||||
@@ -575,8 +634,8 @@ class SimpleCPUOffloadScheduler:
|
||||
if not block_ids_by_group:
|
||||
continue
|
||||
|
||||
gpu_block_ids, advanced_per_group = self._select_eager_blocks_to_store(
|
||||
state, block_ids_by_group
|
||||
gpu_block_ids, advanced_per_group, block_meta = (
|
||||
self._select_eager_blocks_to_store(state, block_ids_by_group)
|
||||
)
|
||||
|
||||
# --- Phase 2: Batch allocate CPU blocks ---
|
||||
@@ -592,6 +651,9 @@ class SimpleCPUOffloadScheduler:
|
||||
merged_gpu_block_ids.extend(gpu_block_ids)
|
||||
merged_cpu_block_ids.extend(cpu_block_ids)
|
||||
in_flight.update(gpu_block_ids)
|
||||
if merged_block_meta is not None:
|
||||
assert block_meta is not None
|
||||
merged_block_meta.extend(block_meta)
|
||||
|
||||
# Touch GPU blocks to prevent freeing during async copy
|
||||
gpu_block_pool.touch(
|
||||
@@ -609,20 +671,26 @@ class SimpleCPUOffloadScheduler:
|
||||
for g in range(num_groups):
|
||||
state.num_stored_blocks[g] += advanced_per_group[g]
|
||||
|
||||
return merged_gpu_block_ids, merged_cpu_block_ids, req_ids
|
||||
return merged_gpu_block_ids, merged_cpu_block_ids, req_ids, merged_block_meta
|
||||
|
||||
def _select_eager_blocks_to_store(
|
||||
self,
|
||||
state: StoreRequestState,
|
||||
block_ids_by_group: tuple[list[int], ...],
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""Return confirmed, uncached eager blocks and per-group cursor advances."""
|
||||
) -> tuple[
|
||||
list[int],
|
||||
list[int],
|
||||
list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None,
|
||||
]:
|
||||
"""Return confirmed eager blocks, cursor advances, and event metadata."""
|
||||
assert self._gpu_block_pool is not None
|
||||
gpu_block_ids: list[int] = []
|
||||
advanced_per_group = [0] * len(self.cpu_kv_cache_config.kv_cache_groups)
|
||||
confirmed_tokens = (
|
||||
state.request.num_computed_tokens - state.request.num_output_placeholders
|
||||
block_meta: list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None = (
|
||||
[] if self.enable_kv_cache_events else None
|
||||
)
|
||||
request = state.request
|
||||
confirmed_tokens = request.num_computed_tokens - request.num_output_placeholders
|
||||
aligned_tokens = confirmed_tokens // self.block_size * self.block_size
|
||||
num_free = self.cpu_block_pool.get_num_free_blocks()
|
||||
|
||||
@@ -637,7 +705,17 @@ class SimpleCPUOffloadScheduler:
|
||||
* self.cp_world_size
|
||||
)
|
||||
ready = min(len(group_gpu_ids), aligned_tokens // group_size)
|
||||
for gpu_block_id in group_gpu_ids[state.num_stored_blocks[g] : ready]:
|
||||
resolved_hashes = (
|
||||
resolve_block_hashes(
|
||||
request.block_hashes, self.hash_block_size, group_size
|
||||
)
|
||||
if block_meta is not None
|
||||
else None
|
||||
)
|
||||
curr_mm_idx = 0
|
||||
secondary_mm_idx = 0
|
||||
start = state.num_stored_blocks[g]
|
||||
for i, gpu_block_id in enumerate(group_gpu_ids[start:ready], start=start):
|
||||
gpu_block = self._gpu_block_pool.blocks[gpu_block_id]
|
||||
if gpu_block.is_null or gpu_block.block_hash is None:
|
||||
advanced_per_group[g] += 1
|
||||
@@ -655,8 +733,65 @@ class SimpleCPUOffloadScheduler:
|
||||
break
|
||||
gpu_block_ids.append(gpu_block_id)
|
||||
advanced_per_group[g] += 1
|
||||
if block_meta is not None:
|
||||
token_start = i * group_size
|
||||
token_end = token_start + group_size
|
||||
assert resolved_hashes is not None
|
||||
parent_hash = (
|
||||
None
|
||||
if i == 0
|
||||
else maybe_convert_block_hash(resolved_hashes[i - 1])
|
||||
)
|
||||
lora_req = request.lora_request
|
||||
extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
|
||||
request, token_start, token_end, curr_mm_idx
|
||||
)
|
||||
meta_by_hash = {
|
||||
gpu_block.block_hash: BlockStoreMeta(
|
||||
token_ids=list(
|
||||
request.all_token_ids[token_start:token_end]
|
||||
),
|
||||
parent_block_hash=parent_hash,
|
||||
lora_id=lora_req.adapter_id if lora_req else None,
|
||||
lora_name=lora_req.name if lora_req else None,
|
||||
extra_keys=extra_keys,
|
||||
)
|
||||
}
|
||||
first_hash_idx = token_start // self.hash_block_size
|
||||
last_hash_idx = token_end // self.hash_block_size
|
||||
for hash_idx in range(first_hash_idx, last_hash_idx):
|
||||
block_hash = make_block_hash_with_group_id(
|
||||
request.block_hashes[hash_idx], g
|
||||
)
|
||||
if block_hash == gpu_block.block_hash:
|
||||
continue
|
||||
hash_start = hash_idx * self.hash_block_size
|
||||
hash_end = hash_start + self.hash_block_size
|
||||
secondary_extra_keys, secondary_mm_idx = (
|
||||
generate_block_hash_extra_keys(
|
||||
request,
|
||||
hash_start,
|
||||
hash_end,
|
||||
secondary_mm_idx,
|
||||
)
|
||||
)
|
||||
meta_by_hash[block_hash] = BlockStoreMeta(
|
||||
token_ids=list(request.all_token_ids[hash_start:hash_end]),
|
||||
parent_block_hash=(
|
||||
None
|
||||
if hash_idx == 0
|
||||
else maybe_convert_block_hash(
|
||||
request.block_hashes[hash_idx - 1]
|
||||
)
|
||||
),
|
||||
lora_id=lora_req.adapter_id if lora_req else None,
|
||||
lora_name=lora_req.name if lora_req else None,
|
||||
extra_keys=secondary_extra_keys,
|
||||
block_size=self.hash_block_size,
|
||||
)
|
||||
block_meta.append(meta_by_hash)
|
||||
|
||||
return gpu_block_ids, advanced_per_group
|
||||
return gpu_block_ids, advanced_per_group, block_meta
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Handle async transfer completions from worker.
|
||||
@@ -695,7 +830,11 @@ class SimpleCPUOffloadScheduler:
|
||||
if not self._lazy_mode:
|
||||
self._in_flight_store_gpu_blocks.difference_update(transfer.gpu_block_ids)
|
||||
|
||||
self._process_store_completion(transfer.gpu_block_ids, transfer.cpu_block_ids)
|
||||
self._process_store_completion(
|
||||
transfer.gpu_block_ids,
|
||||
transfer.cpu_block_ids,
|
||||
transfer.block_meta,
|
||||
)
|
||||
logger.debug(
|
||||
"Store event %d completed: cached %d blocks to CPU",
|
||||
event_idx,
|
||||
@@ -713,15 +852,19 @@ class SimpleCPUOffloadScheduler:
|
||||
self._cleanup_store_request(req_id)
|
||||
|
||||
def _process_store_completion(
|
||||
self, gpu_block_ids: list[int], cpu_block_ids: list[int]
|
||||
self,
|
||||
gpu_block_ids: list[int],
|
||||
cpu_block_ids: list[int],
|
||||
block_meta: list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None = None,
|
||||
) -> None:
|
||||
"""Register copied blocks in the CPU prefix cache and release refs."""
|
||||
assert len(cpu_block_ids) == len(gpu_block_ids)
|
||||
assert block_meta is None or len(block_meta) == len(gpu_block_ids)
|
||||
|
||||
cpu_blocks = [self.cpu_block_pool.blocks[bid] for bid in cpu_block_ids]
|
||||
|
||||
assert self._gpu_block_pool is not None
|
||||
for gpu_block_id, cpu_block in zip(gpu_block_ids, cpu_blocks):
|
||||
for i, (gpu_block_id, cpu_block) in enumerate(zip(gpu_block_ids, cpu_blocks)):
|
||||
gpu_block = self._gpu_block_pool.blocks[gpu_block_id]
|
||||
primary_hash = gpu_block.block_hash
|
||||
assert primary_hash is not None
|
||||
@@ -737,6 +880,76 @@ class SimpleCPUOffloadScheduler:
|
||||
self.cpu_block_pool._insert_block_hash(
|
||||
block_hash, cpu_block, num_tokens=None
|
||||
)
|
||||
if self.enable_kv_cache_events:
|
||||
meta_by_hash = block_meta[i] if block_meta is not None else {}
|
||||
primary_group_idx = get_group_id(primary_hash)
|
||||
primary_spec = self.cpu_kv_cache_config.kv_cache_groups[
|
||||
primary_group_idx
|
||||
].kv_cache_spec
|
||||
primary_block_size = primary_spec.block_size * (
|
||||
self.cp_world_size if not isinstance(primary_spec, MambaSpec) else 1
|
||||
)
|
||||
events_to_emit: list[
|
||||
tuple[BlockHashWithGroupId, BlockStoreMeta | None, int]
|
||||
] = [
|
||||
(
|
||||
primary_hash,
|
||||
meta_by_hash.get(primary_hash),
|
||||
primary_block_size,
|
||||
)
|
||||
]
|
||||
for secondary_hash in secondary_hashes:
|
||||
secondary_meta = meta_by_hash.get(secondary_hash)
|
||||
events_to_emit.append(
|
||||
(
|
||||
secondary_hash,
|
||||
secondary_meta,
|
||||
secondary_meta.block_size
|
||||
if secondary_meta is not None
|
||||
and secondary_meta.block_size is not None
|
||||
else 0,
|
||||
)
|
||||
)
|
||||
|
||||
for block_hash, meta, event_block_size in events_to_emit:
|
||||
group_idx = get_group_id(block_hash)
|
||||
spec = self.cpu_kv_cache_config.kv_cache_groups[
|
||||
group_idx
|
||||
].kv_cache_spec
|
||||
if meta is not None and len(meta.token_ids) != event_block_size:
|
||||
# token_ids were sliced with a different g_block_size
|
||||
# (e.g. Mamba+DCP where capture uses spec.block_size *
|
||||
# cp_world_size but the event emits spec.block_size * 1).
|
||||
# Emit empty metadata until #49962 fixes the sizing.
|
||||
token_ids = []
|
||||
parent_block_hash = None
|
||||
extra_keys = None
|
||||
else:
|
||||
token_ids = meta.token_ids if meta else []
|
||||
parent_block_hash = meta.parent_block_hash if meta else None
|
||||
extra_keys = meta.extra_keys if meta else None
|
||||
self.cpu_block_pool.kv_event_queue.append(
|
||||
BlockStored(
|
||||
block_hashes=[
|
||||
maybe_convert_block_hash(get_block_hash(block_hash))
|
||||
],
|
||||
parent_block_hash=parent_block_hash,
|
||||
token_ids=token_ids,
|
||||
block_size=event_block_size,
|
||||
lora_id=meta.lora_id if meta else None,
|
||||
medium=self.kv_event_medium,
|
||||
lora_name=meta.lora_name if meta else None,
|
||||
extra_keys=(
|
||||
[extra_keys] if extra_keys is not None else None
|
||||
),
|
||||
group_idx=group_idx,
|
||||
kv_cache_spec_kind=get_kv_cache_spec_kind(spec).value,
|
||||
kv_cache_spec_sliding_window=(
|
||||
get_kv_cache_spec_sliding_window(spec)
|
||||
),
|
||||
locality="LOCAL",
|
||||
)
|
||||
)
|
||||
|
||||
# Free CPU and GPU blocks' ref counts to turn them into prefix cache
|
||||
self.cpu_block_pool.free_blocks(cpu_blocks)
|
||||
@@ -811,12 +1024,12 @@ class SimpleCPUOffloadScheduler:
|
||||
gpu_pool = self._gpu_block_pool
|
||||
if state is None or gpu_pool is None:
|
||||
return
|
||||
gpu_ids, _ = self._select_eager_blocks_to_store(state, block_ids)
|
||||
gpu_ids, _, block_meta = self._select_eager_blocks_to_store(state, block_ids)
|
||||
if not gpu_ids:
|
||||
return
|
||||
cpu_blocks = self.cpu_block_pool.get_new_blocks(len(gpu_ids))
|
||||
self._pending_finished_stores.append(
|
||||
TransferMeta(gpu_ids, [b.block_id for b in cpu_blocks])
|
||||
TransferMeta(gpu_ids, [b.block_id for b in cpu_blocks], block_meta)
|
||||
)
|
||||
self._in_flight_store_gpu_blocks.update(gpu_ids)
|
||||
gpu_pool.touch([gpu_pool.blocks[bid] for bid in gpu_ids])
|
||||
@@ -882,7 +1095,12 @@ class SimpleCPUOffloadScheduler:
|
||||
state.store_events.clear()
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
return self.cpu_block_pool.take_events()
|
||||
events = self.cpu_block_pool.take_events()
|
||||
for event in events:
|
||||
if isinstance(event, BlockRemoved):
|
||||
event.medium = self.kv_event_medium
|
||||
event.locality = "LOCAL"
|
||||
return events
|
||||
|
||||
def reset(self) -> bool:
|
||||
"""Abandon pending transfers and reset the CPU cache when safe.
|
||||
|
||||
Reference in New Issue
Block a user