[Bugfix][KV Offload] Emit self-describing CPU events at KV-group block granularity (#51614)
Signed-off-by: Ziqi Fan <ziqif@nvidia.com>
This commit is contained in:
@@ -81,13 +81,18 @@ def _group_config(
|
||||
group_idx: int = 0,
|
||||
block_size: int = 4,
|
||||
blocks_per_chunk: int = 1,
|
||||
tokens_per_hash: int | None = None,
|
||||
sliding_window_size_in_chunks: int | None = None,
|
||||
) -> GroupOffloadConfig:
|
||||
if tokens_per_hash is None:
|
||||
tokens_per_hash = block_size
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
assert tokens_per_chunk % tokens_per_hash == 0
|
||||
return GroupOffloadConfig(
|
||||
group_idx=group_idx,
|
||||
tokens_per_block=block_size,
|
||||
tokens_per_chunk=block_size * blocks_per_chunk,
|
||||
hashes_per_chunk=blocks_per_chunk,
|
||||
tokens_per_chunk=tokens_per_chunk,
|
||||
hashes_per_chunk=tokens_per_chunk // tokens_per_hash,
|
||||
sliding_window_size_in_chunks=sliding_window_size_in_chunks,
|
||||
kv_event_group_spec=_FULL_ATTENTION_EVENT_SPEC,
|
||||
)
|
||||
@@ -334,6 +339,36 @@ def test_promotion_emits_full_cpu_stored_event():
|
||||
assert event.kv_cache_spec_sliding_window is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("blocks_per_chunk", "expected_hash_indices"),
|
||||
[(1, [63]), (2, [63, 127])],
|
||||
)
|
||||
def test_event_hashes_use_group_block_size(
|
||||
blocks_per_chunk: int, expected_hash_indices: list[int]
|
||||
):
|
||||
tokens_per_hash = 4
|
||||
block_size = 256
|
||||
hashes_per_block = block_size // tokens_per_hash
|
||||
tracker = _tracker()
|
||||
group_config = _group_config(
|
||||
block_size=block_size,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
tokens_per_hash=tokens_per_hash,
|
||||
)
|
||||
req = _request(
|
||||
block_hashes=[_hash(i) for i in range(hashes_per_block * blocks_per_chunk)],
|
||||
token_count=block_size * blocks_per_chunk,
|
||||
)
|
||||
[key] = _record_chunks(tracker, req, group_config, num_chunks=1)
|
||||
|
||||
[event] = tracker.take_events([_stored_event([key])])
|
||||
|
||||
assert isinstance(event, BlockStored)
|
||||
assert event.block_hashes == [_wire_hash(_hash(i)) for i in expected_hash_indices]
|
||||
assert event.block_size == block_size
|
||||
assert len(event.token_ids) == block_size * blocks_per_chunk
|
||||
|
||||
|
||||
def test_lookup_promotion_factor_gt_1_store_and_remove():
|
||||
block_size = 4
|
||||
blocks_per_chunk = 2
|
||||
|
||||
@@ -27,7 +27,11 @@ from vllm.distributed.kv_events import (
|
||||
KVCacheEvent,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
maybe_convert_block_hash,
|
||||
resolve_block_hashes,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
get_kv_cache_spec_kind,
|
||||
@@ -255,22 +259,26 @@ class OffloadingEventsTracker:
|
||||
"""Build the payload snapshot for one offloaded chunk: its
|
||||
constituent per-block hashes, the whole chunk's tokens, and the
|
||||
per-block ``block_size``."""
|
||||
hbf = group_config.hashes_per_chunk
|
||||
assert hbf > 0
|
||||
hashes_per_chunk = group_config.hashes_per_chunk
|
||||
assert hashes_per_chunk > 0
|
||||
assert chunk_idx >= 0
|
||||
# per-block token count (= the GPU/hash block size)
|
||||
tokens_per_hash = group_config.tokens_per_chunk // hbf
|
||||
# chunk c covers hash-blocks [c*hbf, (c+1)*hbf); its tail block's hash
|
||||
# is the chunk's OffloadKey.
|
||||
first_hash_idx = chunk_idx * hbf
|
||||
last_hash_idx = first_hash_idx + hbf
|
||||
tokens_per_hash = group_config.tokens_per_chunk // hashes_per_chunk
|
||||
# Each chunk's final raw hash is its OffloadKey.
|
||||
first_hash_idx = chunk_idx * hashes_per_chunk
|
||||
last_hash_idx = first_hash_idx + hashes_per_chunk
|
||||
assert first_hash_idx >= 0
|
||||
assert last_hash_idx <= len(req.block_hashes)
|
||||
chunk_hashes: list[BlockHash] = []
|
||||
for block_hash in req.block_hashes[first_hash_idx:last_hash_idx]:
|
||||
raw_chunk_hashes = req.block_hashes[first_hash_idx:last_hash_idx]
|
||||
chunk_hashes = resolve_block_hashes(
|
||||
raw_chunk_hashes,
|
||||
tokens_per_hash,
|
||||
group_config.tokens_per_block,
|
||||
)
|
||||
for block_hash in chunk_hashes:
|
||||
assert block_hash is not None
|
||||
chunk_hashes.append(block_hash)
|
||||
assert len(chunk_hashes) == hbf
|
||||
assert len(chunk_hashes) == (
|
||||
group_config.tokens_per_chunk // group_config.tokens_per_block
|
||||
)
|
||||
|
||||
if group_config.sliding_window_size_in_chunks is not None:
|
||||
# The recording methods filter these out before calling this helper.
|
||||
@@ -298,7 +306,7 @@ class OffloadingEventsTracker:
|
||||
block_hashes=tuple(chunk_hashes),
|
||||
parent_block_hash=parent_block_hash,
|
||||
token_ids=token_ids,
|
||||
block_size=tokens_per_hash,
|
||||
block_size=group_config.tokens_per_block,
|
||||
lora_id=lora_id,
|
||||
lora_name=lora_name,
|
||||
extra_keys=None,
|
||||
|
||||
Reference in New Issue
Block a user