[Core][KV Events] Echo session_id on GPU BlockStored events (#51381)
Signed-off-by: liuguangli <liuguangli35@gmail.com> Signed-off-by: jinx-machinepulse <2679622408@qq.com> Co-authored-by: liuguangli <liuguangli35@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
liuguangli
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
19c018ec05
commit
d9e2b5238d
@@ -45,6 +45,7 @@ class BlockStored(KVCacheEvent):
|
||||
kv_cache_spec_kind: str | None = None
|
||||
kv_cache_spec_sliding_window: int | None = None
|
||||
locality: str | None = None
|
||||
session_id: str | None = None
|
||||
|
||||
|
||||
class BlockRemoved(KVCacheEvent):
|
||||
|
||||
@@ -46,10 +46,33 @@ class _LegacyBlockRemoved(
|
||||
group_idx: int | None = None
|
||||
|
||||
|
||||
class _PreSessionBlockStored(
|
||||
msgspec.Struct,
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False, # type: ignore[call-arg]
|
||||
tag="BlockStored", # type: ignore[call-arg]
|
||||
):
|
||||
"""BlockStored wire schema before session_id was added."""
|
||||
|
||||
block_hashes: list[bytes]
|
||||
parent_block_hash: bytes | None
|
||||
token_ids: list[int]
|
||||
block_size: int
|
||||
lora_id: int | None
|
||||
medium: str | None
|
||||
lora_name: str | None
|
||||
extra_keys: list[tuple[Any, ...] | None] | None = None
|
||||
group_idx: int | None = None
|
||||
kv_cache_spec_kind: str | None = None
|
||||
kv_cache_spec_sliding_window: int | None = None
|
||||
locality: str | None = None
|
||||
|
||||
|
||||
def _make_block_stored(
|
||||
group_idx: int | None = None,
|
||||
kv_cache_spec_sliding_window: int | None = None,
|
||||
locality: str | None = None,
|
||||
session_id: str | None = None,
|
||||
) -> BlockStored:
|
||||
return BlockStored(
|
||||
block_hashes=[_FAKE_HASH],
|
||||
@@ -62,6 +85,7 @@ def _make_block_stored(
|
||||
group_idx=group_idx,
|
||||
kv_cache_spec_sliding_window=kv_cache_spec_sliding_window,
|
||||
locality=locality,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
|
||||
@@ -105,6 +129,12 @@ def test_block_stored_hash_same_for_equal_group_idx():
|
||||
assert hash(event_a) == hash(event_b)
|
||||
|
||||
|
||||
def test_block_stored_hash_differs_by_session_id():
|
||||
event_a = _make_block_stored(session_id="session-a")
|
||||
event_b = _make_block_stored(session_id="session-b")
|
||||
assert hash(event_a) != hash(event_b)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group_idx", [1, 2, 3])
|
||||
def test_block_removed_hash_differs_by_group_idx(group_idx: int):
|
||||
"""BlockRemoved events that differ only in group_idx must hash differently."""
|
||||
@@ -183,3 +213,38 @@ def test_block_removed_locality_is_wire_compatible():
|
||||
new_payload = msgspec.msgpack.encode(_make_block_removed(locality="REMOTE"))
|
||||
assert msgspec.msgpack.decode(new_payload)["locality"] == "REMOTE"
|
||||
assert msgspec.msgpack.decode(new_payload, type=_LegacyBlockRemoved).medium == "GPU"
|
||||
|
||||
|
||||
def test_block_stored_session_id_is_wire_compatible():
|
||||
pre_session = _PreSessionBlockStored(
|
||||
block_hashes=[_FAKE_HASH],
|
||||
parent_block_hash=None,
|
||||
token_ids=[1, 2, 3, 4],
|
||||
block_size=4,
|
||||
lora_id=None,
|
||||
medium="GPU",
|
||||
lora_name=None,
|
||||
group_idx=2,
|
||||
kv_cache_spec_sliding_window=128,
|
||||
locality="LOCAL",
|
||||
)
|
||||
pre_session_payload = msgspec.msgpack.encode(pre_session)
|
||||
assert (
|
||||
msgspec.msgpack.encode(
|
||||
_make_block_stored(
|
||||
group_idx=2,
|
||||
kv_cache_spec_sliding_window=128,
|
||||
locality="LOCAL",
|
||||
)
|
||||
)
|
||||
== pre_session_payload
|
||||
)
|
||||
assert (
|
||||
msgspec.msgpack.decode(pre_session_payload, type=BlockStored).session_id is None
|
||||
)
|
||||
|
||||
new_payload = msgspec.msgpack.encode(_make_block_stored(session_id="session-1"))
|
||||
assert msgspec.msgpack.decode(new_payload)["session_id"] == "session-1"
|
||||
assert (
|
||||
msgspec.msgpack.decode(new_payload, type=_PreSessionBlockStored).medium == "GPU"
|
||||
)
|
||||
|
||||
@@ -33,6 +33,7 @@ def make_request(
|
||||
prompt_token_ids: list[int],
|
||||
hash_block_size: int,
|
||||
hash_fn: Callable,
|
||||
session_id: str | None = None,
|
||||
) -> Request:
|
||||
sampling_params = SamplingParams(max_tokens=17)
|
||||
sampling_params.update_from_generation_config({}, eos_token_id=100)
|
||||
@@ -42,6 +43,7 @@ def make_request(
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
block_hasher=get_request_block_hasher(hash_block_size, hash_fn),
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
|
||||
@@ -123,6 +125,7 @@ def test_cache_partial_block_kv_cache_events():
|
||||
prompt_token_ids=list(range(hash_block_size * 2)),
|
||||
hash_block_size=hash_block_size,
|
||||
hash_fn=sha256,
|
||||
session_id="agent-session-partial",
|
||||
)
|
||||
|
||||
block = pool.get_new_blocks(1)[0]
|
||||
@@ -148,6 +151,7 @@ def test_cache_partial_block_kv_cache_events():
|
||||
assert stored_event.token_ids == req.all_token_ids[hash_block_size:]
|
||||
assert stored_event.block_size == 4
|
||||
assert stored_event.group_idx == kv_cache_group_id
|
||||
assert stored_event.session_id == "agent-session-partial"
|
||||
|
||||
duplicate_entry_hash = pool.cache_partial_block(
|
||||
request=req,
|
||||
|
||||
@@ -74,6 +74,7 @@ def make_request(
|
||||
prompt_logprobs: int | None = None,
|
||||
cache_salt: str | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
session_id: str | None = None,
|
||||
):
|
||||
mm_features = []
|
||||
if mm_positions is not None:
|
||||
@@ -99,6 +100,7 @@ def make_request(
|
||||
lora_request=lora_request,
|
||||
cache_salt=cache_salt,
|
||||
block_hasher=get_request_block_hasher(block_size, hash_fn),
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
|
||||
@@ -2378,6 +2380,71 @@ def test_block_stored_event_group_idx(group_id: int):
|
||||
)
|
||||
|
||||
|
||||
def test_block_stored_event_session_id():
|
||||
block_size = 4
|
||||
session_id = "agent-session-1"
|
||||
manager = make_kv_cache_manager(
|
||||
make_kv_cache_config(block_size, num_blocks=5),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
enable_kv_cache_events=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
req = make_request(
|
||||
"req_session_id",
|
||||
prompt_token_ids=list(range(block_size * 2)),
|
||||
block_size=block_size,
|
||||
hash_fn=sha256,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
manager.allocate_slots(req, req.num_tokens)
|
||||
events = manager.take_events()
|
||||
|
||||
assert len(events) == 1
|
||||
assert isinstance(events[0], BlockStored)
|
||||
assert events[0].session_id == session_id
|
||||
|
||||
|
||||
def test_session_id_does_not_affect_prefix_cache_identity():
|
||||
block_size = 4
|
||||
prompt_token_ids = list(range(block_size * 2 + 1))
|
||||
manager = make_kv_cache_manager(
|
||||
make_kv_cache_config(block_size, num_blocks=8),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
first_req = make_request(
|
||||
"req_session_a",
|
||||
prompt_token_ids,
|
||||
block_size,
|
||||
sha256,
|
||||
session_id="session-a",
|
||||
)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(first_req)
|
||||
allocated_blocks = manager.allocate_slots(
|
||||
first_req,
|
||||
first_req.num_tokens,
|
||||
num_computed_tokens,
|
||||
computed_blocks,
|
||||
)
|
||||
assert allocated_blocks is not None
|
||||
|
||||
second_req = make_request(
|
||||
"req_session_b",
|
||||
prompt_token_ids,
|
||||
block_size,
|
||||
sha256,
|
||||
session_id="session-b",
|
||||
)
|
||||
assert second_req.block_hashes == first_req.block_hashes
|
||||
|
||||
cached_blocks, num_cached_tokens, _ = manager.get_computed_blocks(second_req)
|
||||
assert num_cached_tokens == block_size * 2
|
||||
assert cached_blocks.get_block_ids() == (allocated_blocks.get_block_ids()[0][:2],)
|
||||
|
||||
|
||||
def test_block_stored_event_group_idx_multiple_groups():
|
||||
"""
|
||||
Test BlockStored events for separate HMA groups that each carry the
|
||||
@@ -2574,6 +2641,7 @@ def test_emit_cached_block_events():
|
||||
prompt_token_ids=list(range(num_tokens)),
|
||||
block_size=block_size,
|
||||
hash_fn=sha256,
|
||||
session_id="agent-session-reuse",
|
||||
)
|
||||
assert len(req.block_hashes) >= num_cached_blocks
|
||||
|
||||
@@ -2611,6 +2679,7 @@ def test_emit_cached_block_events():
|
||||
assert event.medium == MEDIUM_GPU
|
||||
assert event.lora_id is None
|
||||
assert event.lora_name is None
|
||||
assert event.session_id == "agent-session-reuse"
|
||||
|
||||
|
||||
def test_emit_cached_block_events_disabled():
|
||||
|
||||
@@ -78,6 +78,12 @@ class BlockStored(KVCacheEvent):
|
||||
locality: str | None = None
|
||||
"""Secondary offloading tier identifier, if generated by one."""
|
||||
ownership: str | None = None
|
||||
"""Session that triggered this store or reuse report.
|
||||
|
||||
This identifies the request context that emitted the event, not exclusive
|
||||
ownership of the underlying block, which may be shared across sessions.
|
||||
"""
|
||||
session_id: str | None = None
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(
|
||||
@@ -94,6 +100,7 @@ class BlockStored(KVCacheEvent):
|
||||
self.kv_cache_spec_sliding_window,
|
||||
self.locality,
|
||||
self.ownership,
|
||||
self.session_id,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -368,6 +368,7 @@ class BlockPool:
|
||||
lora_name=request.lora_request.name if request.lora_request else None,
|
||||
extra_keys=extra_keys_list if extra_keys_list else None,
|
||||
group_idx=kv_cache_group_id,
|
||||
session_id=request.session_id,
|
||||
)
|
||||
|
||||
def emit_cached_block_events(
|
||||
@@ -539,6 +540,7 @@ class BlockPool:
|
||||
else None,
|
||||
extra_keys=[extra_keys],
|
||||
group_idx=kv_cache_group_id,
|
||||
session_id=request.session_id,
|
||||
)
|
||||
)
|
||||
return block_hash_with_group_id
|
||||
|
||||
Reference in New Issue
Block a user