[Metrics] [KVConnector] Add connector prefix cache hit rate stats (#26245)
Signed-off-by: tovam <tovam@pliops.com>
This commit is contained in:
@@ -1014,6 +1014,66 @@ def test_kv_connector_basic():
|
||||
)
|
||||
|
||||
|
||||
def test_external_prefix_cache_metrics():
|
||||
"""
|
||||
Verify connector prefix cache metrics are updated
|
||||
correctly when the scheduler processes requests with KV connector hits.
|
||||
"""
|
||||
|
||||
# Setup Scheduler.
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=False,
|
||||
use_kv_connector=True,
|
||||
)
|
||||
|
||||
# Mock connector to simulate a partial external cache hit
|
||||
NUM_MATCHED_NEW_TOKENS = 4
|
||||
scheduler.connector.get_num_new_matched_tokens = Mock(name="method")
|
||||
scheduler.connector.get_num_new_matched_tokens.return_value = (
|
||||
NUM_MATCHED_NEW_TOKENS,
|
||||
False,
|
||||
)
|
||||
|
||||
# --- Prepare simple requests ---
|
||||
NUM_REQUESTS = 2
|
||||
NUM_TOKENS = 8
|
||||
MAX_TOKENS = 2
|
||||
requests = create_requests(
|
||||
num_requests=NUM_REQUESTS,
|
||||
num_tokens=NUM_TOKENS,
|
||||
max_tokens=MAX_TOKENS,
|
||||
)
|
||||
|
||||
for req in requests:
|
||||
scheduler.add_request(req)
|
||||
|
||||
# --- Trigger scheduling and simulate model output ---
|
||||
output = scheduler.schedule()
|
||||
MODEL_RUNNER_OUTPUT = ModelRunnerOutput(
|
||||
req_ids=[r.request_id for r in requests],
|
||||
req_id_to_index={r.request_id: i for i, r in enumerate(requests)},
|
||||
sampled_token_ids=[[1000]] * NUM_REQUESTS,
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
)
|
||||
|
||||
# Update scheduler stats
|
||||
ecos = scheduler.update_from_output(output, MODEL_RUNNER_OUTPUT)
|
||||
|
||||
# --- Assertions ---
|
||||
assert ecos is not None and len(ecos) > 0
|
||||
assert ecos[0].scheduler_stats is not None
|
||||
|
||||
external_stats = ecos[0].scheduler_stats.connector_prefix_cache_stats
|
||||
assert external_stats is not None
|
||||
|
||||
assert external_stats.queries == NUM_TOKENS * NUM_REQUESTS
|
||||
assert external_stats.hits == NUM_MATCHED_NEW_TOKENS * NUM_REQUESTS
|
||||
assert external_stats.requests == NUM_REQUESTS
|
||||
assert external_stats.preempted_requests == 0
|
||||
|
||||
|
||||
def test_kv_connector_unable_to_allocate():
|
||||
"""
|
||||
Test whether scheduler with KVConnector is able to handle
|
||||
|
||||
Reference in New Issue
Block a user