From 708c2495c0c6d0fc576224ecfc54efacc6414981 Mon Sep 17 00:00:00 2001 From: Fanli Lin Date: Sun, 20 Sep 2026 10:16:12 +0800 Subject: [PATCH] [Test][Core] Compute the expected hybrid prefix-cache hit instead of hard-coding it (#57467) Signed-off-by: Lin, Fanli --- .buildkite/intel_jobs/test-intel.yaml | 2 +- .../test_hybrid_prefix_cache_hit_rate.py | 62 ++++++++++++++++--- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index 458bef9518..ef2b4ba073 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -98,7 +98,7 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && pip install dpctl && - pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py --ignore=v1/core/prefix_cache/test_hybrid_prefix_cache_hit_rate.py && + pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py && pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py && pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" && pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py && diff --git a/tests/v1/core/prefix_cache/test_hybrid_prefix_cache_hit_rate.py b/tests/v1/core/prefix_cache/test_hybrid_prefix_cache_hit_rate.py index c75591b697..ff62abab8c 100644 --- a/tests/v1/core/prefix_cache/test_hybrid_prefix_cache_hit_rate.py +++ b/tests/v1/core/prefix_cache/test_hybrid_prefix_cache_hit_rate.py @@ -4,6 +4,7 @@ import asyncio import json +from math import lcm from uuid import uuid4 import httpx @@ -12,6 +13,7 @@ from openai.types.chat import ChatCompletionMessageParam from prometheus_client.parser import text_string_to_metric_families from tests.utils import RemoteOpenAIServer +from vllm.utils.math_utils import round_down MODEL = "Qwen/Qwen3.5-0.8B" NUM_CONVERSATIONS = 8 @@ -34,6 +36,29 @@ def _user_turns(conversation: int) -> list[str]: ] +async def _scheduler_block_size(client: httpx.AsyncClient) -> int: + """Return the coarsest boundary an expected cache hit can land on. + + Mirrors ``scheduler_block_size`` in ``resolve_kv_cache_block_sizes``, which + the engine keeps internal: the LCM of the resolved group block sizes. A + finer ``prefix_match_unit`` only adds hits, so this stays conservative. + """ + response = await client.get("/metrics") + response.raise_for_status() + for family in text_string_to_metric_families(response.text): + for sample in family.samples: + if sample.name != "vllm:cache_config_info": + continue + sizes = [ + int(value) + for name in ("block_size", "mamba_block_size") + if (value := sample.labels.get(name, "None")) != "None" + ] + assert sizes, f"no block size in {sample.labels}" + return lcm(*sizes) + raise AssertionError("missing vllm:cache_config_info metric") + + async def _prefix_cache_counters( client: httpx.AsyncClient, ) -> tuple[float, float]: @@ -83,13 +108,13 @@ def server(request): @pytest.mark.asyncio @pytest.mark.parametrize( - "server, min_hit_rate", - # MTP re-prefills an extra block; both floors reject one more missed block. - [pytest.param(False, 0.90, id="base"), pytest.param(True, 0.75, id="mtp")], + "server, dropped_blocks", + # MTP's prefill lookahead pollutes the tail block, so EAGLE drops one. + [pytest.param(False, 0, id="base"), pytest.param(True, 1, id="mtp")], indirect=["server"], ) async def test_prefix_cache_hit_rate( - server: RemoteOpenAIServer, min_hit_rate: float + server: RemoteOpenAIServer, dropped_blocks: int ) -> None: # Isolate sessions so another conversation cannot hide a cache miss. salts = [uuid4().hex for _ in range(NUM_CONVERSATIONS)] @@ -102,6 +127,8 @@ async def test_prefix_cache_hit_rate( server.get_async_client() as client, httpx.AsyncClient(base_url=server.url_root, timeout=METRICS_TIMEOUT) as metrics, ): + block_size = await _scheduler_block_size(metrics) + print(f"{block_size=}") async def run_conversation(session: int, *, replay: bool) -> tuple[int, int]: history = histories[session] @@ -126,15 +153,30 @@ async def test_prefix_cache_hit_rate( assert cached is not None, context assert 0 <= cached < prompt, context assert prompt >= MIN_PROMPT_TOKENS, context - if not replay and turn == 0: - assert cached == 0, context - else: - assert cached / prompt >= min_hit_rate, ( - f"{context}, {min_hit_rate=}" - ) if replay: assert prompt == prompt_lengths[session][turn], context + # The identical request computed this whole prompt, but at + # least one token is always recomputed. + computed = prompt - 1 + elif turn: + computed = prompt_lengths[session][turn - 1] else: + assert cached == 0, context + computed = 0 + # Every token below the last reachable boundary must be reused. + floor = max( + 0, round_down(computed, block_size) - dropped_blocks * block_size + ) + print( + f"{session=} {turn=} {replay=} {prompt=} {cached=} {floor=} " + f"hit_rate={cached / prompt:.4f}" + ) + assert cached >= floor, ( + f"{context}, {computed=}, {block_size=}, " + f"hit_rate={cached / prompt:.4f}, " + f"expected_hit_rate={floor / prompt:.4f}" + ) + if not replay: prompt_lengths[session].append(prompt) history.append( {