From 42919b49c573aa0b06710616a3b9785674f36ef5 Mon Sep 17 00:00:00 2001 From: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Date: Wed, 16 Sep 2026 11:23:31 -0400 Subject: [PATCH] [P/D] Report prefill worker cache hits in prompt_tokens_details (#54222) Signed-off-by: Robert Shaw Co-authored-by: Claude --- tests/v1/engine/test_output_processor.py | 54 ++++++++++++++++++- .../kv_connector/unit/test_nixl_connector.py | 53 ++++++++++++++++++ vllm/v1/core/sched/scheduler.py | 11 ++++ vllm/v1/engine/output_processor.py | 16 ++++++ 4 files changed, 133 insertions(+), 1 deletion(-) diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index f578aae7f0..64ffc6b266 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -23,6 +23,7 @@ from vllm.tokenizers import TokenizerLike from vllm.v1.engine import ( EngineCoreEvent, EngineCoreEventType, + EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest, FinishReason, @@ -32,7 +33,7 @@ from vllm.v1.engine.output_processor import ( RequestOutputCollector, RequestState, ) -from vllm.v1.metrics.stats import IterationStats, SchedulerStats +from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats @pytest.mark.parametrize("flat_logprobs", [False, True]) @@ -171,6 +172,57 @@ def test_incremental_detokenization( assert not output_processor.has_unfinished_requests() +@pytest.mark.parametrize("do_remote_prefill", [True, False]) +def test_remote_prefill_cached_tokens_override(do_remote_prefill: bool): + """P/D disaggregation: num_cached_tokens should report the P worker's + cache hits (passed via kv_transfer_params) instead of the local count, + which sees the KVs pulled from the remote prefill as a ~100% hit.""" + output_processor = OutputProcessor(tokenizer=None, log_stats=False) + + prompt_tokens = [1, 2, 3, 4, 5, 6, 7, 8] + kv_transfer_params = { + "do_remote_prefill": do_remote_prefill, + "remote_prefill_cached_tokens": 5, + } + request = EngineCoreRequest( + request_id="request-0-int", + external_req_id="request-0", + prompt_token_ids=prompt_tokens, + mm_features=None, + arrival_time=0, + lora_request=None, + cache_salt=None, + data_parallel_rank=None, + sampling_params=SamplingParams( + detokenize=False, + extra_args={"kv_transfer_params": kv_transfer_params}, + ), + pooling_params=None, + ) + output_processor.add_request(request, prompt=None) + + prefill_stats = PrefillStats() + prefill_stats.set( + num_prompt_tokens=len(prompt_tokens), + num_local_cached_tokens=0, + num_external_cached_tokens=len(prompt_tokens) - 1, + ) + processed = output_processor.process_outputs( + [ + EngineCoreOutput( + request_id="request-0-int", + new_token_ids=[42], + prefill_stats=prefill_stats, + ) + ] + ) + request_output = processed.request_outputs[0] + if do_remote_prefill: + assert request_output.num_cached_tokens == 5 + else: + assert request_output.num_cached_tokens == len(prompt_tokens) - 1 + + def test_request_stream_interval_raises_but_not_below_engine_default( dummy_test_vectors, ): diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 39fb0bb54f..e41bd2644a 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -74,6 +74,7 @@ from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput from vllm.v1.request import RequestStatus from .utils import ( + create_model_runner_output, create_request, create_scheduler, create_vllm_config, @@ -348,6 +349,58 @@ def test_abort_immediately_remote_prefill_enqueues_empty_recv(): assert req_meta.awaiting_kvs is False +def test_prefill_exports_cached_tokens_in_kv_transfer_params(): + """The P worker reports its own prefix-cache hits in the returned + kv_transfer_params so the D worker can surface them in + prompt_tokens_details instead of the ~100% local hit it measures + when pulling the KVs from the remote.""" + vllm_config = create_vllm_config() + scheduler = create_scheduler(vllm_config) + + BLOCK_SIZE = vllm_config.cache_config.block_size + NUM_TOKENS = BLOCK_SIZE * 3 + + # Warm the prefix cache with a request sharing the full prompt. + warmup = create_request( + request_id=1, + num_tokens=NUM_TOKENS, + common_prefix_len=NUM_TOKENS, + block_size=BLOCK_SIZE, + ) + scheduler.add_request(warmup) + scheduler_output = scheduler.schedule() + model_runner_output = create_model_runner_output([warmup], use_eos=True) + scheduler.update_from_output(scheduler_output, model_runner_output) + + # P-side request with the same prompt hits the local cache on all but + # the last block (recomputed to obtain logits). + request = create_request( + request_id=2, + num_tokens=NUM_TOKENS, + common_prefix_len=NUM_TOKENS, + block_size=BLOCK_SIZE, + do_remote_decode=True, + ) + scheduler.add_request(request) + scheduler_output = scheduler.schedule() + assert request.prefill_stats is not None + assert request.prefill_stats.num_cached_tokens == NUM_TOKENS - BLOCK_SIZE + + # max_tokens=1, so the request finishes and returns kv_transfer_params. + model_runner_output = create_model_runner_output([request]) + engine_core_outputs = scheduler.update_from_output( + scheduler_output, model_runner_output + ) + + output = engine_core_outputs[0].outputs[0] + assert output.finish_reason is not None + assert output.kv_transfer_params is not None + assert ( + output.kv_transfer_params["remote_prefill_cached_tokens"] + == NUM_TOKENS - BLOCK_SIZE + ) + + @patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index c1e956b9ec..b662bcc10b 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -2110,6 +2110,17 @@ class Scheduler(SchedulerInterface): finished = self._handle_stopped_request(request) if finished: kv_transfer_params, ec_transfer_params = self._free_request(request) + if ( + prefill_stats is not None + and kv_transfer_params is not None + and kv_transfer_params.get("do_remote_prefill") + ): + # P-side cache hits, so D can report them in + # prompt_tokens_details instead of its own (~100%) + # hit rate from the KV transfer. + kv_transfer_params["remote_prefill_cached_tokens"] = ( + prefill_stats.num_cached_tokens + ) if status_before_stop == RequestStatus.RUNNING: stopped_running_reqs.add(request) diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index 257287243d..b8fbddf72f 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -155,6 +155,7 @@ class RequestState: n: int | None = None, temperature: float | None = None, stream_input: bool = False, + remote_prefill_cached_tokens: int | None = None, ): self.request_id = request_id self.external_req_id = external_req_id @@ -179,6 +180,7 @@ class RequestState: self.queue = queue self.num_cached_tokens = 0 self.num_cache_creation_tokens = 0 + self.remote_prefill_cached_tokens = remote_prefill_cached_tokens # Per-sequence spec-decode accumulator; arrives once (on finish) via # EngineCoreOutput, then attached to this sequence's CompletionOutput. self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None @@ -230,7 +232,18 @@ class RequestState: log_stats: bool, stream_interval: int, ) -> "RequestState": + remote_prefill_cached_tokens = None if sampling_params := request.sampling_params: + # In a remote prefill scenario, report cached tokens + # as cache hit rate on the remote P worker. + if sampling_params.extra_args: + kv_transfer_params = sampling_params.extra_args.get( + "kv_transfer_params" + ) + if kv_transfer_params and kv_transfer_params.get("do_remote_prefill"): + cached = kv_transfer_params.get("remote_prefill_cached_tokens") + if isinstance(cached, int): + remote_prefill_cached_tokens = cached if not sampling_params.detokenize: tokenizer = None output_kind = sampling_params.output_kind @@ -281,6 +294,7 @@ class RequestState: log_stats=log_stats, stream_interval=stream_interval, stream_input=request.resumable, + remote_prefill_cached_tokens=remote_prefill_cached_tokens, ) def make_request_output( @@ -687,6 +701,8 @@ class OutputProcessor: req_state.num_cache_creation_tokens = ( engine_core_output.prefill_stats.num_cache_creation_tokens ) + if req_state.remote_prefill_cached_tokens is not None: + req_state.num_cached_tokens = req_state.remote_prefill_cached_tokens req_state.is_prefilling = False if engine_core_output.spec_decode_metrics is not None: