[P/D] Report prefill worker cache hits in prompt_tokens_details (#54222)

Signed-off-by: Robert Shaw <robshaw@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Robert Shaw
2026-09-16 15:23:31 +00:00
committed by GitHub
co-authored by Claude
parent dff1bde84d
commit 42919b49c5
4 changed files with 133 additions and 1 deletions
+53 -1
View File
@@ -23,6 +23,7 @@ from vllm.tokenizers import TokenizerLike
from vllm.v1.engine import (
EngineCoreEvent,
EngineCoreEventType,
EngineCoreOutput,
EngineCoreOutputs,
EngineCoreRequest,
FinishReason,
@@ -32,7 +33,7 @@ from vllm.v1.engine.output_processor import (
RequestOutputCollector,
RequestState,
)
from vllm.v1.metrics.stats import IterationStats, SchedulerStats
from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats
@pytest.mark.parametrize("flat_logprobs", [False, True])
@@ -171,6 +172,57 @@ def test_incremental_detokenization(
assert not output_processor.has_unfinished_requests()
@pytest.mark.parametrize("do_remote_prefill", [True, False])
def test_remote_prefill_cached_tokens_override(do_remote_prefill: bool):
"""P/D disaggregation: num_cached_tokens should report the P worker's
cache hits (passed via kv_transfer_params) instead of the local count,
which sees the KVs pulled from the remote prefill as a ~100% hit."""
output_processor = OutputProcessor(tokenizer=None, log_stats=False)
prompt_tokens = [1, 2, 3, 4, 5, 6, 7, 8]
kv_transfer_params = {
"do_remote_prefill": do_remote_prefill,
"remote_prefill_cached_tokens": 5,
}
request = EngineCoreRequest(
request_id="request-0-int",
external_req_id="request-0",
prompt_token_ids=prompt_tokens,
mm_features=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=SamplingParams(
detokenize=False,
extra_args={"kv_transfer_params": kv_transfer_params},
),
pooling_params=None,
)
output_processor.add_request(request, prompt=None)
prefill_stats = PrefillStats()
prefill_stats.set(
num_prompt_tokens=len(prompt_tokens),
num_local_cached_tokens=0,
num_external_cached_tokens=len(prompt_tokens) - 1,
)
processed = output_processor.process_outputs(
[
EngineCoreOutput(
request_id="request-0-int",
new_token_ids=[42],
prefill_stats=prefill_stats,
)
]
)
request_output = processed.request_outputs[0]
if do_remote_prefill:
assert request_output.num_cached_tokens == 5
else:
assert request_output.num_cached_tokens == len(prompt_tokens) - 1
def test_request_stream_interval_raises_but_not_below_engine_default(
dummy_test_vectors,
):
@@ -74,6 +74,7 @@ from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import RequestStatus
from .utils import (
create_model_runner_output,
create_request,
create_scheduler,
create_vllm_config,
@@ -348,6 +349,58 @@ def test_abort_immediately_remote_prefill_enqueues_empty_recv():
assert req_meta.awaiting_kvs is False
def test_prefill_exports_cached_tokens_in_kv_transfer_params():
"""The P worker reports its own prefix-cache hits in the returned
kv_transfer_params so the D worker can surface them in
prompt_tokens_details instead of the ~100% local hit it measures
when pulling the KVs from the remote."""
vllm_config = create_vllm_config()
scheduler = create_scheduler(vllm_config)
BLOCK_SIZE = vllm_config.cache_config.block_size
NUM_TOKENS = BLOCK_SIZE * 3
# Warm the prefix cache with a request sharing the full prompt.
warmup = create_request(
request_id=1,
num_tokens=NUM_TOKENS,
common_prefix_len=NUM_TOKENS,
block_size=BLOCK_SIZE,
)
scheduler.add_request(warmup)
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output([warmup], use_eos=True)
scheduler.update_from_output(scheduler_output, model_runner_output)
# P-side request with the same prompt hits the local cache on all but
# the last block (recomputed to obtain logits).
request = create_request(
request_id=2,
num_tokens=NUM_TOKENS,
common_prefix_len=NUM_TOKENS,
block_size=BLOCK_SIZE,
do_remote_decode=True,
)
scheduler.add_request(request)
scheduler_output = scheduler.schedule()
assert request.prefill_stats is not None
assert request.prefill_stats.num_cached_tokens == NUM_TOKENS - BLOCK_SIZE
# max_tokens=1, so the request finishes and returns kv_transfer_params.
model_runner_output = create_model_runner_output([request])
engine_core_outputs = scheduler.update_from_output(
scheduler_output, model_runner_output
)
output = engine_core_outputs[0].outputs[0]
assert output.finish_reason is not None
assert output.kv_transfer_params is not None
assert (
output.kv_transfer_params["remote_prefill_cached_tokens"]
== NUM_TOKENS - BLOCK_SIZE
)
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
+11
View File
@@ -2110,6 +2110,17 @@ class Scheduler(SchedulerInterface):
finished = self._handle_stopped_request(request)
if finished:
kv_transfer_params, ec_transfer_params = self._free_request(request)
if (
prefill_stats is not None
and kv_transfer_params is not None
and kv_transfer_params.get("do_remote_prefill")
):
# P-side cache hits, so D can report them in
# prompt_tokens_details instead of its own (~100%)
# hit rate from the KV transfer.
kv_transfer_params["remote_prefill_cached_tokens"] = (
prefill_stats.num_cached_tokens
)
if status_before_stop == RequestStatus.RUNNING:
stopped_running_reqs.add(request)
+16
View File
@@ -155,6 +155,7 @@ class RequestState:
n: int | None = None,
temperature: float | None = None,
stream_input: bool = False,
remote_prefill_cached_tokens: int | None = None,
):
self.request_id = request_id
self.external_req_id = external_req_id
@@ -179,6 +180,7 @@ class RequestState:
self.queue = queue
self.num_cached_tokens = 0
self.num_cache_creation_tokens = 0
self.remote_prefill_cached_tokens = remote_prefill_cached_tokens
# Per-sequence spec-decode accumulator; arrives once (on finish) via
# EngineCoreOutput, then attached to this sequence's CompletionOutput.
self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None
@@ -230,7 +232,18 @@ class RequestState:
log_stats: bool,
stream_interval: int,
) -> "RequestState":
remote_prefill_cached_tokens = None
if sampling_params := request.sampling_params:
# In a remote prefill scenario, report cached tokens
# as cache hit rate on the remote P worker.
if sampling_params.extra_args:
kv_transfer_params = sampling_params.extra_args.get(
"kv_transfer_params"
)
if kv_transfer_params and kv_transfer_params.get("do_remote_prefill"):
cached = kv_transfer_params.get("remote_prefill_cached_tokens")
if isinstance(cached, int):
remote_prefill_cached_tokens = cached
if not sampling_params.detokenize:
tokenizer = None
output_kind = sampling_params.output_kind
@@ -281,6 +294,7 @@ class RequestState:
log_stats=log_stats,
stream_interval=stream_interval,
stream_input=request.resumable,
remote_prefill_cached_tokens=remote_prefill_cached_tokens,
)
def make_request_output(
@@ -687,6 +701,8 @@ class OutputProcessor:
req_state.num_cache_creation_tokens = (
engine_core_output.prefill_stats.num_cache_creation_tokens
)
if req_state.remote_prefill_cached_tokens is not None:
req_state.num_cached_tokens = req_state.remote_prefill_cached_tokens
req_state.is_prefilling = False
if engine_core_output.spec_decode_metrics is not None: