[P/D] Report prefill worker cache hits in prompt_tokens_details (#54222)

Signed-off-by: Robert Shaw <robshaw@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Robert Shaw
2026-09-16 15:23:31 +00:00
committed by GitHub
co-authored by Claude
parent dff1bde84d
commit 42919b49c5
4 changed files with 133 additions and 1 deletions
+16
View File
@@ -155,6 +155,7 @@ class RequestState:
n: int | None = None,
temperature: float | None = None,
stream_input: bool = False,
remote_prefill_cached_tokens: int | None = None,
):
self.request_id = request_id
self.external_req_id = external_req_id
@@ -179,6 +180,7 @@ class RequestState:
self.queue = queue
self.num_cached_tokens = 0
self.num_cache_creation_tokens = 0
self.remote_prefill_cached_tokens = remote_prefill_cached_tokens
# Per-sequence spec-decode accumulator; arrives once (on finish) via
# EngineCoreOutput, then attached to this sequence's CompletionOutput.
self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None
@@ -230,7 +232,18 @@ class RequestState:
log_stats: bool,
stream_interval: int,
) -> "RequestState":
remote_prefill_cached_tokens = None
if sampling_params := request.sampling_params:
# In a remote prefill scenario, report cached tokens
# as cache hit rate on the remote P worker.
if sampling_params.extra_args:
kv_transfer_params = sampling_params.extra_args.get(
"kv_transfer_params"
)
if kv_transfer_params and kv_transfer_params.get("do_remote_prefill"):
cached = kv_transfer_params.get("remote_prefill_cached_tokens")
if isinstance(cached, int):
remote_prefill_cached_tokens = cached
if not sampling_params.detokenize:
tokenizer = None
output_kind = sampling_params.output_kind
@@ -281,6 +294,7 @@ class RequestState:
log_stats=log_stats,
stream_interval=stream_interval,
stream_input=request.resumable,
remote_prefill_cached_tokens=remote_prefill_cached_tokens,
)
def make_request_output(
@@ -687,6 +701,8 @@ class OutputProcessor:
req_state.num_cache_creation_tokens = (
engine_core_output.prefill_stats.num_cache_creation_tokens
)
if req_state.remote_prefill_cached_tokens is not None:
req_state.num_cached_tokens = req_state.remote_prefill_cached_tokens
req_state.is_prefilling = False
if engine_core_output.spec_decode_metrics is not None: