[P/D] Report prefill worker cache hits in prompt_tokens_details (#54222)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -155,6 +155,7 @@ class RequestState:
|
||||
n: int | None = None,
|
||||
temperature: float | None = None,
|
||||
stream_input: bool = False,
|
||||
remote_prefill_cached_tokens: int | None = None,
|
||||
):
|
||||
self.request_id = request_id
|
||||
self.external_req_id = external_req_id
|
||||
@@ -179,6 +180,7 @@ class RequestState:
|
||||
self.queue = queue
|
||||
self.num_cached_tokens = 0
|
||||
self.num_cache_creation_tokens = 0
|
||||
self.remote_prefill_cached_tokens = remote_prefill_cached_tokens
|
||||
# Per-sequence spec-decode accumulator; arrives once (on finish) via
|
||||
# EngineCoreOutput, then attached to this sequence's CompletionOutput.
|
||||
self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None
|
||||
@@ -230,7 +232,18 @@ class RequestState:
|
||||
log_stats: bool,
|
||||
stream_interval: int,
|
||||
) -> "RequestState":
|
||||
remote_prefill_cached_tokens = None
|
||||
if sampling_params := request.sampling_params:
|
||||
# In a remote prefill scenario, report cached tokens
|
||||
# as cache hit rate on the remote P worker.
|
||||
if sampling_params.extra_args:
|
||||
kv_transfer_params = sampling_params.extra_args.get(
|
||||
"kv_transfer_params"
|
||||
)
|
||||
if kv_transfer_params and kv_transfer_params.get("do_remote_prefill"):
|
||||
cached = kv_transfer_params.get("remote_prefill_cached_tokens")
|
||||
if isinstance(cached, int):
|
||||
remote_prefill_cached_tokens = cached
|
||||
if not sampling_params.detokenize:
|
||||
tokenizer = None
|
||||
output_kind = sampling_params.output_kind
|
||||
@@ -281,6 +294,7 @@ class RequestState:
|
||||
log_stats=log_stats,
|
||||
stream_interval=stream_interval,
|
||||
stream_input=request.resumable,
|
||||
remote_prefill_cached_tokens=remote_prefill_cached_tokens,
|
||||
)
|
||||
|
||||
def make_request_output(
|
||||
@@ -687,6 +701,8 @@ class OutputProcessor:
|
||||
req_state.num_cache_creation_tokens = (
|
||||
engine_core_output.prefill_stats.num_cache_creation_tokens
|
||||
)
|
||||
if req_state.remote_prefill_cached_tokens is not None:
|
||||
req_state.num_cached_tokens = req_state.remote_prefill_cached_tokens
|
||||
req_state.is_prefilling = False
|
||||
|
||||
if engine_core_output.spec_decode_metrics is not None:
|
||||
|
||||
Reference in New Issue
Block a user