[P/D] Report prefill worker cache hits in prompt_tokens_details (#54222)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -23,6 +23,7 @@ from vllm.tokenizers import TokenizerLike
|
||||
from vllm.v1.engine import (
|
||||
EngineCoreEvent,
|
||||
EngineCoreEventType,
|
||||
EngineCoreOutput,
|
||||
EngineCoreOutputs,
|
||||
EngineCoreRequest,
|
||||
FinishReason,
|
||||
@@ -32,7 +33,7 @@ from vllm.v1.engine.output_processor import (
|
||||
RequestOutputCollector,
|
||||
RequestState,
|
||||
)
|
||||
from vllm.v1.metrics.stats import IterationStats, SchedulerStats
|
||||
from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flat_logprobs", [False, True])
|
||||
@@ -171,6 +172,57 @@ def test_incremental_detokenization(
|
||||
assert not output_processor.has_unfinished_requests()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("do_remote_prefill", [True, False])
|
||||
def test_remote_prefill_cached_tokens_override(do_remote_prefill: bool):
|
||||
"""P/D disaggregation: num_cached_tokens should report the P worker's
|
||||
cache hits (passed via kv_transfer_params) instead of the local count,
|
||||
which sees the KVs pulled from the remote prefill as a ~100% hit."""
|
||||
output_processor = OutputProcessor(tokenizer=None, log_stats=False)
|
||||
|
||||
prompt_tokens = [1, 2, 3, 4, 5, 6, 7, 8]
|
||||
kv_transfer_params = {
|
||||
"do_remote_prefill": do_remote_prefill,
|
||||
"remote_prefill_cached_tokens": 5,
|
||||
}
|
||||
request = EngineCoreRequest(
|
||||
request_id="request-0-int",
|
||||
external_req_id="request-0",
|
||||
prompt_token_ids=prompt_tokens,
|
||||
mm_features=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
sampling_params=SamplingParams(
|
||||
detokenize=False,
|
||||
extra_args={"kv_transfer_params": kv_transfer_params},
|
||||
),
|
||||
pooling_params=None,
|
||||
)
|
||||
output_processor.add_request(request, prompt=None)
|
||||
|
||||
prefill_stats = PrefillStats()
|
||||
prefill_stats.set(
|
||||
num_prompt_tokens=len(prompt_tokens),
|
||||
num_local_cached_tokens=0,
|
||||
num_external_cached_tokens=len(prompt_tokens) - 1,
|
||||
)
|
||||
processed = output_processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id="request-0-int",
|
||||
new_token_ids=[42],
|
||||
prefill_stats=prefill_stats,
|
||||
)
|
||||
]
|
||||
)
|
||||
request_output = processed.request_outputs[0]
|
||||
if do_remote_prefill:
|
||||
assert request_output.num_cached_tokens == 5
|
||||
else:
|
||||
assert request_output.num_cached_tokens == len(prompt_tokens) - 1
|
||||
|
||||
|
||||
def test_request_stream_interval_raises_but_not_below_engine_default(
|
||||
dummy_test_vectors,
|
||||
):
|
||||
|
||||
@@ -74,6 +74,7 @@ from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
from .utils import (
|
||||
create_model_runner_output,
|
||||
create_request,
|
||||
create_scheduler,
|
||||
create_vllm_config,
|
||||
@@ -348,6 +349,58 @@ def test_abort_immediately_remote_prefill_enqueues_empty_recv():
|
||||
assert req_meta.awaiting_kvs is False
|
||||
|
||||
|
||||
def test_prefill_exports_cached_tokens_in_kv_transfer_params():
|
||||
"""The P worker reports its own prefix-cache hits in the returned
|
||||
kv_transfer_params so the D worker can surface them in
|
||||
prompt_tokens_details instead of the ~100% local hit it measures
|
||||
when pulling the KVs from the remote."""
|
||||
vllm_config = create_vllm_config()
|
||||
scheduler = create_scheduler(vllm_config)
|
||||
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
NUM_TOKENS = BLOCK_SIZE * 3
|
||||
|
||||
# Warm the prefix cache with a request sharing the full prompt.
|
||||
warmup = create_request(
|
||||
request_id=1,
|
||||
num_tokens=NUM_TOKENS,
|
||||
common_prefix_len=NUM_TOKENS,
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
scheduler.add_request(warmup)
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output([warmup], use_eos=True)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
|
||||
# P-side request with the same prompt hits the local cache on all but
|
||||
# the last block (recomputed to obtain logits).
|
||||
request = create_request(
|
||||
request_id=2,
|
||||
num_tokens=NUM_TOKENS,
|
||||
common_prefix_len=NUM_TOKENS,
|
||||
block_size=BLOCK_SIZE,
|
||||
do_remote_decode=True,
|
||||
)
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert request.prefill_stats is not None
|
||||
assert request.prefill_stats.num_cached_tokens == NUM_TOKENS - BLOCK_SIZE
|
||||
|
||||
# max_tokens=1, so the request finishes and returns kv_transfer_params.
|
||||
model_runner_output = create_model_runner_output([request])
|
||||
engine_core_outputs = scheduler.update_from_output(
|
||||
scheduler_output, model_runner_output
|
||||
)
|
||||
|
||||
output = engine_core_outputs[0].outputs[0]
|
||||
assert output.finish_reason is not None
|
||||
assert output.kv_transfer_params is not None
|
||||
assert (
|
||||
output.kv_transfer_params["remote_prefill_cached_tokens"]
|
||||
== NUM_TOKENS - BLOCK_SIZE
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
|
||||
@@ -2110,6 +2110,17 @@ class Scheduler(SchedulerInterface):
|
||||
finished = self._handle_stopped_request(request)
|
||||
if finished:
|
||||
kv_transfer_params, ec_transfer_params = self._free_request(request)
|
||||
if (
|
||||
prefill_stats is not None
|
||||
and kv_transfer_params is not None
|
||||
and kv_transfer_params.get("do_remote_prefill")
|
||||
):
|
||||
# P-side cache hits, so D can report them in
|
||||
# prompt_tokens_details instead of its own (~100%)
|
||||
# hit rate from the KV transfer.
|
||||
kv_transfer_params["remote_prefill_cached_tokens"] = (
|
||||
prefill_stats.num_cached_tokens
|
||||
)
|
||||
|
||||
if status_before_stop == RequestStatus.RUNNING:
|
||||
stopped_running_reqs.add(request)
|
||||
|
||||
@@ -155,6 +155,7 @@ class RequestState:
|
||||
n: int | None = None,
|
||||
temperature: float | None = None,
|
||||
stream_input: bool = False,
|
||||
remote_prefill_cached_tokens: int | None = None,
|
||||
):
|
||||
self.request_id = request_id
|
||||
self.external_req_id = external_req_id
|
||||
@@ -179,6 +180,7 @@ class RequestState:
|
||||
self.queue = queue
|
||||
self.num_cached_tokens = 0
|
||||
self.num_cache_creation_tokens = 0
|
||||
self.remote_prefill_cached_tokens = remote_prefill_cached_tokens
|
||||
# Per-sequence spec-decode accumulator; arrives once (on finish) via
|
||||
# EngineCoreOutput, then attached to this sequence's CompletionOutput.
|
||||
self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None
|
||||
@@ -230,7 +232,18 @@ class RequestState:
|
||||
log_stats: bool,
|
||||
stream_interval: int,
|
||||
) -> "RequestState":
|
||||
remote_prefill_cached_tokens = None
|
||||
if sampling_params := request.sampling_params:
|
||||
# In a remote prefill scenario, report cached tokens
|
||||
# as cache hit rate on the remote P worker.
|
||||
if sampling_params.extra_args:
|
||||
kv_transfer_params = sampling_params.extra_args.get(
|
||||
"kv_transfer_params"
|
||||
)
|
||||
if kv_transfer_params and kv_transfer_params.get("do_remote_prefill"):
|
||||
cached = kv_transfer_params.get("remote_prefill_cached_tokens")
|
||||
if isinstance(cached, int):
|
||||
remote_prefill_cached_tokens = cached
|
||||
if not sampling_params.detokenize:
|
||||
tokenizer = None
|
||||
output_kind = sampling_params.output_kind
|
||||
@@ -281,6 +294,7 @@ class RequestState:
|
||||
log_stats=log_stats,
|
||||
stream_interval=stream_interval,
|
||||
stream_input=request.resumable,
|
||||
remote_prefill_cached_tokens=remote_prefill_cached_tokens,
|
||||
)
|
||||
|
||||
def make_request_output(
|
||||
@@ -687,6 +701,8 @@ class OutputProcessor:
|
||||
req_state.num_cache_creation_tokens = (
|
||||
engine_core_output.prefill_stats.num_cache_creation_tokens
|
||||
)
|
||||
if req_state.remote_prefill_cached_tokens is not None:
|
||||
req_state.num_cached_tokens = req_state.remote_prefill_cached_tokens
|
||||
req_state.is_prefilling = False
|
||||
|
||||
if engine_core_output.spec_decode_metrics is not None:
|
||||
|
||||
Reference in New Issue
Block a user