[Core][Metrics][BugFix] Replace num_cached_tokens/num_external_computed_tokens with PrefillStats (#37460)

Related to `Counters can only be incremented by non-negative amounts`
error with the `vllm:prompt_tokens_by_source_total` metric.

Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Co-authored-by: Or Ozeri <or@ozery.com>
This commit is contained in:
Mark McLoughlin
2026-04-14 09:00:45 +01:00
committed by GitHub
co-authored by Or Ozeri
parent 25b3242d8b
commit d3af8c1831
9 changed files with 187 additions and 102 deletions
+11 -12
View File
@@ -629,7 +629,6 @@ class Scheduler(SchedulerInterface):
step_skipped_waiting.prepend_request(request)
continue
request.num_external_computed_tokens = ext_tokens
num_external_computed_tokens = ext_tokens
connector_prefix_cache_queries = (
@@ -642,6 +641,15 @@ class Scheduler(SchedulerInterface):
num_new_local_computed_tokens + num_external_computed_tokens
)
assert num_computed_tokens <= request.num_tokens
# Track first scheduled prefill, not post-preemption repeat prefills
if request.prefill_stats is not None:
assert num_computed_tokens <= request.num_prompt_tokens
request.prefill_stats.set(
num_prompt_tokens=request.num_prompt_tokens,
num_local_cached_tokens=num_new_local_computed_tokens,
num_external_cached_tokens=num_external_computed_tokens,
)
else:
# KVTransfer: WAITING reqs have num_computed_tokens > 0
# after async KV recvs are completed.
@@ -826,9 +834,6 @@ class Scheduler(SchedulerInterface):
token_budget -= num_new_tokens
request.status = RequestStatus.RUNNING
request.num_computed_tokens = num_computed_tokens
# Count the number of prefix cached tokens.
if request.num_cached_tokens < 0:
request.num_cached_tokens = num_computed_tokens
# Encoder-related.
if encoder_inputs_to_schedule:
scheduled_encoder_inputs[request_id] = encoder_inputs_to_schedule
@@ -1466,10 +1471,9 @@ class Scheduler(SchedulerInterface):
pooling_output=pooler_output,
stop_reason=request.stop_reason,
events=request.take_events(),
prefill_stats=request.take_prefill_stats(),
kv_transfer_params=kv_transfer_params,
trace_headers=request.trace_headers,
num_cached_tokens=request.num_cached_tokens,
num_external_computed_tokens=request.num_external_computed_tokens,
routed_experts=routed_experts,
num_nans_in_logits=request.num_nans_in_logits,
)
@@ -1496,7 +1500,6 @@ class Scheduler(SchedulerInterface):
finish_reason=request.get_finished_reason(),
events=request.take_events(),
trace_headers=request.trace_headers,
num_cached_tokens=request.num_cached_tokens,
)
)
@@ -2071,10 +2074,6 @@ class Scheduler(SchedulerInterface):
if request.num_computed_tokens == request.num_tokens:
request.num_computed_tokens = request.num_tokens - 1
# Count the number of prefix cached tokens.
if request.num_cached_tokens < 0:
request.num_cached_tokens = request.num_computed_tokens
self.finished_recving_kv_req_ids.remove(request.request_id)
def _try_promote_blocked_waiting_request(self, request: Request) -> bool:
@@ -2221,7 +2220,7 @@ class Scheduler(SchedulerInterface):
req_num_computed_tokens - request.num_computed_tokens
)
total_affected_tokens += num_affected_tokens
request.num_external_computed_tokens -= num_affected_tokens
# collect invalid block and all downstream dependent blocks
if evict_blocks:
blocks_to_evict.update(req_block_ids[idx:])