[Core][Metrics][BugFix] Replace num_cached_tokens/num_external_computed_tokens with PrefillStats (#37460)

Related to `Counters can only be incremented by non-negative amounts`
error with the `vllm:prompt_tokens_by_source_total` metric.

Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Co-authored-by: Or Ozeri <or@ozery.com>
This commit is contained in:
Mark McLoughlin
2026-04-14 09:00:45 +01:00
committed by GitHub
co-authored by Or Ozeri
parent 25b3242d8b
commit d3af8c1831
9 changed files with 187 additions and 102 deletions
+11 -12
View File
@@ -629,7 +629,6 @@ class Scheduler(SchedulerInterface):
step_skipped_waiting.prepend_request(request)
continue
request.num_external_computed_tokens = ext_tokens
num_external_computed_tokens = ext_tokens
connector_prefix_cache_queries = (
@@ -642,6 +641,15 @@ class Scheduler(SchedulerInterface):
num_new_local_computed_tokens + num_external_computed_tokens
)
assert num_computed_tokens <= request.num_tokens
# Track first scheduled prefill, not post-preemption repeat prefills
if request.prefill_stats is not None:
assert num_computed_tokens <= request.num_prompt_tokens
request.prefill_stats.set(
num_prompt_tokens=request.num_prompt_tokens,
num_local_cached_tokens=num_new_local_computed_tokens,
num_external_cached_tokens=num_external_computed_tokens,
)
else:
# KVTransfer: WAITING reqs have num_computed_tokens > 0
# after async KV recvs are completed.
@@ -826,9 +834,6 @@ class Scheduler(SchedulerInterface):
token_budget -= num_new_tokens
request.status = RequestStatus.RUNNING
request.num_computed_tokens = num_computed_tokens
# Count the number of prefix cached tokens.
if request.num_cached_tokens < 0:
request.num_cached_tokens = num_computed_tokens
# Encoder-related.
if encoder_inputs_to_schedule:
scheduled_encoder_inputs[request_id] = encoder_inputs_to_schedule
@@ -1466,10 +1471,9 @@ class Scheduler(SchedulerInterface):
pooling_output=pooler_output,
stop_reason=request.stop_reason,
events=request.take_events(),
prefill_stats=request.take_prefill_stats(),
kv_transfer_params=kv_transfer_params,
trace_headers=request.trace_headers,
num_cached_tokens=request.num_cached_tokens,
num_external_computed_tokens=request.num_external_computed_tokens,
routed_experts=routed_experts,
num_nans_in_logits=request.num_nans_in_logits,
)
@@ -1496,7 +1500,6 @@ class Scheduler(SchedulerInterface):
finish_reason=request.get_finished_reason(),
events=request.take_events(),
trace_headers=request.trace_headers,
num_cached_tokens=request.num_cached_tokens,
)
)
@@ -2071,10 +2074,6 @@ class Scheduler(SchedulerInterface):
if request.num_computed_tokens == request.num_tokens:
request.num_computed_tokens = request.num_tokens - 1
# Count the number of prefix cached tokens.
if request.num_cached_tokens < 0:
request.num_cached_tokens = request.num_computed_tokens
self.finished_recving_kv_req_ids.remove(request.request_id)
def _try_promote_blocked_waiting_request(self, request: Request) -> bool:
@@ -2221,7 +2220,7 @@ class Scheduler(SchedulerInterface):
req_num_computed_tokens - request.num_computed_tokens
)
total_affected_tokens += num_affected_tokens
request.num_external_computed_tokens -= num_affected_tokens
# collect invalid block and all downstream dependent blocks
if evict_blocks:
blocks_to_evict.update(req_block_ids[idx:])
+4 -5
View File
@@ -15,7 +15,7 @@ from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingParams
from vllm.v1.metrics.stats import SchedulerStats
from vllm.v1.metrics.stats import PrefillStats, SchedulerStats
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
from vllm.v1.serial_utils import UtilityResult
@@ -171,10 +171,9 @@ class EngineCoreOutput(
kv_transfer_params: dict[str, Any] | None = None
trace_headers: Mapping[str, str] | None = None
# The number of tokens with prefix cache hits (local + external).
num_cached_tokens: int = 0
# The number of tokens computed remotely (original count from connector).
num_external_computed_tokens: int = 0
prefill_stats: PrefillStats | None = None
routed_experts: np.ndarray | None = None
# The number of NaNs in logits.
# A value greater than 0 indicates that the output is corrupted.
+7 -3
View File
@@ -617,8 +617,13 @@ class OutputProcessor:
stop_reason = engine_core_output.stop_reason
kv_transfer_params = engine_core_output.kv_transfer_params
routed_experts = engine_core_output.routed_experts
req_state.num_cached_tokens = engine_core_output.num_cached_tokens
req_state.is_prefilling = False
if req_state.is_prefilling:
if engine_core_output.prefill_stats is not None:
req_state.num_cached_tokens = (
engine_core_output.prefill_stats.num_cached_tokens
)
req_state.is_prefilling = False
if pooling_output is None:
assert req_state.detokenizer is not None
@@ -776,7 +781,6 @@ class OutputProcessor:
engine_core_output,
engine_core_timestamp,
req_state.is_prefilling,
req_state.prompt_len,
req_state.stats,
self.lora_states,
req_state.lora_name,
+43 -27
View File
@@ -239,6 +239,40 @@ class FinishedRequestStats:
num_cached_tokens: int = 0
@dataclass
class PrefillStats:
"""Breakdown of a scheduled prefill computation.
Fields:
num_prompt_tokens: Total number of tokens to be prefilled.
num_computed_tokens: Tokens to be prefilled locally (actual compute work).
num_cached_tokens: Tokens to be prefilled without actual compute work.
num_local_cached_tokens: Tokens to be prefilled from local prefix cache.
num_external_cached_tokens: Tokens to be prefilled from external KV transfer.
"""
num_prompt_tokens: int = 0
num_computed_tokens: int = 0
num_cached_tokens: int = 0
num_local_cached_tokens: int = 0
num_external_cached_tokens: int = 0
def set(
self,
num_prompt_tokens: int,
num_local_cached_tokens: int,
num_external_cached_tokens: int,
):
num_cached_tokens = num_local_cached_tokens + num_external_cached_tokens
assert num_cached_tokens <= num_prompt_tokens
self.num_prompt_tokens = num_prompt_tokens
self.num_computed_tokens = num_prompt_tokens - num_cached_tokens
self.num_cached_tokens = num_cached_tokens
self.num_local_cached_tokens = num_local_cached_tokens
self.num_external_cached_tokens = num_external_cached_tokens
@dataclass
class PromptTokenStats:
"""Breakdown of prompt tokens by source.
@@ -267,28 +301,14 @@ class PromptTokenStats:
cached_tokens: int = 0
total: int = 0
def update_from_output(
self,
num_cached_tokens: int,
num_external_computed_tokens: int,
prompt_len: int,
) -> None:
def update_from_output(self, prefill_stats: PrefillStats) -> None:
"""Update stats from a prefill output."""
self.computed += prompt_len - num_cached_tokens
self.external_kv_transfer += num_external_computed_tokens
# FIXME(yifan): local_cache_hit can go negative after preemption.
# num_cached_tokens is a one-time snapshot from first scheduling and
# is never reset on preemption, while num_external_computed_tokens is
# overwritten on re-scheduling. If CPU offload finds more tokens on
# the second pass than the original total, the subtraction underflows.
# A fundamental fix is to track the first-time num_external_computed_tokens
# as a separate metric rather than reusing num_external_computed_tokens
# for metric directly.
self.local_cache_hit += max(
0, (num_cached_tokens - num_external_computed_tokens)
)
self.cached_tokens += num_cached_tokens
self.total += prompt_len
self.computed += prefill_stats.num_computed_tokens
self.cached_tokens += prefill_stats.num_cached_tokens
self.total += prefill_stats.num_prompt_tokens
self.local_cache_hit += prefill_stats.num_local_cached_tokens
self.external_kv_transfer += prefill_stats.num_external_cached_tokens
def get_by_source(self, source: str) -> int:
"""Get token count by source label."""
@@ -335,7 +355,6 @@ class IterationStats:
output: "EngineCoreOutput",
engine_core_timestamp: float,
is_prefilling: bool,
prompt_len: int,
req_stats: RequestStateStats,
lora_states: "LoRARequestStates",
lora_name: str | None,
@@ -344,11 +363,8 @@ class IterationStats:
self.num_generation_tokens += num_new_generation_tokens
if is_prefilling:
self.prompt_token_stats.update_from_output(
num_cached_tokens=output.num_cached_tokens,
num_external_computed_tokens=output.num_external_computed_tokens,
prompt_len=prompt_len,
)
if output.prefill_stats is not None:
self.prompt_token_stats.update_from_output(output.prefill_stats)
first_token_latency = self._time_since(req_stats.arrival_time)
self.time_to_first_tokens_iter.append(first_token_latency)
+9 -5
View File
@@ -20,6 +20,7 @@ from vllm.v1.engine import (
EngineCoreRequest,
FinishReason,
)
from vllm.v1.metrics.stats import PrefillStats
from vllm.v1.structured_output.request import StructuredOutputRequest
from vllm.v1.utils import ConstantList
@@ -145,9 +146,6 @@ class Request:
self.all_token_ids = ConstantList(self._all_token_ids)
# trace_headers
self.trace_headers = trace_headers
# State
# The number of tokens with prefix cache hits.
self.num_cached_tokens = -1
# True if this request is scheduled as a non-final prefill chunk.
self.is_prefill_chunk = False
@@ -159,8 +157,7 @@ class Request:
# The number of times this request has been preempted by the scheduler.
self.num_preemptions = 0
# The number of tokens that have been computed remotely.
self.num_external_computed_tokens = 0
self.prefill_stats: PrefillStats | None = PrefillStats()
self.block_hashes: list[BlockHash] = []
# Store the block hasher without binding self to avoid creating a
@@ -278,6 +275,13 @@ class Request:
events, self.events = self.events, []
return events
def take_prefill_stats(self) -> PrefillStats | None:
if self.prefill_stats is None:
return None
prefill_stats = self.prefill_stats
self.prefill_stats = None
return prefill_stats
def __lt__(self, other: "Request") -> bool:
"""
Compare two requests based on priority, arrival time, and request ID.