[Core][Metrics][BugFix] Replace num_cached_tokens/num_external_computed_tokens with PrefillStats (#37460)
Related to `Counters can only be incremented by non-negative amounts` error with the `vllm:prompt_tokens_by_source_total` metric. Signed-off-by: Mark McLoughlin <markmc@redhat.com> Co-authored-by: Or Ozeri <or@ozery.com>
This commit is contained in:
co-authored by
Or Ozeri
parent
25b3242d8b
commit
d3af8c1831
@@ -629,7 +629,6 @@ class Scheduler(SchedulerInterface):
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
continue
|
||||
|
||||
request.num_external_computed_tokens = ext_tokens
|
||||
num_external_computed_tokens = ext_tokens
|
||||
|
||||
connector_prefix_cache_queries = (
|
||||
@@ -642,6 +641,15 @@ class Scheduler(SchedulerInterface):
|
||||
num_new_local_computed_tokens + num_external_computed_tokens
|
||||
)
|
||||
assert num_computed_tokens <= request.num_tokens
|
||||
|
||||
# Track first scheduled prefill, not post-preemption repeat prefills
|
||||
if request.prefill_stats is not None:
|
||||
assert num_computed_tokens <= request.num_prompt_tokens
|
||||
request.prefill_stats.set(
|
||||
num_prompt_tokens=request.num_prompt_tokens,
|
||||
num_local_cached_tokens=num_new_local_computed_tokens,
|
||||
num_external_cached_tokens=num_external_computed_tokens,
|
||||
)
|
||||
else:
|
||||
# KVTransfer: WAITING reqs have num_computed_tokens > 0
|
||||
# after async KV recvs are completed.
|
||||
@@ -826,9 +834,6 @@ class Scheduler(SchedulerInterface):
|
||||
token_budget -= num_new_tokens
|
||||
request.status = RequestStatus.RUNNING
|
||||
request.num_computed_tokens = num_computed_tokens
|
||||
# Count the number of prefix cached tokens.
|
||||
if request.num_cached_tokens < 0:
|
||||
request.num_cached_tokens = num_computed_tokens
|
||||
# Encoder-related.
|
||||
if encoder_inputs_to_schedule:
|
||||
scheduled_encoder_inputs[request_id] = encoder_inputs_to_schedule
|
||||
@@ -1466,10 +1471,9 @@ class Scheduler(SchedulerInterface):
|
||||
pooling_output=pooler_output,
|
||||
stop_reason=request.stop_reason,
|
||||
events=request.take_events(),
|
||||
prefill_stats=request.take_prefill_stats(),
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
trace_headers=request.trace_headers,
|
||||
num_cached_tokens=request.num_cached_tokens,
|
||||
num_external_computed_tokens=request.num_external_computed_tokens,
|
||||
routed_experts=routed_experts,
|
||||
num_nans_in_logits=request.num_nans_in_logits,
|
||||
)
|
||||
@@ -1496,7 +1500,6 @@ class Scheduler(SchedulerInterface):
|
||||
finish_reason=request.get_finished_reason(),
|
||||
events=request.take_events(),
|
||||
trace_headers=request.trace_headers,
|
||||
num_cached_tokens=request.num_cached_tokens,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2071,10 +2074,6 @@ class Scheduler(SchedulerInterface):
|
||||
if request.num_computed_tokens == request.num_tokens:
|
||||
request.num_computed_tokens = request.num_tokens - 1
|
||||
|
||||
# Count the number of prefix cached tokens.
|
||||
if request.num_cached_tokens < 0:
|
||||
request.num_cached_tokens = request.num_computed_tokens
|
||||
|
||||
self.finished_recving_kv_req_ids.remove(request.request_id)
|
||||
|
||||
def _try_promote_blocked_waiting_request(self, request: Request) -> bool:
|
||||
@@ -2221,7 +2220,7 @@ class Scheduler(SchedulerInterface):
|
||||
req_num_computed_tokens - request.num_computed_tokens
|
||||
)
|
||||
total_affected_tokens += num_affected_tokens
|
||||
request.num_external_computed_tokens -= num_affected_tokens
|
||||
|
||||
# collect invalid block and all downstream dependent blocks
|
||||
if evict_blocks:
|
||||
blocks_to_evict.update(req_block_ids[idx:])
|
||||
|
||||
@@ -15,7 +15,7 @@ from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.metrics.stats import SchedulerStats
|
||||
from vllm.v1.metrics.stats import PrefillStats, SchedulerStats
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
|
||||
from vllm.v1.serial_utils import UtilityResult
|
||||
|
||||
@@ -171,10 +171,9 @@ class EngineCoreOutput(
|
||||
kv_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
trace_headers: Mapping[str, str] | None = None
|
||||
# The number of tokens with prefix cache hits (local + external).
|
||||
num_cached_tokens: int = 0
|
||||
# The number of tokens computed remotely (original count from connector).
|
||||
num_external_computed_tokens: int = 0
|
||||
|
||||
prefill_stats: PrefillStats | None = None
|
||||
|
||||
routed_experts: np.ndarray | None = None
|
||||
# The number of NaNs in logits.
|
||||
# A value greater than 0 indicates that the output is corrupted.
|
||||
|
||||
@@ -617,8 +617,13 @@ class OutputProcessor:
|
||||
stop_reason = engine_core_output.stop_reason
|
||||
kv_transfer_params = engine_core_output.kv_transfer_params
|
||||
routed_experts = engine_core_output.routed_experts
|
||||
req_state.num_cached_tokens = engine_core_output.num_cached_tokens
|
||||
req_state.is_prefilling = False
|
||||
|
||||
if req_state.is_prefilling:
|
||||
if engine_core_output.prefill_stats is not None:
|
||||
req_state.num_cached_tokens = (
|
||||
engine_core_output.prefill_stats.num_cached_tokens
|
||||
)
|
||||
req_state.is_prefilling = False
|
||||
|
||||
if pooling_output is None:
|
||||
assert req_state.detokenizer is not None
|
||||
@@ -776,7 +781,6 @@ class OutputProcessor:
|
||||
engine_core_output,
|
||||
engine_core_timestamp,
|
||||
req_state.is_prefilling,
|
||||
req_state.prompt_len,
|
||||
req_state.stats,
|
||||
self.lora_states,
|
||||
req_state.lora_name,
|
||||
|
||||
+43
-27
@@ -239,6 +239,40 @@ class FinishedRequestStats:
|
||||
num_cached_tokens: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class PrefillStats:
|
||||
"""Breakdown of a scheduled prefill computation.
|
||||
|
||||
Fields:
|
||||
num_prompt_tokens: Total number of tokens to be prefilled.
|
||||
num_computed_tokens: Tokens to be prefilled locally (actual compute work).
|
||||
num_cached_tokens: Tokens to be prefilled without actual compute work.
|
||||
num_local_cached_tokens: Tokens to be prefilled from local prefix cache.
|
||||
num_external_cached_tokens: Tokens to be prefilled from external KV transfer.
|
||||
"""
|
||||
|
||||
num_prompt_tokens: int = 0
|
||||
num_computed_tokens: int = 0
|
||||
num_cached_tokens: int = 0
|
||||
num_local_cached_tokens: int = 0
|
||||
num_external_cached_tokens: int = 0
|
||||
|
||||
def set(
|
||||
self,
|
||||
num_prompt_tokens: int,
|
||||
num_local_cached_tokens: int,
|
||||
num_external_cached_tokens: int,
|
||||
):
|
||||
num_cached_tokens = num_local_cached_tokens + num_external_cached_tokens
|
||||
assert num_cached_tokens <= num_prompt_tokens
|
||||
|
||||
self.num_prompt_tokens = num_prompt_tokens
|
||||
self.num_computed_tokens = num_prompt_tokens - num_cached_tokens
|
||||
self.num_cached_tokens = num_cached_tokens
|
||||
self.num_local_cached_tokens = num_local_cached_tokens
|
||||
self.num_external_cached_tokens = num_external_cached_tokens
|
||||
|
||||
|
||||
@dataclass
|
||||
class PromptTokenStats:
|
||||
"""Breakdown of prompt tokens by source.
|
||||
@@ -267,28 +301,14 @@ class PromptTokenStats:
|
||||
cached_tokens: int = 0
|
||||
total: int = 0
|
||||
|
||||
def update_from_output(
|
||||
self,
|
||||
num_cached_tokens: int,
|
||||
num_external_computed_tokens: int,
|
||||
prompt_len: int,
|
||||
) -> None:
|
||||
def update_from_output(self, prefill_stats: PrefillStats) -> None:
|
||||
"""Update stats from a prefill output."""
|
||||
self.computed += prompt_len - num_cached_tokens
|
||||
self.external_kv_transfer += num_external_computed_tokens
|
||||
# FIXME(yifan): local_cache_hit can go negative after preemption.
|
||||
# num_cached_tokens is a one-time snapshot from first scheduling and
|
||||
# is never reset on preemption, while num_external_computed_tokens is
|
||||
# overwritten on re-scheduling. If CPU offload finds more tokens on
|
||||
# the second pass than the original total, the subtraction underflows.
|
||||
# A fundamental fix is to track the first-time num_external_computed_tokens
|
||||
# as a separate metric rather than reusing num_external_computed_tokens
|
||||
# for metric directly.
|
||||
self.local_cache_hit += max(
|
||||
0, (num_cached_tokens - num_external_computed_tokens)
|
||||
)
|
||||
self.cached_tokens += num_cached_tokens
|
||||
self.total += prompt_len
|
||||
self.computed += prefill_stats.num_computed_tokens
|
||||
self.cached_tokens += prefill_stats.num_cached_tokens
|
||||
self.total += prefill_stats.num_prompt_tokens
|
||||
|
||||
self.local_cache_hit += prefill_stats.num_local_cached_tokens
|
||||
self.external_kv_transfer += prefill_stats.num_external_cached_tokens
|
||||
|
||||
def get_by_source(self, source: str) -> int:
|
||||
"""Get token count by source label."""
|
||||
@@ -335,7 +355,6 @@ class IterationStats:
|
||||
output: "EngineCoreOutput",
|
||||
engine_core_timestamp: float,
|
||||
is_prefilling: bool,
|
||||
prompt_len: int,
|
||||
req_stats: RequestStateStats,
|
||||
lora_states: "LoRARequestStates",
|
||||
lora_name: str | None,
|
||||
@@ -344,11 +363,8 @@ class IterationStats:
|
||||
|
||||
self.num_generation_tokens += num_new_generation_tokens
|
||||
if is_prefilling:
|
||||
self.prompt_token_stats.update_from_output(
|
||||
num_cached_tokens=output.num_cached_tokens,
|
||||
num_external_computed_tokens=output.num_external_computed_tokens,
|
||||
prompt_len=prompt_len,
|
||||
)
|
||||
if output.prefill_stats is not None:
|
||||
self.prompt_token_stats.update_from_output(output.prefill_stats)
|
||||
|
||||
first_token_latency = self._time_since(req_stats.arrival_time)
|
||||
self.time_to_first_tokens_iter.append(first_token_latency)
|
||||
|
||||
+9
-5
@@ -20,6 +20,7 @@ from vllm.v1.engine import (
|
||||
EngineCoreRequest,
|
||||
FinishReason,
|
||||
)
|
||||
from vllm.v1.metrics.stats import PrefillStats
|
||||
from vllm.v1.structured_output.request import StructuredOutputRequest
|
||||
from vllm.v1.utils import ConstantList
|
||||
|
||||
@@ -145,9 +146,6 @@ class Request:
|
||||
self.all_token_ids = ConstantList(self._all_token_ids)
|
||||
# trace_headers
|
||||
self.trace_headers = trace_headers
|
||||
# State
|
||||
# The number of tokens with prefix cache hits.
|
||||
self.num_cached_tokens = -1
|
||||
|
||||
# True if this request is scheduled as a non-final prefill chunk.
|
||||
self.is_prefill_chunk = False
|
||||
@@ -159,8 +157,7 @@ class Request:
|
||||
# The number of times this request has been preempted by the scheduler.
|
||||
self.num_preemptions = 0
|
||||
|
||||
# The number of tokens that have been computed remotely.
|
||||
self.num_external_computed_tokens = 0
|
||||
self.prefill_stats: PrefillStats | None = PrefillStats()
|
||||
|
||||
self.block_hashes: list[BlockHash] = []
|
||||
# Store the block hasher without binding self to avoid creating a
|
||||
@@ -278,6 +275,13 @@ class Request:
|
||||
events, self.events = self.events, []
|
||||
return events
|
||||
|
||||
def take_prefill_stats(self) -> PrefillStats | None:
|
||||
if self.prefill_stats is None:
|
||||
return None
|
||||
prefill_stats = self.prefill_stats
|
||||
self.prefill_stats = None
|
||||
return prefill_stats
|
||||
|
||||
def __lt__(self, other: "Request") -> bool:
|
||||
"""
|
||||
Compare two requests based on priority, arrival time, and request ID.
|
||||
|
||||
Reference in New Issue
Block a user