[Core][Metrics][BugFix] Replace num_cached_tokens/num_external_computed_tokens with PrefillStats (#37460)
Related to `Counters can only be incremented by non-negative amounts` error with the `vllm:prompt_tokens_by_source_total` metric. Signed-off-by: Mark McLoughlin <markmc@redhat.com> Co-authored-by: Or Ozeri <or@ozery.com>
This commit is contained in:
co-authored by
Or Ozeri
parent
25b3242d8b
commit
d3af8c1831
@@ -629,7 +629,6 @@ class Scheduler(SchedulerInterface):
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
continue
|
||||
|
||||
request.num_external_computed_tokens = ext_tokens
|
||||
num_external_computed_tokens = ext_tokens
|
||||
|
||||
connector_prefix_cache_queries = (
|
||||
@@ -642,6 +641,15 @@ class Scheduler(SchedulerInterface):
|
||||
num_new_local_computed_tokens + num_external_computed_tokens
|
||||
)
|
||||
assert num_computed_tokens <= request.num_tokens
|
||||
|
||||
# Track first scheduled prefill, not post-preemption repeat prefills
|
||||
if request.prefill_stats is not None:
|
||||
assert num_computed_tokens <= request.num_prompt_tokens
|
||||
request.prefill_stats.set(
|
||||
num_prompt_tokens=request.num_prompt_tokens,
|
||||
num_local_cached_tokens=num_new_local_computed_tokens,
|
||||
num_external_cached_tokens=num_external_computed_tokens,
|
||||
)
|
||||
else:
|
||||
# KVTransfer: WAITING reqs have num_computed_tokens > 0
|
||||
# after async KV recvs are completed.
|
||||
@@ -826,9 +834,6 @@ class Scheduler(SchedulerInterface):
|
||||
token_budget -= num_new_tokens
|
||||
request.status = RequestStatus.RUNNING
|
||||
request.num_computed_tokens = num_computed_tokens
|
||||
# Count the number of prefix cached tokens.
|
||||
if request.num_cached_tokens < 0:
|
||||
request.num_cached_tokens = num_computed_tokens
|
||||
# Encoder-related.
|
||||
if encoder_inputs_to_schedule:
|
||||
scheduled_encoder_inputs[request_id] = encoder_inputs_to_schedule
|
||||
@@ -1466,10 +1471,9 @@ class Scheduler(SchedulerInterface):
|
||||
pooling_output=pooler_output,
|
||||
stop_reason=request.stop_reason,
|
||||
events=request.take_events(),
|
||||
prefill_stats=request.take_prefill_stats(),
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
trace_headers=request.trace_headers,
|
||||
num_cached_tokens=request.num_cached_tokens,
|
||||
num_external_computed_tokens=request.num_external_computed_tokens,
|
||||
routed_experts=routed_experts,
|
||||
num_nans_in_logits=request.num_nans_in_logits,
|
||||
)
|
||||
@@ -1496,7 +1500,6 @@ class Scheduler(SchedulerInterface):
|
||||
finish_reason=request.get_finished_reason(),
|
||||
events=request.take_events(),
|
||||
trace_headers=request.trace_headers,
|
||||
num_cached_tokens=request.num_cached_tokens,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2071,10 +2074,6 @@ class Scheduler(SchedulerInterface):
|
||||
if request.num_computed_tokens == request.num_tokens:
|
||||
request.num_computed_tokens = request.num_tokens - 1
|
||||
|
||||
# Count the number of prefix cached tokens.
|
||||
if request.num_cached_tokens < 0:
|
||||
request.num_cached_tokens = request.num_computed_tokens
|
||||
|
||||
self.finished_recving_kv_req_ids.remove(request.request_id)
|
||||
|
||||
def _try_promote_blocked_waiting_request(self, request: Request) -> bool:
|
||||
@@ -2221,7 +2220,7 @@ class Scheduler(SchedulerInterface):
|
||||
req_num_computed_tokens - request.num_computed_tokens
|
||||
)
|
||||
total_affected_tokens += num_affected_tokens
|
||||
request.num_external_computed_tokens -= num_affected_tokens
|
||||
|
||||
# collect invalid block and all downstream dependent blocks
|
||||
if evict_blocks:
|
||||
blocks_to_evict.update(req_block_ids[idx:])
|
||||
|
||||
Reference in New Issue
Block a user