[Core][Metrics][BugFix] Replace num_cached_tokens/num_external_computed_tokens with PrefillStats (#37460)

Related to `Counters can only be incremented by non-negative amounts`
error with the `vllm:prompt_tokens_by_source_total` metric.

Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Co-authored-by: Or Ozeri <or@ozery.com>
This commit is contained in:
Mark McLoughlin
2026-04-14 09:00:45 +01:00
committed by GitHub
co-authored by Or Ozeri
parent 25b3242d8b
commit d3af8c1831
9 changed files with 187 additions and 102 deletions
+10 -4
View File
@@ -84,6 +84,7 @@ def test_incremental_detokenization(
engine_core = MockEngineCore(
tokens_list=dummy_test_vectors.generation_tokens,
prompts_list=dummy_test_vectors.prompt_tokens,
request_ids=[req.request_id for req in requests],
)
@@ -506,6 +507,7 @@ def test_logprobs_processor(
engine_core = MockEngineCore(
tokens_list=dummy_test_vectors.generation_tokens,
prompts_list=dummy_test_vectors.prompt_tokens,
generated_logprobs_raw=None
if num_sample_logprobs is None
else dummy_test_vectors.generation_logprobs,
@@ -691,6 +693,7 @@ def test_stop_token(
engine_core = MockEngineCore(
tokens_list=[generation_tokens],
prompts_list=dummy_test_vectors.prompt_tokens,
generated_logprobs_raw=[generation_logprobs] if do_logprobs else None,
prompt_logprobs_raw=None,
eos_token_id=sampling_params.eos_token_id,
@@ -794,6 +797,7 @@ def test_stop_string(
engine_core = MockEngineCore(
tokens_list=dummy_test_vectors.generation_tokens,
prompts_list=dummy_test_vectors.prompt_tokens,
generated_logprobs_raw=dummy_test_vectors.generation_logprobs
if num_sample_logprobs
else None,
@@ -917,6 +921,7 @@ def test_iteration_stats(dummy_test_vectors):
engine_core = MockEngineCore(
dummy_test_vectors.generation_tokens,
dummy_test_vectors.prompt_tokens,
request_ids=[req.request_id for req in requests],
)
@@ -927,7 +932,7 @@ def test_iteration_stats(dummy_test_vectors):
inactive_request = requests[num_active]
# First iteration has 2 prefills.
outputs = engine_core.get_outputs()[:num_active]
outputs = engine_core.get_outputs(num_active)
iteration_stats = IterationStats()
output_processor.process_outputs(outputs, engine_core_timestamp, iteration_stats)
total_prompt_tokens = sum(
@@ -941,7 +946,7 @@ def test_iteration_stats(dummy_test_vectors):
assert iteration_stats.num_generation_tokens == num_active
# Just decodes in this step.
outputs = engine_core.get_outputs()[:num_active]
outputs = engine_core.get_outputs(num_active)
iteration_stats = IterationStats()
output_processor.process_outputs(outputs, engine_core_timestamp, iteration_stats)
@@ -951,7 +956,7 @@ def test_iteration_stats(dummy_test_vectors):
# Add a new request - prefill and 2 decodes in this step.
output_processor.add_request(inactive_request, None)
num_active += 1
outputs = engine_core.get_outputs()[:num_active]
outputs = engine_core.get_outputs(num_active)
iteration_stats = IterationStats()
output_processor.process_outputs(outputs, engine_core_timestamp, iteration_stats)
total_prompt_tokens = len(dummy_test_vectors.prompt_tokens[num_active - 1])
@@ -960,7 +965,7 @@ def test_iteration_stats(dummy_test_vectors):
assert iteration_stats.num_generation_tokens == num_active
# Just decodes in this step.
outputs = engine_core.get_outputs()[:num_active]
outputs = engine_core.get_outputs(num_active)
iteration_stats = IterationStats()
output_processor.process_outputs(outputs, engine_core_timestamp, iteration_stats)
@@ -1003,6 +1008,7 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
engine_core = MockEngineCore(
dummy_test_vectors.generation_tokens,
dummy_test_vectors.prompt_tokens,
request_ids=[req.request_id for req in requests],
)
+27 -6
View File
@@ -11,6 +11,7 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
from vllm.engine.arg_utils import EngineArgs
from vllm.v1.engine import EngineCoreOutput, FinishReason
from vllm.v1.metrics.stats import PrefillStats
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
GeneralTokenizerType: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast
@@ -330,6 +331,7 @@ class MockEngineCore:
def __init__(
self,
tokens_list: list[list[int]],
prompts_list: list[list[int]],
# For each request, for each sampled token offset,
# a tuple of
# (list of topk token ids, list of sample logprob vals, rank)
@@ -346,12 +348,13 @@ class MockEngineCore:
) -> None:
self.num_requests = len(tokens_list)
self.tokens_list = tokens_list
self.current_idx = 0
self.prompts_list = prompts_list
self.generated_logprobs_raw = generated_logprobs_raw
self.do_logprobs = generated_logprobs_raw is not None
self.prompt_logprobs_raw = prompt_logprobs_raw
self.do_prompt_logprobs = prompt_logprobs_raw is not None
self.request_finished = [False for _ in range(self.num_requests)]
self.request_token_idx = [0 for _ in range(self.num_requests)]
self.eos_token_id = eos_token_id
self.stop_token_ids = stop_token_ids
self.request_ids = (
@@ -360,14 +363,18 @@ class MockEngineCore:
else [f"request-{i}" for i in range(self.num_requests)]
)
def get_outputs(self) -> list[EngineCoreOutput]:
def get_outputs(self, num_active: int = -1) -> list[EngineCoreOutput]:
do_logprobs = self.do_logprobs
do_prompt_logprobs = self.do_prompt_logprobs
token_idx = self.current_idx
outputs = []
for req_idx, token_ids in enumerate(self.tokens_list):
for req_idx, (token_ids, prompt_token_ids) in enumerate(
zip(self.tokens_list, self.prompts_list)
):
if num_active != -1 and req_idx >= num_active:
break
if not self.request_finished[req_idx]:
token_idx = self.request_token_idx[req_idx]
if do_logprobs:
assert self.generated_logprobs_raw is not None
(logprobs_token_ids_, logprobs_, sampled_token_ranks_) = (
@@ -381,19 +388,32 @@ class MockEngineCore:
else:
logprobs = None
if do_prompt_logprobs:
if self.current_idx == 0:
if token_idx == 0:
assert self.prompt_logprobs_raw is not None
prompt_logprobs = self.prompt_logprobs_raw[req_idx]
else:
prompt_logprobs = None
else:
prompt_logprobs = None
# Add prefill_stats on first output (prefill) for this request
if token_idx == 0:
prefill_stats = PrefillStats()
prefill_stats.set(
num_prompt_tokens=len(prompt_token_ids),
num_local_cached_tokens=0,
num_external_cached_tokens=0,
)
else:
prefill_stats = None
new_token_id = token_ids[token_idx]
output = EngineCoreOutput(
request_id=self.request_ids[req_idx],
new_token_ids=[new_token_id],
new_logprobs=logprobs,
new_prompt_logprobs_tensors=prompt_logprobs,
prefill_stats=prefill_stats,
)
if token_idx == len(token_ids) - 1:
output.finish_reason = FinishReason.LENGTH
@@ -407,5 +427,6 @@ class MockEngineCore:
self.request_finished[req_idx] = True
outputs.append(output)
self.current_idx += 1
self.request_token_idx[req_idx] += 1
return outputs