[Core][Metrics][BugFix] Replace num_cached_tokens/num_external_computed_tokens with PrefillStats (#37460)
Related to `Counters can only be incremented by non-negative amounts` error with the `vllm:prompt_tokens_by_source_total` metric. Signed-off-by: Mark McLoughlin <markmc@redhat.com> Co-authored-by: Or Ozeri <or@ozery.com>
This commit is contained in:
co-authored by
Or Ozeri
parent
25b3242d8b
commit
d3af8c1831
@@ -11,6 +11,7 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.v1.engine import EngineCoreOutput, FinishReason
|
||||
from vllm.v1.metrics.stats import PrefillStats
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
|
||||
|
||||
GeneralTokenizerType: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast
|
||||
@@ -330,6 +331,7 @@ class MockEngineCore:
|
||||
def __init__(
|
||||
self,
|
||||
tokens_list: list[list[int]],
|
||||
prompts_list: list[list[int]],
|
||||
# For each request, for each sampled token offset,
|
||||
# a tuple of
|
||||
# (list of topk token ids, list of sample logprob vals, rank)
|
||||
@@ -346,12 +348,13 @@ class MockEngineCore:
|
||||
) -> None:
|
||||
self.num_requests = len(tokens_list)
|
||||
self.tokens_list = tokens_list
|
||||
self.current_idx = 0
|
||||
self.prompts_list = prompts_list
|
||||
self.generated_logprobs_raw = generated_logprobs_raw
|
||||
self.do_logprobs = generated_logprobs_raw is not None
|
||||
self.prompt_logprobs_raw = prompt_logprobs_raw
|
||||
self.do_prompt_logprobs = prompt_logprobs_raw is not None
|
||||
self.request_finished = [False for _ in range(self.num_requests)]
|
||||
self.request_token_idx = [0 for _ in range(self.num_requests)]
|
||||
self.eos_token_id = eos_token_id
|
||||
self.stop_token_ids = stop_token_ids
|
||||
self.request_ids = (
|
||||
@@ -360,14 +363,18 @@ class MockEngineCore:
|
||||
else [f"request-{i}" for i in range(self.num_requests)]
|
||||
)
|
||||
|
||||
def get_outputs(self) -> list[EngineCoreOutput]:
|
||||
def get_outputs(self, num_active: int = -1) -> list[EngineCoreOutput]:
|
||||
do_logprobs = self.do_logprobs
|
||||
do_prompt_logprobs = self.do_prompt_logprobs
|
||||
token_idx = self.current_idx
|
||||
|
||||
outputs = []
|
||||
for req_idx, token_ids in enumerate(self.tokens_list):
|
||||
for req_idx, (token_ids, prompt_token_ids) in enumerate(
|
||||
zip(self.tokens_list, self.prompts_list)
|
||||
):
|
||||
if num_active != -1 and req_idx >= num_active:
|
||||
break
|
||||
if not self.request_finished[req_idx]:
|
||||
token_idx = self.request_token_idx[req_idx]
|
||||
if do_logprobs:
|
||||
assert self.generated_logprobs_raw is not None
|
||||
(logprobs_token_ids_, logprobs_, sampled_token_ranks_) = (
|
||||
@@ -381,19 +388,32 @@ class MockEngineCore:
|
||||
else:
|
||||
logprobs = None
|
||||
if do_prompt_logprobs:
|
||||
if self.current_idx == 0:
|
||||
if token_idx == 0:
|
||||
assert self.prompt_logprobs_raw is not None
|
||||
prompt_logprobs = self.prompt_logprobs_raw[req_idx]
|
||||
else:
|
||||
prompt_logprobs = None
|
||||
else:
|
||||
prompt_logprobs = None
|
||||
|
||||
# Add prefill_stats on first output (prefill) for this request
|
||||
if token_idx == 0:
|
||||
prefill_stats = PrefillStats()
|
||||
prefill_stats.set(
|
||||
num_prompt_tokens=len(prompt_token_ids),
|
||||
num_local_cached_tokens=0,
|
||||
num_external_cached_tokens=0,
|
||||
)
|
||||
else:
|
||||
prefill_stats = None
|
||||
|
||||
new_token_id = token_ids[token_idx]
|
||||
output = EngineCoreOutput(
|
||||
request_id=self.request_ids[req_idx],
|
||||
new_token_ids=[new_token_id],
|
||||
new_logprobs=logprobs,
|
||||
new_prompt_logprobs_tensors=prompt_logprobs,
|
||||
prefill_stats=prefill_stats,
|
||||
)
|
||||
if token_idx == len(token_ids) - 1:
|
||||
output.finish_reason = FinishReason.LENGTH
|
||||
@@ -407,5 +427,6 @@ class MockEngineCore:
|
||||
self.request_finished[req_idx] = True
|
||||
outputs.append(output)
|
||||
|
||||
self.current_idx += 1
|
||||
self.request_token_idx[req_idx] += 1
|
||||
|
||||
return outputs
|
||||
|
||||
Reference in New Issue
Block a user