[Frontend] Add max_tokens prometheus metric (#9881)

Signed-off-by: Tomer Asida <tomera@ai21.com>
This commit is contained in:
tomeras91
2024-11-04 22:53:24 +00:00
committed by GitHub
parent 9a5664d4a4
commit ac04a97a9f
5 changed files with 23 additions and 2 deletions
+4
View File
@@ -1685,6 +1685,7 @@ class LLMEngine:
num_prompt_tokens_requests: List[int] = []
num_generation_tokens_requests: List[int] = []
n_requests: List[int] = []
max_tokens_requests: List[int] = []
finished_reason_requests: List[str] = []
# Lora requests
@@ -1792,6 +1793,8 @@ class LLMEngine:
])
if seq_group.sampling_params is not None:
n_requests.append(seq_group.sampling_params.n)
max_tokens_requests.append(
seq_group.sampling_params.max_tokens)
finished_reason_requests.extend([
SequenceStatus.get_finished_reason(seq.status)
for seq in seq_group.get_finished_seqs()
@@ -1847,6 +1850,7 @@ class LLMEngine:
num_prompt_tokens_requests=num_prompt_tokens_requests,
num_generation_tokens_requests=num_generation_tokens_requests,
n_requests=n_requests,
max_tokens_requests=max_tokens_requests,
finished_reason_requests=finished_reason_requests,
max_lora=str(max_lora_stat),
waiting_lora_adapters=list(waiting_lora_adapters.keys()),
+8
View File
@@ -179,6 +179,12 @@ class Metrics:
labelnames=labelnames,
buckets=[1, 2, 5, 10, 20],
)
self.histogram_max_tokens_request = self._histogram_cls(
name="vllm:request_params_max_tokens",
documentation="Histogram of the max_tokens request parameter.",
labelnames=labelnames,
buckets=build_1_2_5_buckets(max_model_len),
)
self.counter_request_success = self._counter_cls(
name="vllm:request_success_total",
documentation="Count of successfully processed requests.",
@@ -547,6 +553,8 @@ class PrometheusStatLogger(StatLoggerBase):
self.metrics.histogram_num_generation_tokens_request,
stats.num_generation_tokens_requests)
self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
self._log_histogram(self.metrics.histogram_max_tokens_request,
stats.max_tokens_requests)
def _log_prometheus_interval(self, prompt_throughput: float,
generation_throughput: float) -> None:
+1
View File
@@ -53,6 +53,7 @@ class Stats:
num_prompt_tokens_requests: List[int]
num_generation_tokens_requests: List[int]
n_requests: List[int]
max_tokens_requests: List[int]
finished_reason_requests: List[str]
waiting_lora_adapters: List[str]
running_lora_adapters: List[str]