[V1][Metrics] Implement vllm:lora_requests_info metric (#13504)

This commit is contained in:
Mark McLoughlin
2025-02-24 20:01:33 -08:00
committed by GitHub
parent ab1091d5f2
commit bc32bc73aa
3 changed files with 121 additions and 10 deletions
+19 -4
View File
@@ -11,7 +11,8 @@ from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
from vllm.v1.engine.detokenizer import IncrementalDetokenizer
from vllm.v1.engine.logprobs import LogprobsProcessor
from vllm.v1.metrics.stats import IterationStats, RequestStateStats
from vllm.v1.metrics.stats import (IterationStats, LoRARequestStates,
RequestStateStats)
@dataclass
@@ -26,6 +27,7 @@ class RequestState:
def __init__(
self,
request_id: str,
lora_name: Optional[str],
output_kind: RequestOutputKind,
prompt: Optional[str],
prompt_token_ids: List[int],
@@ -36,6 +38,7 @@ class RequestState:
log_stats: bool,
):
self.request_id = request_id
self.lora_name = lora_name
self.output_kind = output_kind
self.prompt = prompt
self.prompt_token_ids = prompt_token_ids
@@ -58,6 +61,8 @@ class RequestState:
) -> "RequestState":
return cls(
request_id=request.request_id,
lora_name=(request.lora_request.name
if request.lora_request is not None else None),
output_kind=request.sampling_params.output_kind,
prompt=request.prompt,
prompt_token_ids=request.prompt_token_ids,
@@ -86,6 +91,7 @@ class OutputProcessor:
self.log_stats = log_stats
self.tokenizer = tokenizer
self.request_states: Dict[str, RequestState] = {}
self.lora_states = LoRARequestStates()
def is_request_active(self, request_id: str) -> bool:
return request_id in self.request_states
@@ -101,7 +107,9 @@ class OutputProcessor:
request_ids: List[str],
) -> None:
for request_id in request_ids:
self.request_states.pop(request_id, None)
req_state = self.request_states.pop(request_id, None)
if req_state is not None:
self.lora_states.abort_request(req_state)
def add_request(
self,
@@ -112,11 +120,13 @@ class OutputProcessor:
if request_id in self.request_states:
raise ValueError(f"Request id {request_id} already running.")
self.request_states[request_id] = RequestState.from_new_request(
req_state = RequestState.from_new_request(
tokenizer=self.tokenizer.get_lora_tokenizer(request.lora_request),
request=request,
queue=queue,
log_stats=self.log_stats)
self.request_states[request_id] = req_state
self.lora_states.add_request(req_state)
def process_outputs(
self,
@@ -214,6 +224,8 @@ class OutputProcessor:
finish_reason,
iteration_stats)
self.lora_states.update_iteration_stats(iteration_stats)
return OutputProcessorOutput(
request_outputs=request_outputs,
reqs_to_abort=reqs_to_abort,
@@ -226,13 +238,15 @@ class OutputProcessor:
if iteration_stats is None:
return
lora_stats = self.lora_states.get_stats(req_state)
assert engine_core_timestamp is not None
assert req_state.stats is not None
iteration_stats.update_from_output(engine_core_output,
engine_core_timestamp,
req_state.is_prefilling,
req_state.prompt_len,
req_state.stats)
req_state.stats, lora_stats)
def _update_stats_from_finished(self, req_state: RequestState,
request_output: RequestOutput,
@@ -246,6 +260,7 @@ class OutputProcessor:
iteration_stats.update_from_finished_request(finish_reason,
request_output,
req_state.stats)
self.lora_states.finish_request(req_state)
@staticmethod
def _make_request_output(