[V1][Metrics] Implement vllm:lora_requests_info metric (#13504)
This commit is contained in:
@@ -11,7 +11,8 @@ from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
|
||||
from vllm.v1.engine.detokenizer import IncrementalDetokenizer
|
||||
from vllm.v1.engine.logprobs import LogprobsProcessor
|
||||
from vllm.v1.metrics.stats import IterationStats, RequestStateStats
|
||||
from vllm.v1.metrics.stats import (IterationStats, LoRARequestStates,
|
||||
RequestStateStats)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -26,6 +27,7 @@ class RequestState:
|
||||
def __init__(
|
||||
self,
|
||||
request_id: str,
|
||||
lora_name: Optional[str],
|
||||
output_kind: RequestOutputKind,
|
||||
prompt: Optional[str],
|
||||
prompt_token_ids: List[int],
|
||||
@@ -36,6 +38,7 @@ class RequestState:
|
||||
log_stats: bool,
|
||||
):
|
||||
self.request_id = request_id
|
||||
self.lora_name = lora_name
|
||||
self.output_kind = output_kind
|
||||
self.prompt = prompt
|
||||
self.prompt_token_ids = prompt_token_ids
|
||||
@@ -58,6 +61,8 @@ class RequestState:
|
||||
) -> "RequestState":
|
||||
return cls(
|
||||
request_id=request.request_id,
|
||||
lora_name=(request.lora_request.name
|
||||
if request.lora_request is not None else None),
|
||||
output_kind=request.sampling_params.output_kind,
|
||||
prompt=request.prompt,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
@@ -86,6 +91,7 @@ class OutputProcessor:
|
||||
self.log_stats = log_stats
|
||||
self.tokenizer = tokenizer
|
||||
self.request_states: Dict[str, RequestState] = {}
|
||||
self.lora_states = LoRARequestStates()
|
||||
|
||||
def is_request_active(self, request_id: str) -> bool:
|
||||
return request_id in self.request_states
|
||||
@@ -101,7 +107,9 @@ class OutputProcessor:
|
||||
request_ids: List[str],
|
||||
) -> None:
|
||||
for request_id in request_ids:
|
||||
self.request_states.pop(request_id, None)
|
||||
req_state = self.request_states.pop(request_id, None)
|
||||
if req_state is not None:
|
||||
self.lora_states.abort_request(req_state)
|
||||
|
||||
def add_request(
|
||||
self,
|
||||
@@ -112,11 +120,13 @@ class OutputProcessor:
|
||||
if request_id in self.request_states:
|
||||
raise ValueError(f"Request id {request_id} already running.")
|
||||
|
||||
self.request_states[request_id] = RequestState.from_new_request(
|
||||
req_state = RequestState.from_new_request(
|
||||
tokenizer=self.tokenizer.get_lora_tokenizer(request.lora_request),
|
||||
request=request,
|
||||
queue=queue,
|
||||
log_stats=self.log_stats)
|
||||
self.request_states[request_id] = req_state
|
||||
self.lora_states.add_request(req_state)
|
||||
|
||||
def process_outputs(
|
||||
self,
|
||||
@@ -214,6 +224,8 @@ class OutputProcessor:
|
||||
finish_reason,
|
||||
iteration_stats)
|
||||
|
||||
self.lora_states.update_iteration_stats(iteration_stats)
|
||||
|
||||
return OutputProcessorOutput(
|
||||
request_outputs=request_outputs,
|
||||
reqs_to_abort=reqs_to_abort,
|
||||
@@ -226,13 +238,15 @@ class OutputProcessor:
|
||||
if iteration_stats is None:
|
||||
return
|
||||
|
||||
lora_stats = self.lora_states.get_stats(req_state)
|
||||
|
||||
assert engine_core_timestamp is not None
|
||||
assert req_state.stats is not None
|
||||
iteration_stats.update_from_output(engine_core_output,
|
||||
engine_core_timestamp,
|
||||
req_state.is_prefilling,
|
||||
req_state.prompt_len,
|
||||
req_state.stats)
|
||||
req_state.stats, lora_stats)
|
||||
|
||||
def _update_stats_from_finished(self, req_state: RequestState,
|
||||
request_output: RequestOutput,
|
||||
@@ -246,6 +260,7 @@ class OutputProcessor:
|
||||
iteration_stats.update_from_finished_request(finish_reason,
|
||||
request_output,
|
||||
req_state.stats)
|
||||
self.lora_states.finish_request(req_state)
|
||||
|
||||
@staticmethod
|
||||
def _make_request_output(
|
||||
|
||||
Reference in New Issue
Block a user