[Feature][V1]: suupports cached_tokens in response usage (#18149)

Co-authored-by: simon-mo <xmo@berkeley.edu>
This commit is contained in:
Chauncey
2025-05-23 01:41:03 -07:00
committed by GitHub
co-authored by simon-mo
parent 54af915949
commit b046cf792d
5 changed files with 27 additions and 5 deletions
+6 -3
View File
@@ -147,6 +147,7 @@ class RequestState:
finish_reason: Optional[FinishReason],
stop_reason: Union[int, str, None],
kv_transfer_params: Optional[dict[str, Any]] = None,
num_cached_tokens: int = 0,
) -> Optional[RequestOutput]:
finished = finish_reason is not None
@@ -169,7 +170,7 @@ class RequestState:
return None
return self._new_request_output(request_id, outputs, finished,
kv_transfer_params)
kv_transfer_params, num_cached_tokens)
def _new_request_output(
self,
@@ -177,6 +178,7 @@ class RequestState:
outputs: list[CompletionOutput],
finished: bool,
kv_transfer_params: Optional[dict[str, Any]] = None,
num_cached_tokens: int = 0,
) -> RequestOutput:
if self.output_kind == RequestOutputKind.DELTA:
@@ -193,6 +195,7 @@ class RequestState:
outputs=outputs,
finished=finished,
kv_transfer_params=kv_transfer_params,
num_cached_tokens=num_cached_tokens,
)
def _new_completion_output(
@@ -340,7 +343,7 @@ class OutputProcessor:
finish_reason = engine_core_output.finish_reason
stop_reason = engine_core_output.stop_reason
kv_transfer_params = engine_core_output.kv_transfer_params
num_cached_tokens = engine_core_output.num_cached_tokens
req_state.is_prefilling = False
# 2) Detokenize the token ids into text and perform stop checks.
@@ -356,7 +359,7 @@ class OutputProcessor:
# 4) Create and handle RequestOutput objects.
if request_output := req_state.make_request_output(
new_token_ids, finish_reason, stop_reason,
kv_transfer_params):
kv_transfer_params, num_cached_tokens):
if req_state.queue is not None:
# AsyncLLM: put into queue for handling by generate().
req_state.queue.put(request_output)