[KV Offload] Fix num_tokens_after_batch for different termination types (#49285)
Signed-off-by: Alex <jihuihuang@example.com> Signed-off-by: Alex <jihui.huang@daocloud.io> Signed-off-by: Alex <alex.tech.lab@outlook.com> Signed-off-by: Alex <jihuihuang@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Claude
Or Ozeri
parent
0da6e7f3d6
commit
3f1d40960f
@@ -50,7 +50,7 @@ from vllm.v1.kv_offload.base import (
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -976,7 +976,9 @@ class OffloadingConnectorScheduler:
|
||||
continue
|
||||
req = req_status.req
|
||||
|
||||
if req.is_finished():
|
||||
if req.status is RequestStatus.FINISHED_ABORTED:
|
||||
num_tokens_after_batch = req.num_computed_tokens
|
||||
elif req.is_finished():
|
||||
num_tokens_after_batch = req.num_tokens
|
||||
else:
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
|
||||
Reference in New Issue
Block a user