[KV Offload] Pass ScheduleEndContext to on_schedule_end hook (#46450)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Ronen Schaffer
2026-06-30 17:07:12 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 62c7d8009f
commit 0fc2512094
11 changed files with 42 additions and 14 deletions
@@ -42,6 +42,7 @@ from vllm.v1.kv_offload.base import (
OffloadPolicy,
ReqContext,
RequestOffloadingContext,
ScheduleEndContext,
make_offload_key,
)
from vllm.v1.outputs import KVConnectorOutput
@@ -1027,7 +1028,11 @@ class OffloadingConnectorScheduler:
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
self._update_req_states(scheduler_output)
self.manager.on_schedule_end()
schedule_end_context = ScheduleEndContext(
new_req_ids=[req.req_id for req in scheduler_output.scheduled_new_reqs],
preempted_req_ids=scheduler_output.preempted_req_ids or (),
)
self.manager.on_schedule_end(schedule_end_context)
# Flush jobs for preempted requests.
for req_id in scheduler_output.preempted_req_ids or ():
+11 -2
View File
@@ -8,7 +8,7 @@ from abc import ABC, abstractmethod
from collections.abc import Collection, Iterable, Sequence
from dataclasses import dataclass
from enum import Enum, auto
from typing import TYPE_CHECKING, Any, NewType
from typing import TYPE_CHECKING, Any, NamedTuple, NewType
import numpy as np
import torch
@@ -76,6 +76,15 @@ class RequestOffloadingContext:
policy: OffloadPolicy = OffloadPolicy.BLOCK_LEVEL
class ScheduleEndContext(NamedTuple):
"""Per-step scheduling info passed to on_schedule_end()."""
# Request IDs scheduled for the first time this step.
new_req_ids: Collection[str]
# Request IDs preempted this step.
preempted_req_ids: Collection[str]
class LoadStoreSpec(ABC):
"""
Abstract metadata that encapsulates information allowing a worker
@@ -309,7 +318,7 @@ class OffloadingManager(ABC):
"""
return ()
def on_schedule_end(self) -> None:
def on_schedule_end(self, context: ScheduleEndContext) -> None:
"""Called once at the end of each scheduler step.
Managers may override this to flush deferred work accumulated
+2 -1
View File
@@ -17,6 +17,7 @@ from vllm.v1.kv_offload.base import (
OffloadKey,
ReqContext,
RequestOffloadingContext,
ScheduleEndContext,
)
if TYPE_CHECKING:
@@ -209,7 +210,7 @@ class SecondaryTierManager(ABC):
"""
return
def on_schedule_end(self) -> None:
def on_schedule_end(self, context: ScheduleEndContext) -> None:
"""Called once at the end of each scheduler step.
Secondary tiers may override this for per-step cleanup or
+2 -1
View File
@@ -38,6 +38,7 @@ from vllm.v1.kv_offload.tiering.base import (
JobMetadata,
JobResult,
RequestOffloadingContext,
ScheduleEndContext,
SecondaryTierManager,
)
from vllm.v1.kv_offload.tiering.fs.io import load_block, store_block
@@ -201,7 +202,7 @@ class FileSystemTierManager(SecondaryTierManager):
self._lookup_manager.cleanup(req_context.req_id)
@override
def on_schedule_end(self) -> None:
def on_schedule_end(self, context: ScheduleEndContext) -> None:
self._lookup_manager.flush()
@override
+3 -2
View File
@@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import (
PrepareStoreOutput,
ReqContext,
RequestOffloadingContext,
ScheduleEndContext,
)
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
@@ -603,7 +604,7 @@ class TieringOffloadingManager(OffloadingManager):
del self._req_state[req_id]
@override
def on_schedule_end(self) -> None:
def on_schedule_end(self, context: ScheduleEndContext) -> None:
"""End-of-schedule hook: process finished jobs, flush deferred
promotions, and reset the per-step gate.
@@ -614,7 +615,7 @@ class TieringOffloadingManager(OffloadingManager):
self._processed_jobs_this_step = False
self._flush_pending_promotions()
for tier in self.secondary_tiers:
tier.on_schedule_end()
tier.on_schedule_end(context)
@override
def has_pending_work(self) -> bool:
+2 -1
View File
@@ -17,6 +17,7 @@ from vllm.v1.kv_offload.tiering.base import (
JobMetadata,
JobResult,
RequestOffloadingContext,
ScheduleEndContext,
SecondaryTierManager,
)
from vllm.v1.kv_offload.tiering.obj.config import ObjStoreConfig
@@ -242,7 +243,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
def on_request_finished(self, req_context: ReqContext) -> None:
self._lookup_manager.cleanup(req_context.req_id)
def on_schedule_end(self) -> None:
def on_schedule_end(self, context: ScheduleEndContext) -> None:
self._lookup_manager.flush()
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
+2 -1
View File
@@ -26,6 +26,7 @@ from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.base import (
JobMetadata,
JobResult,
ScheduleEndContext,
SecondaryTierManager,
)
from vllm.v1.kv_offload.tiering.p2p.control import ControlTransport, ZmqTransport
@@ -426,7 +427,7 @@ class P2PSecondaryTierManager(SecondaryTierManager):
time.sleep(_DRAIN_SLEEP_S)
@override
def on_schedule_end(self) -> None:
def on_schedule_end(self, context: ScheduleEndContext) -> None:
return
# ------------------------------------------------------------------