[KV Offload] Pass ScheduleEndContext to on_schedule_end hook (#46450)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Or Ozeri
parent
62c7d8009f
commit
0fc2512094
@@ -13,7 +13,7 @@ from types import SimpleNamespace
|
|||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
from vllm.v1.kv_offload.base import LookupResult, ReqContext
|
from vllm.v1.kv_offload.base import LookupResult, ReqContext, ScheduleEndContext
|
||||||
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
|
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
|
||||||
from vllm.v1.kv_offload.tiering.p2p import manager as manager_module
|
from vllm.v1.kv_offload.tiering.p2p import manager as manager_module
|
||||||
from vllm.v1.kv_offload.tiering.p2p.manager import (
|
from vllm.v1.kv_offload.tiering.p2p.manager import (
|
||||||
@@ -1279,7 +1279,12 @@ class TestOnScheduleEnd:
|
|||||||
mgr = _make_manager()
|
mgr = _make_manager()
|
||||||
before_sessions = dict(mgr._sessions)
|
before_sessions = dict(mgr._sessions)
|
||||||
before_jobs = list(mgr._finished_jobs)
|
before_jobs = list(mgr._finished_jobs)
|
||||||
assert mgr.on_schedule_end() is None
|
assert (
|
||||||
|
mgr.on_schedule_end(
|
||||||
|
ScheduleEndContext(new_req_ids=[], preempted_req_ids=())
|
||||||
|
)
|
||||||
|
is None
|
||||||
|
)
|
||||||
assert mgr._sessions == before_sessions
|
assert mgr._sessions == before_sessions
|
||||||
assert mgr._finished_jobs == before_jobs
|
assert mgr._finished_jobs == before_jobs
|
||||||
|
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ from vllm.v1.kv_offload.base import (
|
|||||||
LookupResult,
|
LookupResult,
|
||||||
OffloadKey,
|
OffloadKey,
|
||||||
ReqContext,
|
ReqContext,
|
||||||
|
ScheduleEndContext,
|
||||||
make_offload_key,
|
make_offload_key,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.base import JobMetadata
|
from vllm.v1.kv_offload.tiering.base import JobMetadata
|
||||||
@@ -107,7 +108,7 @@ def lookup_and_wait(
|
|||||||
"""Perform a full async lookup cycle and return resolved results."""
|
"""Perform a full async lookup cycle and return resolved results."""
|
||||||
for k in keys:
|
for k in keys:
|
||||||
tier.lookup(k, ctx)
|
tier.lookup(k, ctx)
|
||||||
tier.on_schedule_end()
|
tier.on_schedule_end(ScheduleEndContext(new_req_ids=[], preempted_req_ids=()))
|
||||||
deadline = time.monotonic() + timeout
|
deadline = time.monotonic() + timeout
|
||||||
while time.monotonic() < deadline:
|
while time.monotonic() < deadline:
|
||||||
if not tier._lookup_manager._pending_results.empty():
|
if not tier._lookup_manager._pending_results.empty():
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from vllm.v1.kv_offload.base import (
|
|||||||
LookupResult,
|
LookupResult,
|
||||||
OffloadKey,
|
OffloadKey,
|
||||||
ReqContext,
|
ReqContext,
|
||||||
|
ScheduleEndContext,
|
||||||
make_offload_key,
|
make_offload_key,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
|
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
|
||||||
@@ -222,7 +223,7 @@ def lookup_and_wait(
|
|||||||
"""Perform a full async lookup cycle and return resolved results."""
|
"""Perform a full async lookup cycle and return resolved results."""
|
||||||
for k in keys:
|
for k in keys:
|
||||||
tier.lookup(k, ctx)
|
tier.lookup(k, ctx)
|
||||||
tier.on_schedule_end()
|
tier.on_schedule_end(ScheduleEndContext(new_req_ids=[], preempted_req_ids=()))
|
||||||
deadline = time.monotonic() + timeout
|
deadline = time.monotonic() + timeout
|
||||||
while time.monotonic() < deadline:
|
while time.monotonic() < deadline:
|
||||||
if not tier._lookup_manager._pending_results.empty():
|
if not tier._lookup_manager._pending_results.empty():
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ from vllm.v1.kv_offload.base import (
|
|||||||
OffloadPolicy,
|
OffloadPolicy,
|
||||||
ReqContext,
|
ReqContext,
|
||||||
RequestOffloadingContext,
|
RequestOffloadingContext,
|
||||||
|
ScheduleEndContext,
|
||||||
make_offload_key,
|
make_offload_key,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.base import (
|
from vllm.v1.kv_offload.tiering.base import (
|
||||||
@@ -233,7 +234,8 @@ class TestTieringOffloadingManager:
|
|||||||
|
|
||||||
def _simulate_on_schedule_end(self):
|
def _simulate_on_schedule_end(self):
|
||||||
"""Simulate end of scheduler step: lifecycle flush + drain events."""
|
"""Simulate end of scheduler step: lifecycle flush + drain events."""
|
||||||
self.manager.on_schedule_end()
|
ctx = ScheduleEndContext(new_req_ids=[], preempted_req_ids=())
|
||||||
|
self.manager.on_schedule_end(ctx)
|
||||||
list(self.manager.take_events())
|
list(self.manager.take_events())
|
||||||
|
|
||||||
def _start_request(self, req_context: ReqContext = _CTX):
|
def _start_request(self, req_context: ReqContext = _CTX):
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ from vllm.v1.kv_offload.base import (
|
|||||||
OffloadPolicy,
|
OffloadPolicy,
|
||||||
ReqContext,
|
ReqContext,
|
||||||
RequestOffloadingContext,
|
RequestOffloadingContext,
|
||||||
|
ScheduleEndContext,
|
||||||
make_offload_key,
|
make_offload_key,
|
||||||
)
|
)
|
||||||
from vllm.v1.outputs import KVConnectorOutput
|
from vllm.v1.outputs import KVConnectorOutput
|
||||||
@@ -1027,7 +1028,11 @@ class OffloadingConnectorScheduler:
|
|||||||
self, scheduler_output: SchedulerOutput
|
self, scheduler_output: SchedulerOutput
|
||||||
) -> KVConnectorMetadata:
|
) -> KVConnectorMetadata:
|
||||||
self._update_req_states(scheduler_output)
|
self._update_req_states(scheduler_output)
|
||||||
self.manager.on_schedule_end()
|
schedule_end_context = ScheduleEndContext(
|
||||||
|
new_req_ids=[req.req_id for req in scheduler_output.scheduled_new_reqs],
|
||||||
|
preempted_req_ids=scheduler_output.preempted_req_ids or (),
|
||||||
|
)
|
||||||
|
self.manager.on_schedule_end(schedule_end_context)
|
||||||
|
|
||||||
# Flush jobs for preempted requests.
|
# Flush jobs for preempted requests.
|
||||||
for req_id in scheduler_output.preempted_req_ids or ():
|
for req_id in scheduler_output.preempted_req_ids or ():
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from abc import ABC, abstractmethod
|
|||||||
from collections.abc import Collection, Iterable, Sequence
|
from collections.abc import Collection, Iterable, Sequence
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from enum import Enum, auto
|
from enum import Enum, auto
|
||||||
from typing import TYPE_CHECKING, Any, NewType
|
from typing import TYPE_CHECKING, Any, NamedTuple, NewType
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
@@ -76,6 +76,15 @@ class RequestOffloadingContext:
|
|||||||
policy: OffloadPolicy = OffloadPolicy.BLOCK_LEVEL
|
policy: OffloadPolicy = OffloadPolicy.BLOCK_LEVEL
|
||||||
|
|
||||||
|
|
||||||
|
class ScheduleEndContext(NamedTuple):
|
||||||
|
"""Per-step scheduling info passed to on_schedule_end()."""
|
||||||
|
|
||||||
|
# Request IDs scheduled for the first time this step.
|
||||||
|
new_req_ids: Collection[str]
|
||||||
|
# Request IDs preempted this step.
|
||||||
|
preempted_req_ids: Collection[str]
|
||||||
|
|
||||||
|
|
||||||
class LoadStoreSpec(ABC):
|
class LoadStoreSpec(ABC):
|
||||||
"""
|
"""
|
||||||
Abstract metadata that encapsulates information allowing a worker
|
Abstract metadata that encapsulates information allowing a worker
|
||||||
@@ -309,7 +318,7 @@ class OffloadingManager(ABC):
|
|||||||
"""
|
"""
|
||||||
return ()
|
return ()
|
||||||
|
|
||||||
def on_schedule_end(self) -> None:
|
def on_schedule_end(self, context: ScheduleEndContext) -> None:
|
||||||
"""Called once at the end of each scheduler step.
|
"""Called once at the end of each scheduler step.
|
||||||
|
|
||||||
Managers may override this to flush deferred work accumulated
|
Managers may override this to flush deferred work accumulated
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from vllm.v1.kv_offload.base import (
|
|||||||
OffloadKey,
|
OffloadKey,
|
||||||
ReqContext,
|
ReqContext,
|
||||||
RequestOffloadingContext,
|
RequestOffloadingContext,
|
||||||
|
ScheduleEndContext,
|
||||||
)
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -209,7 +210,7 @@ class SecondaryTierManager(ABC):
|
|||||||
"""
|
"""
|
||||||
return
|
return
|
||||||
|
|
||||||
def on_schedule_end(self) -> None:
|
def on_schedule_end(self, context: ScheduleEndContext) -> None:
|
||||||
"""Called once at the end of each scheduler step.
|
"""Called once at the end of each scheduler step.
|
||||||
|
|
||||||
Secondary tiers may override this for per-step cleanup or
|
Secondary tiers may override this for per-step cleanup or
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ from vllm.v1.kv_offload.tiering.base import (
|
|||||||
JobMetadata,
|
JobMetadata,
|
||||||
JobResult,
|
JobResult,
|
||||||
RequestOffloadingContext,
|
RequestOffloadingContext,
|
||||||
|
ScheduleEndContext,
|
||||||
SecondaryTierManager,
|
SecondaryTierManager,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.fs.io import load_block, store_block
|
from vllm.v1.kv_offload.tiering.fs.io import load_block, store_block
|
||||||
@@ -201,7 +202,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
|||||||
self._lookup_manager.cleanup(req_context.req_id)
|
self._lookup_manager.cleanup(req_context.req_id)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def on_schedule_end(self) -> None:
|
def on_schedule_end(self, context: ScheduleEndContext) -> None:
|
||||||
self._lookup_manager.flush()
|
self._lookup_manager.flush()
|
||||||
|
|
||||||
@override
|
@override
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import (
|
|||||||
PrepareStoreOutput,
|
PrepareStoreOutput,
|
||||||
ReqContext,
|
ReqContext,
|
||||||
RequestOffloadingContext,
|
RequestOffloadingContext,
|
||||||
|
ScheduleEndContext,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||||
@@ -603,7 +604,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
del self._req_state[req_id]
|
del self._req_state[req_id]
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def on_schedule_end(self) -> None:
|
def on_schedule_end(self, context: ScheduleEndContext) -> None:
|
||||||
"""End-of-schedule hook: process finished jobs, flush deferred
|
"""End-of-schedule hook: process finished jobs, flush deferred
|
||||||
promotions, and reset the per-step gate.
|
promotions, and reset the per-step gate.
|
||||||
|
|
||||||
@@ -614,7 +615,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
self._processed_jobs_this_step = False
|
self._processed_jobs_this_step = False
|
||||||
self._flush_pending_promotions()
|
self._flush_pending_promotions()
|
||||||
for tier in self.secondary_tiers:
|
for tier in self.secondary_tiers:
|
||||||
tier.on_schedule_end()
|
tier.on_schedule_end(context)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def has_pending_work(self) -> bool:
|
def has_pending_work(self) -> bool:
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from vllm.v1.kv_offload.tiering.base import (
|
|||||||
JobMetadata,
|
JobMetadata,
|
||||||
JobResult,
|
JobResult,
|
||||||
RequestOffloadingContext,
|
RequestOffloadingContext,
|
||||||
|
ScheduleEndContext,
|
||||||
SecondaryTierManager,
|
SecondaryTierManager,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.obj.config import ObjStoreConfig
|
from vllm.v1.kv_offload.tiering.obj.config import ObjStoreConfig
|
||||||
@@ -242,7 +243,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
|||||||
def on_request_finished(self, req_context: ReqContext) -> None:
|
def on_request_finished(self, req_context: ReqContext) -> None:
|
||||||
self._lookup_manager.cleanup(req_context.req_id)
|
self._lookup_manager.cleanup(req_context.req_id)
|
||||||
|
|
||||||
def on_schedule_end(self) -> None:
|
def on_schedule_end(self, context: ScheduleEndContext) -> None:
|
||||||
self._lookup_manager.flush()
|
self._lookup_manager.flush()
|
||||||
|
|
||||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ from vllm.v1.kv_offload.file_mapper import FileMapper
|
|||||||
from vllm.v1.kv_offload.tiering.base import (
|
from vllm.v1.kv_offload.tiering.base import (
|
||||||
JobMetadata,
|
JobMetadata,
|
||||||
JobResult,
|
JobResult,
|
||||||
|
ScheduleEndContext,
|
||||||
SecondaryTierManager,
|
SecondaryTierManager,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.p2p.control import ControlTransport, ZmqTransport
|
from vllm.v1.kv_offload.tiering.p2p.control import ControlTransport, ZmqTransport
|
||||||
@@ -426,7 +427,7 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
|||||||
time.sleep(_DRAIN_SLEEP_S)
|
time.sleep(_DRAIN_SLEEP_S)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def on_schedule_end(self) -> None:
|
def on_schedule_end(self, context: ScheduleEndContext) -> None:
|
||||||
return
|
return
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|||||||
Reference in New Issue
Block a user