Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
195 lines
6.6 KiB
Python
195 lines
6.6 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Thread pool:
|
|
Two queues (load, store) and two sets of threads:
|
|
- Load-priority threads: drain the load queue first, then the store queue.
|
|
- Store-priority threads: drain the store queue first, then the load queue.
|
|
Load jobs are enqueued to the load queue; store jobs to the store queue.
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
from collections import deque
|
|
from collections.abc import Callable, Iterable
|
|
|
|
from vllm.logger import init_logger
|
|
from vllm.v1.kv_offload.tiering.base import JobId
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class JobState:
|
|
"""Thread-safe completion tracker for a set of per-block I/O tasks.
|
|
|
|
Each task calls task_done(success) when it finishes.
|
|
"""
|
|
|
|
__slots__ = (
|
|
"_job_id",
|
|
"_n_tasks",
|
|
"_completed",
|
|
"_success",
|
|
"_transfer_time",
|
|
"_lock",
|
|
)
|
|
|
|
def __init__(self, job_id: JobId, n_tasks: int) -> None:
|
|
self._job_id: JobId = job_id
|
|
self._n_tasks = n_tasks
|
|
self._completed = 0
|
|
self._success = True
|
|
self._transfer_time = 0.0
|
|
self._lock = threading.Lock()
|
|
|
|
@property
|
|
def job_id(self) -> JobId:
|
|
return self._job_id
|
|
|
|
def task_done(
|
|
self, success: bool, transfer_time: float
|
|
) -> tuple[bool, bool, float]:
|
|
"""Returns if job completed and success flag."""
|
|
with self._lock:
|
|
self._completed += 1
|
|
self._transfer_time += transfer_time
|
|
if not success:
|
|
self._success = False
|
|
return self._completed == self._n_tasks, self._success, self._transfer_time
|
|
|
|
|
|
class DualQueueThreadPool:
|
|
"""Thread pool with two task queues (load and store) and two thread groups.
|
|
|
|
Load-priority threads drain the load queue first, then fall back to the
|
|
store queue. Store-priority threads do the reverse. Both queues share
|
|
a single condition variable.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
n_read_threads: int,
|
|
n_write_threads: int,
|
|
thread_name_prefix: str = "fs_secondary_tier",
|
|
) -> None:
|
|
self._load_q: deque = deque()
|
|
self._store_q: deque = deque()
|
|
self._condition = threading.Condition(threading.Lock())
|
|
self._stop = False
|
|
self._threads: list[threading.Thread] = []
|
|
self._finished_q: deque[tuple[JobId, bool, float]] = deque()
|
|
self._inflight_jobs = 0 # guarded by _condition
|
|
|
|
for i in range(n_read_threads):
|
|
t = threading.Thread(
|
|
target=self._worker,
|
|
args=(True,),
|
|
name=f"{thread_name_prefix}_l{i}",
|
|
daemon=True,
|
|
)
|
|
t.start()
|
|
self._threads.append(t)
|
|
|
|
for i in range(n_write_threads):
|
|
t = threading.Thread(
|
|
target=self._worker,
|
|
args=(False,),
|
|
name=f"{thread_name_prefix}_s{i}",
|
|
daemon=True,
|
|
)
|
|
t.start()
|
|
self._threads.append(t)
|
|
|
|
def enqueue_load(
|
|
self,
|
|
job_id: JobId,
|
|
n_tasks: int,
|
|
tasks: Iterable[Callable],
|
|
) -> None:
|
|
"""Enqueue load tasks for a job (high-priority for load-priority threads)."""
|
|
state = JobState(job_id, n_tasks)
|
|
with self._condition:
|
|
self._inflight_jobs += 1
|
|
for fn in tasks:
|
|
self._load_q.append((fn, state))
|
|
self._condition.notify(n_tasks)
|
|
|
|
def enqueue_store(
|
|
self,
|
|
job_id: JobId,
|
|
n_tasks: int,
|
|
tasks: Iterable[Callable],
|
|
) -> None:
|
|
"""Enqueue store tasks for a job (high-priority for store-priority threads)."""
|
|
state = JobState(job_id, n_tasks)
|
|
with self._condition:
|
|
self._inflight_jobs += 1
|
|
for fn in tasks:
|
|
self._store_q.append((fn, state))
|
|
self._condition.notify(n_tasks)
|
|
|
|
def get_finished(self) -> list[tuple[JobId, bool, float]]:
|
|
# No lock needed: deque is thread-safe for concurrent append/popleft,
|
|
# and the manager is the sole popper.
|
|
jobs = []
|
|
while self._finished_q:
|
|
jobs.append(self._finished_q.popleft())
|
|
return jobs
|
|
|
|
def wait_idle(self) -> None:
|
|
"""Block until there are no in-flight jobs.
|
|
|
|
After this returns, every submitted job has had its last task
|
|
finish, so no worker thread is still copying data. Note:
|
|
completed jobs may still be sitting in ``_finished_q`` waiting
|
|
for ``get_finished()`` to drain them.
|
|
"""
|
|
with self._condition:
|
|
self._condition.wait_for(lambda: self._inflight_jobs == 0)
|
|
|
|
def shutdown(self, wait: bool = True) -> None:
|
|
with self._condition:
|
|
self._stop = True
|
|
self._load_q.clear()
|
|
self._store_q.clear()
|
|
# Cancelled tasks will not decrement _inflight_jobs; reset it so a
|
|
# subsequent wait_idle() returns instead of hanging.
|
|
self._inflight_jobs = 0
|
|
self._condition.notify_all()
|
|
if wait:
|
|
for t in self._threads:
|
|
t.join()
|
|
|
|
def _worker(self, load_priority: bool) -> None:
|
|
# Wait for tasks, process from primary queue first, fall back to secondary.
|
|
while True:
|
|
with self._condition:
|
|
self._condition.wait_for(
|
|
lambda: self._stop or self._load_q or self._store_q
|
|
)
|
|
if self._stop:
|
|
return
|
|
primary = self._load_q if load_priority else self._store_q
|
|
secondary = self._store_q if load_priority else self._load_q
|
|
task, state = primary.popleft() if primary else secondary.popleft()
|
|
try:
|
|
start_time = time.monotonic()
|
|
task()
|
|
transfer_time = time.monotonic() - start_time
|
|
job_finished, success, total_time = state.task_done(True, transfer_time)
|
|
except Exception as exc:
|
|
transfer_time = time.monotonic() - start_time
|
|
logger.error(
|
|
"Job %s block I/O failed: %s",
|
|
state.job_id,
|
|
exc,
|
|
)
|
|
job_finished, success, total_time = state.task_done(
|
|
False, transfer_time
|
|
)
|
|
|
|
if job_finished:
|
|
with self._condition:
|
|
self._finished_q.append((state.job_id, success, total_time))
|
|
self._inflight_jobs -= 1
|
|
self._condition.notify_all()
|