[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers
- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs, moriio, hisparse, example and hidden-states connectors - Reduce weight_transfer to the base API used by model loading - Remove kv_offload tiering, simple_kv_offload, ec cpu/example, snapshot command, engine snapshot server - Remove transformers model fallback, dsv4 tokenizers, dead platforms cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn ops, dead layers (short_conv, fused_embed_norm, indexer_topk, relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar - Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant - Rebuild parser hubs around Qwen3 adapters only (36 dead adapters) - Remove dead tests with the code they covered Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -150,48 +150,12 @@ class KVConnectorFactory:
|
||||
# The registration should not be done in each individual file, as we want to
|
||||
# only load the files corresponding to the current connector.
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"ExampleHiddenStatesConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.example_hidden_states_connector",
|
||||
"ExampleHiddenStatesConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"NixlConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
|
||||
"NixlConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"NixlPullConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
|
||||
"NixlPullConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"NixlPushConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
|
||||
"NixlPushConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"MultiConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.multi_connector",
|
||||
"MultiConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"HiSparseConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector",
|
||||
"HiSparseConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"MoRIIOConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector",
|
||||
"MoRIIOConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"OffloadingConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector",
|
||||
@@ -203,14 +167,3 @@ KVConnectorFactory.register_connector(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.decode_bench_connector",
|
||||
"DecodeBenchConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"SimpleCPUOffloadConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector",
|
||||
"SimpleCPUOffloadConnector",
|
||||
)
|
||||
KVConnectorFactory.register_connector(
|
||||
"HF3FSKVConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_connector",
|
||||
"HF3FSKVConnector",
|
||||
)
|
||||
|
||||
@@ -1,302 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import threading
|
||||
from functools import wraps
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import torch.utils.cpp_extension
|
||||
from torch.utils.cpp_extension import load
|
||||
|
||||
root = Path(__file__).parent.resolve()
|
||||
cuda_include_path = os.path.join(torch.utils.cpp_extension.CUDA_HOME, "include")
|
||||
hf3fs_utils = load(
|
||||
name="hf3fs_utils",
|
||||
sources=[f"{root}/utils/hf3fs_utils.cpp"],
|
||||
extra_include_paths=[cuda_include_path],
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HF3FS_AVAILABLE = True
|
||||
try:
|
||||
from hf3fs_fuse.io import (
|
||||
deregister_fd,
|
||||
extract_mount_point,
|
||||
make_ioring,
|
||||
make_iovec,
|
||||
register_fd,
|
||||
)
|
||||
except ImportError:
|
||||
HF3FS_AVAILABLE = False
|
||||
|
||||
|
||||
def rsynchronized():
|
||||
def _decorator(func):
|
||||
@wraps(func)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
with self.rlock:
|
||||
return func(self, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return _decorator
|
||||
|
||||
|
||||
def wsynchronized():
|
||||
def _decorator(func):
|
||||
@wraps(func)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
with self.wlock:
|
||||
return func(self, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return _decorator
|
||||
|
||||
|
||||
class Hf3fsClient:
|
||||
def __init__(self, path: str, size: int, bytes_per_page: int, entries: int):
|
||||
"""Initialize the HF3FS client with hf3fs_fuse.
|
||||
|
||||
Args:
|
||||
path: Path to the file used for storage
|
||||
size: Total size of the storage file in bytes
|
||||
bytes_per_page: Size of each page in bytes
|
||||
entries: Maximum number of concurrent operations
|
||||
|
||||
"""
|
||||
if not HF3FS_AVAILABLE:
|
||||
raise ImportError(
|
||||
"hf3fs_fuse.io is not available. Please install the hf3fs_fuse package."
|
||||
)
|
||||
|
||||
self.path = path
|
||||
self.size = size
|
||||
self.bytes_per_page = bytes_per_page
|
||||
self.entries = entries
|
||||
|
||||
self._closed = False
|
||||
|
||||
self.file = None
|
||||
self.shm_r = None
|
||||
self.shm_w = None
|
||||
self.ior_r = None
|
||||
self.ior_w = None
|
||||
self.iov_r = None
|
||||
self.iov_w = None
|
||||
try:
|
||||
# Create the file if it doesn't exist and set its size
|
||||
self.file = os.open(self.path, os.O_RDWR | os.O_CREAT)
|
||||
os.ftruncate(self.file, size)
|
||||
register_fd(self.file)
|
||||
|
||||
self.hf3fs_mount_point = extract_mount_point(path)
|
||||
self.bs = self.bytes_per_page
|
||||
self.shm_r = multiprocessing.shared_memory.SharedMemory(
|
||||
size=self.bs * self.entries, create=True
|
||||
)
|
||||
self.shm_w = multiprocessing.shared_memory.SharedMemory(
|
||||
size=self.bs * self.entries, create=True
|
||||
)
|
||||
|
||||
self.shm_r_tensor = torch.frombuffer(self.shm_r.buf, dtype=torch.uint8)
|
||||
self.shm_w_tensor = torch.frombuffer(self.shm_w.buf, dtype=torch.uint8)
|
||||
|
||||
numel = self.bs * self.entries
|
||||
self.r_pinned = torch.empty(
|
||||
numel,
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
)
|
||||
self.w_pinned = torch.empty(
|
||||
numel,
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
)
|
||||
|
||||
self.numa = -1
|
||||
self.ior_r = make_ioring(
|
||||
self.hf3fs_mount_point,
|
||||
self.entries,
|
||||
for_read=True,
|
||||
timeout=1,
|
||||
numa=self.numa,
|
||||
)
|
||||
self.ior_w = make_ioring(
|
||||
self.hf3fs_mount_point,
|
||||
self.entries,
|
||||
for_read=False,
|
||||
timeout=1,
|
||||
numa=self.numa,
|
||||
)
|
||||
self.iov_r = make_iovec(self.shm_r, self.hf3fs_mount_point)
|
||||
self.iov_w = make_iovec(self.shm_w, self.hf3fs_mount_point)
|
||||
self.shm_r.unlink()
|
||||
self.shm_w.unlink()
|
||||
|
||||
self.rlock = threading.RLock()
|
||||
self.wlock = threading.RLock()
|
||||
|
||||
self.stream = torch.cuda.Stream()
|
||||
self.stream_ptr_int = self.stream.cuda_stream
|
||||
|
||||
except Exception:
|
||||
self._release_resources()
|
||||
raise
|
||||
|
||||
logger.debug(
|
||||
"Initialized HF3FS client with file: %s, size: %s bytes", path, size
|
||||
)
|
||||
|
||||
def _release_resources(self) -> None:
|
||||
"""Release all acquired resources safely."""
|
||||
# iov must be released before ioring and shm
|
||||
for attr in ("iov_r", "iov_w", "ior_r", "ior_w"):
|
||||
obj = getattr(self, attr, None)
|
||||
if obj is not None:
|
||||
del obj
|
||||
setattr(self, attr, None)
|
||||
|
||||
for attr in ("shm_r", "shm_w"):
|
||||
shm = getattr(self, attr, None)
|
||||
if shm is not None:
|
||||
try:
|
||||
shm.close()
|
||||
except Exception as e:
|
||||
logger.warning("Failed to close %s: %s", attr, e)
|
||||
setattr(self, attr, None)
|
||||
|
||||
if self.file is not None:
|
||||
try:
|
||||
deregister_fd(self.file)
|
||||
except Exception as e:
|
||||
logger.warning("deregister_fd failed: %s", e)
|
||||
try:
|
||||
os.close(self.file)
|
||||
except OSError as e:
|
||||
logger.warning("os.close failed: %s", e)
|
||||
self.file = None
|
||||
|
||||
@rsynchronized()
|
||||
def batch_read(self, offsets: list[int], tensors: list[torch.Tensor]) -> list[int]:
|
||||
"""Read data from the file at specified offsets into tensors.
|
||||
|
||||
Args:
|
||||
offsets: List of byte offsets to read from
|
||||
tensors: List of tensors to read data into
|
||||
|
||||
Returns:
|
||||
List of operation results (0 for success, non-zero for error)
|
||||
|
||||
"""
|
||||
self.check(offsets, tensors)
|
||||
assert self.ior_r is not None
|
||||
assert self.iov_r is not None
|
||||
|
||||
# prepare
|
||||
current = 0
|
||||
for offset, tensor in zip(offsets, tensors):
|
||||
size = tensor.numel() * tensor.itemsize
|
||||
self.ior_r.prepare(
|
||||
self.iov_r[current : current + size], True, self.file, offset
|
||||
)
|
||||
current += size
|
||||
|
||||
# submit
|
||||
ionum = len(offsets)
|
||||
resv = self.ior_r.submit().wait(min_results=ionum)
|
||||
|
||||
# results
|
||||
with torch.cuda.stream(self.stream):
|
||||
hf3fs_utils.read_shm(
|
||||
self.shm_r_tensor, self.r_pinned, tensors, self.stream_ptr_int
|
||||
)
|
||||
results = [res.result for res in resv]
|
||||
|
||||
return results
|
||||
|
||||
@wsynchronized()
|
||||
def batch_write(
|
||||
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
|
||||
) -> list[int]:
|
||||
"""Write data from tensors to the file at specified offsets.
|
||||
|
||||
Args:
|
||||
offsets: List of byte offsets to write to
|
||||
tensors: List of tensors containing data to write
|
||||
event: CUDA event to synchronize on before reading the tensors
|
||||
|
||||
Returns:
|
||||
List of operation results (0 for success, non-zero for error)
|
||||
|
||||
"""
|
||||
self.check(offsets, tensors)
|
||||
assert self.ior_w is not None
|
||||
assert self.iov_w is not None
|
||||
|
||||
# prepare
|
||||
with torch.cuda.stream(self.stream):
|
||||
self.stream.wait_event(event)
|
||||
hf3fs_utils.write_shm(
|
||||
tensors, self.shm_w_tensor, self.w_pinned, self.stream_ptr_int
|
||||
)
|
||||
|
||||
current = 0
|
||||
for offset, tensor in zip(offsets, tensors):
|
||||
size = tensor.numel() * tensor.itemsize
|
||||
self.ior_w.prepare(
|
||||
self.iov_w[current : current + size], False, self.file, offset
|
||||
)
|
||||
current += size
|
||||
|
||||
# submit
|
||||
ionum = len(offsets)
|
||||
resv = self.ior_w.submit().wait(min_results=ionum)
|
||||
|
||||
# results
|
||||
results = [res.result for res in resv]
|
||||
|
||||
return results
|
||||
|
||||
def check(self, offsets: list[int], tensors: list[torch.Tensor]) -> None:
|
||||
sizes = [t.numel() * t.itemsize for t in tensors]
|
||||
if any(
|
||||
[
|
||||
len(offsets) > self.entries,
|
||||
len(offsets) != len(sizes),
|
||||
any(
|
||||
offset < 0 or offset + size > self.size
|
||||
for offset, size in zip(offsets, sizes)
|
||||
),
|
||||
any(size > self.bytes_per_page for size in sizes),
|
||||
]
|
||||
):
|
||||
self.close()
|
||||
raise ValueError("Hf3fsClient.check Failed")
|
||||
|
||||
def get_size(self) -> int:
|
||||
"""Get the total size of the storage file.
|
||||
|
||||
Returns:
|
||||
Size of the file in bytes
|
||||
|
||||
"""
|
||||
return self.size
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the client and clean up resources."""
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
self._release_resources()
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Flush any pending writes to disk."""
|
||||
if not self._closed and self.file is not None:
|
||||
os.fsync(self.file)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,532 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HF3FS Metadata Server with key-based organization."""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import threading
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
|
||||
try:
|
||||
import orjson
|
||||
|
||||
HAS_ORJSON = True
|
||||
except ImportError:
|
||||
import json as orjson # type: ignore
|
||||
|
||||
HAS_ORJSON = False
|
||||
|
||||
import requests
|
||||
from fastapi import FastAPI, HTTPException, Request, Response
|
||||
from fastapi.responses import ORJSONResponse
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RankFileMetadata:
|
||||
"""Manages file page allocation for a single rank."""
|
||||
|
||||
rank_id: int
|
||||
num_pages: int
|
||||
free_pages: list[int]
|
||||
|
||||
def allocate_pages(self, num_pages: int) -> list[int]:
|
||||
"""Allocate specified number of free pages."""
|
||||
if len(self.free_pages) < num_pages:
|
||||
return []
|
||||
|
||||
allocated = self.free_pages[:num_pages]
|
||||
self.free_pages = self.free_pages[num_pages:]
|
||||
return allocated
|
||||
|
||||
def release_pages(self, page_indices: list[int]) -> None:
|
||||
"""Release pages back to free pool."""
|
||||
for page_idx in page_indices:
|
||||
if page_idx not in self.free_pages:
|
||||
self.free_pages.append(page_idx)
|
||||
|
||||
def get_free_page_count(self) -> int:
|
||||
"""Get current number of free pages."""
|
||||
return len(self.free_pages)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KeyMetadata:
|
||||
"""Manages metadata for a single key across multiple ranks."""
|
||||
|
||||
key: str
|
||||
rank_to_page: dict[int, int] # rank -> allocated page index
|
||||
tp_world_size: int
|
||||
|
||||
def add_rank_page(self, rank: int, page_index: int) -> None:
|
||||
"""Add page allocation for a specific rank."""
|
||||
self.rank_to_page[rank] = page_index
|
||||
|
||||
def get_all_pages(self) -> list[tuple[int, int]]:
|
||||
"""Get all (rank, page) pairs for this key."""
|
||||
return [(rank, page) for rank, page in self.rank_to_page.items()]
|
||||
|
||||
def get_rank_page(self, rank: int) -> int | None:
|
||||
"""Get page index for a specific rank."""
|
||||
return self.rank_to_page.get(rank)
|
||||
|
||||
def is_complete(self) -> bool:
|
||||
"""Check if all ranks in the TP world have allocated pages."""
|
||||
return len(self.rank_to_page) == self.tp_world_size
|
||||
|
||||
|
||||
class GlobalMetadataState:
|
||||
"""Manages global metadata state across all ranks and keys."""
|
||||
|
||||
def __init__(self):
|
||||
self.global_lock = threading.RLock()
|
||||
self.rank_metadata: dict[int, RankFileMetadata] = {}
|
||||
self.key_metadata: dict[str, KeyMetadata] = {}
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all metadata state."""
|
||||
with self.global_lock:
|
||||
self.rank_metadata.clear()
|
||||
self.key_metadata.clear()
|
||||
logger.info("Cleared all metadata state")
|
||||
|
||||
def initialize_rank(self, rank: int, num_pages: int) -> None:
|
||||
"""Initialize a new rank with specified number of pages."""
|
||||
with self.global_lock:
|
||||
if rank not in self.rank_metadata:
|
||||
self.rank_metadata[rank] = RankFileMetadata(
|
||||
rank, num_pages, list(range(num_pages))
|
||||
)
|
||||
logger.info("Initialized rank %s with %s pages", rank, num_pages)
|
||||
|
||||
def allocate_pages_for_keys(
|
||||
self, rank: int, keys: list[tuple[str, str]]
|
||||
) -> dict[str, int]:
|
||||
"""Allocate one page for each key on the specified rank.
|
||||
|
||||
Args:
|
||||
rank: Rank ID to allocate pages on
|
||||
keys: List of keys to allocate pages for
|
||||
|
||||
Returns:
|
||||
Dictionary mapping key -> allocated page index
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
if rank not in self.rank_metadata:
|
||||
raise ValueError(f"Rank {rank} not initialized")
|
||||
|
||||
# Batch allocate pages for all keys
|
||||
num_pages_needed = len(keys)
|
||||
allocated_pages = self.rank_metadata[rank].allocate_pages(num_pages_needed)
|
||||
|
||||
if len(allocated_pages) < num_pages_needed:
|
||||
logger.warning(
|
||||
"Rank %s only allocated %s pages for %s keys",
|
||||
rank,
|
||||
len(allocated_pages),
|
||||
num_pages_needed,
|
||||
)
|
||||
|
||||
allocation_results = {}
|
||||
for i, (key, prefix_key) in enumerate(keys):
|
||||
if key in self.key_metadata:
|
||||
key_meta = self.key_metadata[key]
|
||||
if key_meta.is_complete() and rank in key_meta.rank_to_page:
|
||||
# key is already fully written, reuse the existing page
|
||||
# and release the allocated pages back to the free pool.
|
||||
if i < len(allocated_pages):
|
||||
self.rank_metadata[rank].release_pages([allocated_pages[i]])
|
||||
allocation_results[key] = key_meta.rank_to_page[rank]
|
||||
continue
|
||||
|
||||
if i < len(allocated_pages):
|
||||
allocation_results[key] = allocated_pages[i]
|
||||
else:
|
||||
allocation_results[key] = -1 # No pages available
|
||||
|
||||
return allocation_results
|
||||
|
||||
def confirm_write_for_keys(
|
||||
self,
|
||||
rank: int,
|
||||
key_confirmations: list[tuple[str, int]],
|
||||
pages_to_release: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Confirm write operations for keys and update metadata.
|
||||
|
||||
Args:
|
||||
rank: Rank ID that confirmed the writes
|
||||
key_confirmations: List of (key, page_index) tuples
|
||||
pages_to_release: List of page indices to release back to free pool
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
# Confirm successful writes
|
||||
for key, page_index in key_confirmations:
|
||||
if key not in self.key_metadata:
|
||||
# Need to determine tp_world_size from rank_metadata
|
||||
tp_world_size = len(self.rank_metadata)
|
||||
self.key_metadata[key] = KeyMetadata(key, {}, tp_world_size)
|
||||
|
||||
# Add confirmed page to key metadata
|
||||
self.key_metadata[key].add_rank_page(rank, page_index)
|
||||
|
||||
# Release specified pages back to free pool
|
||||
if pages_to_release:
|
||||
self.rank_metadata[rank].release_pages(pages_to_release)
|
||||
logger.debug(
|
||||
"Released %s pages on rank %s: %s",
|
||||
len(pages_to_release),
|
||||
rank,
|
||||
pages_to_release,
|
||||
)
|
||||
|
||||
def batch_key_exists(self, keys: list[str]) -> list[bool]:
|
||||
"""Check if keys exist in metadata and all ranks have confirmed writes.
|
||||
|
||||
Args:
|
||||
keys: List of keys to check
|
||||
|
||||
Returns:
|
||||
List of boolean values indicating key existence and completion
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
results = []
|
||||
for key in keys:
|
||||
if key not in self.key_metadata:
|
||||
results.append(False)
|
||||
else:
|
||||
# Check if all ranks in the TP world have confirmed writes
|
||||
key_meta = self.key_metadata[key]
|
||||
results.append(key_meta.is_complete())
|
||||
return results
|
||||
|
||||
def get_key_locations(self, rank: int, keys: list[str]) -> list[int | None]:
|
||||
"""Get page indices for keys on a specific rank.
|
||||
|
||||
Args:
|
||||
rank: Rank ID to query
|
||||
keys: List of keys to look up
|
||||
|
||||
Returns:
|
||||
List of page indices in the same order as input keys (None if key not found)
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
if rank not in self.rank_metadata:
|
||||
raise ValueError(f"Rank {rank} not initialized")
|
||||
|
||||
results = []
|
||||
for key in keys:
|
||||
if key in self.key_metadata:
|
||||
key_meta = self.key_metadata[key]
|
||||
if key_meta.is_complete():
|
||||
page_index = key_meta.get_rank_page(rank)
|
||||
else:
|
||||
page_index = None
|
||||
|
||||
results.append(page_index)
|
||||
else:
|
||||
results.append(None)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
class Hf3fsMetadataServer:
|
||||
"""HF3FS Metadata Server with improved key-based organization."""
|
||||
|
||||
def __init__(self, persistence_path: str | None = None, save_interval: int = 60):
|
||||
self.state = GlobalMetadataState()
|
||||
if HAS_ORJSON:
|
||||
self.app = FastAPI(default_response_class=ORJSONResponse)
|
||||
else:
|
||||
self.app = FastAPI()
|
||||
self._setup_routes()
|
||||
|
||||
async def _read_json(self, request: Request) -> dict:
|
||||
"""Parse request JSON using orjson if available."""
|
||||
body = await request.body()
|
||||
return orjson.loads(body)
|
||||
|
||||
def _json_response(self, content: dict):
|
||||
"""Return ORJSONResponse when available to bypass jsonable_encoder."""
|
||||
if HAS_ORJSON:
|
||||
return ORJSONResponse(content)
|
||||
else:
|
||||
return content
|
||||
|
||||
def _setup_routes(self):
|
||||
"""Setup FastAPI routes for new API design."""
|
||||
self.app.post("/rank/{rank}/initialize")(self.initialize_rank)
|
||||
self.app.post("/keys/batch_allocate")(self.batch_allocate_pages_for_keys)
|
||||
self.app.post("/keys/confirm_write")(self.confirm_write_for_keys)
|
||||
self.app.post("/keys/batch_exists")(self.batch_key_exists)
|
||||
self.app.post("/keys/get_locations")(self.get_key_locations)
|
||||
self.app.post("/clear")(self.clear)
|
||||
|
||||
async def initialize_rank(self, rank: int, request: Request):
|
||||
"""Initialize a rank with specified number of pages."""
|
||||
data = await self._read_json(request)
|
||||
role = data.get("role", "worker")
|
||||
num_pages = data.get("num_pages", 0)
|
||||
|
||||
if role == "scheduler":
|
||||
return self._json_response(
|
||||
{"message": "Scheduler role does not require initialization"}
|
||||
)
|
||||
|
||||
if role == "worker" and num_pages > 0:
|
||||
self.state.initialize_rank(rank, num_pages)
|
||||
return self._json_response(
|
||||
{"message": f"Rank {rank} initialized with {num_pages} pages"}
|
||||
)
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Invalid initialization parameters"
|
||||
)
|
||||
|
||||
async def batch_allocate_pages_for_keys(self, request: Request):
|
||||
"""Allocate one page for each key on a specific rank."""
|
||||
data = await self._read_json(request)
|
||||
rank = data.get("rank")
|
||||
keys = data.get("keys", [])
|
||||
|
||||
# Validate input format
|
||||
if rank is None or not isinstance(keys, list):
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Invalid request format: need 'rank' and 'keys'"
|
||||
)
|
||||
|
||||
try:
|
||||
# Perform allocation
|
||||
results = self.state.allocate_pages_for_keys(rank, keys)
|
||||
|
||||
# Convert results to response format
|
||||
response = {"rank": rank, "results": list(results.items())}
|
||||
return self._json_response(response)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Allocation failed: {str(e)}"
|
||||
) from e
|
||||
|
||||
async def confirm_write_for_keys(self, request: Request):
|
||||
"""Confirm write operations for keys."""
|
||||
data = await self._read_json(request)
|
||||
rank = data.get("rank")
|
||||
confirmations = data.get("confirmations", [])
|
||||
pages_to_release = data.get("pages_to_release", [])
|
||||
|
||||
# Validate input format
|
||||
if rank is None or not isinstance(confirmations, list):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Invalid request format: need 'rank' and 'confirmations'",
|
||||
)
|
||||
|
||||
try:
|
||||
self.state.confirm_write_for_keys(rank, confirmations, pages_to_release)
|
||||
|
||||
return Response(status_code=204)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Confirm write for keys failed: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Confirmation failed: {str(e)}"
|
||||
) from e
|
||||
|
||||
async def batch_key_exists(self, request: Request):
|
||||
"""Check if multiple keys exist in metadata."""
|
||||
data = await self._read_json(request)
|
||||
keys = data.get("keys", [])
|
||||
|
||||
if not isinstance(keys, list):
|
||||
raise HTTPException(status_code=400, detail="Invalid keys format")
|
||||
|
||||
try:
|
||||
exists_results = self.state.batch_key_exists(keys)
|
||||
return self._json_response({"exists": exists_results})
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Key existence check failed: {str(e)}"
|
||||
) from e
|
||||
|
||||
async def get_key_locations(self, request: Request):
|
||||
"""Get page indices for keys on a specific rank."""
|
||||
data = await self._read_json(request)
|
||||
rank = data.get("rank")
|
||||
keys = data.get("keys", [])
|
||||
|
||||
# Validate input format
|
||||
if rank is None or not isinstance(keys, list):
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Invalid request format: need 'rank' and 'keys'"
|
||||
)
|
||||
|
||||
try:
|
||||
# Get key locations
|
||||
locations = self.state.get_key_locations(rank, keys)
|
||||
return self._json_response({"locations": locations})
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Failed to get key locations: {str(e)}"
|
||||
) from e
|
||||
|
||||
async def clear(self, request: Request):
|
||||
"""Clear the metadata server."""
|
||||
self.state.clear()
|
||||
return Response(status_code=204)
|
||||
|
||||
def run(self, host: str = "0.0.0.0", port: int = 18000):
|
||||
"""Run the metadata server."""
|
||||
import uvicorn
|
||||
|
||||
logger.info("Starting improved metadata server on http://%s:%s", host, port)
|
||||
uvicorn.run(self.app, host=host, port=port)
|
||||
|
||||
|
||||
# --- Client implementation ---
|
||||
class Hf3fsMetadataInterface(ABC):
|
||||
"""Interface for HF3FS metadata operations."""
|
||||
|
||||
@abstractmethod
|
||||
def initialize(self, rank: int, num_pages: int = 0, role: str = "worker") -> None:
|
||||
"""Initialize the metadata service with specified number of pages."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def allocate_pages_for_keys(
|
||||
self, rank: int, keys: list[tuple[str, str]]
|
||||
) -> list[tuple[str, int]]:
|
||||
"""Allocate one page for each key on the specified rank."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def confirm_write_for_keys(
|
||||
self,
|
||||
rank: int,
|
||||
key_confirmations: list[tuple[str, int]],
|
||||
pages_to_release: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Confirm write operations for keys and optionally release pages."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def batch_key_exists(self, keys: list[str]) -> list[bool]:
|
||||
"""Check if keys exist and are complete across all ranks."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_key_locations(self, rank: int, keys: list[str]) -> list[int]:
|
||||
"""Get page indices for keys on a specific rank."""
|
||||
pass
|
||||
|
||||
|
||||
class Hf3fsGlobalMetadataClient(Hf3fsMetadataInterface):
|
||||
"""Global HTTP metadata client for HF3FS."""
|
||||
|
||||
def __init__(self, base_url: str = "http://localhost:18000", max_retries: int = 3):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self._session = requests.Session()
|
||||
|
||||
retry_strategy = Retry(
|
||||
total=max_retries,
|
||||
backoff_factor=0.3,
|
||||
status_forcelist=[500, 502, 503, 504],
|
||||
allowed_methods=["GET", "POST"],
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry_strategy)
|
||||
self._session.mount("http://", adapter)
|
||||
|
||||
def _post(self, endpoint: str, json_data: dict) -> dict:
|
||||
"""Make POST request to metadata server."""
|
||||
try:
|
||||
url = f"{self.base_url}/{endpoint}"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if HAS_ORJSON:
|
||||
payload = orjson.dumps(json_data)
|
||||
else:
|
||||
import json
|
||||
|
||||
payload = json.dumps(json_data).encode("utf-8")
|
||||
response = self._session.post(url, data=payload, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
if response.status_code == 204 or not response.content:
|
||||
return {}
|
||||
if HAS_ORJSON:
|
||||
return orjson.loads(response.content)
|
||||
else:
|
||||
return response.json()
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error("Failed to POST to %s after retries: %s", endpoint, e)
|
||||
raise RuntimeError(f"Failed to connect to metadata server: {e}") from e
|
||||
|
||||
def initialize(self, rank: int, num_pages: int = 0, role: str = "worker") -> None:
|
||||
"""Initialize a rank with specified number of pages."""
|
||||
self._post(f"rank/{rank}/initialize", {"num_pages": num_pages, "role": role})
|
||||
|
||||
def allocate_pages_for_keys(
|
||||
self, rank: int, keys: list[tuple[str, str]]
|
||||
) -> list[tuple[str, int]]:
|
||||
"""Allocate pages for keys on the specified rank."""
|
||||
response = self._post("keys/batch_allocate", {"rank": rank, "keys": keys})
|
||||
|
||||
# Convert response to expected format
|
||||
return response.get("results", {})
|
||||
|
||||
def confirm_write_for_keys(
|
||||
self,
|
||||
rank: int,
|
||||
key_confirmations: list[tuple[str, int]],
|
||||
pages_to_release: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Confirm write operations for keys and optionally release pages."""
|
||||
payload = {
|
||||
"rank": rank,
|
||||
"confirmations": key_confirmations,
|
||||
"pages_to_release": pages_to_release or [],
|
||||
}
|
||||
|
||||
self._post("keys/confirm_write", payload)
|
||||
|
||||
def batch_key_exists(self, keys: list[str]) -> list[bool]:
|
||||
"""Check if keys exist and are complete across all ranks."""
|
||||
response = self._post("keys/batch_exists", {"keys": keys})
|
||||
return response.get("exists", [])
|
||||
|
||||
def get_key_locations(self, rank: int, keys: list[str]) -> list[int]:
|
||||
"""Get page indices for keys on a specific rank."""
|
||||
response = self._post("keys/get_locations", {"rank": rank, "keys": keys})
|
||||
return response.get("locations", [])
|
||||
|
||||
|
||||
def run_metadata_server(
|
||||
host: str = "0.0.0.0",
|
||||
port: int = 18000,
|
||||
):
|
||||
"""Run the improved HF3FS metadata server."""
|
||||
server = Hf3fsMetadataServer()
|
||||
server.run(host=host, port=port)
|
||||
|
||||
|
||||
# --- Main Execution ---
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Improved HF3FS Metadata Server")
|
||||
parser.add_argument(
|
||||
"--host", type=str, default="0.0.0.0", help="Host to bind the server to."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=18000, help="Port to run the server on."
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
run_metadata_server(args.host, args.port)
|
||||
@@ -1,139 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.v1.request import Request
|
||||
|
||||
|
||||
class AtomicCounter:
|
||||
"""Thread-safe atomic counter for round-robin operations."""
|
||||
|
||||
def __init__(self, n: int):
|
||||
assert n > 0, "Counter size must be positive"
|
||||
self._n = n
|
||||
self._value = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def next(self) -> int:
|
||||
"""Get next value in round-robin fashion."""
|
||||
with self._lock:
|
||||
current = self._value
|
||||
self._value = (current + 1) % self._n
|
||||
return current
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadBlockInfo:
|
||||
"""Operation for loading blocks from external storage."""
|
||||
|
||||
num_computed_blocks: int
|
||||
num_blocks_to_load: int
|
||||
need_fetch_block_ids: list[int]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SaveBlockInfo:
|
||||
"""Operation for saving blocks to external storage."""
|
||||
|
||||
skip_leading_blocks: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestSchedulingState:
|
||||
"""Unified request scheduling state management."""
|
||||
|
||||
request_id: str
|
||||
request: Request | None = None
|
||||
|
||||
# Token and block tracking
|
||||
token_ids: list[int] = field(default_factory=list)
|
||||
allocated_block_ids: list[int] = field(default_factory=list)
|
||||
num_saved_blocks: int = 0
|
||||
|
||||
# Load operation info
|
||||
load_op: LoadBlockInfo | None = None
|
||||
|
||||
# Scheduling phase
|
||||
phase: str = "NEW" # NEW -> WAITING_TO_LOAD -> ACTIVE -> FINISHED
|
||||
|
||||
def needs_loading(self) -> bool:
|
||||
"""Check if request needs loading."""
|
||||
return self.load_op is not None and self.load_op.num_blocks_to_load > 0
|
||||
|
||||
def is_ready_to_load(self) -> bool:
|
||||
"""Check if request is ready for loading."""
|
||||
return self.phase == "WAITING_TO_LOAD" and self.needs_loading()
|
||||
|
||||
def update_tokens_and_blocks(self, new_token_ids: list[int], new_block_ids) -> None:
|
||||
"""Update with new tokens and blocks."""
|
||||
if new_token_ids:
|
||||
self.token_ids.extend(new_token_ids)
|
||||
|
||||
if new_block_ids is not None:
|
||||
normalized_block_ids = self._normalize_block_ids(new_block_ids)
|
||||
self.allocated_block_ids.extend(normalized_block_ids)
|
||||
|
||||
def _normalize_block_ids(self, block_ids) -> list[int]:
|
||||
"""Normalize block_ids to list format."""
|
||||
if not block_ids:
|
||||
return []
|
||||
if isinstance(block_ids, tuple):
|
||||
return block_ids[0] if block_ids else []
|
||||
if isinstance(block_ids, list):
|
||||
return block_ids
|
||||
return []
|
||||
|
||||
|
||||
@dataclass
|
||||
class HF3FSRequestMetadata:
|
||||
"""Metadata for a single request in HF3FS connector."""
|
||||
|
||||
request_id: str
|
||||
token_ids: list[int]
|
||||
block_ids: list[int]
|
||||
load_block_op: LoadBlockInfo | None = None
|
||||
save_block_op: SaveBlockInfo | None = None
|
||||
|
||||
@staticmethod
|
||||
def from_scheduling_state(
|
||||
state: "RequestSchedulingState",
|
||||
block_size: int,
|
||||
load_op: LoadBlockInfo | None = None,
|
||||
skip_leading_blocks: int | None = None,
|
||||
) -> Optional["HF3FSRequestMetadata"]:
|
||||
"""Create request metadata from scheduling state."""
|
||||
token_count = len(state.token_ids)
|
||||
total_blocks = token_count // block_size
|
||||
|
||||
skip_blocks = (
|
||||
state.num_saved_blocks
|
||||
if skip_leading_blocks is None
|
||||
else skip_leading_blocks
|
||||
)
|
||||
|
||||
new_blocks_to_save = total_blocks - state.num_saved_blocks
|
||||
if new_blocks_to_save <= 0 and load_op is None:
|
||||
return None
|
||||
|
||||
state.num_saved_blocks = total_blocks
|
||||
return HF3FSRequestMetadata(
|
||||
request_id=state.request_id,
|
||||
token_ids=state.token_ids,
|
||||
block_ids=state.allocated_block_ids,
|
||||
load_block_op=load_op,
|
||||
save_block_op=SaveBlockInfo(skip_leading_blocks=skip_blocks),
|
||||
)
|
||||
|
||||
|
||||
class HF3FSConnectorMetadata(KVConnectorMetadata):
|
||||
"""Container for HF3FS connector metadata."""
|
||||
|
||||
def __init__(self):
|
||||
self.requests: list[HF3FSRequestMetadata] = []
|
||||
|
||||
def add_request(self, request_metadata: HF3FSRequestMetadata) -> None:
|
||||
"""Add request to metadata."""
|
||||
self.requests.append(request_metadata)
|
||||
@@ -1,262 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
|
||||
@triton.jit
|
||||
def kv_cache_scatter_kernel(
|
||||
kv_cache_ptrs_ptr,
|
||||
source_ptr,
|
||||
token_indices_ptr,
|
||||
num_tokens_in_block,
|
||||
total_token_in_kvcache,
|
||||
num_layers,
|
||||
tokens_per_block,
|
||||
block_stride,
|
||||
head_stride,
|
||||
token_stride,
|
||||
content_stride,
|
||||
num_heads: tl.constexpr,
|
||||
content_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
layer_idx = tl.program_id(0)
|
||||
token_pos = tl.program_id(1)
|
||||
|
||||
if layer_idx >= num_layers or token_pos >= num_tokens_in_block:
|
||||
return
|
||||
|
||||
token_idx = tl.load(token_indices_ptr + token_pos)
|
||||
kv_cache_ptr = tl.cast(tl.load(kv_cache_ptrs_ptr + layer_idx), source_ptr.dtype)
|
||||
|
||||
if token_idx >= total_token_in_kvcache:
|
||||
return
|
||||
|
||||
block_idx = token_idx // tokens_per_block
|
||||
state_idx = token_idx % tokens_per_block
|
||||
hidden_size = num_heads * content_size
|
||||
for i in range(0, hidden_size, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < hidden_size
|
||||
head_idx = offset // content_size
|
||||
content_idx = offset % content_size
|
||||
source_offset = (
|
||||
(layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos
|
||||
) * content_size + content_idx
|
||||
target_offset = (
|
||||
block_idx * block_stride
|
||||
+ head_idx * head_stride
|
||||
+ state_idx * token_stride
|
||||
+ content_idx * content_stride
|
||||
)
|
||||
value = tl.load(source_ptr + source_offset, mask=mask)
|
||||
tl.store(kv_cache_ptr + target_offset, value, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def kv_cache_gather_kernel(
|
||||
kv_cache_ptrs_ptr,
|
||||
dst_ptr,
|
||||
token_indices_ptr,
|
||||
num_tokens_in_block,
|
||||
total_token_in_kvcache,
|
||||
num_layers,
|
||||
tokens_per_block,
|
||||
block_stride,
|
||||
head_stride,
|
||||
token_stride,
|
||||
content_stride,
|
||||
num_heads: tl.constexpr,
|
||||
content_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
layer_idx = tl.program_id(0)
|
||||
token_pos = tl.program_id(1)
|
||||
|
||||
if layer_idx >= num_layers or token_pos >= num_tokens_in_block:
|
||||
return
|
||||
|
||||
token_idx = tl.load(token_indices_ptr + token_pos)
|
||||
kv_cache_ptr = tl.cast(tl.load(kv_cache_ptrs_ptr + layer_idx), dst_ptr.dtype)
|
||||
|
||||
if token_idx >= total_token_in_kvcache:
|
||||
return
|
||||
|
||||
block_idx = token_idx // tokens_per_block
|
||||
state_idx = token_idx % tokens_per_block
|
||||
hidden_size = num_heads * content_size
|
||||
for i in range(0, hidden_size, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < hidden_size
|
||||
head_idx = offset // content_size
|
||||
content_idx = offset % content_size
|
||||
kvcache_offset = (
|
||||
block_idx * block_stride
|
||||
+ head_idx * head_stride
|
||||
+ state_idx * token_stride
|
||||
+ content_idx * content_stride
|
||||
)
|
||||
dst_offset = (
|
||||
(layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos
|
||||
) * content_size + content_idx
|
||||
value = tl.load(kv_cache_ptr + kvcache_offset, mask=mask)
|
||||
tl.store(dst_ptr + dst_offset, value, mask=mask)
|
||||
|
||||
|
||||
def scatter_kv_caches(
|
||||
kv_caches_ptrs: torch.Tensor,
|
||||
total_token_in_kvcache: int,
|
||||
src_tensor: torch.Tensor,
|
||||
token_indices: list[int],
|
||||
tokens_per_block: int,
|
||||
num_heads: int,
|
||||
content_size: int,
|
||||
kv_cache_strides: tuple[int, ...],
|
||||
) -> None:
|
||||
"""Scatter KV cache data from source tensor to KV cache storage.
|
||||
|
||||
Args:
|
||||
kv_caches_ptrs: Tensor of KV cache pointers (one per layer)
|
||||
total_token_in_kvcache: Total number of tokens in KV cache
|
||||
src_tensor: Source ``[L, H, N, C]`` tensor containing data to scatter
|
||||
token_indices: List of token positions to update
|
||||
tokens_per_block: Number of stored states in each cache block
|
||||
num_heads: Size of the H axis
|
||||
content_size: Size of the C axis
|
||||
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
|
||||
|
||||
"""
|
||||
num_layers = len(kv_caches_ptrs)
|
||||
num_tokens_in_block = len(token_indices)
|
||||
|
||||
assert src_tensor.shape == (
|
||||
num_layers,
|
||||
num_heads,
|
||||
num_tokens_in_block,
|
||||
content_size,
|
||||
)
|
||||
assert len(kv_cache_strides) == 4
|
||||
|
||||
device = src_tensor.device
|
||||
token_indices_tensor = async_tensor_h2d(
|
||||
token_indices, device=device, dtype=torch.int32
|
||||
)
|
||||
|
||||
grid = (num_layers, num_tokens_in_block)
|
||||
BLOCK_SIZE = 128
|
||||
|
||||
kv_cache_scatter_kernel[grid](
|
||||
kv_caches_ptrs,
|
||||
src_tensor,
|
||||
token_indices_tensor,
|
||||
num_tokens_in_block,
|
||||
total_token_in_kvcache,
|
||||
num_layers,
|
||||
tokens_per_block,
|
||||
*kv_cache_strides,
|
||||
num_heads=num_heads,
|
||||
content_size=content_size,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
|
||||
def gather_kv_caches(
|
||||
kv_caches_ptrs: torch.Tensor,
|
||||
total_token_in_kvcache: int,
|
||||
dst_tensor: torch.Tensor,
|
||||
token_indices: list[int],
|
||||
tokens_per_block: int,
|
||||
num_heads: int,
|
||||
content_size: int,
|
||||
kv_cache_strides: tuple[int, ...],
|
||||
) -> None:
|
||||
"""Gather KV cache data from KV cache storage to destination tensor.
|
||||
|
||||
Args:
|
||||
kv_caches_ptrs: Tensor of KV cache pointers (one per layer)
|
||||
total_token_in_kvcache: Total number of tokens in KV cache
|
||||
dst_tensor: Destination ``[L, H, N, C]`` tensor
|
||||
token_indices: List of token positions to gather
|
||||
tokens_per_block: Number of stored states in each cache block
|
||||
num_heads: Size of the H axis
|
||||
content_size: Size of the C axis
|
||||
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
|
||||
|
||||
"""
|
||||
num_layers = kv_caches_ptrs.shape[0]
|
||||
num_tokens_in_block = len(token_indices)
|
||||
|
||||
assert dst_tensor.shape == (
|
||||
num_layers,
|
||||
num_heads,
|
||||
num_tokens_in_block,
|
||||
content_size,
|
||||
)
|
||||
assert len(kv_cache_strides) == 4
|
||||
|
||||
device = dst_tensor.device
|
||||
token_indices_tensor = async_tensor_h2d(
|
||||
token_indices, device=device, dtype=torch.int32
|
||||
)
|
||||
|
||||
grid = (num_layers, num_tokens_in_block)
|
||||
BLOCK_SIZE = 128
|
||||
|
||||
kv_cache_gather_kernel[grid](
|
||||
kv_caches_ptrs,
|
||||
dst_tensor,
|
||||
token_indices_tensor,
|
||||
num_tokens_in_block,
|
||||
total_token_in_kvcache,
|
||||
num_layers,
|
||||
tokens_per_block,
|
||||
*kv_cache_strides,
|
||||
num_heads=num_heads,
|
||||
content_size=content_size,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
|
||||
class CopyBufferAllocator:
|
||||
"""Memory pool for tensor buffers to avoid frequent allocation/deallocation."""
|
||||
|
||||
def __init__(
|
||||
self, device: torch.device, dtype: torch.dtype, shape: list, max_count: int
|
||||
):
|
||||
self._shape = shape
|
||||
self._max_count = max_count
|
||||
self._device = device
|
||||
self._free_buffers = [
|
||||
torch.empty(shape, dtype=dtype, device=device) for _ in range(max_count)
|
||||
]
|
||||
self._inuse_count = 0
|
||||
|
||||
def alloc_buffer(self, count: int) -> list[torch.Tensor] | None:
|
||||
"""Allocate buffers from the pool."""
|
||||
if count == 0:
|
||||
return []
|
||||
|
||||
if self._inuse_count + count <= self._max_count:
|
||||
self._inuse_count += count
|
||||
result = self._free_buffers[-count:]
|
||||
del self._free_buffers[-count:]
|
||||
return result
|
||||
return None
|
||||
|
||||
def free_buffer(self, buffers: list[torch.Tensor]) -> None:
|
||||
"""Return buffers to the pool."""
|
||||
if not buffers:
|
||||
return
|
||||
|
||||
if self._inuse_count >= len(buffers):
|
||||
self._inuse_count -= len(buffers)
|
||||
self._free_buffers.extend(buffers)
|
||||
else:
|
||||
raise RuntimeError("Attempted to free more buffers than allocated")
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -1,133 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
HF3FS_AVAILABLE = True
|
||||
|
||||
|
||||
class Hf3fsClient:
|
||||
"""Mock HF3FS client using file backend for debugging and testing."""
|
||||
|
||||
def __init__(self, path: str, size: int, bytes_per_page: int, entries: int):
|
||||
self._size = size
|
||||
self._bytes_per_page = bytes_per_page
|
||||
self._entries = entries
|
||||
self._file_path = path
|
||||
|
||||
self._ensure_file_exists()
|
||||
logger.debug("Initialized mock HF3FS client: %s (%d bytes)", path, size)
|
||||
|
||||
def _ensure_file_exists(self) -> None:
|
||||
"""Create file if it doesn't exist."""
|
||||
if not os.path.exists(self._file_path):
|
||||
with open(self._file_path, "w+b") as f:
|
||||
f.truncate(self._size)
|
||||
|
||||
def batch_read(self, offsets: list[int], tensors: list[torch.Tensor]) -> list[int]:
|
||||
"""Read data from file at specified offsets into tensors."""
|
||||
results = []
|
||||
|
||||
try:
|
||||
with open(self._file_path, "rb") as f:
|
||||
for offset, tensor in zip(offsets, tensors):
|
||||
num_bytes = tensor.numel() * tensor.element_size()
|
||||
|
||||
if offset < 0 or offset + num_bytes > self._size:
|
||||
results.append(-1)
|
||||
continue
|
||||
|
||||
f.seek(offset)
|
||||
buffer_data = f.read(num_bytes)
|
||||
|
||||
if len(buffer_data) == num_bytes == self._bytes_per_page:
|
||||
tensor_data = self._convert_buffer_to_tensor(
|
||||
buffer_data, tensor.dtype
|
||||
)
|
||||
tensor.copy_(
|
||||
tensor_data.reshape(tensor.shape).to(tensor.device)
|
||||
)
|
||||
results.append(self._bytes_per_page)
|
||||
else:
|
||||
logger.error(
|
||||
"Read size mismatch: got %d, expected %d",
|
||||
len(buffer_data),
|
||||
num_bytes,
|
||||
)
|
||||
results.append(-1)
|
||||
except Exception as e:
|
||||
logger.error("Batch read error: %s", e)
|
||||
results.extend([-1] * (len(offsets) - len(results)))
|
||||
|
||||
return results
|
||||
|
||||
def _convert_buffer_to_tensor(
|
||||
self, buffer_data: bytes, dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
"""Convert buffer data to tensor with proper dtype handling."""
|
||||
if dtype == torch.bfloat16:
|
||||
tensor_data = torch.frombuffer(buffer_data, dtype=torch.uint16)
|
||||
return tensor_data.view(dtype=torch.bfloat16)
|
||||
else:
|
||||
return torch.frombuffer(buffer_data, dtype=dtype)
|
||||
|
||||
def batch_write(
|
||||
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
|
||||
) -> list[int]:
|
||||
"""Write data from tensors to file at specified offsets."""
|
||||
results = []
|
||||
|
||||
try:
|
||||
torch.cuda.current_stream().wait_event(event)
|
||||
|
||||
# Convert tensors to bytes
|
||||
data_bytes_list = [self._tensor_to_bytes(tensor) for tensor in tensors]
|
||||
|
||||
# Write to file
|
||||
with open(self._file_path, "r+b") as f:
|
||||
for offset, data_bytes in zip(offsets, data_bytes_list):
|
||||
if offset < 0 or offset + len(data_bytes) > self._size:
|
||||
results.append(-1)
|
||||
continue
|
||||
|
||||
f.seek(offset)
|
||||
bytes_written = f.write(data_bytes)
|
||||
|
||||
if bytes_written == len(data_bytes) == self._bytes_per_page:
|
||||
results.append(self._bytes_per_page)
|
||||
else:
|
||||
logger.error(
|
||||
"Write size mismatch: wrote %d, expected %d",
|
||||
bytes_written,
|
||||
self._bytes_per_page,
|
||||
)
|
||||
results.append(-1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Batch write error: %s", e)
|
||||
results.extend([-1] * (len(offsets) - len(results)))
|
||||
|
||||
return results
|
||||
|
||||
def _tensor_to_bytes(self, tensor: torch.Tensor) -> bytes:
|
||||
"""Convert tensor to bytes with proper dtype handling."""
|
||||
cpu_tensor = tensor.cpu()
|
||||
if cpu_tensor.dtype == torch.bfloat16:
|
||||
return cpu_tensor.view(dtype=torch.uint16).numpy().tobytes()
|
||||
else:
|
||||
return cpu_tensor.numpy().tobytes()
|
||||
|
||||
def get_size(self) -> int:
|
||||
"""Get the total size of the storage file."""
|
||||
return self._size
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the client (no-op for file backend)."""
|
||||
pass
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Flush any pending writes (no-op for file backend)."""
|
||||
pass
|
||||
@@ -1,57 +0,0 @@
|
||||
#include <cuda_runtime.h>
|
||||
#include <torch/extension.h>
|
||||
#include <cstring>
|
||||
#include <vector>
|
||||
|
||||
void read_shm(const torch::Tensor& shm, const torch::Tensor& pin,
|
||||
std::vector<torch::Tensor> dst, uint64_t stream_ptr) {
|
||||
py::gil_scoped_release release;
|
||||
|
||||
cudaStream_t stream = reinterpret_cast<cudaStream_t>(stream_ptr);
|
||||
|
||||
// Copy from shared memory to pinned memory
|
||||
char* shm_ptr = static_cast<char*>(shm.data_ptr());
|
||||
char* src_ptr = static_cast<char*>(pin.data_ptr());
|
||||
std::memcpy(src_ptr, shm_ptr, shm.numel() * shm.element_size());
|
||||
|
||||
// Copy from pinned memory to GPU tensors
|
||||
size_t current = 0;
|
||||
for (size_t i = 0; i < dst.size(); ++i) {
|
||||
auto& t = dst[i];
|
||||
size_t t_bytes = t.numel() * t.element_size();
|
||||
char* dst_ptr = static_cast<char*>(t.data_ptr());
|
||||
cudaMemcpyAsync(dst_ptr, src_ptr + current, t_bytes, cudaMemcpyHostToDevice,
|
||||
stream);
|
||||
current += t_bytes;
|
||||
}
|
||||
cudaStreamSynchronize(stream);
|
||||
}
|
||||
|
||||
void write_shm(const std::vector<torch::Tensor> src, torch::Tensor& shm,
|
||||
const torch::Tensor& pin, uint64_t stream_ptr) {
|
||||
py::gil_scoped_release release;
|
||||
|
||||
cudaStream_t stream = reinterpret_cast<cudaStream_t>(stream_ptr);
|
||||
|
||||
// Copy from GPU tensors to pinned memory
|
||||
char* dst_ptr = static_cast<char*>(pin.data_ptr());
|
||||
size_t current = 0;
|
||||
for (size_t i = 0; i < src.size(); ++i) {
|
||||
auto& t = src[i];
|
||||
size_t t_bytes = t.numel() * t.element_size();
|
||||
char* src_ptr = static_cast<char*>(t.data_ptr());
|
||||
cudaMemcpyAsync(dst_ptr + current, src_ptr, t_bytes, cudaMemcpyDeviceToHost,
|
||||
stream);
|
||||
current += t_bytes;
|
||||
}
|
||||
cudaStreamSynchronize(stream);
|
||||
|
||||
// Copy from pinned memory to shared memory
|
||||
char* shm_ptr = static_cast<char*>(shm.data_ptr());
|
||||
std::memcpy(shm_ptr, dst_ptr, shm.numel() * shm.element_size());
|
||||
}
|
||||
|
||||
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
|
||||
m.def("read_shm", &read_shm, "Read tensors from shared memory");
|
||||
m.def("write_shm", &write_shm, "Write tensors to shared memory");
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,408 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Connector boundary for HiSparse scheduler and worker state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparseKVConnectorStats,
|
||||
HiSparsePromMetrics,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
|
||||
HiSparseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.hisparse.layout import get_hisparse_steady_state_concurrency
|
||||
from vllm.v1.hisparse.types import (
|
||||
SparseKVOffloadCommand,
|
||||
SparseKVResidencyUpdate,
|
||||
SparseKVRowMirror,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.hisparse.coordinator import HiSparseCoordinator
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HiSparseConnectorMetadata(KVConnectorMetadata):
|
||||
command: SparseKVOffloadCommand | None
|
||||
host_block_copies: tuple[KVCacheBlockCopy, ...]
|
||||
source_block_ids: tuple[int, ...]
|
||||
row_mirrors: dict[str, tuple[SparseKVRowMirror, ...]]
|
||||
all_context_pages_resident: bool
|
||||
residency_updates: dict[str, SparseKVResidencyUpdate]
|
||||
|
||||
|
||||
@dataclass
|
||||
class HiSparseConnectorWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
enqueued_transfer_counts: dict[int, int]
|
||||
completed_transfer_counts: dict[int, int]
|
||||
completed_host_copy_dst_ids: tuple[int, ...] = ()
|
||||
|
||||
def aggregate(self, other: KVConnectorWorkerMetadata) -> KVConnectorWorkerMetadata:
|
||||
assert isinstance(other, HiSparseConnectorWorkerMetadata)
|
||||
|
||||
def add_counts(first: dict[int, int], second: dict[int, int]) -> dict[int, int]:
|
||||
combined = first.copy()
|
||||
for transfer_id, count in second.items():
|
||||
combined[transfer_id] = combined.get(transfer_id, 0) + count
|
||||
return combined
|
||||
|
||||
return HiSparseConnectorWorkerMetadata(
|
||||
enqueued_transfer_counts=add_counts(
|
||||
self.enqueued_transfer_counts, other.enqueued_transfer_counts
|
||||
),
|
||||
completed_transfer_counts=add_counts(
|
||||
self.completed_transfer_counts, other.completed_transfer_counts
|
||||
),
|
||||
# Every worker runs the same copies, so a union is the ack.
|
||||
completed_host_copy_dst_ids=tuple(
|
||||
{*self.completed_host_copy_dst_ids, *other.completed_host_copy_dst_ids}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class HiSparseConnectorScheduler:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
async_speculative: bool,
|
||||
draft_kv_lookahead: int = 0,
|
||||
) -> None:
|
||||
self.coordinator: HiSparseCoordinator | None = None
|
||||
self.async_speculative = async_speculative
|
||||
self.draft_kv_lookahead = draft_kv_lookahead
|
||||
self.requests: dict[str, Request] = {}
|
||||
|
||||
def bind_coordinator(self, coordinator: HiSparseCoordinator) -> None:
|
||||
assert self.coordinator is None
|
||||
self.coordinator = coordinator
|
||||
|
||||
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
|
||||
if self.coordinator is None:
|
||||
return None
|
||||
host_pool = self.coordinator.get_host_block_pool()
|
||||
assert host_pool is not None
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_host_usage(
|
||||
host_pool.get_usage(), len(self.coordinator.pending_spills)
|
||||
)
|
||||
if host_pool.metrics_collector is not None:
|
||||
stats.record_host_evictions(host_pool.metrics_collector.drain_events())
|
||||
return stats
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> HiSparseConnectorMetadata:
|
||||
assert self.coordinator is not None
|
||||
# HiSparse rebinds worker state every step, so the load must start
|
||||
# before the forward rather than being deferred to post-forward.
|
||||
scheduler_output.has_sync_kv_loads = True
|
||||
num_computed_tokens = {
|
||||
request.req_id: request.num_computed_tokens
|
||||
for request in scheduler_output.scheduled_new_reqs
|
||||
}
|
||||
num_computed_tokens.update(
|
||||
zip(
|
||||
scheduler_output.scheduled_cached_reqs.req_ids,
|
||||
scheduler_output.scheduled_cached_reqs.num_computed_tokens,
|
||||
)
|
||||
)
|
||||
scheduled_requests = tuple(
|
||||
(
|
||||
request_id,
|
||||
num_computed_tokens[request_id],
|
||||
scheduled_count,
|
||||
)
|
||||
for request_id, scheduled_count in (
|
||||
scheduler_output.num_scheduled_tokens.items()
|
||||
)
|
||||
)
|
||||
# Runs first: it plans this step's transfers and residency changes.
|
||||
self.coordinator.advance_scheduled(
|
||||
(
|
||||
request_id,
|
||||
min(start + count, self.requests[request_id].num_tokens),
|
||||
)
|
||||
for request_id, start, count in scheduled_requests
|
||||
)
|
||||
command = self.coordinator.build_offload_command()
|
||||
host_block_copies = self.coordinator.take_host_block_copies()
|
||||
source_group_id = self.coordinator.host_group_id
|
||||
assert source_group_id is not None
|
||||
source_block_ids = [
|
||||
block_id
|
||||
for request in scheduler_output.scheduled_new_reqs
|
||||
for block_id in request.block_ids[source_group_id]
|
||||
]
|
||||
for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids:
|
||||
if new_block_ids is not None:
|
||||
source_block_ids.extend(new_block_ids[source_group_id])
|
||||
row_mirrors = {}
|
||||
for request_id, scheduled_start, scheduled_count in scheduled_requests:
|
||||
mirror_start = scheduled_start
|
||||
if self.async_speculative:
|
||||
mirror_start = max(
|
||||
0,
|
||||
scheduled_start - self.requests[request_id].num_output_placeholders,
|
||||
)
|
||||
row_mirrors[request_id] = self.coordinator.build_row_mirrors(
|
||||
(
|
||||
(
|
||||
request_id,
|
||||
mirror_start,
|
||||
scheduled_count
|
||||
+ scheduled_start
|
||||
- mirror_start
|
||||
+ self.draft_kv_lookahead,
|
||||
),
|
||||
),
|
||||
)
|
||||
return HiSparseConnectorMetadata(
|
||||
command,
|
||||
host_block_copies,
|
||||
tuple(source_block_ids),
|
||||
row_mirrors,
|
||||
self.coordinator.all_context_pages_resident(scheduled_requests),
|
||||
self.coordinator.take_residency_updates(
|
||||
scheduler_output.num_scheduled_tokens
|
||||
),
|
||||
)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
assert self.coordinator is not None
|
||||
for request_id in connector_output.finished_recving or ():
|
||||
self.coordinator.finish_host_import(
|
||||
request_id, failed=request_id in connector_output.failed_recving
|
||||
)
|
||||
metadata = connector_output.kv_connector_worker_meta
|
||||
if metadata is None:
|
||||
return
|
||||
assert isinstance(metadata, HiSparseConnectorWorkerMetadata)
|
||||
self.coordinator.release_completed_host_copies(
|
||||
metadata.completed_host_copy_dst_ids
|
||||
)
|
||||
self.coordinator.update_spills(
|
||||
metadata.enqueued_transfer_counts,
|
||||
metadata.completed_transfer_counts,
|
||||
)
|
||||
|
||||
|
||||
class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""Join the scheduler coordinator to the worker's transfer engine."""
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str:
|
||||
return "BLHNC"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
if kv_cache_config.hisparse_host_num_blocks is None:
|
||||
raise ValueError("HiSparseConnector requires a HiSparse host pool")
|
||||
self.connector_scheduler: HiSparseConnectorScheduler | None = None
|
||||
self.connector_worker: HiSparseConnectorWorker | None = None
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
self.connector_scheduler = HiSparseConnectorScheduler(
|
||||
async_speculative=bool(
|
||||
vllm_config.scheduler_config.async_scheduling
|
||||
and speculative_config is not None
|
||||
),
|
||||
draft_kv_lookahead=vllm_config.num_lookahead_tokens,
|
||||
)
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
steady_concurrency = get_hisparse_steady_state_concurrency(
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
logger.info_once(
|
||||
"HiSparse steady-state maximum concurrency for %s tokens per "
|
||||
"request: %.2fx (running requests reading from host).",
|
||||
f"{max_model_len:,}",
|
||||
steady_concurrency,
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.connector_worker = HiSparseConnectorWorker(
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported KV connector role: {role}")
|
||||
|
||||
def bind_kv_cache_manager(self, kv_cache_manager: KVCacheManager) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.bind_coordinator(
|
||||
get_hisparse_coordinator(kv_cache_manager)
|
||||
)
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
return False
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
assert self.connector_scheduler.coordinator is not None
|
||||
return self.connector_scheduler.coordinator.has_pending_work()
|
||||
|
||||
def has_pending_block_frees(self) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
assert self.connector_scheduler.coordinator is not None
|
||||
return self.connector_scheduler.coordinator.has_pending_reclamation()
|
||||
|
||||
def finish_forward(self) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.finish_forward()
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def reset_capture_state(self) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.reset_hot_state()
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
if self.connector_worker is not None:
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
if self.connector_scheduler is not None:
|
||||
return self.connector_scheduler.get_kv_connector_stats()
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
HiSparseKVConnectorStats(data=data)
|
||||
if data is not None
|
||||
else HiSparseKVConnectorStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return HiSparsePromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, HiSparseConnectorMetadata)
|
||||
request_state_indices = kwargs.get("request_state_indices")
|
||||
assert request_state_indices is None or isinstance(
|
||||
request_state_indices, torch.Tensor
|
||||
)
|
||||
request_ids = kwargs.get("request_ids")
|
||||
assert request_ids is None or isinstance(request_ids, list)
|
||||
attn_metadata = kwargs.get("attn_metadata")
|
||||
assert attn_metadata is None or isinstance(attn_metadata, dict)
|
||||
self.connector_worker.start_step(
|
||||
metadata,
|
||||
request_state_indices,
|
||||
request_ids,
|
||||
num_tokens=int(kwargs.get("num_tokens") or 0),
|
||||
)
|
||||
self.connector_worker.prepare_forward(attn_metadata)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
return
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
def wait_for_save(self) -> None:
|
||||
return
|
||||
|
||||
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
|
||||
assert self.connector_worker is not None
|
||||
enqueued, completed = self.connector_worker.take_transfer_updates()
|
||||
host_copies = self.connector_worker.take_completed_host_copies()
|
||||
if not enqueued and not completed and not host_copies:
|
||||
return None
|
||||
return HiSparseConnectorWorkerMetadata(
|
||||
enqueued_transfer_counts={transfer_id: 1 for transfer_id in enqueued},
|
||||
completed_transfer_counts={transfer_id: 1 for transfer_id in completed},
|
||||
completed_host_copy_dst_ids=tuple(host_copies),
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.shutdown()
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
return 0, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.requests[request.request_id] = request
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.requests.pop(request.request_id, None)
|
||||
return False, None
|
||||
@@ -1,214 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stats and Prometheus metrics for the HiSparse connector."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.v1.metrics.buckets import histogram_buckets
|
||||
from vllm.v1.metrics.stats import KVCacheEvictionEvent
|
||||
from vllm.v1.metrics.utils import create_metric_per_engine
|
||||
|
||||
_HISPARSE_LEVEL_KEYS = frozenset({"host_cache_usage_perc", "pending_page_transfers"})
|
||||
|
||||
|
||||
@dataclass
|
||||
class HiSparseKVConnectorStats(KVConnectorStats):
|
||||
"""Container for HiSparse hot-buffer residency metrics.
|
||||
|
||||
Each list entry is the delta recorded over one device counter snapshot
|
||||
interval, so a list can hold multiple snapshots per logging interval.
|
||||
The host-tier gauges are level values, not deltas: aggregation keeps
|
||||
only the most recent observation.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
# Empty container init, no data is passed in.
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
# Must be serializable
|
||||
self.data: dict[str, list[int | float]] = {
|
||||
"cache_hits": [],
|
||||
"cache_misses": [],
|
||||
"host_to_device_bytes": [],
|
||||
"host_cache_usage_perc": [],
|
||||
"pending_page_transfers": [],
|
||||
"host_block_lifetime_seconds": [],
|
||||
"host_block_idle_before_evict_seconds": [],
|
||||
"host_block_reuse_gap_seconds": [],
|
||||
}
|
||||
|
||||
def record_snapshot(self, hits: int, misses: int, host_to_device_bytes: int):
|
||||
self.data["cache_hits"].append(hits)
|
||||
self.data["cache_misses"].append(misses)
|
||||
self.data["host_to_device_bytes"].append(host_to_device_bytes)
|
||||
|
||||
def record_host_usage(self, usage: float, pending_page_transfers: int) -> None:
|
||||
self.data["host_cache_usage_perc"].append(usage)
|
||||
self.data["pending_page_transfers"].append(pending_page_transfers)
|
||||
|
||||
def record_host_evictions(self, events: list[KVCacheEvictionEvent]) -> None:
|
||||
for event in events:
|
||||
self.data["host_block_lifetime_seconds"].append(event.lifetime_seconds)
|
||||
self.data["host_block_idle_before_evict_seconds"].append(event.idle_seconds)
|
||||
self.data["host_block_reuse_gap_seconds"].extend(event.reuse_gaps_seconds)
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if not other.is_empty():
|
||||
for k, v in other.data.items():
|
||||
accumulator = self.data[k]
|
||||
assert isinstance(accumulator, list)
|
||||
if k in _HISPARSE_LEVEL_KEYS:
|
||||
accumulator.extend(v[-1:])
|
||||
else:
|
||||
accumulator.extend(v)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
# Compute compact representative stats suitable for CLI logging.
|
||||
reduced: dict[str, int | float] = {
|
||||
"HiSparse hot-buffer hits": sum(self.data["cache_hits"]),
|
||||
"HiSparse hot-buffer misses": sum(self.data["cache_misses"]),
|
||||
"HiSparse host-to-device bytes": sum(self.data["host_to_device_bytes"]),
|
||||
}
|
||||
if self.data["host_cache_usage_perc"]:
|
||||
usage = self.data["host_cache_usage_perc"][-1]
|
||||
reduced["HiSparse host KV cache usage %"] = round(100 * usage, 1)
|
||||
return reduced
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return all(len(values) == 0 for values in self.data.values())
|
||||
|
||||
|
||||
class HiSparsePromMetrics(KVConnectorPromMetrics):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
|
||||
counter_cache_hits = self._counter_cls(
|
||||
name="vllm:hisparse_cache_hits",
|
||||
documentation="Number of HiSparse device hot-buffer hits.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
counter_cache_misses = self._counter_cls(
|
||||
name="vllm:hisparse_cache_misses",
|
||||
documentation="Number of HiSparse device hot-buffer misses.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
counter_host_to_device_bytes = self._counter_cls(
|
||||
name="vllm:hisparse_host_to_device_bytes",
|
||||
documentation=(
|
||||
"Bytes transferred from host KV storage to HiSparse hot buffers."
|
||||
),
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.hisparse_counters: dict[str, Any] = {
|
||||
"cache_hits": create_metric_per_engine(
|
||||
counter_cache_hits, self.per_engine_labelvalues
|
||||
),
|
||||
"cache_misses": create_metric_per_engine(
|
||||
counter_cache_misses, self.per_engine_labelvalues
|
||||
),
|
||||
"host_to_device_bytes": create_metric_per_engine(
|
||||
counter_host_to_device_bytes, self.per_engine_labelvalues
|
||||
),
|
||||
}
|
||||
|
||||
gauge_host_cache_usage = self._gauge_cls(
|
||||
name="vllm:hisparse_host_cache_usage_perc",
|
||||
documentation=(
|
||||
"HiSparse host KV-cache usage. 1 means 100 percent usage. "
|
||||
"Evictable cached blocks count as free."
|
||||
),
|
||||
multiprocess_mode="mostrecent",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
gauge_pending_page_transfers = self._gauge_cls(
|
||||
name="vllm:hisparse_pending_page_transfers",
|
||||
documentation=(
|
||||
"HiSparse page transfers (write-backs and restores) not yet completed."
|
||||
),
|
||||
multiprocess_mode="mostrecent",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.hisparse_gauges: dict[str, Any] = {
|
||||
"host_cache_usage_perc": create_metric_per_engine(
|
||||
gauge_host_cache_usage, self.per_engine_labelvalues
|
||||
),
|
||||
"pending_page_transfers": create_metric_per_engine(
|
||||
gauge_pending_page_transfers, self.per_engine_labelvalues
|
||||
),
|
||||
}
|
||||
|
||||
self.hisparse_histograms: dict[str, Any] = {}
|
||||
if vllm_config.observability_config.kv_cache_metrics:
|
||||
residency_buckets = histogram_buckets(
|
||||
"kv_cache_residency",
|
||||
overrides=vllm_config.observability_config.custom_histogram_buckets,
|
||||
)
|
||||
histogram_host_block_lifetime = self._histogram_cls(
|
||||
name="vllm:hisparse_host_block_lifetime_seconds",
|
||||
documentation=(
|
||||
"Histogram of HiSparse host KV block lifetime from allocation "
|
||||
"to eviction. Sampled metrics (controlled by "
|
||||
"--kv-cache-metrics-sample)."
|
||||
),
|
||||
buckets=residency_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
histogram_host_block_idle_before_evict = self._histogram_cls(
|
||||
name="vllm:hisparse_host_block_idle_before_evict_seconds",
|
||||
documentation=(
|
||||
"Histogram of HiSparse host KV block idle time before eviction. "
|
||||
"Sampled metrics (controlled by --kv-cache-metrics-sample)."
|
||||
),
|
||||
buckets=residency_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
histogram_host_block_reuse_gap = self._histogram_cls(
|
||||
name="vllm:hisparse_host_block_reuse_gap_seconds",
|
||||
documentation=(
|
||||
"Histogram of time gaps between consecutive HiSparse host KV "
|
||||
"block accesses. Sampled metrics (controlled by "
|
||||
"--kv-cache-metrics-sample)."
|
||||
),
|
||||
buckets=residency_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.hisparse_histograms = {
|
||||
"host_block_lifetime_seconds": create_metric_per_engine(
|
||||
histogram_host_block_lifetime, self.per_engine_labelvalues
|
||||
),
|
||||
"host_block_idle_before_evict_seconds": create_metric_per_engine(
|
||||
histogram_host_block_idle_before_evict, self.per_engine_labelvalues
|
||||
),
|
||||
"host_block_reuse_gap_seconds": create_metric_per_engine(
|
||||
histogram_host_block_reuse_gap, self.per_engine_labelvalues
|
||||
),
|
||||
}
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
for name, counter in self.hisparse_counters.items():
|
||||
for value in transfer_stats_data.get(name, []):
|
||||
counter[engine_idx].inc(value)
|
||||
for name, gauge in self.hisparse_gauges.items():
|
||||
values = transfer_stats_data.get(name, [])
|
||||
if values:
|
||||
gauge[engine_idx].set(values[-1])
|
||||
for name, histogram in self.hisparse_histograms.items():
|
||||
for value in transfer_stats_data.get(name, []):
|
||||
histogram[engine_idx].observe(value)
|
||||
@@ -1,978 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HiSparse worker-side host/hot state and data movement."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparseKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
HiSparseCacheHandle,
|
||||
drop_hisparse_host_pool,
|
||||
release_pinned_state,
|
||||
update_hisparse_residency,
|
||||
)
|
||||
from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
HiSparseHotSpec,
|
||||
KVCacheConfig,
|
||||
)
|
||||
from vllm.v1.worker.utils import copy_kv_cache_blocks_inplace
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnectorMetadata,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
|
||||
|
||||
class _DMADescriptors(NamedTuple):
|
||||
src: torch.Tensor
|
||||
dst: torch.Tensor
|
||||
sizes: torch.Tensor
|
||||
src_np: np.ndarray
|
||||
dst_np: np.ndarray
|
||||
sizes_np: np.ndarray
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SlotMappingStaging:
|
||||
stream: torch.Stream
|
||||
event: torch.Event
|
||||
slots: torch.Tensor
|
||||
candidates: tuple[SparseKVRowMirror, ...] = ()
|
||||
num_tokens: int = 0
|
||||
source_index: int = 0
|
||||
|
||||
|
||||
def _allocate_dma_descriptors(size: int) -> _DMADescriptors:
|
||||
src, dst, sizes = (torch.empty(size, dtype=torch.int64) for _ in range(3))
|
||||
return _DMADescriptors(
|
||||
src,
|
||||
dst,
|
||||
sizes,
|
||||
src.numpy(),
|
||||
dst.numpy(),
|
||||
sizes.numpy(),
|
||||
)
|
||||
|
||||
|
||||
_METRICS_INTERVAL = 2000
|
||||
|
||||
|
||||
def _get_hisparse_cache(
|
||||
forward_context: dict[str, Any], layer_name: str
|
||||
) -> HiSparseCacheHandle:
|
||||
attention_layer = forward_context[layer_name]
|
||||
hisparse_cache = attention_layer.hisparse_cache
|
||||
assert hisparse_cache is not None
|
||||
return hisparse_cache
|
||||
|
||||
|
||||
def _flatten_row_mirrors(
|
||||
row_mirrors: Mapping[str, tuple[SparseKVRowMirror, ...]],
|
||||
request_ids: Sequence[str] | None,
|
||||
) -> tuple[SparseKVRowMirror, ...]:
|
||||
ordered_ids = row_mirrors if request_ids is None else request_ids
|
||||
return tuple(
|
||||
mirror
|
||||
for request_id in ordered_ids
|
||||
for mirror in row_mirrors.get(request_id, ())
|
||||
)
|
||||
|
||||
|
||||
def _select_written_row_mirrors(
|
||||
candidates: tuple[SparseKVRowMirror, ...],
|
||||
source_slots: np.ndarray,
|
||||
source_index: int,
|
||||
) -> tuple[SparseKVRowMirror, ...]:
|
||||
"""Select and coalesce GPU-written rows from a scheduler-owned envelope."""
|
||||
source_slots = source_slots[source_slots >= 0]
|
||||
if source_slots.size == 0 or not candidates:
|
||||
return ()
|
||||
starts = np.asarray([mirror.source_starts for mirror in candidates], dtype=np.int64)
|
||||
counts = np.fromiter((mirror.num_rows for mirror in candidates), dtype=np.int64)
|
||||
destinations = np.fromiter(
|
||||
[mirror.destination_start for mirror in candidates], dtype=np.int64
|
||||
)
|
||||
if source_index >= starts.shape[1]:
|
||||
raise RuntimeError("HiSparse row-mirror source index is out of range.")
|
||||
source_starts = starts[:, source_index]
|
||||
order = np.argsort(source_starts)
|
||||
matches = np.searchsorted(source_starts[order], source_slots, side="right") - 1
|
||||
candidate_ids = order[np.maximum(matches, 0)]
|
||||
offsets = source_slots - source_starts[candidate_ids]
|
||||
valid = (matches >= 0) & (offsets < counts[candidate_ids])
|
||||
if not np.all(valid):
|
||||
# A page the scheduler could not map has no host destination, so its
|
||||
# rows are simply not mirrorable this step.
|
||||
source_slots = source_slots[valid]
|
||||
candidate_ids = candidate_ids[valid]
|
||||
offsets = offsets[valid]
|
||||
if source_slots.size == 0:
|
||||
return ()
|
||||
sources = starts[candidate_ids] + offsets[:, None]
|
||||
destinations = destinations[candidate_ids] + offsets
|
||||
boundaries = (
|
||||
np.flatnonzero(
|
||||
(candidate_ids[1:] != candidate_ids[:-1])
|
||||
| (np.diff(destinations) != 1)
|
||||
| np.any(np.diff(sources, axis=0) != 1, axis=1)
|
||||
)
|
||||
+ 1
|
||||
)
|
||||
run_starts = np.concatenate(([0], boundaries))
|
||||
run_ends = np.concatenate((boundaries, [destinations.size]))
|
||||
return tuple(
|
||||
SparseKVRowMirror(
|
||||
source_starts=tuple(int(value) for value in sources[start]),
|
||||
destination_start=int(destinations[start]),
|
||||
num_rows=int(end - start),
|
||||
)
|
||||
for start, end in zip(run_starts, run_ends, strict=True)
|
||||
)
|
||||
|
||||
|
||||
def _is_hisparse_host_writer(
|
||||
shared_host_region: SharedOffloadRegion | None,
|
||||
) -> bool:
|
||||
return shared_host_region is None or get_tensor_model_parallel_rank() == 0
|
||||
|
||||
|
||||
def _create_hisparse_host_events(
|
||||
shared_host_region: SharedOffloadRegion | None,
|
||||
is_host_writer: bool,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Event, torch.Event]:
|
||||
if shared_host_region is None:
|
||||
return torch.Event(), torch.Event()
|
||||
|
||||
events: tuple[torch.Event, torch.Event] | None = None
|
||||
ipc_handles = None
|
||||
if is_host_writer:
|
||||
events = (
|
||||
torch.cuda.Event(interprocess=True),
|
||||
torch.cuda.Event(interprocess=True),
|
||||
)
|
||||
stream = current_stream()
|
||||
for event in events:
|
||||
event.record(stream)
|
||||
ipc_handles = tuple(event.ipc_handle() for event in events)
|
||||
ipc_handles = get_tp_group().broadcast_object(ipc_handles, src=0)
|
||||
if events is None:
|
||||
events = (
|
||||
torch.cuda.Event.from_ipc_handle(device, ipc_handles[0]),
|
||||
torch.cuda.Event.from_ipc_handle(device, ipc_handles[1]),
|
||||
)
|
||||
return events
|
||||
|
||||
|
||||
class HiSparseConnectorWorker:
|
||||
"""Own HiSparse host/hot state and execute its worker-side transfers."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> None:
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self._initialized = False
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
forward_context = self.vllm_config.compilation_config.static_forward_context
|
||||
cache_handles: list[HiSparseCacheHandle] = []
|
||||
cache_layer_names: list[str] = []
|
||||
for group in self.kv_cache_config.kv_cache_groups:
|
||||
if not isinstance(group.kv_cache_spec, HiSparseHotSpec):
|
||||
continue
|
||||
for cache_name in group.layer_names:
|
||||
assert cache_name.endswith(HISPARSE_HOT_SUFFIX)
|
||||
layer_name = cache_name[: -len(HISPARSE_HOT_SUFFIX)]
|
||||
cache_handles.append(_get_hisparse_cache(forward_context, layer_name))
|
||||
cache_layer_names.append(layer_name)
|
||||
|
||||
if not cache_handles:
|
||||
raise RuntimeError("HiSparse connector found no hot-cache handles.")
|
||||
hot_backings: dict[int, torch.Tensor] = {}
|
||||
registered_host_pools: dict[int, torch.Tensor] = {}
|
||||
shared_host_regions: dict[int, SharedOffloadRegion] = {}
|
||||
for cache in cache_handles:
|
||||
hot_backing = cache.runtime.hot_backing
|
||||
hot_backings[hot_backing.untyped_storage().data_ptr()] = hot_backing
|
||||
registered_pool = cache.runtime.registered_host_pool
|
||||
registered_host_pools[registered_pool.data_ptr()] = registered_pool
|
||||
if (region := cache.runtime.shared_host_region) is not None:
|
||||
shared_host_regions[id(region)] = region
|
||||
if len(hot_backings) != 1:
|
||||
raise RuntimeError("HiSparse hot tensors must share one GPU backing.")
|
||||
hot_backing = next(iter(hot_backings.values()))
|
||||
if len(shared_host_regions) > 1:
|
||||
raise RuntimeError("HiSparse caches must share one host region.")
|
||||
shared_host_region = next(iter(shared_host_regions.values()), None)
|
||||
pinned_host_pools = (
|
||||
[]
|
||||
if shared_host_region is not None
|
||||
else list(registered_host_pools.values())
|
||||
)
|
||||
is_host_writer = _is_hisparse_host_writer(shared_host_region)
|
||||
|
||||
resident = cache_handles[0].view
|
||||
assert resident is not None
|
||||
host_num_blocks = self.kv_cache_config.hisparse_host_num_blocks
|
||||
assert host_num_blocks is not None
|
||||
try:
|
||||
self.initialize(
|
||||
cache_handles,
|
||||
cache_layer_names,
|
||||
hot_backing,
|
||||
self.vllm_config.scheduler_config.max_num_seqs,
|
||||
host_num_blocks,
|
||||
hot_backing.device,
|
||||
pinned_host_pools,
|
||||
shared_host_region=shared_host_region,
|
||||
is_host_writer=is_host_writer,
|
||||
)
|
||||
except Exception:
|
||||
release_pinned_state(
|
||||
[cache.runtime for cache in cache_handles],
|
||||
pinned_host_pools,
|
||||
shared_host_region,
|
||||
)
|
||||
drop_hisparse_host_pool(self.vllm_config)
|
||||
raise
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
cache_handles: list[HiSparseCacheHandle],
|
||||
cache_layer_names: list[str],
|
||||
hot_backing: torch.Tensor,
|
||||
max_num_reqs: int,
|
||||
host_num_blocks: int,
|
||||
device: torch.device,
|
||||
pinned_host_pools: list[torch.Tensor],
|
||||
*,
|
||||
shared_host_region: SharedOffloadRegion | None = None,
|
||||
is_host_writer: bool = True,
|
||||
) -> None:
|
||||
if self._initialized:
|
||||
raise RuntimeError("HiSparse connector worker is already initialized.")
|
||||
resident = cache_handles[0].view
|
||||
assert resident is not None
|
||||
self.kernel_block_size = resident.block_size
|
||||
self.host_num_blocks = host_num_blocks
|
||||
self.pinned_host_pools = pinned_host_pools
|
||||
self.shared_host_region = shared_host_region
|
||||
self.is_host_writer = is_host_writer
|
||||
self.dma_stream = (
|
||||
torch.cuda.Stream(device=device) if self.is_host_writer else None
|
||||
)
|
||||
self._slot_mapping_staging = None
|
||||
if self.is_host_writer:
|
||||
max_mirror_rows = (
|
||||
self.vllm_config.scheduler_config.max_num_batched_tokens
|
||||
+ max_num_reqs * (self.vllm_config.num_lookahead_tokens + 1)
|
||||
)
|
||||
self._slot_mapping_staging = _SlotMappingStaging(
|
||||
stream=torch.Stream(device=device),
|
||||
event=torch.Event(),
|
||||
slots=torch.empty(
|
||||
max_mirror_rows,
|
||||
dtype=torch.int64,
|
||||
pin_memory=True,
|
||||
),
|
||||
)
|
||||
self._dma_free_descriptors: list[_DMADescriptors] = []
|
||||
self._pending_dma_descriptors: deque[tuple[torch.Event, _DMADescriptors]] = (
|
||||
deque()
|
||||
)
|
||||
self._dma_submitted = False
|
||||
self._per_layer_mirrored: set[int] = set()
|
||||
self._submitted_mirror_layers: set[int] = set()
|
||||
self._layer_ready_events = tuple(torch.Event() for _ in cache_handles)
|
||||
self._forward_ready_event = torch.Event()
|
||||
self._draft_layers = tuple(
|
||||
index for index, cache in enumerate(cache_handles) if cache.draft_layer
|
||||
)
|
||||
self._draft_mirror_pending = False
|
||||
self._set_row_mirrors(())
|
||||
self.cache_layer_names = cache_layer_names
|
||||
self._group_leaders = tuple(
|
||||
(layer_name, cache)
|
||||
for layer_name, cache in zip(cache_layer_names, cache_handles, strict=True)
|
||||
if cache.runtime.is_group_leader
|
||||
)
|
||||
self.host_write_events = _create_hisparse_host_events(
|
||||
shared_host_region, is_host_writer, device
|
||||
)
|
||||
self.host_write_event = self.host_write_events[1]
|
||||
self._next_host_write_event = 0
|
||||
self.cache_handles = cache_handles
|
||||
self.leader_runtimes = [cache.runtime for _, cache in self._group_leaders]
|
||||
request_state_indices = {
|
||||
indices.data_ptr(): indices
|
||||
for cache in cache_handles
|
||||
if (indices := cache.runtime.request_state_indices) is not None
|
||||
}
|
||||
if len(request_state_indices) != 1:
|
||||
raise RuntimeError(
|
||||
"HiSparse runtimes must share one request-state mapping."
|
||||
)
|
||||
self.request_state_indices = next(iter(request_state_indices.values()))
|
||||
if self.request_state_indices.numel() != max_num_reqs:
|
||||
raise RuntimeError(
|
||||
"HiSparse request-state mapping does not match max_num_seqs."
|
||||
)
|
||||
self.hot_backing = hot_backing
|
||||
self.residency: torch.Tensor | None = next(
|
||||
(cache.residency for cache in cache_handles if cache.residency is not None),
|
||||
None,
|
||||
)
|
||||
self._pending_invalid_block_ids: list[int] = []
|
||||
# Destination block ids of host copies this worker has run.
|
||||
self._completed_host_copy_dst_ids: list[int] = []
|
||||
self._post_forward_transfers: list[SparseKVPageTransfer] = []
|
||||
self._enqueued_transfer_ids: list[int] = []
|
||||
self._pending_transfer_events: deque[tuple[torch.Event, tuple[int, ...]]] = (
|
||||
deque()
|
||||
)
|
||||
self._metrics_calls = 0
|
||||
self._metrics_event = torch.Event()
|
||||
self._metrics_pending = False
|
||||
self._init_dma()
|
||||
if self.is_host_writer:
|
||||
for layer_index, handle in enumerate(cache_handles):
|
||||
# Draft layers are mirrored once in the next start_step instead.
|
||||
if not handle.draft_layer:
|
||||
handle.submit_layer_mirror = partial(
|
||||
self._enqueue_layer_mirror, layer_index
|
||||
)
|
||||
self._initialized = True
|
||||
|
||||
def set_request_state_indices(self, indices: torch.Tensor) -> None:
|
||||
if indices.numel() > self.request_state_indices.numel():
|
||||
raise ValueError(
|
||||
"HiSparse request-state mapping exceeds max_num_seqs: "
|
||||
f"{indices.numel()} > {self.request_state_indices.numel()}."
|
||||
)
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
return
|
||||
# Attention indexes persistent request state by input-batch row. Refresh
|
||||
# that indirection after every batch compaction or reorder.
|
||||
self.request_state_indices.fill_(-1)
|
||||
self.request_state_indices[: indices.numel()].copy_(indices)
|
||||
if self._pending_invalid_block_ids:
|
||||
self.invalidate_blocks(self._pending_invalid_block_ids, indices)
|
||||
self._pending_invalid_block_ids.clear()
|
||||
|
||||
def _init_dma(self) -> None:
|
||||
host_caches = tuple(cache.runtime.host_cache for cache in self.cache_handles)
|
||||
resident_caches = []
|
||||
for cache in self.cache_handles:
|
||||
assert cache.view is not None and cache.slot_mapping is not None
|
||||
resident_caches.append(cache.view.cache)
|
||||
self.host_caches = host_caches
|
||||
self.resident_caches = tuple(resident_caches)
|
||||
|
||||
for resident_cache, host_cache in zip(resident_caches, host_caches):
|
||||
row_bytes = resident_cache.shape[-1] * resident_cache.element_size()
|
||||
if (
|
||||
resident_cache.ndim != 3
|
||||
or resident_cache.shape[1] != self.kernel_block_size
|
||||
or resident_cache.stride(1) * resident_cache.element_size() != row_bytes
|
||||
):
|
||||
raise RuntimeError("HiSparse DMA requires contiguous resident rows.")
|
||||
if (
|
||||
host_cache.ndim != 2
|
||||
or not host_cache.is_contiguous()
|
||||
or host_cache.shape[1] * host_cache.element_size() != row_bytes
|
||||
):
|
||||
raise RuntimeError("HiSparse DMA requires contiguous host rows.")
|
||||
|
||||
def start_step(
|
||||
self,
|
||||
metadata: HiSparseConnectorMetadata,
|
||||
request_state_indices: torch.Tensor | None,
|
||||
request_ids: list[str] | None = None,
|
||||
num_tokens: int = 0,
|
||||
) -> None:
|
||||
self._stage_row_mirror_mapping(num_tokens)
|
||||
self._finish_previous_step()
|
||||
previous_host_write_event = self.host_write_event
|
||||
self.host_write_event = self.host_write_events[self._next_host_write_event]
|
||||
self._next_host_write_event ^= 1
|
||||
current_stream().wait_event(previous_host_write_event)
|
||||
if self.shared_host_region is not None and not num_tokens:
|
||||
# Without a forward, no TP collective keeps rank 0 from recording
|
||||
# a shared event again before every rank has issued its wait.
|
||||
get_tp_group().barrier()
|
||||
self._release_completed_dma_descriptors()
|
||||
self._dma_submitted = False
|
||||
mirrors = _flatten_row_mirrors(metadata.row_mirrors, request_ids)
|
||||
if self._slot_mapping_staging is not None:
|
||||
self._slot_mapping_staging.candidates = mirrors
|
||||
self._set_row_mirrors(mirrors)
|
||||
self._clear_forward_mirror_state()
|
||||
for handle in self.cache_handles:
|
||||
handle.all_context_pages_resident = metadata.all_context_pages_resident
|
||||
self._copy_host_blocks(metadata.host_block_copies, previous_host_write_event)
|
||||
transfers = (
|
||||
metadata.command.page_transfers if metadata.command is not None else []
|
||||
)
|
||||
self._restore_pages([transfer for transfer in transfers if transfer.restore])
|
||||
self._post_forward_transfers = [
|
||||
transfer
|
||||
for transfer in transfers
|
||||
if not transfer.restore and transfer.after_forward
|
||||
]
|
||||
self._submit_transfers(
|
||||
[
|
||||
transfer
|
||||
for transfer in transfers
|
||||
if not transfer.restore and not transfer.after_forward
|
||||
]
|
||||
)
|
||||
self._pending_invalid_block_ids.extend(metadata.source_block_ids)
|
||||
if request_state_indices is not None:
|
||||
self.set_request_state_indices(request_state_indices)
|
||||
if metadata.residency_updates:
|
||||
assert self.residency is not None
|
||||
assert request_state_indices is not None and request_ids is not None
|
||||
update_hisparse_residency(
|
||||
self.residency,
|
||||
metadata.residency_updates,
|
||||
request_ids,
|
||||
request_state_indices,
|
||||
)
|
||||
|
||||
def _clear_forward_mirror_state(self) -> None:
|
||||
self._per_layer_mirrored.clear()
|
||||
self._submitted_mirror_layers.clear()
|
||||
for handle in self.cache_handles:
|
||||
handle.decode_batch = False
|
||||
handle.host_mirror_required = False
|
||||
handle.num_actual_tokens = 0
|
||||
handle.num_decode_tokens = 0
|
||||
handle.req_id_per_token = None
|
||||
|
||||
def prepare_forward(self, attn_metadata: Mapping[str, Any] | None) -> None:
|
||||
if attn_metadata is None:
|
||||
return
|
||||
for layer_name, handle in self._group_leaders:
|
||||
metadata = attn_metadata.get(layer_name)
|
||||
if metadata is not None:
|
||||
handle.prepare_group_for_batch(metadata)
|
||||
|
||||
def _stage_row_mirror_mapping(self, num_tokens: int) -> None:
|
||||
"""Snapshot the rows this forward will write, off the compute stream.
|
||||
|
||||
The resident slot mapping is a persistent view bound at registration
|
||||
time, so the rows are already staged by the time the forward launches.
|
||||
"""
|
||||
state = self._slot_mapping_staging
|
||||
if state is None or not num_tokens:
|
||||
return
|
||||
handle = self.cache_handles[0]
|
||||
slots = handle.slot_mapping
|
||||
assert slots is not None
|
||||
source_index = handle.runtime.resident_source_index
|
||||
start = state.num_tokens
|
||||
end = start + num_tokens
|
||||
if end > state.slots.shape[0]:
|
||||
raise ValueError(
|
||||
"HiSparse row mapping exceeds staging capacity: "
|
||||
f"{end} > {state.slots.shape[0]}."
|
||||
)
|
||||
main_stream = current_stream()
|
||||
state.stream.wait_stream(main_stream)
|
||||
with torch.cuda.stream(state.stream):
|
||||
state.slots[start:end].copy_(
|
||||
slots[:num_tokens],
|
||||
non_blocking=True,
|
||||
)
|
||||
state.event.record(state.stream)
|
||||
state.num_tokens = end
|
||||
state.source_index = source_index
|
||||
|
||||
def _resolve_row_mirrors(self, state: _SlotMappingStaging) -> None:
|
||||
"""Compute the final row mirrors from the staged GPU slot mapping."""
|
||||
self._set_row_mirrors(
|
||||
_select_written_row_mirrors(
|
||||
state.candidates,
|
||||
state.slots[: state.num_tokens].numpy(),
|
||||
state.source_index,
|
||||
)
|
||||
)
|
||||
state.num_tokens = 0
|
||||
|
||||
def _copy_host_blocks(
|
||||
self,
|
||||
host_block_copies: Sequence[KVCacheBlockCopy],
|
||||
previous_host_write_event: torch.Event,
|
||||
) -> None:
|
||||
if not host_block_copies:
|
||||
return
|
||||
self._completed_host_copy_dst_ids.extend(
|
||||
copy.dst_block_id for copy in host_block_copies
|
||||
)
|
||||
if self.shared_host_region is None or get_tensor_model_parallel_rank() == 0:
|
||||
if self.host_caches:
|
||||
previous_host_write_event.synchronize()
|
||||
copy_kv_cache_blocks_inplace(
|
||||
self.host_caches,
|
||||
self.host_num_blocks,
|
||||
host_block_copies,
|
||||
)
|
||||
if self.shared_host_region is not None:
|
||||
get_tp_group().barrier()
|
||||
|
||||
def invalidate_blocks(
|
||||
self, block_ids: list[int], request_state_indices: torch.Tensor
|
||||
) -> None:
|
||||
"""Invalidate recycled host slots in this worker's leader runtimes."""
|
||||
if not block_ids:
|
||||
return
|
||||
device = self.cache_handles[0].runtime.device
|
||||
staging = torch.tensor(block_ids, dtype=torch.int32, pin_memory=True)
|
||||
blocks = staging.to(device, dtype=torch.int32, non_blocking=True)
|
||||
offsets = torch.arange(self.kernel_block_size, dtype=torch.int32, device=device)
|
||||
slots = (blocks[:, None] * self.kernel_block_size + offsets[None, :]).flatten()
|
||||
sorted_slots = torch.sort(slots).values
|
||||
state_indices = request_state_indices.to(device=device, dtype=torch.long)
|
||||
for runtime in self.leader_runtimes:
|
||||
runtime.invalidate_sorted_slots(sorted_slots, state_indices)
|
||||
|
||||
def reset_hot_state(self) -> None:
|
||||
for runtime in self.leader_runtimes:
|
||||
runtime.reset_hot_state()
|
||||
|
||||
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
|
||||
stats = None
|
||||
if self._metrics_pending and self._metrics_event.query():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
for runtime in self.leader_runtimes:
|
||||
group = runtime.index_group
|
||||
hits, misses = group.swap_stats_host.tolist()
|
||||
if hits or misses:
|
||||
stats.record_snapshot(hits, misses, misses * group.stats_row_bytes)
|
||||
self._metrics_pending = False
|
||||
if stats.is_empty():
|
||||
stats = None
|
||||
|
||||
self._metrics_calls += 1
|
||||
if (
|
||||
self._metrics_calls % _METRICS_INTERVAL == 0
|
||||
and not self._metrics_pending
|
||||
and not torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
compute_stream = current_stream()
|
||||
for runtime in self.leader_runtimes:
|
||||
group = runtime.index_group
|
||||
compute_stream.wait_stream(group.copy_stream)
|
||||
group.swap_stats_host.copy_(group.swap_stats, non_blocking=True)
|
||||
group.swap_stats.zero_()
|
||||
group.copy_stream.wait_stream(compute_stream)
|
||||
self._metrics_event.record()
|
||||
self._metrics_pending = True
|
||||
return stats
|
||||
|
||||
def _release_completed_dma_descriptors(self) -> None:
|
||||
pending = self._pending_dma_descriptors
|
||||
while pending and pending[0][0].query():
|
||||
_, descriptors = pending.popleft()
|
||||
self._dma_free_descriptors.append(descriptors)
|
||||
|
||||
def _acquire_dma_descriptors(self, size: int) -> _DMADescriptors:
|
||||
for index, descriptors in enumerate(self._dma_free_descriptors):
|
||||
if descriptors.src.numel() >= size:
|
||||
return self._dma_free_descriptors.pop(index)
|
||||
return _allocate_dma_descriptors(size)
|
||||
|
||||
def _submit_dma_descriptors(
|
||||
self,
|
||||
descriptors: _DMADescriptors,
|
||||
descriptor_count: int,
|
||||
transfer_ids: tuple[int, ...] = (),
|
||||
ready_event: torch.Event | None = None,
|
||||
) -> None:
|
||||
stream = self.dma_stream
|
||||
assert stream is not None
|
||||
if ready_event is None:
|
||||
stream.wait_stream(current_stream())
|
||||
else:
|
||||
stream.wait_event(ready_event)
|
||||
completion_event = torch.Event()
|
||||
with torch.cuda.stream(stream):
|
||||
ops.swap_blocks_batch(
|
||||
descriptors.src[:descriptor_count],
|
||||
descriptors.dst[:descriptor_count],
|
||||
descriptors.sizes[:descriptor_count],
|
||||
)
|
||||
self.host_write_event.record(stream)
|
||||
completion_event.record(stream)
|
||||
self._pending_dma_descriptors.append((completion_event, descriptors))
|
||||
self._dma_submitted = True
|
||||
if transfer_ids:
|
||||
self._pending_transfer_events.append((completion_event, transfer_ids))
|
||||
self._enqueued_transfer_ids.extend(transfer_ids)
|
||||
|
||||
def _set_row_mirrors(self, mirrors: tuple[SparseKVRowMirror, ...]) -> None:
|
||||
self._row_mirrors = mirrors
|
||||
self._row_mirror_destination_starts = np.fromiter(
|
||||
(mirror.destination_start for mirror in mirrors),
|
||||
dtype=np.int64,
|
||||
count=len(mirrors),
|
||||
)
|
||||
self._row_mirror_counts = np.fromiter(
|
||||
(mirror.num_rows for mirror in mirrors),
|
||||
dtype=np.int64,
|
||||
count=len(mirrors),
|
||||
)
|
||||
if mirrors:
|
||||
self._row_mirror_source_starts = np.asarray(
|
||||
[mirror.source_starts for mirror in mirrors], dtype=np.int64
|
||||
)
|
||||
if self._row_mirror_source_starts.ndim != 2:
|
||||
raise RuntimeError("HiSparse DMA source mappings must be rectangular.")
|
||||
else:
|
||||
self._row_mirror_source_starts = np.empty((0, 0), dtype=np.int64)
|
||||
self._row_mirror_num_rows = int(self._row_mirror_counts.sum())
|
||||
|
||||
def _enqueue_row_dma(
|
||||
self, layer_indices: Sequence[int], ready_event: torch.Event | None = None
|
||||
) -> None:
|
||||
if (
|
||||
not layer_indices
|
||||
or not self._row_mirror_num_rows
|
||||
or not self.is_host_writer
|
||||
):
|
||||
return
|
||||
mirrors = self._row_mirrors
|
||||
num_layers = len(layer_indices)
|
||||
descriptor_count = len(mirrors) * num_layers
|
||||
descriptors = self._acquire_dma_descriptors(descriptor_count)
|
||||
destination_starts = self._row_mirror_destination_starts
|
||||
row_counts = self._row_mirror_counts
|
||||
for descriptor_offset, layer_index in enumerate(layer_indices):
|
||||
cache = self.cache_handles[layer_index]
|
||||
source_index = cache.runtime.resident_source_index
|
||||
if source_index >= self._row_mirror_source_starts.shape[1]:
|
||||
raise RuntimeError("HiSparse row DMA source index is out of range.")
|
||||
source_rows = self._row_mirror_source_starts[:, source_index]
|
||||
source = self.resident_caches[layer_index]
|
||||
destination = self.host_caches[layer_index]
|
||||
row_bytes = source.shape[-1] * source.element_size()
|
||||
if (
|
||||
source.stride(1) * source.element_size() != row_bytes
|
||||
or destination.shape[1] * destination.element_size() != row_bytes
|
||||
):
|
||||
raise RuntimeError("HiSparse row DMA requires contiguous rows.")
|
||||
source_blocks, source_row_offsets = np.divmod(
|
||||
source_rows, self.kernel_block_size
|
||||
)
|
||||
source_out_of_range = (
|
||||
np.any(source_blocks < 0)
|
||||
or np.any(source_blocks >= source.shape[0])
|
||||
or np.any(source_row_offsets + row_counts > self.kernel_block_size)
|
||||
)
|
||||
if source_out_of_range:
|
||||
raise RuntimeError("HiSparse row DMA source is out of range.")
|
||||
if np.any(destination_starts < 0) or np.any(
|
||||
destination_starts + row_counts > destination.shape[0]
|
||||
):
|
||||
raise RuntimeError("HiSparse row DMA index is out of range.")
|
||||
descriptor_slice = slice(descriptor_offset, descriptor_count, num_layers)
|
||||
descriptors.src_np[descriptor_slice] = (
|
||||
source.data_ptr()
|
||||
+ source_blocks * source.stride(0) * source.element_size()
|
||||
+ source_row_offsets * row_bytes
|
||||
)
|
||||
descriptors.dst_np[descriptor_slice] = (
|
||||
destination.data_ptr() + destination_starts * row_bytes
|
||||
)
|
||||
descriptors.sizes_np[descriptor_slice] = row_counts * row_bytes
|
||||
|
||||
self._submit_dma_descriptors(
|
||||
descriptors, descriptor_count, ready_event=ready_event
|
||||
)
|
||||
|
||||
def _enqueue_layer_mirror(self, layer_index: int) -> None:
|
||||
handle = self.cache_handles[layer_index]
|
||||
if not handle.host_mirror_required:
|
||||
return
|
||||
if layer_index in self._per_layer_mirrored:
|
||||
raise RuntimeError(f"HiSparse layer {layer_index} mirrored twice.")
|
||||
state = self._slot_mapping_staging
|
||||
if state is not None and state.num_tokens:
|
||||
# The staging stream waited on the compute stream before the
|
||||
# forward launched, so this only drains pre-forward work that is
|
||||
# already queued ahead of the layer kernels: no GPU bubble.
|
||||
state.event.synchronize()
|
||||
self._resolve_row_mirrors(state)
|
||||
self._per_layer_mirrored.add(layer_index)
|
||||
next_layer = layer_index + 1
|
||||
if (
|
||||
next_layer < len(self.cache_handles)
|
||||
and self.cache_handles[next_layer].runtime.resident_source_index
|
||||
== handle.runtime.resident_source_index
|
||||
):
|
||||
return
|
||||
ready_event = self._layer_ready_events[layer_index]
|
||||
ready_event.record()
|
||||
pending_layers = tuple(
|
||||
sorted(self._per_layer_mirrored - self._submitted_mirror_layers)
|
||||
)
|
||||
self._enqueue_row_dma(
|
||||
pending_layers,
|
||||
ready_event=ready_event,
|
||||
)
|
||||
self._submitted_mirror_layers.update(pending_layers)
|
||||
|
||||
def _record_transfer_completion(
|
||||
self,
|
||||
transfers: list[SparseKVPageTransfer],
|
||||
*,
|
||||
stream: torch.Stream | None = None,
|
||||
) -> None:
|
||||
if not transfers:
|
||||
return
|
||||
if stream is None:
|
||||
if not self.is_host_writer:
|
||||
return
|
||||
stream = self.dma_stream
|
||||
assert stream is not None
|
||||
completion_event = torch.Event()
|
||||
completion_event.record(stream)
|
||||
transfer_ids = tuple(transfer.transfer_id for transfer in transfers)
|
||||
self._pending_transfer_events.append((completion_event, transfer_ids))
|
||||
self._enqueued_transfer_ids.extend(transfer_ids)
|
||||
|
||||
def _restore_pages(self, transfers: list[SparseKVPageTransfer]) -> None:
|
||||
"""Restore imported tails on every rank before its forward or graph replay."""
|
||||
if not transfers:
|
||||
return
|
||||
for layer_index, cache in enumerate(self.cache_handles):
|
||||
source_index = cache.runtime.resident_source_index
|
||||
source = self.host_caches[layer_index]
|
||||
destination = self.resident_caches[layer_index]
|
||||
for transfer in transfers:
|
||||
host_start = transfer.host_block_id * self.kernel_block_size
|
||||
resident_block = transfer.resident_block_ids[source_index]
|
||||
# Each import pays one H2D page copy and enqueue per layer/rank
|
||||
# instead of tail prefill on D. Restore the prompt rows before
|
||||
# decode appends to this page; unused rows do not extend the
|
||||
# logical sequence.
|
||||
# cudaHostRegister pins this pool outside PyTorch's allocator,
|
||||
# so the sync checker cannot recognize it via is_pinned().
|
||||
with gpu_sync_allowed():
|
||||
destination[resident_block].copy_(
|
||||
source[host_start : host_start + self.kernel_block_size],
|
||||
non_blocking=True,
|
||||
)
|
||||
self._record_transfer_completion(transfers, stream=current_stream())
|
||||
|
||||
def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
|
||||
if self.cache_handles[0].runtime.eager_host_mirror:
|
||||
self._record_transfer_completion(transfers)
|
||||
else:
|
||||
self._enqueue_transfers(transfers)
|
||||
|
||||
def _enqueue_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
|
||||
if not transfers or not self.is_host_writer:
|
||||
return
|
||||
num_layers = len(self.cache_handles)
|
||||
descriptor_count = len(transfers) * num_layers
|
||||
descriptors = self._acquire_dma_descriptors(descriptor_count)
|
||||
destination_rows = np.fromiter(
|
||||
(transfer.host_block_id * self.kernel_block_size for transfer in transfers),
|
||||
dtype=np.int64,
|
||||
count=len(transfers),
|
||||
)
|
||||
source_blocks_by_transfer = np.asarray(
|
||||
[transfer.resident_block_ids for transfer in transfers], dtype=np.int64
|
||||
)
|
||||
if source_blocks_by_transfer.ndim != 2:
|
||||
raise RuntimeError(
|
||||
"HiSparse spill DMA source mappings must be rectangular."
|
||||
)
|
||||
for layer_index, cache in enumerate(self.cache_handles):
|
||||
source_index = cache.runtime.resident_source_index
|
||||
if source_index >= source_blocks_by_transfer.shape[1]:
|
||||
raise RuntimeError("HiSparse spill DMA source index is out of range.")
|
||||
source_blocks = source_blocks_by_transfer[:, source_index]
|
||||
source = self.resident_caches[layer_index]
|
||||
destination = self.host_caches[layer_index]
|
||||
if np.any(source_blocks < 0) or np.any(source_blocks >= source.shape[0]):
|
||||
raise RuntimeError("HiSparse spill DMA source is out of range.")
|
||||
if np.any(destination_rows < 0) or np.any(
|
||||
destination_rows + self.kernel_block_size > destination.shape[0]
|
||||
):
|
||||
raise RuntimeError("HiSparse spill DMA destination is out of range.")
|
||||
row_bytes = source.shape[-1] * source.element_size()
|
||||
descriptor_slice = slice(layer_index, descriptor_count, num_layers)
|
||||
descriptors.src_np[descriptor_slice] = (
|
||||
source.data_ptr()
|
||||
+ source_blocks * source.stride(0) * source.element_size()
|
||||
)
|
||||
descriptors.dst_np[descriptor_slice] = (
|
||||
destination.data_ptr() + destination_rows * row_bytes
|
||||
)
|
||||
descriptors.sizes_np[descriptor_slice] = self.kernel_block_size * row_bytes
|
||||
self._submit_dma_descriptors(
|
||||
descriptors,
|
||||
descriptor_count,
|
||||
transfer_ids=tuple(transfer.transfer_id for transfer in transfers),
|
||||
)
|
||||
|
||||
def _enqueue_host_mirror(
|
||||
self,
|
||||
ready_event: torch.Event | None = None,
|
||||
) -> None:
|
||||
active = [
|
||||
(index, handle)
|
||||
for index, handle in enumerate(self.cache_handles)
|
||||
if handle.num_actual_tokens != 0
|
||||
]
|
||||
if not active:
|
||||
return
|
||||
|
||||
def mirror_key(handle: HiSparseCacheHandle) -> tuple:
|
||||
slots = handle.mirror_slot_mapping
|
||||
return (
|
||||
handle.num_actual_tokens,
|
||||
handle.num_decode_tokens,
|
||||
handle.decode_batch,
|
||||
handle.host_mirror_required,
|
||||
handle.runtime.eager_host_mirror,
|
||||
None if slots is None else slots.data_ptr(),
|
||||
)
|
||||
|
||||
keys = {mirror_key(handle) for _, handle in active}
|
||||
if len(keys) > 1:
|
||||
raise RuntimeError(
|
||||
f"HiSparse cache layers disagree on mirror metadata: {list(keys)}."
|
||||
)
|
||||
cache = active[0][1]
|
||||
if not cache.host_mirror_required:
|
||||
return
|
||||
dst_slots = cache.mirror_slot_mapping
|
||||
if dst_slots is None:
|
||||
raise RuntimeError("HiSparse host mirror has no source slot mapping.")
|
||||
num_rows = min(cache.num_actual_tokens, dst_slots.numel())
|
||||
if num_rows == 0:
|
||||
return
|
||||
if self.is_host_writer:
|
||||
# The drafter has not written its layers' rows yet; the next
|
||||
# start_step mirrors them after it runs.
|
||||
self._draft_mirror_pending = bool(self._draft_layers)
|
||||
expected_layers = {index for index, _ in active}.difference(
|
||||
self._draft_layers
|
||||
)
|
||||
if self._per_layer_mirrored and self._per_layer_mirrored != expected_layers:
|
||||
raise RuntimeError(
|
||||
"HiSparse per-layer DMA did not mirror every active layer: "
|
||||
f"expected {sorted(expected_layers)}, got "
|
||||
f"{sorted(self._per_layer_mirrored)}."
|
||||
)
|
||||
pending_layers = tuple(
|
||||
sorted(expected_layers - self._submitted_mirror_layers)
|
||||
)
|
||||
if pending_layers:
|
||||
self._enqueue_row_dma(pending_layers, ready_event=ready_event)
|
||||
self._submitted_mirror_layers.update(pending_layers)
|
||||
assert cache.req_id_per_token is not None
|
||||
for _, handle in active:
|
||||
if handle.runtime.is_group_leader:
|
||||
handle.runtime.invalidate_written_slots(
|
||||
dst_slots[:num_rows],
|
||||
cache.req_id_per_token[:num_rows],
|
||||
)
|
||||
|
||||
def _finish_mirror_phase(self, ready_event: torch.Event | None = None) -> None:
|
||||
state = self._slot_mapping_staging
|
||||
if state is not None and state.num_tokens:
|
||||
# Measured free: blocking here costs no throughput (26.7 vs 27.6
|
||||
# gen tok/s) and lets every step mirror exactly the written rows.
|
||||
state.event.synchronize()
|
||||
self._resolve_row_mirrors(state)
|
||||
self._enqueue_host_mirror(ready_event)
|
||||
self._clear_forward_mirror_state()
|
||||
|
||||
def finish_forward(self) -> None:
|
||||
compute_stream = current_stream()
|
||||
self._forward_ready_event.record()
|
||||
self._finish_mirror_phase(self._forward_ready_event)
|
||||
if self.is_host_writer:
|
||||
if self._dma_submitted:
|
||||
compute_stream.wait_event(self.host_write_event)
|
||||
self._dma_submitted = False
|
||||
else:
|
||||
self.host_write_event.record(compute_stream)
|
||||
self._release_completed_dma_descriptors()
|
||||
|
||||
def _finish_previous_step(self) -> None:
|
||||
"""Mirror the previous step's draft-layer rows, then hand its pages over.
|
||||
|
||||
Runs before this step's work, so after the previous step's drafter.
|
||||
Post-forward transfers are ordered behind the draft mirror, so a page is
|
||||
never reported clean, and its resident block never released, before
|
||||
its draft-layer rows reach the host. Both land in the previous step's
|
||||
host write event, which this step waits on before its forward and its
|
||||
host copies, since the scheduler may already have reused the
|
||||
finished requests' blocks they read and write.
|
||||
"""
|
||||
if self._draft_mirror_pending:
|
||||
self._draft_mirror_pending = False
|
||||
self._enqueue_row_dma(self._draft_layers)
|
||||
transfers = self._post_forward_transfers
|
||||
self._post_forward_transfers = []
|
||||
self._submit_transfers(transfers)
|
||||
|
||||
def take_completed_host_copies(self) -> list[int]:
|
||||
"""Drain host copies this worker has enqueued for this step."""
|
||||
completed = self._completed_host_copy_dst_ids
|
||||
self._completed_host_copy_dst_ids = []
|
||||
return completed
|
||||
|
||||
def take_transfer_updates(self) -> tuple[list[int], list[int]]:
|
||||
enqueued = self._enqueued_transfer_ids
|
||||
self._enqueued_transfer_ids = []
|
||||
completed: list[int] = []
|
||||
while self._pending_transfer_events:
|
||||
event, transfer_ids = self._pending_transfer_events[0]
|
||||
if not event.query():
|
||||
break
|
||||
self._pending_transfer_events.popleft()
|
||||
completed.extend(transfer_ids)
|
||||
return enqueued, completed
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if not self._initialized:
|
||||
return
|
||||
if self._slot_mapping_staging is not None:
|
||||
self._slot_mapping_staging.stream.synchronize()
|
||||
if self.dma_stream is not None:
|
||||
self.dma_stream.synchronize()
|
||||
release_pinned_state(
|
||||
[cache.runtime for cache in self.cache_handles],
|
||||
self.pinned_host_pools,
|
||||
self.shared_host_region,
|
||||
)
|
||||
drop_hisparse_host_pool(self.vllm_config)
|
||||
self._initialized = False
|
||||
@@ -1,656 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
import msgspec
|
||||
import regex as re
|
||||
import torch
|
||||
import zmq
|
||||
|
||||
from vllm.config import KVTransferConfig, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import (
|
||||
get_ip,
|
||||
get_open_port,
|
||||
make_zmq_socket,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
from dataclasses import field
|
||||
from enum import Enum
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
Transfer = tuple[int, float]
|
||||
EngineId = str
|
||||
ReqId = str
|
||||
TransferId = str
|
||||
TransferOffsetsKey = tuple[str, tuple[int, ...], tuple[int, ...], torch.dtype]
|
||||
|
||||
|
||||
def _positive_finite_timeout(name: str, value: Any) -> float:
|
||||
timeout = float(value)
|
||||
if not math.isfinite(timeout) or timeout <= 0:
|
||||
raise ValueError(f"{name} must be finite and greater than zero")
|
||||
return timeout
|
||||
|
||||
|
||||
class MoRIIOTransferAck(NamedTuple):
|
||||
transfer_id: TransferId
|
||||
consumer_tp_size: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class WriteTask:
|
||||
request_id: ReqId
|
||||
transfer_id: TransferId
|
||||
dst_engine_id: str
|
||||
local_block_ids: BlockIds
|
||||
remote_block_ids_hint: BlockIds | None
|
||||
layer_name: str
|
||||
event: torch.cuda.Event
|
||||
remote_notify_port: int
|
||||
remote_ip: str
|
||||
remote_tp_size: int | None = None
|
||||
multi_pod_hosts: list[str] = field(default_factory=list)
|
||||
remote_dp_size_local: int = 0
|
||||
remote_dp_size: int = 1
|
||||
enqueue_time: float = field(default_factory=time.perf_counter)
|
||||
retried: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class LayerTransferPlan:
|
||||
"""Plan for transferring a single layer."""
|
||||
|
||||
request_id: ReqId
|
||||
transfer_id: TransferId
|
||||
layer_name: str
|
||||
sess_idx: int
|
||||
transfer_local_offsets: list[int]
|
||||
transfer_remote_offsets: list[int]
|
||||
transfer_sizes: list[int]
|
||||
use_batch: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class RemoteAllocInfo:
|
||||
"""Information about remote block allocation."""
|
||||
|
||||
block_ids: BlockIds | None
|
||||
writes_done: int = 0
|
||||
writes_expected: int | None = None
|
||||
decode_dp_rank: int = 0
|
||||
completion_request_id: str | None = None
|
||||
completion_notify_port: int | None = None
|
||||
completion_remote_ip: str | None = None
|
||||
completion_notified: bool = False
|
||||
transfer_statuses: list[Any] = field(default_factory=list)
|
||||
transfer_offsets: dict[
|
||||
TransferOffsetsKey,
|
||||
tuple[list[int], list[int], list[int]],
|
||||
] = field(default_factory=dict)
|
||||
|
||||
|
||||
class ROLE(Enum):
|
||||
PRODUCER = "producer"
|
||||
CONSUMER = "consumer"
|
||||
NOTINIT = "notinit"
|
||||
|
||||
|
||||
class MoRIIOAgentMetadata(
|
||||
msgspec.Struct,
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
# required for @cached_property.d
|
||||
dict=True,
|
||||
):
|
||||
engine_id: str
|
||||
agent_metadata: bytes
|
||||
kv_caches_base_addr: list[int]
|
||||
num_blocks: int
|
||||
block_len: int
|
||||
attn_backend_name: str
|
||||
|
||||
|
||||
class RoleManager:
|
||||
"""Manages role state across the connector."""
|
||||
|
||||
_instance: "RoleManager | None" = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._role: ROLE = ROLE.NOTINIT
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls) -> "RoleManager":
|
||||
if cls._instance is None:
|
||||
with cls._lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
return cls._instance
|
||||
|
||||
def set_role(self, role: ROLE) -> None:
|
||||
"""Set the current role."""
|
||||
with self._lock:
|
||||
self._role = role
|
||||
|
||||
def get_role(self) -> ROLE:
|
||||
"""Get the current role."""
|
||||
return self._role
|
||||
|
||||
|
||||
def set_role(role: ROLE):
|
||||
"""Set the global role."""
|
||||
RoleManager.get_instance().set_role(role)
|
||||
|
||||
|
||||
def get_role() -> ROLE:
|
||||
"""Get the global role."""
|
||||
return RoleManager.get_instance().get_role()
|
||||
|
||||
|
||||
class MoRIIOMode(Enum):
|
||||
READ = "read"
|
||||
WRITE = "write"
|
||||
|
||||
|
||||
class TransferBatchState(Enum):
|
||||
"""Verdict for a group of transfer statuses that belong to one request.
|
||||
|
||||
A request's KV transfer is spread over one status per layer (two for a KDA
|
||||
layer), so a single status never decides the request: PENDING means at
|
||||
least one is still in flight and none has failed, FAILED means at least
|
||||
one failed, DONE means all succeeded.
|
||||
"""
|
||||
|
||||
DONE = "done"
|
||||
FAILED = "failed"
|
||||
PENDING = "pending"
|
||||
|
||||
|
||||
class MoRIIOError(Exception):
|
||||
"""Base exception for MoRIIO operations."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class HandshakeError(MoRIIOError):
|
||||
"""Exception raised when handshake fails."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TransferError(MoRIIOError):
|
||||
"""Exception raised when transfer fails."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def get_moriio_mode(kv_transfer_config: KVTransferConfig) -> MoRIIOMode:
|
||||
read_mode = str(
|
||||
kv_transfer_config.kv_connector_extra_config.get("read_mode", "false")
|
||||
).lower().strip() in ("true", "1")
|
||||
logger.debug("MoRIIO Connector read_mode: %s", read_mode)
|
||||
if read_mode:
|
||||
return MoRIIOMode.READ
|
||||
else:
|
||||
return MoRIIOMode.WRITE
|
||||
|
||||
|
||||
def get_port_offset(dp_rank: int, tp_rank: int, tp_size: int = 1) -> int:
|
||||
return (dp_rank) * tp_size + tp_rank
|
||||
|
||||
|
||||
def fold_local_rank(global_dp_rank: int, dp_size_local: int) -> int:
|
||||
"""Fold a global DP rank into its pod-local rank [0, dp_size_local).
|
||||
|
||||
``dp_size_local == 0`` is the external-DP sentinel (local size unknown):
|
||||
return the rank unchanged, since a global DP rank is always < the global
|
||||
DP size so no folding is needed and the modulo is skipped (never divides
|
||||
by zero).
|
||||
"""
|
||||
return global_dp_rank % dp_size_local if dp_size_local else global_dp_rank
|
||||
|
||||
|
||||
def pod_index(global_dp_rank: int, dp_size_local: int) -> int:
|
||||
"""Pod index (0-based) a global DP rank lives on for Wide-EP multi-pod.
|
||||
|
||||
``dp_size_local == 0`` (external-DP sentinel) collapses to a single pod.
|
||||
"""
|
||||
return global_dp_rank // dp_size_local if dp_size_local else 0
|
||||
|
||||
|
||||
def resolve_host_ip(extra_config: dict) -> str:
|
||||
"""The IP this MoRIIO process advertises for KV transfer.
|
||||
|
||||
Honors an explicit ``host_ip`` in ``kv_connector_extra_config`` before
|
||||
falling back to ``get_ip()``. An external router/orchestrator can set it to
|
||||
the node's routable address; this is required under frameworks (e.g. Ray)
|
||||
where ``get_ip()`` resolves to an unroutable public IP and ``VLLM_HOST_IP``
|
||||
cannot be propagated to the worker processes that bind the transfer engine.
|
||||
"""
|
||||
return extra_config.get("host_ip") or get_ip()
|
||||
|
||||
|
||||
_DEPRECATED_ENV_VARS: dict[str, str] = {
|
||||
"VLLM_MORIIO_CONNECTOR_READ_MODE": "read_mode",
|
||||
"VLLM_MORIIO_QP_PER_TRANSFER": "qp_per_transfer",
|
||||
"VLLM_MORIIO_POST_BATCH_SIZE": "post_batch_size",
|
||||
"VLLM_MORIIO_NUM_WORKERS": "num_workers",
|
||||
}
|
||||
|
||||
|
||||
def _warn_deprecated_env_vars() -> None:
|
||||
for env_var, new_key in _DEPRECATED_ENV_VARS.items():
|
||||
if env_var in os.environ:
|
||||
logger.warning_once(
|
||||
"The environment variable %s is deprecated and ignored. "
|
||||
"Set %r inside kv_transfer_config.kv_connector_extra_config "
|
||||
"instead.",
|
||||
env_var,
|
||||
new_key,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MoRIIOConfig:
|
||||
local_ip: str
|
||||
local_kv_port: int
|
||||
proxy_ip: str
|
||||
local_ping_port: int
|
||||
proxy_ping_port: int
|
||||
http_port: int
|
||||
handshake_port: int
|
||||
notify_port: int
|
||||
tp_rank: int
|
||||
dp_rank: int
|
||||
dp_size: int
|
||||
tp_size: int
|
||||
transfer_timeout: float
|
||||
defer_timeout: float
|
||||
recv_abort_timeout: float
|
||||
read_mode: bool = False
|
||||
qp_per_transfer: int = 1
|
||||
post_batch_size: int = -1
|
||||
num_workers: int = 1
|
||||
backend: str = "rdma"
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(cls, vllm_config: VllmConfig) -> "MoRIIOConfig":
|
||||
# Port Configuration:
|
||||
# local_ping_port -> Outgoing heartbeat to proxy
|
||||
# proxy_ping_port -> Remote proxy's heartbeat ingress port
|
||||
# http_port -> Instance's HTTP service endpoint
|
||||
# local_kv_port -> service port for mori engine
|
||||
# notify_port -> For synchronizing stages between prefill and decode
|
||||
# handshake_port -> For initial handshake between mori engine
|
||||
|
||||
# Optional tuning knobs
|
||||
# read_mode -> If true, run the connector in READ mode (consumer
|
||||
# pulls KV from producer) instead of the default
|
||||
# WRITE mode.
|
||||
# transfer_timeout -> Timeout for waiting_for_transfer_complete before
|
||||
# raising TransferError (sec).
|
||||
# defer_timeout -> Timeout for deferred WRITE allocation or a missing
|
||||
# READ release notification (sec).
|
||||
# recv_abort_timeout -> Timeout before an in-flight recv whose RDMA
|
||||
# completion never arrived is aborted (sec).
|
||||
|
||||
# Knobs for RDMA transfers, ignored if on xgmi backend
|
||||
# qp_per_transfer -> Number of RDMA Queue Pairs per KV transfer.
|
||||
# post_batch_size -> Batch size for posting transfer work requests
|
||||
# (-1 lets the MoRI backend choose).
|
||||
# num_workers -> Number of background worker threads the MoRI
|
||||
# engine uses for transfer processing.
|
||||
|
||||
# TODO : merge notify_port and handshake_port to simplify port management
|
||||
# supports non-contiguous ports
|
||||
assert vllm_config.kv_transfer_config is not None, (
|
||||
"kv_transfer_config must be set for MoRIIOConnector"
|
||||
)
|
||||
_warn_deprecated_env_vars()
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
# Fold the global data_parallel_rank back to [0, dp_size_local) for
|
||||
# per-node port allocation (handles the external-DP sentinel).
|
||||
pc = vllm_config.parallel_config
|
||||
dp_rank = fold_local_rank(pc.data_parallel_rank, pc.data_parallel_size_local)
|
||||
base_notify_port = int(extra_config["notify_port"])
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
port_offset = get_port_offset(dp_rank, tp_rank)
|
||||
backend = str(extra_config.get("backend", "rdma")).lower()
|
||||
if backend not in ("rdma", "xgmi"):
|
||||
raise ValueError(
|
||||
f"Invalid MoRIIO backend {backend!r} in kv_connector_extra_config; "
|
||||
"must be one of 'rdma' or 'xgmi'."
|
||||
)
|
||||
|
||||
transfer_timeout = _positive_finite_timeout(
|
||||
"kv_connector_extra_config.transfer_timeout",
|
||||
extra_config.get(
|
||||
"transfer_timeout", MoRIIOConstants.DEFAULT_TRANSFER_TIMEOUT
|
||||
),
|
||||
)
|
||||
defer_timeout = _positive_finite_timeout(
|
||||
"kv_connector_extra_config.defer_timeout",
|
||||
extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT),
|
||||
)
|
||||
legacy_recv_abort_timeout = os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S")
|
||||
if legacy_recv_abort_timeout is not None:
|
||||
logger.warning_once(
|
||||
"The environment variable VLLM_MORIIO_TRANSFER_TIMEOUT_S is "
|
||||
"deprecated. Set 'recv_abort_timeout' inside "
|
||||
"kv_transfer_config.kv_connector_extra_config instead."
|
||||
)
|
||||
recv_abort_timeout = _positive_finite_timeout(
|
||||
"kv_connector_extra_config.recv_abort_timeout",
|
||||
extra_config.get(
|
||||
"recv_abort_timeout",
|
||||
legacy_recv_abort_timeout or MoRIIOConstants.DEFAULT_RECV_ABORT_TIMEOUT,
|
||||
),
|
||||
)
|
||||
|
||||
return cls(
|
||||
local_ip=resolve_host_ip(extra_config),
|
||||
local_kv_port=get_open_port(),
|
||||
proxy_ip=extra_config.get("proxy_ip", ""),
|
||||
local_ping_port=get_open_port(),
|
||||
proxy_ping_port=int(extra_config.get("proxy_ping_port", 0)),
|
||||
http_port=int(extra_config["http_port"]),
|
||||
handshake_port=int(extra_config["handshake_port"]),
|
||||
notify_port=base_notify_port + port_offset,
|
||||
tp_rank=tp_rank,
|
||||
dp_rank=dp_rank,
|
||||
# Advertise the GLOBAL DP world size to the routing proxy so it can
|
||||
# address every rank across pods; dp_size_local above is only for
|
||||
# per-pod port folding, not for routing.
|
||||
dp_size=pc.data_parallel_size,
|
||||
tp_size=tp_size,
|
||||
read_mode=get_moriio_mode(kv_transfer_config) == MoRIIOMode.READ,
|
||||
qp_per_transfer=int(extra_config.get("qp_per_transfer", 1)),
|
||||
post_batch_size=int(extra_config.get("post_batch_size", -1)),
|
||||
num_workers=int(extra_config.get("num_workers", 1)),
|
||||
backend=backend,
|
||||
transfer_timeout=transfer_timeout,
|
||||
defer_timeout=defer_timeout,
|
||||
recv_abort_timeout=recv_abort_timeout,
|
||||
)
|
||||
|
||||
|
||||
class MoRIIOConstants:
|
||||
"""Constants for MoRIIO connector."""
|
||||
|
||||
# ZMQ message types
|
||||
GET_META_MSG = b"get_meta_msg"
|
||||
POP_DONE_RECV = b"pop_done_recv"
|
||||
OVER = b"OVER"
|
||||
COMPLETION_PREFIX = "cmpl"
|
||||
TRANSFER_PREFIX = "tx"
|
||||
|
||||
PING_INTERVAL = 3
|
||||
MAX_PING_RETRIES = 100
|
||||
DEFAULT_HANDSHAKE_PORT = "6301"
|
||||
DEFAULT_NOTIFY_PORT = "61005"
|
||||
|
||||
VLLM_MORI_READ_ABORT_REQUEST_TIMEOUT = 3600
|
||||
|
||||
# Timeout (seconds) for waiting_for_transfer_complete before raising TransferError.
|
||||
# Overridable via kv_connector_extra_config["transfer_timeout"].
|
||||
DEFAULT_TRANSFER_TIMEOUT = 30.0
|
||||
# Timeout (seconds) for deferred WRITE allocation or a missing READ release
|
||||
# notification.
|
||||
# Overridable via kv_connector_extra_config["defer_timeout"].
|
||||
DEFAULT_DEFER_TIMEOUT = 60.0
|
||||
# Timeout (seconds) before an in-flight recv whose RDMA completion was lost
|
||||
# is aborted, so the decode worker does not hang on it forever.
|
||||
# Overridable via kv_connector_extra_config["recv_abort_timeout"].
|
||||
DEFAULT_RECV_ABORT_TIMEOUT = 120.0
|
||||
|
||||
|
||||
# The router embeds both zmq_addresses in the request_id:
|
||||
# "___prefill_addr_{zmq}___decode_addr_{zmq}_{32-hex-uuid}"
|
||||
# MoRIIO zmq_address format: "host:IP,handshake:PORT,notify:PORT"
|
||||
#
|
||||
# This lets each connector side parse the peer's connection info without
|
||||
# requiring the router to pass it explicitly in kv_transfer_params.
|
||||
_PREFILL_ZMQ_RE = re.compile(r"___prefill_addr_(.+?)___decode_addr_")
|
||||
# vLLM wraps the router's X-Request-Id as "cmpl-<id>-<seq>-<hex>" so there may
|
||||
# be a trailing "-<seq>-<hex>" suffix after the 32-char UUID. Allow it.
|
||||
_DECODE_ZMQ_RE = re.compile(r"___decode_addr_(.+)_[0-9a-f]{32}(?:-.*)?$")
|
||||
|
||||
|
||||
def parse_moriio_zmq_address(
|
||||
zmq_address: str,
|
||||
) -> tuple[str, int, int]:
|
||||
"""Parse the MoRI-IO zmq address into its components.
|
||||
|
||||
Parses ``"host:IP,handshake:PORT,notify:PORT"`` into
|
||||
(host, handshake_port, notify_port).
|
||||
|
||||
Each key-value pair is split on the *first* colon so that IPv6 addresses
|
||||
(e.g. ``host:::1``) are handled correctly. Raises ``ValueError`` if any
|
||||
of ``host``, ``handshake``, or ``notify`` keys are absent or if the port
|
||||
values are non-numeric.
|
||||
"""
|
||||
parts: dict[str, str] = {}
|
||||
for segment in zmq_address.split(","):
|
||||
key, _, val = segment.partition(":")
|
||||
parts[key.strip()] = val.strip()
|
||||
try:
|
||||
host = parts["host"]
|
||||
handshake_port = int(parts["handshake"])
|
||||
notify_port = int(parts["notify"])
|
||||
except (KeyError, ValueError) as e:
|
||||
raise ValueError(
|
||||
f"Malformed zmq_address {zmq_address!r}: expected "
|
||||
f"'host:IP,handshake:PORT,notify:PORT' format"
|
||||
) from e
|
||||
return host, handshake_port, notify_port
|
||||
|
||||
|
||||
def get_peer_zmq_from_request_id(request_id: str, is_producer: bool) -> str | None:
|
||||
"""Extract the *peer's* zmq_address from the vLLM router request_id.
|
||||
|
||||
The producer (prefill) needs the decode's address; the consumer (decode)
|
||||
needs the prefill's address.
|
||||
|
||||
Returns ``None`` when the request_id does not encode peer info. The
|
||||
llm-d routing sidecar (``llm-d-inference-scheduler``) does not embed
|
||||
addresses in ``request_id``; instead it passes ``remote_host``,
|
||||
``remote_handshake_port`` and ``remote_notify_port`` explicitly in
|
||||
``kv_transfer_params``. Callers must handle the ``None`` return by
|
||||
falling back to those fields. See ``add_new_req`` for the canonical
|
||||
fallback path.
|
||||
"""
|
||||
if is_producer:
|
||||
m = _DECODE_ZMQ_RE.search(request_id)
|
||||
else:
|
||||
m = _PREFILL_ZMQ_RE.search(request_id)
|
||||
if m is None:
|
||||
return None
|
||||
return m.group(1)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReqMeta:
|
||||
"""Metadata for a single request."""
|
||||
|
||||
transfer_id: TransferId
|
||||
local_block_ids: BlockIds
|
||||
remote_block_ids: BlockIds
|
||||
remote_host: str
|
||||
remote_port: int
|
||||
remote_handshake_port: int
|
||||
remote_notify_port: int
|
||||
remote_engine_id: str
|
||||
tp_size: int
|
||||
remote_dp_size: int
|
||||
# Prefill DP rank that owns this request's KV (forwarded by the proxy). The
|
||||
# read must target this rank's memory registration; the default 0 preserves
|
||||
# the symmetric single-DP behaviour.
|
||||
remote_dp_rank: int = 0
|
||||
# Multi-pod: list of remote pod IPs indexed by pod_idx.
|
||||
multi_pod_hosts: list[str] = field(default_factory=list)
|
||||
# Per-pod DP size; 0 means fallback to remote_dp_size.
|
||||
remote_dp_size_local: int = 0
|
||||
|
||||
|
||||
class MoRIIOConnectorMetadata(KVConnectorMetadata):
|
||||
def __init__(self):
|
||||
self.reqs_to_recv: dict[ReqId, ReqMeta] = {}
|
||||
self.reqs_to_save: dict[ReqId, ReqMeta] = {}
|
||||
self.reqs_to_send: dict[ReqId, float] = {}
|
||||
self.transfer_id_to_request_id: dict[TransferId, ReqId] = {}
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"MoRIIOConnectorMetadata: reqs_to_recv={self.reqs_to_recv}, "
|
||||
f"reqs_to_save={self.reqs_to_save}, "
|
||||
f"reqs_to_send={self.reqs_to_send}, "
|
||||
f"transfer_id_to_request_id={self.transfer_id_to_request_id}"
|
||||
)
|
||||
|
||||
def add_new_req(
|
||||
self,
|
||||
request_id: ReqId,
|
||||
local_block_ids: BlockIds,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
write_mode=False,
|
||||
):
|
||||
"""Ingest a peer's ``kv_transfer_params`` into a typed ``ReqMeta``.
|
||||
|
||||
This is the single place peer info enters the connector. The
|
||||
``kv_transfer_params`` contract (as produced by the llm-d sidecar /
|
||||
vLLM router, or echoed by the prefill leg's ``request_finished``):
|
||||
|
||||
Required (always):
|
||||
* ``transfer_id`` -- stable id shared by both legs.
|
||||
* ``remote_engine_id`` -- peer engine id for the handshake table.
|
||||
* ``remote_block_ids`` -- peer block ids (may be [] in WRITE mode,
|
||||
where decode allocates its own blocks).
|
||||
|
||||
Peer address -- ONE of the following two must resolve:
|
||||
* embedded in ``request_id`` (vLLM-router PD id form), OR
|
||||
* explicit ``remote_host`` + ``remote_handshake_port`` +
|
||||
``remote_notify_port`` (llm-d sidecar / returnable path).
|
||||
If neither resolves we raise -- there is no safe default host/port.
|
||||
|
||||
Optional (defaulted):
|
||||
* ``tp_size`` (default 1) -- peer TP size.
|
||||
* ``remote_dp_size`` (default 1) -- peer GLOBAL DP size.
|
||||
* ``remote_dp_size_local`` (default = ``remote_dp_size``) -- per-pod
|
||||
DP size for Wide-EP multi-pod port/host
|
||||
folding; 0/absent means single-pod.
|
||||
* ``remote_hosts`` (default [remote_host]) -- per-pod IP list
|
||||
indexed by ``pod_idx = rank // dp_local``.
|
||||
|
||||
Routing keys consumed elsewhere (NOT here): ``remote_dp_rank`` /
|
||||
``remote_dp_rank_override`` gate the decode->prefill notify target in
|
||||
MoRIIOConnectorScheduler; they are router-authoritative and never
|
||||
self-derived (see that class's request routing contract).
|
||||
"""
|
||||
transfer_id = kv_transfer_params["transfer_id"]
|
||||
|
||||
# Try request_id embedded address first, fallback to explicit params.
|
||||
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode)
|
||||
if peer_zmq is not None:
|
||||
remote_host, remote_handshake_port, remote_notify_port = (
|
||||
parse_moriio_zmq_address(peer_zmq)
|
||||
)
|
||||
else:
|
||||
try:
|
||||
remote_host = kv_transfer_params["remote_host"]
|
||||
if not remote_host:
|
||||
raise ValueError(
|
||||
f"request_id {request_id!r} does not embed a peer "
|
||||
f"zmq_address and kv_transfer_params['remote_host'] is "
|
||||
f"empty; cannot route MoRI-IO transfer"
|
||||
)
|
||||
remote_handshake_port = int(kv_transfer_params["remote_handshake_port"])
|
||||
remote_notify_port = int(kv_transfer_params["remote_notify_port"])
|
||||
except (KeyError, TypeError, ValueError) as e:
|
||||
raise ValueError(
|
||||
f"request_id {request_id!r} does not embed a peer "
|
||||
f"zmq_address and kv_transfer_params is missing one or "
|
||||
f"more sidecar-fallback keys (need remote_host, "
|
||||
f"remote_handshake_port, remote_notify_port): {e}"
|
||||
) from e
|
||||
|
||||
# Multi-pod: use multi_pod_hosts list or fallback to single host.
|
||||
_pod_hosts = kv_transfer_params.get("remote_hosts") or [remote_host]
|
||||
if not isinstance(_pod_hosts, list):
|
||||
_pod_hosts = [_pod_hosts]
|
||||
_pod_hosts = [str(h) for h in _pod_hosts]
|
||||
_remote_dp_size_local = int(
|
||||
kv_transfer_params.get(
|
||||
"remote_dp_size_local",
|
||||
kv_transfer_params.get("remote_dp_size", 1),
|
||||
)
|
||||
)
|
||||
|
||||
_req = ReqMeta(
|
||||
transfer_id=transfer_id,
|
||||
local_block_ids=local_block_ids,
|
||||
remote_block_ids=kv_transfer_params["remote_block_ids"],
|
||||
remote_engine_id=kv_transfer_params["remote_engine_id"],
|
||||
remote_host=remote_host,
|
||||
remote_port=int(remote_handshake_port),
|
||||
remote_handshake_port=int(remote_handshake_port),
|
||||
remote_notify_port=int(remote_notify_port),
|
||||
# Remote peer TP degree (used as remote_tp_size downstream). The
|
||||
# decode side must know the prefiller's TP degree to map producer
|
||||
# ranks onto the correct decode rank (and vice versa). The old plain
|
||||
# "tp_size" key (#46332) is absent on WRITE producer requests and
|
||||
# defaulted to 1, collapsing every producer rank onto decode rank 0
|
||||
# and corrupting the transfer; prefer "remote_tp_size". 0 == unknown
|
||||
# (treated as homogeneous downstream).
|
||||
tp_size=int(
|
||||
kv_transfer_params.get("remote_tp_size")
|
||||
or kv_transfer_params.get("tp_size")
|
||||
or 0
|
||||
),
|
||||
remote_dp_size=kv_transfer_params.get("remote_dp_size", 1),
|
||||
remote_dp_rank=kv_transfer_params.get("remote_dp_rank", 0),
|
||||
multi_pod_hosts=_pod_hosts,
|
||||
remote_dp_size_local=_remote_dp_size_local,
|
||||
)
|
||||
if write_mode:
|
||||
self.reqs_to_save[request_id] = _req
|
||||
else:
|
||||
self.reqs_to_recv[request_id] = _req
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
|
||||
"""Context manager for a ZMQ socket."""
|
||||
if socket_type not in (zmq.ROUTER, zmq.REQ, zmq.DEALER):
|
||||
raise ValueError(f"Unexpected socket type: {socket_type}")
|
||||
|
||||
ctx: zmq.Context | None = None
|
||||
try:
|
||||
ctx = zmq.Context() # type: ignore[attr-defined]
|
||||
yield make_zmq_socket(
|
||||
ctx=ctx, path=addr, socket_type=socket_type, bind=socket_type == zmq.ROUTER
|
||||
)
|
||||
finally:
|
||||
if ctx is not None:
|
||||
ctx.destroy(linger=0)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,96 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Keep discovery heartbeats independent of the model worker's GIL."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import select
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import msgpack
|
||||
import zmq
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MoRIIOHeartbeat:
|
||||
def __init__(
|
||||
self,
|
||||
endpoint: str,
|
||||
payload: dict[str, Any],
|
||||
interval: float,
|
||||
max_retries: int = 100,
|
||||
):
|
||||
self._process = subprocess.Popen(
|
||||
[sys.executable, __file__, endpoint, str(interval), str(max_retries)],
|
||||
stdin=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
assert self._process.stdin is not None
|
||||
try:
|
||||
self._process.stdin.write(json.dumps(payload) + "\n")
|
||||
self._process.stdin.flush()
|
||||
except BaseException:
|
||||
self.shutdown()
|
||||
raise
|
||||
|
||||
def check_health(self) -> None:
|
||||
returncode = self._process.poll()
|
||||
if returncode is not None:
|
||||
raise RuntimeError(
|
||||
f"MoRIIO discovery heartbeat exited with code {returncode}"
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
assert self._process.stdin is not None
|
||||
try:
|
||||
if not self._process.stdin.closed:
|
||||
self._process.stdin.close()
|
||||
except BrokenPipeError:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
self._process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
self._process.terminate()
|
||||
try:
|
||||
self._process.wait(timeout=1)
|
||||
except subprocess.TimeoutExpired:
|
||||
self._process.kill()
|
||||
self._process.wait(timeout=1)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
endpoint, interval_arg, max_retries_arg = sys.argv[1:]
|
||||
initial = sys.stdin.readline()
|
||||
if not initial:
|
||||
return
|
||||
payload = msgpack.dumps(json.loads(initial))
|
||||
interval = float(interval_arg)
|
||||
max_retries = int(max_retries_arg)
|
||||
failures = 0
|
||||
with zmq.Context() as context, context.socket(zmq.DEALER) as sock:
|
||||
sock.setsockopt(zmq.LINGER, 0)
|
||||
sock.setsockopt(zmq.SNDTIMEO, 1000)
|
||||
sock.connect(endpoint)
|
||||
while not select.select([sys.stdin], [], [], 0)[0]:
|
||||
try:
|
||||
sock.send(payload)
|
||||
failures = 0
|
||||
except zmq.Again:
|
||||
failures = 0
|
||||
logger.warning("MoRIIO discovery heartbeat send failed; retrying")
|
||||
except zmq.ZMQError:
|
||||
failures += 1
|
||||
logger.exception("MoRIIO discovery heartbeat send failed")
|
||||
if failures >= max_retries:
|
||||
raise
|
||||
# EOF also stops the child when the worker exits without cleanup.
|
||||
if select.select([sys.stdin], [], [], interval)[0]:
|
||||
break
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,615 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import NamedTuple, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import (
|
||||
MambaConvSplitInfo,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
|
||||
class LayerTransferGeometry(NamedTuple):
|
||||
num_blocks: int
|
||||
block_size: int
|
||||
block_len: int
|
||||
slot_size_bytes: int
|
||||
block_stride: int
|
||||
local_kv_stride: int | None
|
||||
remote_kv_stride: int | None
|
||||
transfers_per_block: int
|
||||
regions_per_block: int
|
||||
split_kv_regions: bool
|
||||
|
||||
|
||||
class MambaTransferGeometry(NamedTuple):
|
||||
"""Geometry of a hybrid (mamba/KDA) layer's recurrent state.
|
||||
|
||||
A KDA layer's packed page is exposed as ``conv_state`` and ``ssm_state``
|
||||
views. Each is slot-strided, so a slot's bytes live at
|
||||
``slot * slot_stride * element_size`` and span ``slot_bytes``. The two
|
||||
tensors are registered as two separate MoRIIO regions per layer.
|
||||
"""
|
||||
|
||||
conv_slot_stride: int
|
||||
ssm_slot_stride: int
|
||||
ssm_slot_bytes: int
|
||||
conv_element_size: int
|
||||
ssm_element_size: int
|
||||
conv_region_len: int
|
||||
ssm_region_len: int
|
||||
|
||||
|
||||
def get_mamba_transfer_geometry(
|
||||
conv: torch.Tensor,
|
||||
ssm: torch.Tensor,
|
||||
) -> MambaTransferGeometry:
|
||||
"""Derive the slot-strided geometry of a KDA layer's conv + ssm state.
|
||||
|
||||
``conv``/``ssm`` are zero-copy views over the layer's packed page. Slot byte
|
||||
offset is ``slot * slot_stride * element_size`` and each
|
||||
slot spans ``subtensor[0].numel() * element_size`` bytes; the whole tensor
|
||||
is registered as one region (``*_region_len`` bytes).
|
||||
"""
|
||||
if conv.ndim == 0 or ssm.ndim == 0 or conv.shape[0] == 0 or ssm.shape[0] == 0:
|
||||
raise ValueError("KDA conv/ssm caches must contain at least one slot")
|
||||
if conv.shape[0] != ssm.shape[0]:
|
||||
raise ValueError(
|
||||
f"KDA conv/ssm slot count mismatch: {conv.shape[0]} != {ssm.shape[0]}"
|
||||
)
|
||||
conv_slot_bytes = conv[0].numel() * conv.element_size()
|
||||
ssm_slot_bytes = ssm[0].numel() * ssm.element_size()
|
||||
conv_available_bytes = (
|
||||
conv.untyped_storage().nbytes() - conv.storage_offset() * conv.element_size()
|
||||
)
|
||||
ssm_available_bytes = (
|
||||
ssm.untyped_storage().nbytes() - ssm.storage_offset() * ssm.element_size()
|
||||
)
|
||||
if conv.shape[0]:
|
||||
conv_required_bytes = (conv.shape[0] - 1) * conv.stride(
|
||||
0
|
||||
) * conv.element_size() + conv_slot_bytes
|
||||
ssm_required_bytes = (ssm.shape[0] - 1) * ssm.stride(
|
||||
0
|
||||
) * ssm.element_size() + ssm_slot_bytes
|
||||
if conv_required_bytes > conv_available_bytes:
|
||||
raise ValueError("KDA conv slots exceed the registered storage extent")
|
||||
if ssm_required_bytes > ssm_available_bytes:
|
||||
raise ValueError("KDA ssm slots exceed the registered storage extent")
|
||||
|
||||
return MambaTransferGeometry(
|
||||
conv_slot_stride=conv.stride(0),
|
||||
ssm_slot_stride=ssm.stride(0),
|
||||
ssm_slot_bytes=ssm_slot_bytes,
|
||||
conv_element_size=conv.element_size(),
|
||||
ssm_element_size=ssm.element_size(),
|
||||
# Registered region must span the full slot-strided extent: stride(0)
|
||||
# may exceed the per-slot element count, so numel() would under-size the
|
||||
# region and truncate the highest-slot transfers. shape[0]*stride(0)
|
||||
# covers every byte a ``slot * slot_stride * elem`` offset can address.
|
||||
# Cap at the bytes remaining from each view's storage offset. A packed
|
||||
# conv+ssm page places ssm at a non-zero intra-buffer offset,
|
||||
# so do not register past the end of the shared backing allocation.
|
||||
conv_region_len=min(
|
||||
conv.shape[0] * conv.stride(0) * conv.element_size(),
|
||||
conv_available_bytes,
|
||||
),
|
||||
ssm_region_len=min(
|
||||
ssm.shape[0] * ssm.stride(0) * ssm.element_size(),
|
||||
ssm_available_bytes,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def build_layer_to_spec(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheSpec]:
|
||||
layer_to_spec: dict[str, KVCacheSpec] = {}
|
||||
for group in kv_cache_config.kv_cache_groups:
|
||||
group_spec = group.kv_cache_spec
|
||||
if isinstance(group_spec, UniformTypeKVCacheSpecs):
|
||||
layer_to_spec.update(
|
||||
{
|
||||
layer_name: group_spec.kv_cache_specs[layer_name]
|
||||
for layer_name in group.layer_names
|
||||
}
|
||||
)
|
||||
else:
|
||||
layer_to_spec.update(
|
||||
{layer_name: group_spec for layer_name in group.layer_names}
|
||||
)
|
||||
return layer_to_spec
|
||||
|
||||
|
||||
def is_mla_cache_layer(
|
||||
layer_to_spec: Mapping[str, KVCacheSpec], layer_name: str
|
||||
) -> bool:
|
||||
try:
|
||||
spec = layer_to_spec[layer_name]
|
||||
except KeyError as e:
|
||||
raise ValueError(f"Missing KV cache spec for layer {layer_name}") from e
|
||||
return isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec))
|
||||
|
||||
|
||||
def _spec_dim_matches(value: int, expected: int | None) -> bool:
|
||||
return expected is None or value == expected
|
||||
|
||||
|
||||
def _kernel_layout_matches(
|
||||
spec: KVCacheSpec, kernel_block_size: int, num_kv_heads: int, head_dim: int
|
||||
) -> bool:
|
||||
if kernel_block_size <= 0 or spec.block_size % kernel_block_size != 0:
|
||||
return False
|
||||
return _spec_dim_matches(
|
||||
num_kv_heads, getattr(spec, "num_kv_heads", None)
|
||||
) and _spec_dim_matches(head_dim, getattr(spec, "head_size", None))
|
||||
|
||||
|
||||
def _select_kernel_block_layout(
|
||||
layer_name: str, shape: torch.Size, spec: KVCacheSpec
|
||||
) -> tuple[int, int, int]:
|
||||
axis2_matches = _kernel_layout_matches(spec, shape[2], shape[3], shape[4])
|
||||
axis3_matches = _kernel_layout_matches(spec, shape[3], shape[2], shape[4])
|
||||
|
||||
if axis2_matches and axis3_matches and shape[2] != shape[3]:
|
||||
raise ValueError(
|
||||
f"Ambiguous MoRIIO kernel-block K/V cache shape for layer "
|
||||
f"{layer_name}: {tuple(shape)}"
|
||||
)
|
||||
if axis2_matches:
|
||||
return shape[2], shape[3], shape[4]
|
||||
if axis3_matches:
|
||||
return shape[3], shape[2], shape[4]
|
||||
|
||||
raise ValueError(
|
||||
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
|
||||
f"{tuple(shape)} does not contain block size {spec.block_size}"
|
||||
)
|
||||
|
||||
|
||||
def get_layer_transfer_geometry(
|
||||
layer_name: str,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_to_spec: Mapping[str, KVCacheSpec],
|
||||
remote_num_blocks: int | None = None,
|
||||
) -> LayerTransferGeometry:
|
||||
spec = layer_to_spec[layer_name]
|
||||
shape = kv_cache.shape
|
||||
stride = kv_cache.stride()
|
||||
element_size = kv_cache.element_size()
|
||||
is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name)
|
||||
|
||||
if not is_mla_cache and len(shape) == 5 and shape[0] == 2:
|
||||
_, num_blocks = shape[:2]
|
||||
kernel_blocks_per_block = 1
|
||||
if shape[2] == spec.block_size:
|
||||
block_size, num_kv_heads, head_dim = shape[2:]
|
||||
elif shape[3] == spec.block_size:
|
||||
num_kv_heads, block_size, head_dim = shape[2:]
|
||||
else:
|
||||
kernel_num_blocks = num_blocks
|
||||
kernel_block_size, num_kv_heads, head_dim = _select_kernel_block_layout(
|
||||
layer_name, shape, spec
|
||||
)
|
||||
kernel_blocks_per_block = spec.block_size // kernel_block_size
|
||||
if kernel_num_blocks % kernel_blocks_per_block != 0:
|
||||
raise ValueError(
|
||||
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
|
||||
f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, "
|
||||
f"not divisible by {kernel_blocks_per_block}"
|
||||
)
|
||||
num_blocks = kernel_num_blocks // kernel_blocks_per_block
|
||||
block_size = spec.block_size
|
||||
slot_size_bytes = num_kv_heads * head_dim * element_size
|
||||
block_len = block_size * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[1] * kernel_blocks_per_block,
|
||||
local_kv_stride=stride[0],
|
||||
remote_kv_stride=(
|
||||
stride[1] * kernel_blocks_per_block * (remote_num_blocks or num_blocks)
|
||||
),
|
||||
transfers_per_block=2,
|
||||
regions_per_block=1,
|
||||
split_kv_regions=True,
|
||||
)
|
||||
|
||||
if not is_mla_cache and len(shape) == 5 and shape[1] == 2:
|
||||
num_blocks = shape[0]
|
||||
if shape[2] == spec.block_size:
|
||||
block_size, num_kv_heads, head_dim = shape[2:]
|
||||
slot_size_bytes = num_kv_heads * head_dim * element_size
|
||||
block_len = block_size * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[0],
|
||||
local_kv_stride=stride[1],
|
||||
remote_kv_stride=stride[1],
|
||||
transfers_per_block=2,
|
||||
regions_per_block=2,
|
||||
split_kv_regions=False,
|
||||
)
|
||||
elif shape[3] == spec.block_size:
|
||||
num_kv_heads, block_size, head_dim = shape[2:]
|
||||
else:
|
||||
kernel_num_blocks = num_blocks
|
||||
kernel_block_size, _, _ = _select_kernel_block_layout(
|
||||
layer_name, shape, spec
|
||||
)
|
||||
kernel_blocks_per_block = spec.block_size // kernel_block_size
|
||||
if kernel_num_blocks % kernel_blocks_per_block != 0:
|
||||
raise ValueError(
|
||||
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
|
||||
f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, "
|
||||
f"not divisible by {kernel_blocks_per_block}"
|
||||
)
|
||||
num_blocks = kernel_num_blocks // kernel_blocks_per_block
|
||||
block_size = spec.block_size
|
||||
block_stride = stride[0] * kernel_blocks_per_block
|
||||
block_len = block_stride * element_size
|
||||
slot_size_bytes = block_len // block_size
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=block_stride,
|
||||
local_kv_stride=None,
|
||||
remote_kv_stride=None,
|
||||
transfers_per_block=1,
|
||||
regions_per_block=1,
|
||||
split_kv_regions=False,
|
||||
)
|
||||
slot_size_bytes = num_kv_heads * head_dim * element_size
|
||||
block_len = block_size * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[0],
|
||||
local_kv_stride=stride[1],
|
||||
remote_kv_stride=stride[1],
|
||||
transfers_per_block=2,
|
||||
regions_per_block=2,
|
||||
split_kv_regions=False,
|
||||
)
|
||||
|
||||
if (
|
||||
isinstance(spec, AttentionSpec)
|
||||
and len(shape) == 4
|
||||
and shape[1] == spec.num_heads
|
||||
and shape[2] == spec.num_states
|
||||
and shape[3] * element_size == spec.state_content_size_bytes
|
||||
):
|
||||
# Standardized per-layer [B, H, N, C] view (MLA is just H == 1).
|
||||
num_blocks, num_heads, num_states, content_dim = shape
|
||||
slot_size_bytes = num_heads * content_dim * element_size
|
||||
block_len = num_states * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=spec.block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[0],
|
||||
local_kv_stride=None,
|
||||
remote_kv_stride=None,
|
||||
transfers_per_block=1,
|
||||
regions_per_block=1,
|
||||
split_kv_regions=False,
|
||||
)
|
||||
|
||||
cache_kind = "MLA" if is_mla_cache else "K/V"
|
||||
raise ValueError(
|
||||
f"Unsupported MoRIIO {cache_kind} cache shape for layer "
|
||||
f"{layer_name}: {tuple(shape)}"
|
||||
)
|
||||
|
||||
|
||||
def kda_conv_ssm(
|
||||
kv_cache: torch.Tensor,
|
||||
spec: MambaSpec,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Return zero-copy conv and SSM views over a packed two-state page."""
|
||||
from math import prod
|
||||
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
|
||||
if len(spec.shapes) != 2 or len(spec.dtypes) != 2:
|
||||
raise ValueError(
|
||||
"MoRIIO KDA transfer requires exactly two states (conv and SSM)"
|
||||
)
|
||||
if kv_cache.element_size() != 1:
|
||||
raise ValueError("MoRIIO KDA packed cache must use a byte-sized dtype")
|
||||
if kv_cache.ndim != 4 or kv_cache.shape[1:3] != (1, 1) or kv_cache.stride(3) != 1:
|
||||
raise ValueError(
|
||||
"MoRIIO KDA packed cache must have shape [B, 1, 1, C] with "
|
||||
"a contiguous byte dimension"
|
||||
)
|
||||
# Mirror MambaBase.bind_kv_cache: blocks may be strided within a shared
|
||||
# multi-layer allocation, while every individual byte page is contiguous.
|
||||
pages = kv_cache.squeeze(dim=(1, 2))
|
||||
states: list[torch.Tensor] = []
|
||||
offset = 0
|
||||
for shp, dt in zip(spec.shapes, spec.dtypes, strict=True):
|
||||
nbytes = prod(shp) * get_dtype_size(dt)
|
||||
if offset + nbytes > pages.shape[1]:
|
||||
raise ValueError(
|
||||
"MoRIIO KDA state layout exceeds the packed cache page size"
|
||||
)
|
||||
state = pages[:, offset : offset + nbytes].view(dt)
|
||||
states.append(state.view(-1, *shp))
|
||||
offset += nbytes
|
||||
return states[0], states[1]
|
||||
|
||||
|
||||
def iter_layer_registration_regions(
|
||||
layer_name: str,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_to_spec: Mapping[str, KVCacheSpec],
|
||||
) -> list[tuple[torch.Tensor, int]]:
|
||||
spec = layer_to_spec[layer_name]
|
||||
if isinstance(spec, MambaSpec):
|
||||
# KDA layer: register the conv and ssm tensors as two whole-tensor
|
||||
# regions (the conv 3-subprojection split is expressed as transfer
|
||||
# offset triples, not extra registrations).
|
||||
conv, ssm = kda_conv_ssm(kv_cache, spec)
|
||||
geom = get_mamba_transfer_geometry(conv, ssm)
|
||||
return [(conv, geom.conv_region_len), (ssm, geom.ssm_region_len)]
|
||||
geometry = cast(
|
||||
LayerTransferGeometry,
|
||||
get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec),
|
||||
)
|
||||
region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len
|
||||
if geometry.regions_per_block == 1:
|
||||
# Padded/interleaved pages stride past block_len: register the strided
|
||||
# span, ending at the last block's bytes so it stays in-allocation.
|
||||
block_stride_bytes = geometry.block_stride * kv_cache.element_size()
|
||||
region_len = max(
|
||||
region_len,
|
||||
(geometry.num_blocks - 1) * block_stride_bytes + geometry.block_len,
|
||||
)
|
||||
if geometry.split_kv_regions:
|
||||
return [(cache, region_len) for cache in kv_cache]
|
||||
return [(kv_cache, region_len)]
|
||||
|
||||
|
||||
def merge_contiguous_offsets(
|
||||
offsets_local: list[int],
|
||||
offsets_remote: list[int],
|
||||
sizes: list[int],
|
||||
) -> tuple[list[int], list[int], list[int]]:
|
||||
if not offsets_local:
|
||||
return [], [], []
|
||||
if not (len(offsets_local) == len(offsets_remote) == len(sizes)):
|
||||
raise ValueError("Input list lengths mismatch")
|
||||
|
||||
rows = sorted(zip(offsets_local, offsets_remote, sizes), key=lambda row: row[0])
|
||||
merged: list[list[int]] = []
|
||||
for local, remote, size in rows:
|
||||
if (
|
||||
merged
|
||||
and local == merged[-1][0] + merged[-1][2]
|
||||
and remote == merged[-1][1] + merged[-1][2]
|
||||
):
|
||||
merged[-1][2] += size
|
||||
else:
|
||||
merged.append([local, remote, size])
|
||||
|
||||
return (
|
||||
[row[0] for row in merged],
|
||||
[row[1] for row in merged],
|
||||
[row[2] for row in merged],
|
||||
)
|
||||
|
||||
|
||||
def compute_block_transfer_offsets(
|
||||
layer_name: str,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_to_spec: Mapping[str, KVCacheSpec],
|
||||
local_block_ids: list[int],
|
||||
remote_block_ids: list[int],
|
||||
remote_num_blocks: int,
|
||||
merge_fn: Callable[
|
||||
[list[int], list[int], list[int]], tuple[list[int], list[int], list[int]]
|
||||
] = merge_contiguous_offsets,
|
||||
) -> tuple[list[int], list[int], list[int]]:
|
||||
# A shorter/empty local list transfers only what decode allocated (READ-mode
|
||||
# partial or full-prefix-hit); a longer local list is a genuine bug.
|
||||
if len(local_block_ids) > len(remote_block_ids):
|
||||
raise ValueError(
|
||||
"local_block_ids longer than remote_block_ids: "
|
||||
f"{len(local_block_ids)} > {len(remote_block_ids)}"
|
||||
)
|
||||
geometry = cast(
|
||||
LayerTransferGeometry,
|
||||
get_layer_transfer_geometry(
|
||||
layer_name, kv_cache, layer_to_spec, remote_num_blocks
|
||||
),
|
||||
)
|
||||
invalid_local = next(
|
||||
(
|
||||
block_id
|
||||
for block_id in local_block_ids
|
||||
if not 0 <= block_id < geometry.num_blocks
|
||||
),
|
||||
None,
|
||||
)
|
||||
if invalid_local is not None:
|
||||
raise ValueError(
|
||||
f"local block id {invalid_local} is outside [0, {geometry.num_blocks})"
|
||||
)
|
||||
invalid_remote = next(
|
||||
(
|
||||
block_id
|
||||
for block_id in remote_block_ids
|
||||
if not 0 <= block_id < remote_num_blocks
|
||||
),
|
||||
None,
|
||||
)
|
||||
if invalid_remote is not None:
|
||||
raise ValueError(
|
||||
f"remote block id {invalid_remote} is outside [0, {remote_num_blocks})"
|
||||
)
|
||||
element_size = kv_cache.element_size()
|
||||
transfer_size_byte = geometry.block_len
|
||||
per_block = geometry.transfers_per_block
|
||||
total = len(local_block_ids) * per_block
|
||||
offset_local = [0] * total
|
||||
offset_remote = [0] * total
|
||||
sizes = [transfer_size_byte] * total
|
||||
|
||||
w = 0
|
||||
for lb, rb in zip(local_block_ids, remote_block_ids):
|
||||
offset_local[w] = element_size * (lb * geometry.block_stride)
|
||||
offset_remote[w] = element_size * (rb * geometry.block_stride)
|
||||
w += 1
|
||||
if per_block == 2:
|
||||
assert geometry.local_kv_stride is not None
|
||||
assert geometry.remote_kv_stride is not None
|
||||
offset_local[w] = element_size * (
|
||||
geometry.local_kv_stride + lb * geometry.block_stride
|
||||
)
|
||||
offset_remote[w] = element_size * (
|
||||
geometry.remote_kv_stride + rb * geometry.block_stride
|
||||
)
|
||||
w += 1
|
||||
|
||||
return merge_fn(offset_local, offset_remote, sizes)
|
||||
|
||||
|
||||
class MambaOffsetTemplate(NamedTuple):
|
||||
"""Slot-independent conv+ssm offset decomposition for a KDA layer.
|
||||
|
||||
Homogeneous-TP geometry (conv/ssm slot strides, conv sub-projection
|
||||
offsets, ssm per-slot size) is identical across requests and across the
|
||||
homogeneous GDN layers, so it is computed once and reused; only the
|
||||
per-request slot bases vary (see ``apply_mamba_offset_template``).
|
||||
"""
|
||||
|
||||
num_slots: int
|
||||
conv_slot_stride_bytes: int
|
||||
ssm_slot_stride_bytes: int
|
||||
ssm_read_bytes: int
|
||||
conv_subprojs: tuple[tuple[int, int, int], ...]
|
||||
|
||||
|
||||
def build_mamba_offset_template(
|
||||
conv: torch.Tensor,
|
||||
ssm: torch.Tensor,
|
||||
split_info: MambaConvSplitInfo,
|
||||
tp_ratio: int,
|
||||
) -> MambaOffsetTemplate:
|
||||
"""Precompute the slot-independent conv+ssm offset decomposition.
|
||||
|
||||
Only homogeneous TP (``tp_ratio == 1``) is supported; heterogeneous TP is
|
||||
gated with ``NotImplementedError`` (see the design doc). The returned
|
||||
template is a pure function of the layer geometry and ``split_info``, so it
|
||||
can be cached and reused across requests without recomputation.
|
||||
"""
|
||||
if tp_ratio != 1:
|
||||
# Heterogeneous-TP recurrent-state transfer needs the remote page's
|
||||
# slot stride (which differs from the local page) and, for
|
||||
# P_TP > D_TP, a multi-rank gather of conv/ssm slices. Both are
|
||||
# follow-ups. The mamba path bypasses the attention KV-head validator,
|
||||
# so fail loudly here instead of silently transferring wrong bytes.
|
||||
raise NotImplementedError(
|
||||
"heterogeneous-TP mamba/KDA transfer is not supported yet "
|
||||
f"(tp_ratio={tp_ratio}); use equal prefill/decode TP for "
|
||||
"hybrid models"
|
||||
)
|
||||
geom = get_mamba_transfer_geometry(conv, ssm)
|
||||
conv_local_offsets = split_info.local_conv_offsets
|
||||
conv_remote_offsets = split_info.remote_conv_offsets(0, 1)
|
||||
ssm_read_bytes = geom.ssm_slot_bytes
|
||||
conv_subprojs = tuple(
|
||||
(loff, roff, rsz)
|
||||
for (loff, _lsz), (roff, rsz) in zip(conv_local_offsets, conv_remote_offsets)
|
||||
)
|
||||
return MambaOffsetTemplate(
|
||||
num_slots=conv.shape[0],
|
||||
conv_slot_stride_bytes=geom.conv_slot_stride * geom.conv_element_size,
|
||||
ssm_slot_stride_bytes=geom.ssm_slot_stride * geom.ssm_element_size,
|
||||
ssm_read_bytes=ssm_read_bytes,
|
||||
conv_subprojs=conv_subprojs,
|
||||
)
|
||||
|
||||
|
||||
def apply_mamba_offset_template(
|
||||
template: MambaOffsetTemplate,
|
||||
local_slots: list[int],
|
||||
remote_slots: list[int],
|
||||
) -> tuple[list[int], list[int], list[int]]:
|
||||
"""Apply per-request slot bases to a cached ``MambaOffsetTemplate``.
|
||||
|
||||
Conv sub-projection entries (all slots) come first, followed by one ssm
|
||||
entry per slot.
|
||||
"""
|
||||
if len(local_slots) != len(remote_slots):
|
||||
raise ValueError(
|
||||
"local and remote mamba slot counts must match: "
|
||||
f"{len(local_slots)} != {len(remote_slots)}"
|
||||
)
|
||||
invalid_local = next(
|
||||
(slot for slot in local_slots if not 0 <= slot < template.num_slots), None
|
||||
)
|
||||
if invalid_local is not None:
|
||||
raise ValueError(
|
||||
f"local mamba slot {invalid_local} is outside [0, {template.num_slots})"
|
||||
)
|
||||
invalid_remote = next((slot for slot in remote_slots if slot < 0), None)
|
||||
if invalid_remote is not None:
|
||||
raise ValueError(f"remote mamba slot {invalid_remote} must be non-negative")
|
||||
# The peer's KDA slot count is not part of MoRIIOAgentMetadata yet. Do not
|
||||
# compare remote slot IDs against the local pool size: two otherwise
|
||||
# compatible engines may reserve different numbers of slots.
|
||||
local_offs: list[int] = []
|
||||
remote_offs: list[int] = []
|
||||
sizes: list[int] = []
|
||||
|
||||
# Conv sub-projections (region 0), all slots first.
|
||||
conv_slot_stride_bytes = template.conv_slot_stride_bytes
|
||||
for ls, rs in zip(local_slots, remote_slots):
|
||||
lbase = ls * conv_slot_stride_bytes
|
||||
rbase = rs * conv_slot_stride_bytes
|
||||
for loff, roff, rsz in template.conv_subprojs:
|
||||
local_offs.append(lbase + loff)
|
||||
remote_offs.append(rbase + roff)
|
||||
sizes.append(rsz)
|
||||
|
||||
# SSM temporal state (region 1), one entry per slot.
|
||||
ssm_slot_stride_bytes = template.ssm_slot_stride_bytes
|
||||
ssm_read_bytes = template.ssm_read_bytes
|
||||
for ls, rs in zip(local_slots, remote_slots):
|
||||
local_offs.append(ls * ssm_slot_stride_bytes)
|
||||
remote_offs.append(rs * ssm_slot_stride_bytes)
|
||||
sizes.append(ssm_read_bytes)
|
||||
|
||||
return local_offs, remote_offs, sizes
|
||||
|
||||
|
||||
def compute_mamba_conv_split_count(
|
||||
local_slots: list[int],
|
||||
split_info: MambaConvSplitInfo,
|
||||
) -> int:
|
||||
"""Number of leading conv entries in a KDA layer transfer plan.
|
||||
|
||||
Entries ``[:count]`` are conv sub-projections (conv region/session);
|
||||
entries ``[count:]`` are the per-slot ssm state (ssm region/session).
|
||||
"""
|
||||
return len(local_slots) * len(split_info.local_conv_offsets)
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NIXL KV-cache transfer connector (disaggregated prefill / decode)."""
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
|
||||
NixlBaseConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
|
||||
NixlBaseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector import (
|
||||
NixlBaseConnector,
|
||||
NixlConnector,
|
||||
NixlPullConnector,
|
||||
NixlPushConnector,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlAgentMetadata,
|
||||
NixlConnectorMetadata,
|
||||
NixlHandshakePayload,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
|
||||
NixlPullConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import (
|
||||
NixlPushConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
NixlPushConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import (
|
||||
NixlConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
|
||||
NixlKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"NixlAgentMetadata",
|
||||
"NixlBaseConnector",
|
||||
"NixlBaseConnectorScheduler",
|
||||
"NixlBaseConnectorWorker",
|
||||
"NixlConnector",
|
||||
"NixlConnectorMetadata",
|
||||
"NixlConnectorScheduler",
|
||||
"NixlConnectorWorker",
|
||||
"NixlHandshakePayload",
|
||||
"NixlKVConnectorStats",
|
||||
"NixlPullConnector",
|
||||
"NixlPullConnectorScheduler",
|
||||
"NixlPullConnectorWorker",
|
||||
"NixlPushConnector",
|
||||
"NixlPushConnectorScheduler",
|
||||
"NixlPushConnectorWorker",
|
||||
]
|
||||
@@ -1,549 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Base scheduler-side logic for the NIXL connector."""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import msgspec
|
||||
import zmq
|
||||
|
||||
from vllm import envs
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
BlockIds,
|
||||
EngineId,
|
||||
clip_ssm_state_blocks,
|
||||
yield_req_data,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorHandshakeMetadata,
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
GET_META_MSG,
|
||||
HeartbeatInfo,
|
||||
NixlConnectorMetadata,
|
||||
NixlHandshakePayload,
|
||||
ReqId,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import zmq_ctx
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.network_utils import make_zmq_path
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NixlBaseConnectorScheduler:
|
||||
"""Base implementation of Scheduler side methods shared by pull and push."""
|
||||
|
||||
# Emitted in kv_transfer_params so an external router can distinguish a
|
||||
# pull (READ) producer from a push (WRITE) one. Overridden by the push
|
||||
# scheduler.
|
||||
_TRANSFER_MODE: str = "pull"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
engine_id: str,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
# TP1 PCP+DCP exposes its DCP shards as transfer ranks.
|
||||
self.transfer_tp_size = max(
|
||||
parallel_config.tensor_parallel_size,
|
||||
parallel_config.decode_context_parallel_size,
|
||||
)
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.engine_id: EngineId = engine_id
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.side_channel_host = envs.VLLM_NIXL_SIDE_CHANNEL_HOST
|
||||
self.side_channel_port = (
|
||||
envs.VLLM_NIXL_SIDE_CHANNEL_PORT
|
||||
+ vllm_config.parallel_config.data_parallel_index
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
self._kv_lease_duration: int = (
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"kv_lease_duration", 30
|
||||
)
|
||||
)
|
||||
# NOTE (NickLucche): For now we use a hardcoded value for a simpler interface.
|
||||
self._heartbeat_interval = self._kv_lease_duration // 6
|
||||
if current_platform.device_type == "cpu":
|
||||
self.use_host_buffer = False
|
||||
else:
|
||||
self.use_host_buffer = (
|
||||
vllm_config.kv_transfer_config.kv_buffer_device == "cpu"
|
||||
)
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
# Also handle unlikely SW-only model case instead of checking num_groups>1.
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
self._has_mamba = kv_cache_config.has_mamba_layers
|
||||
self._bounded_replay = any(
|
||||
g.kv_cache_spec.prefix_replay_tokens > 0
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
|
||||
logger.info("Initializing NIXL Scheduler %s", engine_id)
|
||||
if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager:
|
||||
logger.info("Hybrid Memory Allocator is enabled with NIXL")
|
||||
|
||||
# Background thread for handling new handshake requests.
|
||||
self._nixl_handshake_listener_t: threading.Thread | None = None
|
||||
self._stop_event = threading.Event()
|
||||
|
||||
# Requests that need to start recv/send.
|
||||
# New requests are added by update_state_after_alloc in
|
||||
# the scheduler. Used to make metadata passed to Worker.
|
||||
self._reqs_need_recv: dict[
|
||||
ReqId, tuple[Request, BlockIds, tuple[int, ...], bool]
|
||||
] = {}
|
||||
self._reqs_need_save: dict[ReqId, Request] = {}
|
||||
# Reqs to send and their expiration time
|
||||
self._reqs_need_send: dict[ReqId, float] = {}
|
||||
self._reqs_in_batch: set[ReqId] = set()
|
||||
# Reqs to remove from processed set because they're not to send after
|
||||
# remote prefill or aborted.
|
||||
self._reqs_not_processed: set[ReqId] = set()
|
||||
|
||||
# Heartbeat tracking: requests needing periodic lease-renewal heartbeats to
|
||||
# remote P-side, stored as ready-to-send HeartbeatInfo grouped by remote engine
|
||||
self._heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {}
|
||||
# Reverse lookup: local req_id -> (engine_id, remote_req_id) for O(1) removal
|
||||
self._heartbeat_req_engine: dict[ReqId, tuple[EngineId, ReqId]] = {}
|
||||
self._last_heartbeat_time: float = 0.0
|
||||
|
||||
# Gather Sliding Window sizes for each kv cache group (if any) in number of
|
||||
# blocks per KV cache group. This is used to clip the local attention window.
|
||||
sw_sizes_tokens: list[tuple[int, int]] = [
|
||||
(g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size)
|
||||
if isinstance(g.kv_cache_spec, SlidingWindowSpec)
|
||||
else (0, self.block_size)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
]
|
||||
# cdiv(n_tokens, block_size) gives blocks/window; add 1 to conservatively
|
||||
# account for boundary overlap eg window isn't fully aligned with blocks.
|
||||
self.blocks_per_sw = [
|
||||
cdiv(n_tokens, block_size) + 1 if n_tokens else 0
|
||||
for n_tokens, block_size in sw_sizes_tokens
|
||||
]
|
||||
|
||||
# Trailing scratch slots that mamba managers co-allocate per request
|
||||
# for speculative decoding; None for non-SSM groups.
|
||||
self._ssm_spec_blocks = [
|
||||
g.kv_cache_spec.num_speculative_blocks
|
||||
if isinstance(g.kv_cache_spec, MambaSpec)
|
||||
else None
|
||||
for g in kv_cache_config.transfer_groups
|
||||
]
|
||||
|
||||
# Threshold to decide whether to compute kv cache locally
|
||||
# or pull from a remote node: minimum number of remote
|
||||
# tokens to amortize the xfer latencies
|
||||
self.kv_recompute_threshold: int = int(
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"kv_recompute_threshold", 64
|
||||
)
|
||||
)
|
||||
|
||||
# Bi-directional KV transfer feature supports KV block
|
||||
# transfers from D node to P node
|
||||
self.is_bidirectional_kv_xfer_enabled = (
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"bidirectional_kv_xfer", False
|
||||
)
|
||||
)
|
||||
self.decoder_kv_blocks_ttl = (
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"decoder_kv_blocks_ttl", 480
|
||||
)
|
||||
)
|
||||
|
||||
if self.is_bidirectional_kv_xfer_enabled and self.kv_recompute_threshold > 0:
|
||||
logger.info(
|
||||
"Bidirectional KV transfer is enabled and the kv "
|
||||
"recompute threshold is set to %d tokens."
|
||||
"KV blocks on D are released after a TTL of %d seconds.",
|
||||
self.kv_recompute_threshold,
|
||||
self.decoder_kv_blocks_ttl,
|
||||
)
|
||||
|
||||
def shutdown(self):
|
||||
self._stop_event.set()
|
||||
if self._nixl_handshake_listener_t is not None:
|
||||
self._nixl_handshake_listener_t.join()
|
||||
self._nixl_handshake_listener_t = None
|
||||
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
"""Track a request that may need heartbeats."""
|
||||
params = request.kv_transfer_params
|
||||
if params is not None and params.get("do_remote_decode"):
|
||||
self._truncate_request_for_prefill(request)
|
||||
|
||||
# NOTE (NickLucche) This excludes request meant for P, ie heartbeats are
|
||||
# effectively disabled for Bidirectional KV transfer.
|
||||
if params is None or not params.get("do_remote_prefill"):
|
||||
return
|
||||
# Only track if all required remote fields are present.
|
||||
remote_engine_id = params.get("remote_engine_id")
|
||||
remote_request_id = params.get("remote_request_id")
|
||||
host = params.get("remote_host")
|
||||
port = params.get("remote_port")
|
||||
tp_size = params.get("tp_size")
|
||||
dcp_size = params.get("dcp_size", 1)
|
||||
pp_size = params.get("pp_size", 1)
|
||||
if (
|
||||
remote_engine_id is None
|
||||
or remote_request_id is None
|
||||
or host is None
|
||||
or port is None
|
||||
or tp_size is None
|
||||
):
|
||||
return
|
||||
if remote_engine_id not in self._heartbeat_by_engine:
|
||||
self._heartbeat_by_engine[remote_engine_id] = HeartbeatInfo(
|
||||
req_ids=set(),
|
||||
host=host,
|
||||
port=port,
|
||||
tp_size=tp_size,
|
||||
dcp_size=dcp_size,
|
||||
pp_size=pp_size,
|
||||
)
|
||||
self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id)
|
||||
self._heartbeat_req_engine[request.request_id] = (
|
||||
remote_engine_id,
|
||||
remote_request_id,
|
||||
)
|
||||
|
||||
def _stop_heartbeat(self, req_id: ReqId) -> None:
|
||||
"""Remove *req_id* from heartbeat tracking (if tracked)."""
|
||||
if key := self._heartbeat_req_engine.pop(req_id, None):
|
||||
engine_id, remote_id = key
|
||||
if info := self._heartbeat_by_engine.get(engine_id):
|
||||
info.req_ids.discard(remote_id)
|
||||
if not info.req_ids:
|
||||
# Clean up empty engines so we don't leak a key when remote dies.
|
||||
del self._heartbeat_by_engine[engine_id]
|
||||
|
||||
def get_exchange_clipped_blocks(
|
||||
self, block_ids: BlockIds, clip_ssm: bool = True
|
||||
) -> BlockIds:
|
||||
"""Clip a request's block lists down to the transferable blocks.
|
||||
|
||||
Sliding-window groups keep only the in-window tail: the KV cache
|
||||
manager allocates blocks for the entire sequence length and cleans up
|
||||
out-of-window blocks only prior to the `request_finished_all_groups`
|
||||
hook.
|
||||
|
||||
SSM groups keep only their state-bearing slot: the trailing
|
||||
speculative scratch slots and everything before the running state
|
||||
(null placeholders and the previous step's superseded state) go.
|
||||
|
||||
Use this at every block-id exchange point. Pass ``clip_ssm=False``
|
||||
for per-step partial lists (host-buffer save), where the SSM strip
|
||||
does not apply.
|
||||
"""
|
||||
if len(block_ids) == 0:
|
||||
# No blocks to clip, e.g. a full prefix cache hit.
|
||||
return block_ids
|
||||
block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids)
|
||||
if not self._is_hma_required:
|
||||
return block_ids
|
||||
# NOTE (NickLucche) This logic is currently handled at the connector level
|
||||
# because offloading connectors might want to receive the whole sequence even
|
||||
# for SWA groups. We will abstract this logic once the interface is more stable
|
||||
assert len(block_ids) == len(self.blocks_per_sw), (
|
||||
"Number of KV cache groups must match"
|
||||
)
|
||||
clipped = []
|
||||
for i, blocks in enumerate(block_ids):
|
||||
if n_sw := self.blocks_per_sw[i]:
|
||||
blocks = blocks[-n_sw:]
|
||||
elif (
|
||||
clip_ssm
|
||||
and blocks
|
||||
and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None
|
||||
):
|
||||
blocks = clip_ssm_state_blocks(blocks, n_spec_blocks)
|
||||
clipped.append(blocks)
|
||||
return tuple(clipped)
|
||||
|
||||
def set_xfer_handshake_metadata(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""Set the KV connector handshake metadata for this connector.
|
||||
|
||||
Args:
|
||||
metadata (dict): the handshake metadata to set.
|
||||
|
||||
"""
|
||||
encoded_data: dict[tuple[int, int], bytes] = {}
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
for (pp_rank, tp_rank), rank_metadata in metadata.items():
|
||||
if not isinstance(rank_metadata, NixlHandshakePayload):
|
||||
raise ValueError(
|
||||
"NixlConnectorScheduler expects NixlHandshakePayload for "
|
||||
"handshake metadata."
|
||||
)
|
||||
encoded_data[(pp_rank, tp_rank)] = encoder.encode(rank_metadata)
|
||||
logger.debug(
|
||||
"PP rank %d, TP rank %d: encoded NixlHandshakePayload size: %s bytes",
|
||||
pp_rank,
|
||||
tp_rank,
|
||||
str(len(encoded_data[(pp_rank, tp_rank)])),
|
||||
)
|
||||
|
||||
# Only start the listener when we have metadata to serve.
|
||||
if self._nixl_handshake_listener_t is None:
|
||||
ready_event = threading.Event()
|
||||
self._nixl_handshake_listener_t = threading.Thread(
|
||||
target=self._nixl_handshake_listener,
|
||||
args=(
|
||||
encoded_data,
|
||||
ready_event,
|
||||
self._stop_event,
|
||||
self.side_channel_host,
|
||||
self.side_channel_port,
|
||||
),
|
||||
daemon=True,
|
||||
name="nixl_handshake_listener",
|
||||
)
|
||||
self._nixl_handshake_listener_t.start()
|
||||
ready_event.wait() # Wait for listener ZMQ socket to be ready.
|
||||
|
||||
@staticmethod
|
||||
def _nixl_handshake_listener(
|
||||
encoded_data: dict[tuple[int, int], Any],
|
||||
ready_event: threading.Event,
|
||||
stop_event: threading.Event,
|
||||
host: str,
|
||||
port: int,
|
||||
):
|
||||
"""Background thread for getting new NIXL handshakes."""
|
||||
# NOTE(rob): this is a simple implementation. We will move
|
||||
# to a better approach via HTTP endpoint soon.
|
||||
|
||||
# Listen for new requests for metadata.
|
||||
path = make_zmq_path("tcp", host, port)
|
||||
logger.debug("Starting listening on path: %s", path)
|
||||
with zmq_ctx(zmq.ROUTER, path) as sock:
|
||||
sock.setsockopt(zmq.RCVTIMEO, 1000)
|
||||
ready_event.set()
|
||||
while True:
|
||||
try:
|
||||
identity, _, msg = sock.recv_multipart()
|
||||
except zmq.Again:
|
||||
if stop_event.is_set():
|
||||
break
|
||||
continue
|
||||
# Decode (GET_META_MSG, pp_rank, tp_rank).
|
||||
msg, target_pp_rank, target_tp_rank = msgspec.msgpack.decode(msg)
|
||||
logger.debug(
|
||||
"Received message for pp rank %s, tp rank %s",
|
||||
target_pp_rank,
|
||||
target_tp_rank,
|
||||
)
|
||||
if msg != GET_META_MSG:
|
||||
logger.warning("Connection listener got unexpected message %s", msg)
|
||||
# Echo our perf_counter so P can estimate the clock offset.
|
||||
# perf_counter is only comparable within a process, so this
|
||||
# listener must run in the same process that stamps the block
|
||||
# expiry deadline (`_reqs_need_send`).
|
||||
ts = msgspec.msgpack.encode(time.perf_counter())
|
||||
sock.send_multipart(
|
||||
(identity, b"", encoded_data[(target_pp_rank, target_tp_rank)], ts)
|
||||
)
|
||||
|
||||
def _prefill_backoff(self) -> int:
|
||||
"""Trailing prompt tokens the prefiller must not compute; the decoder
|
||||
recomputes them locally.
|
||||
|
||||
Mamba needs h(N-1) so the decoder can derive h(N) itself. DSV41's sliding
|
||||
window under bounded replay is not replayed after a P/D load, so it must
|
||||
be laid out for the token the decoder recomputes. Multi-module
|
||||
MTP needs to keep its whole lookahead window off of the prefiller, which
|
||||
would otherwise embed the unverified drafts in the MTP layer's KV cache. The
|
||||
decoder would never rebuild them, because the update is sized by the rejection
|
||||
count, which is zero for the first decode.
|
||||
"""
|
||||
return max(
|
||||
1 if self._has_mamba or self._bounded_replay else 0,
|
||||
self.vllm_config.num_prefill_lookahead_tokens - 1,
|
||||
)
|
||||
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
"""D-side only. The number of prompt tokens to load from the prefiller.
|
||||
Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder
|
||||
will recompute locally."""
|
||||
backoff = self._prefill_backoff()
|
||||
if backoff and num_prompt_tokens > backoff:
|
||||
return num_prompt_tokens - backoff
|
||||
return num_prompt_tokens
|
||||
|
||||
def _truncate_request_for_prefill(self, request: "Request") -> None:
|
||||
"""P-side only: drop the trailing ``_prefill_backoff()`` prompt tokens
|
||||
so the prefiller stops short of what the decoder recomputes locally.
|
||||
For Mamba that is the single token needed to yield h(N-1); for
|
||||
multi-module MTP it is the drafter's whole lookahead window.
|
||||
|
||||
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
|
||||
request is preempted and rescheduled."""
|
||||
backoff = self._prefill_backoff()
|
||||
params = request.kv_transfer_params
|
||||
if (
|
||||
backoff
|
||||
and params is not None
|
||||
# Guard against repeated truncation after preemption/reschedule.
|
||||
and not params.get("_p_side_truncated")
|
||||
and request.num_prompt_tokens > backoff
|
||||
# A request that skips reading the prefix cache (prompt logprobs)
|
||||
# loads nothing on the decoder, which recomputes its whole prompt.
|
||||
and not request.get_skip_reading_prefix_cache()
|
||||
):
|
||||
if request.prompt_token_ids is not None:
|
||||
del request.prompt_token_ids[-backoff:]
|
||||
elif request.prompt_embeds is not None:
|
||||
request.prompt_embeds = request.prompt_embeds[:-backoff]
|
||||
else:
|
||||
return
|
||||
|
||||
del request._all_token_ids[-backoff:]
|
||||
request.num_prompt_tokens -= backoff
|
||||
request.max_tokens = 1
|
||||
params["_p_side_truncated"] = True
|
||||
|
||||
def _build_save_meta(
|
||||
self,
|
||||
meta: NixlConnectorMetadata,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> None:
|
||||
# only called when use_host_buffer is True to build the save metadata
|
||||
|
||||
# NOTE: For the prefill side, there might be a chance that an early added
|
||||
# request is a chunked prefill, so we need to check if new blocks are added
|
||||
for req_id, new_block_id_groups, _ in yield_req_data(scheduler_output):
|
||||
req_to_save = self._reqs_need_save.get(req_id)
|
||||
if req_to_save is None or new_block_id_groups is None:
|
||||
continue
|
||||
req = req_to_save
|
||||
|
||||
assert req.kv_transfer_params is not None
|
||||
clipped_block_id_groups = self.get_exchange_clipped_blocks(
|
||||
new_block_id_groups, clip_ssm=False
|
||||
)
|
||||
meta.add_new_req_to_save(
|
||||
request_id=req_id,
|
||||
local_block_ids=clipped_block_id_groups,
|
||||
kv_transfer_params=req.kv_transfer_params,
|
||||
)
|
||||
assert scheduler_output.num_scheduled_tokens is not None
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
is_partial = (
|
||||
req.num_computed_tokens + num_scheduled_tokens
|
||||
) < req.num_prompt_tokens
|
||||
if not is_partial:
|
||||
# For non-partial prefills, once new req_meta is scheduled, it
|
||||
# can be removed from _reqs_need_save.
|
||||
# For partial prefill case, we will retain the request in
|
||||
# _reqs_need_save until all blocks are scheduled with req_meta.
|
||||
# Therefore, only pop if `not is_partial`.
|
||||
self._reqs_need_save.pop(req_id)
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
meta = NixlConnectorMetadata()
|
||||
|
||||
# Loop through scheduled reqs and convert to ReqMeta.
|
||||
for req_id, (
|
||||
req,
|
||||
block_ids,
|
||||
cached,
|
||||
awaiting_kvs,
|
||||
) in self._reqs_need_recv.items():
|
||||
assert req.kv_transfer_params is not None
|
||||
meta.add_new_req_to_recv(
|
||||
request_id=req_id,
|
||||
local_block_ids=block_ids,
|
||||
kv_transfer_params=req.kv_transfer_params,
|
||||
local_num_computed_blocks=cached,
|
||||
awaiting_kvs=awaiting_kvs,
|
||||
)
|
||||
|
||||
if self.use_host_buffer:
|
||||
self._build_save_meta(meta, scheduler_output)
|
||||
|
||||
meta.reqs_to_send = self._reqs_need_send
|
||||
# Clock reference for reqs_to_send: deadlines above are in this
|
||||
# process's perf_counter domain; workers (possibly on other nodes,
|
||||
# where perf_counter has a different epoch) rebase against this.
|
||||
meta.scheduler_clock = time.perf_counter()
|
||||
meta.reqs_in_batch = self._reqs_in_batch
|
||||
meta.reqs_not_processed = self._reqs_not_processed
|
||||
|
||||
# Package heartbeats, throttled by heartbeat_interval.
|
||||
if self._heartbeat_by_engine:
|
||||
now = time.perf_counter()
|
||||
if now - self._last_heartbeat_time >= self._heartbeat_interval:
|
||||
self._last_heartbeat_time = now
|
||||
meta.heartbeat_by_engine = self._heartbeat_by_engine
|
||||
|
||||
# Clear the list once workers start the transfers
|
||||
self._reqs_need_recv.clear()
|
||||
self._reqs_in_batch = set()
|
||||
self._reqs_not_processed = set()
|
||||
self._reqs_need_send = {}
|
||||
|
||||
return meta
|
||||
|
||||
def update_connector_output(self, connector_output: "KVConnectorOutput") -> None:
|
||||
"""Stop heartbeating for requests whose KV transfer completed."""
|
||||
for req_id in connector_output.finished_recving or ():
|
||||
self._stop_heartbeat(req_id)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
return False
|
||||
|
||||
############################################################
|
||||
# Abstract methods that subclasses must implement
|
||||
############################################################
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int, bool]:
|
||||
raise NotImplementedError
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: BlockIds,
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
raise NotImplementedError
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,426 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NIXL connector facades.
|
||||
|
||||
This module hosts the thin facade classes that vLLM's KV-connector layer
|
||||
instantiates. Almost all the real work lives in the per-mode scheduler
|
||||
and worker classes; the connector classes here only forward calls.
|
||||
|
||||
* :class:`NixlBaseConnector` – common logic shared by pull and push.
|
||||
* :class:`NixlPullConnector` – pull-based (READ) KV transfer.
|
||||
* :class:`NixlPushConnector` – push-based (WRITE) KV transfer.
|
||||
* ``NixlConnector`` – backward-compatible alias for :class:`NixlPullConnector`.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import KVTransferConfig, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineId,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
CopyBlocksOp,
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorHandshakeMetadata,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
|
||||
NixlPullConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import (
|
||||
NixlPushConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
NixlPushConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
|
||||
NixlKVConnectorStats,
|
||||
NixlPromMetrics,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
|
||||
NixlBaseConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
|
||||
NixlBaseConnectorWorker,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""Base connector with common logic shared by pull and push modes."""
|
||||
|
||||
_cache_hit_source = CacheHitSource.P2P
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
return True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
assert vllm_config.kv_transfer_config.engine_id is not None
|
||||
|
||||
if vllm_config.kv_transfer_config.kv_role == "kv_both":
|
||||
logger.warning_once(
|
||||
"Using kv_role='kv_both' with NixlConnector is deprecated "
|
||||
"and will be removed in a future release. Please set "
|
||||
"kv_role='kv_producer' for prefill instances and "
|
||||
"kv_role='kv_consumer' for decode instances. "
|
||||
)
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
pcp_size = parallel_config.prefill_context_parallel_size
|
||||
kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
if pcp_size > 1 and kv_role in ("kv_consumer", "kv_both"):
|
||||
raise NotImplementedError(
|
||||
"NixlConnector PCP currently supports kv_producer only. "
|
||||
"Consumers and kv_both require "
|
||||
"prefill_context_parallel_size=1."
|
||||
)
|
||||
dcp_size = parallel_config.decode_context_parallel_size
|
||||
if (
|
||||
pcp_size > 1
|
||||
and dcp_size > 1
|
||||
and (parallel_config.tensor_parallel_size != 1 or dcp_size != pcp_size)
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"NixlConnector PCP+DCP requires TP1 with DCP spanning "
|
||||
"the full PCP group."
|
||||
)
|
||||
# TODO: Support PCP with bidirectional KV transfer by tracking separate
|
||||
# send and receive completion counts.
|
||||
if pcp_size > 1 and vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"bidirectional_kv_xfer", False
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"NixlConnector PCP producers do not support bidirectional KV transfer."
|
||||
)
|
||||
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id
|
||||
self.kv_transfer_config = vllm_config.kv_transfer_config
|
||||
# Subclasses must set self.connector_scheduler and self.connector_worker
|
||||
self.connector_scheduler: NixlBaseConnectorScheduler | None = None
|
||||
self.connector_worker: NixlBaseConnectorWorker | None = None
|
||||
|
||||
############################################################
|
||||
# Class Methods
|
||||
############################################################
|
||||
@classmethod
|
||||
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: VllmConfig):
|
||||
if vllm_config.model_config is None:
|
||||
logger.warning_once(
|
||||
"Unable to detect current VLLM config. "
|
||||
"Fallback to default kv cache layout."
|
||||
)
|
||||
return None
|
||||
use_mla = vllm_config.model_config.use_mla
|
||||
if use_mla:
|
||||
# return None when we have mla
|
||||
# as the layout should not matter in that case,
|
||||
# which fallback to the default behavior.
|
||||
return None
|
||||
logger.info_once(
|
||||
"NixlConnector setting KV cache layout to LBHNC for "
|
||||
"better xfer performance."
|
||||
)
|
||||
return "LBHNC"
|
||||
|
||||
############################################################
|
||||
# Scheduler Side Methods
|
||||
############################################################
|
||||
|
||||
def get_finished_count(self) -> int | None:
|
||||
return None
|
||||
|
||||
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
|
||||
return self._kv_cache_config.transfer_group_ids
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.update_state_after_alloc(
|
||||
request, blocks, num_external_tokens
|
||||
)
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.on_new_request(request)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, (block_ids,))
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, block_ids)
|
||||
|
||||
def set_xfer_handshake_metadata_pp_aware(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""Set handshake metadata keyed by (pp_rank, tp_rank) so the side
|
||||
channel can serve every PP stage's agent metadata.
|
||||
|
||||
Args:
|
||||
metadata (dict): the handshake metadata to set.
|
||||
|
||||
"""
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.set_xfer_handshake_metadata(metadata)
|
||||
|
||||
############################################################
|
||||
# Worker Side Methods
|
||||
############################################################
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.set_host_xfer_buffer_ops(copy_operation)
|
||||
|
||||
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
|
||||
"""Get the finished recving and sending requests."""
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_finished()
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
assert self.connector_worker is not None
|
||||
results = self.connector_worker.get_transfer_results()
|
||||
if (
|
||||
self.kv_transfer_config.kv_role == "kv_producer"
|
||||
and self.connector_worker.pcp_rank > 0
|
||||
and not self.connector_worker.pcp_dcp_sharded
|
||||
):
|
||||
results.finished_sending.clear()
|
||||
return results
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get block IDs that failed to load via NIXL."""
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_block_ids_with_load_errors()
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
NixlKVConnectorStats(data=data)
|
||||
if data is not None
|
||||
else NixlKVConnectorStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return NixlPromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""NixlConnector does not do layerwise saving."""
|
||||
pass
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""NixlConnector does not save explicitly."""
|
||||
pass
|
||||
|
||||
def wait_for_save(self):
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(self._connector_metadata, NixlConnectorMetadata)
|
||||
if self.connector_worker.use_host_buffer and self.connector_worker.copy_blocks:
|
||||
self.connector_worker.save_kv_to_host(self._connector_metadata)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
if self.connector_scheduler is not None:
|
||||
return self.connector_scheduler.has_pending_push_work()
|
||||
return False
|
||||
|
||||
def shutdown(self):
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.shutdown()
|
||||
if self.connector_scheduler is not None:
|
||||
self.connector_scheduler.shutdown()
|
||||
|
||||
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
|
||||
"""Get the KVConnector handshake metadata for this connector.
|
||||
This metadata is used for out-of-band connector handshake
|
||||
between P/D workers.
|
||||
|
||||
Returns:
|
||||
KVConnectorHandshakeMetadata: the handshake metadata.
|
||||
None if no handshake metadata is available.
|
||||
|
||||
"""
|
||||
assert self.connector_worker is not None
|
||||
if (
|
||||
self.kv_transfer_config.kv_role == "kv_producer"
|
||||
and self.connector_worker.pcp_rank > 0
|
||||
and not self.connector_worker.pcp_dcp_sharded
|
||||
):
|
||||
return None
|
||||
return self.connector_worker.xfer_handshake_metadata
|
||||
|
||||
|
||||
class NixlPullConnector(NixlBaseConnector):
|
||||
"""Pull-based (READ) NIXL KV transfer connector."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = NixlPullConnectorScheduler(
|
||||
vllm_config, self.engine_id, kv_cache_config
|
||||
)
|
||||
self.connector_worker = None
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.connector_scheduler = None
|
||||
self.connector_worker = NixlPullConnectorWorker(
|
||||
vllm_config, self.engine_id, kv_cache_config
|
||||
)
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(self.connector_worker, NixlPullConnectorWorker)
|
||||
assert isinstance(self._connector_metadata, NixlConnectorMetadata)
|
||||
self.connector_worker.start_load_kv(self._connector_metadata)
|
||||
|
||||
|
||||
class NixlPushConnector(NixlBaseConnector):
|
||||
"""Push-based (WRITE) NIXL KV transfer connector."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
if vllm_config.parallel_config.decode_context_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"NixlPushConnector does not support decode_context_parallel_size > 1."
|
||||
)
|
||||
self.connector_scheduler: NixlPushConnectorScheduler | None = None
|
||||
self.connector_worker: NixlPushConnectorWorker | None = None
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = NixlPushConnectorScheduler(
|
||||
vllm_config, self.engine_id, kv_cache_config
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.connector_worker = NixlPushConnectorWorker(
|
||||
vllm_config, self.engine_id, kv_cache_config
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported KVConnectorRole: {role}")
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
|
||||
"""Drive push processing on the worker.
|
||||
|
||||
The worker enqueues registrations / finished blocks for the
|
||||
background ``nixl-push-writer`` thread; the writer issues the
|
||||
WRITE transfers and polls NIXL notifs without further
|
||||
engine-thread involvement.
|
||||
"""
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(self._connector_metadata, NixlConnectorMetadata)
|
||||
self.connector_worker.start_load_kv(self._connector_metadata)
|
||||
|
||||
|
||||
# Backward compatibility: NixlConnector is the pull-based connector.
|
||||
NixlConnector = NixlPullConnector
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NixlBaseConnector",
|
||||
"NixlConnector",
|
||||
"NixlPullConnector",
|
||||
"NixlPushConnector",
|
||||
]
|
||||
@@ -1,340 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Metadata dataclasses and helpers for the NIXL connector."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds, EngineId
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorHandshakeMetadata,
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
TransferHandle = int
|
||||
ReqId = str
|
||||
|
||||
GET_META_MSG = b"get_meta_msg"
|
||||
|
||||
# Push-mode (WRITE-based) registration notification.
|
||||
# Sent worker-to-worker over NIXL: D worker -> P worker, encoded as
|
||||
# PUSH_REG_NOTIF_PREFIX + msgpack(registration_data).
|
||||
PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
|
||||
#
|
||||
# NIXL Connector Version
|
||||
#
|
||||
# Increment this version whenever there is an incompatible change to:
|
||||
# - NixlAgentMetadata schema
|
||||
# - kv_transfer_params schema or semantics
|
||||
# - NIXL transfer protocol or wire format
|
||||
# - KV cache memory layout or block organization
|
||||
# - Any other change that breaks P/D interoperability
|
||||
#
|
||||
# Version History:
|
||||
# 1: Initial version with compatibility checking
|
||||
# 2: Add remote_request_id to kv_transfer_params
|
||||
# 3: Add physical_blocks_per_logical_kv_block to NixlAgentMetadata
|
||||
# 4: Add KV block lease renewal through heartbeats
|
||||
# 5: Add remote_blocks_expiry_time to kv_transfer_params + handshake
|
||||
# clock-sync timestamp
|
||||
# 6: Validate EAGLE/MTP speculative configuration compatibility
|
||||
# 7: Include NIXL transfer mode (push vs pull) in the compatibility hash
|
||||
# 8: Add dcp_size and pcp_size to NixlAgentMetadata
|
||||
# 9: Add block_strides
|
||||
# 10: Add dense virtual transfer pages for compressed MLA caches
|
||||
# 11: Add per-region transfer geometry and memory types to NixlAgentMetadata
|
||||
# 12: Add per-region member names for PP push
|
||||
# 13: Add packed-member layouts and order-independent packed-push backend hashes
|
||||
# 14: Add Qwen3.8-Flash-Next PLE page length - #59997
|
||||
#
|
||||
NIXL_CONNECTOR_VERSION: int = 14
|
||||
|
||||
|
||||
@dataclass
|
||||
class NixlAgentMetadata:
|
||||
engine_id: str
|
||||
agent_metadata: bytes
|
||||
kv_caches_base_addr: list[int]
|
||||
device_id: int
|
||||
num_blocks: int
|
||||
block_lens: list[int]
|
||||
block_strides: list[int]
|
||||
kv_cache_layout: str
|
||||
block_size: int
|
||||
ssm_sizes: tuple[int, int]
|
||||
attn_backend_name: str
|
||||
physical_blocks_per_logical_kv_block: int
|
||||
region_num_blocks: list[int] | None = None
|
||||
region_group_ids: list[int] | None = None
|
||||
region_names: list[str] | None = None
|
||||
region_mem_types: list[str] | None = None
|
||||
dcp_size: int = 1
|
||||
pcp_size: int = 1
|
||||
# Page length of the PLE short-conv state (None when the model has none).
|
||||
ple_block_len: int | None = None
|
||||
# Layer names sharing each advertised region, in region order.
|
||||
region_members: list[list[str]] = field(default_factory=list)
|
||||
# Packed member -> (byte offset in its region's block, bytes per page).
|
||||
packed_member_layouts: dict[str, tuple[int, int]] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NixlHandshakePayload(KVConnectorHandshakeMetadata):
|
||||
"""Wrapper for NIXL handshake sent over the wire.
|
||||
|
||||
Enables two-phase decoding for graceful compatibility checking:
|
||||
1. Decode NixlHandshakePayload to get compatibility_hash
|
||||
2. Compute local hash and compare
|
||||
3. Only if hashes match, decode agent_metadata_bytes
|
||||
|
||||
This prevents decoder errors when NixlAgentMetadata schema is
|
||||
incompatible, allowing graceful failure with clear error message.
|
||||
"""
|
||||
|
||||
compatibility_hash: str
|
||||
agent_metadata_bytes: bytes # NixlAgentMetadata encoded
|
||||
|
||||
|
||||
def _get_speculative_compatibility_factors(
|
||||
vllm_config: VllmConfig,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Return NIXL compatibility factors for hidden-state-based speculators."""
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None or not speculative_config.use_eagle():
|
||||
return None
|
||||
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
assert draft_model_config is not None
|
||||
auxiliary_layer_ids = getattr(
|
||||
draft_model_config.hf_config,
|
||||
"eagle_aux_hidden_state_layer_ids",
|
||||
None,
|
||||
)
|
||||
|
||||
# kv_cache_dtype is a user override that defaults to None, meaning "inherit
|
||||
# the target's --kv-cache-dtype". Resolve it to the effective value so an
|
||||
# explicit setting on one side and inheritance on the other (same effective
|
||||
# dtype) don't spuriously mismatch.
|
||||
kv_cache_dtype = (
|
||||
speculative_config.kv_cache_dtype or vllm_config.cache_config.cache_dtype
|
||||
)
|
||||
|
||||
# Note: the draft attention_backend is intentionally not hashed. Its only
|
||||
# transfer-relevant effect is the KV block layout/size, which is validated
|
||||
# per region at runtime in _validate_remote_agent_handshake. The connector
|
||||
# only sees the raw override here (usually None = auto-select), never the
|
||||
# resolved backend, so hashing it would cause false mismatches without
|
||||
# catching anything the runtime layout check misses.
|
||||
return {
|
||||
"method": speculative_config.method,
|
||||
"model": draft_model_config.model,
|
||||
"revision": draft_model_config.revision,
|
||||
"code_revision": draft_model_config.code_revision,
|
||||
"parallel_drafting": speculative_config.parallel_drafting,
|
||||
"kv_cache_dtype": str(kv_cache_dtype),
|
||||
"auxiliary_layer_ids": (
|
||||
tuple(auxiliary_layer_ids) if auxiliary_layer_ids is not None else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def compute_nixl_compatibility_hash(
|
||||
vllm_config: VllmConfig,
|
||||
attn_backend_name: str,
|
||||
transfer_mode: str = "pull",
|
||||
) -> str:
|
||||
"""Compute compatibility hash for NIXL KV transfer.
|
||||
|
||||
Hash only the factors that affect whether two NIXL instances can
|
||||
successfully transfer KV cache data.
|
||||
|
||||
Factors included:
|
||||
- vLLM version and NIXL connector version
|
||||
- Model architecture (name, dtype, KV heads, layers)
|
||||
- KV cache format (dtype, sliding window)
|
||||
- Attention backend
|
||||
- EAGLE/MTP configuration that affects transferred state
|
||||
- Transfer mode (push vs pull)
|
||||
|
||||
The transfer mode is included because the push (WRITE) and pull (READ)
|
||||
connectors use incompatible transfer protocols; a push connector and a
|
||||
pull connector must never complete a handshake with each other.
|
||||
|
||||
Note: Factors like tensor_parallel_size, block_size, and kv_cache_layout
|
||||
are validated at runtime in _validate_remote_agent_handshake and are not
|
||||
included in this hash to support heterogeneous deployments.
|
||||
|
||||
Note - the set of factors are likely to evolve significantly over
|
||||
time to be more or less permissive.
|
||||
|
||||
Returns:
|
||||
SHA-256 hex digest
|
||||
|
||||
"""
|
||||
from vllm import __version__ as vllm_version
|
||||
from vllm.config.utils import hash_factors
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
is_hma_enabled = not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
|
||||
factors = {
|
||||
# Version compatibility
|
||||
"vllm_version": vllm_version,
|
||||
"nixl_connector_version": NIXL_CONNECTOR_VERSION,
|
||||
# Model architecture - affects KV cache shape
|
||||
"model": model_config.model,
|
||||
"dtype": str(model_config.dtype),
|
||||
"num_kv_heads": model_config.get_total_num_kv_heads(),
|
||||
"head_size": model_config.get_head_size(),
|
||||
"num_hidden_layers": model_config.get_total_num_hidden_layers(),
|
||||
# Attention backend and KV cache dtype affect memory layout
|
||||
"attn_backend_name": attn_backend_name,
|
||||
"cache_dtype": str(cache_config.cache_dtype),
|
||||
"is_hma_enabled": is_hma_enabled,
|
||||
"speculative_config": _get_speculative_compatibility_factors(vllm_config),
|
||||
# push (WRITE) and pull (READ) connectors are protocol-incompatible
|
||||
"transfer_mode": transfer_mode,
|
||||
}
|
||||
|
||||
compat_hash = hash_factors(factors)
|
||||
logger.debug(
|
||||
"NIXL compatibility hash: %s (model=%s, dtype=%s, num_kv_heads=%d, "
|
||||
"cache_dtype=%s, attn_backend=%s)",
|
||||
compat_hash,
|
||||
factors["model"],
|
||||
factors["dtype"],
|
||||
factors["num_kv_heads"],
|
||||
factors["cache_dtype"],
|
||||
attn_backend_name,
|
||||
)
|
||||
return compat_hash
|
||||
|
||||
|
||||
@dataclass
|
||||
class HeartbeatInfo:
|
||||
"""Heartbeat data for a single remote engine, sent from D worker to P."""
|
||||
|
||||
req_ids: set[ReqId]
|
||||
host: str
|
||||
port: int
|
||||
tp_size: int
|
||||
dcp_size: int = 1
|
||||
pp_size: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class RemoteMeta:
|
||||
block_ids: BlockIds
|
||||
host: str
|
||||
port: int
|
||||
engine_id: str
|
||||
request_id: str
|
||||
blocks_expiry_time: float | None = None
|
||||
num_tokens: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReqMeta:
|
||||
local_block_ids: BlockIds
|
||||
# To be used when logical block size does not match the kernel block size
|
||||
local_physical_block_ids: BlockIds
|
||||
tp_size: int
|
||||
dcp_size: int = 1
|
||||
# Per-KV-cache-group logical blocks this rank already holds, i.e. its
|
||||
# prefix-cache hit. Fixes where this rank's DCP slice starts relative to
|
||||
# the remote's; kept per-group since hybrid models (e.g. SWA+FA) can have
|
||||
# different cache-hit counts per group.
|
||||
local_num_computed_blocks: tuple[int, ...] = ()
|
||||
remote: RemoteMeta | None = None
|
||||
# Remote block size, discovered during NIXL handshake (push mode).
|
||||
remote_block_size: int | None = None
|
||||
# Remote producer pipeline-parallel size (push mode, D side).
|
||||
pp_size: int = 1
|
||||
# True only when the scheduler parked the request in WAITING_FOR_REMOTE_KVS
|
||||
# and expects it in finished_recving; notify-only recvs must not be reported.
|
||||
awaiting_kvs: bool = False
|
||||
# Worker-only, per-region physical pages to zero after a successful pull.
|
||||
# None selects group-based completion; empty lists mean no zeroing.
|
||||
region_blocks_to_zero: BlockIds | None = None
|
||||
|
||||
|
||||
class NixlConnectorMetadata(KVConnectorMetadata):
|
||||
def __init__(self):
|
||||
self.reqs_to_recv: dict[ReqId, ReqMeta] = {}
|
||||
self.reqs_to_save: dict[ReqId, ReqMeta] = {}
|
||||
self.reqs_to_send: dict[ReqId, float] = {}
|
||||
# The scheduler process's time.perf_counter() when this metadata was
|
||||
# built. reqs_to_send deadlines are stamped with the scheduler's
|
||||
# clock, which is NOT comparable across processes (perf_counter is
|
||||
# process/boot-local): workers must rebase the remaining TTL onto
|
||||
# their own clock via this reference. 0.0 = unset (legacy metadata).
|
||||
self.scheduler_clock: float = 0.0
|
||||
self.reqs_in_batch: set[ReqId] = set()
|
||||
self.reqs_not_processed: set[ReqId] = set()
|
||||
# Heartbeat data grouped by remote engine, sent by D worker to P.
|
||||
self.heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {}
|
||||
# Push mode (D side): registration data the D worker should send to
|
||||
# P workers via NIXL notification on this step.
|
||||
self.push_registrations: dict[ReqId, dict[str, Any]] = {}
|
||||
# Push mode (P side): newly finished request blocks to be matched
|
||||
# against pending D registrations on the P worker.
|
||||
self.push_finished_blocks: dict[ReqId, BlockIds] = {}
|
||||
|
||||
def _add_new_req(
|
||||
self,
|
||||
local_block_ids: BlockIds,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
local_num_computed_blocks: tuple[int, ...] = (),
|
||||
awaiting_kvs: bool = False,
|
||||
) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
local_block_ids=local_block_ids,
|
||||
local_physical_block_ids=local_block_ids,
|
||||
# P workers don't need to receive these from proxy here.
|
||||
tp_size=kv_transfer_params.get("tp_size", 1),
|
||||
dcp_size=kv_transfer_params.get("dcp_size", 1),
|
||||
remote_block_size=kv_transfer_params.get("remote_block_size"),
|
||||
pp_size=kv_transfer_params.get("pp_size", 1),
|
||||
local_num_computed_blocks=local_num_computed_blocks,
|
||||
awaiting_kvs=awaiting_kvs,
|
||||
)
|
||||
|
||||
def add_new_req_to_save(
|
||||
self,
|
||||
request_id: ReqId,
|
||||
local_block_ids: BlockIds,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
):
|
||||
self.reqs_to_save[request_id] = self._add_new_req(
|
||||
local_block_ids, kv_transfer_params
|
||||
)
|
||||
|
||||
def add_new_req_to_recv(
|
||||
self,
|
||||
request_id: ReqId,
|
||||
local_block_ids: BlockIds,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
local_num_computed_blocks: tuple[int, ...] = (),
|
||||
awaiting_kvs: bool = False,
|
||||
):
|
||||
req = self._add_new_req(
|
||||
local_block_ids,
|
||||
kv_transfer_params,
|
||||
local_num_computed_blocks,
|
||||
awaiting_kvs,
|
||||
)
|
||||
req.remote = RemoteMeta(
|
||||
block_ids=kv_transfer_params["remote_block_ids"],
|
||||
engine_id=kv_transfer_params["remote_engine_id"],
|
||||
request_id=kv_transfer_params["remote_request_id"],
|
||||
host=kv_transfer_params["remote_host"],
|
||||
port=kv_transfer_params["remote_port"],
|
||||
blocks_expiry_time=kv_transfer_params.get("remote_blocks_expiry_time"),
|
||||
num_tokens=kv_transfer_params.get("remote_num_tokens"),
|
||||
)
|
||||
self.reqs_to_recv[request_id] = req
|
||||
@@ -1,292 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pull-specific scheduler-side logic for the NIXL connector."""
|
||||
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
|
||||
NixlBaseConnectorScheduler,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
"""Pull-specific scheduler logic (READ-based KV transfer)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
engine_id: str,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, engine_id, kv_cache_config)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int, bool]:
|
||||
"""For remote prefill, pull all prompt blocks from remote
|
||||
asynchronously relative to engine execution.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
num_computed_tokens (int): the number of locally
|
||||
computed tokens for this request
|
||||
Returns:
|
||||
* the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
* true if the external KV cache tokens will be loaded
|
||||
asynchronously (between scheduler steps).
|
||||
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NIXLConnector get_num_new_matched_tokens: "
|
||||
"num_computed_tokens=%s, kv_transfer_params=%s",
|
||||
num_computed_tokens,
|
||||
params,
|
||||
)
|
||||
|
||||
if params is not None and params.get("do_remote_prefill"):
|
||||
# Remote prefill: get all prompt blocks from remote.
|
||||
token_ids = request.prompt_token_ids or []
|
||||
actual = self._get_remote_prefill_token_count(len(token_ids))
|
||||
count = actual - num_computed_tokens
|
||||
if count > 0:
|
||||
return count, True
|
||||
|
||||
if (
|
||||
params is not None
|
||||
and params.get("do_remote_decode")
|
||||
and params.get("remote_block_ids")
|
||||
and all(
|
||||
p in params
|
||||
for p in (
|
||||
"remote_engine_id",
|
||||
"remote_request_id",
|
||||
"remote_host",
|
||||
"remote_port",
|
||||
)
|
||||
)
|
||||
):
|
||||
# Decode node has kv blocks for part of prefill request, so, provide them
|
||||
# as an external token count to scheduler.
|
||||
# The tokens will be loaded if not already present
|
||||
# in the prefill node local cache
|
||||
remote_num_tokens = params.get("remote_num_tokens") or 0
|
||||
count = (
|
||||
min(remote_num_tokens, request.num_prompt_tokens) - num_computed_tokens
|
||||
)
|
||||
if count > 0:
|
||||
# Check kv_recompute_threshold: skip pull if
|
||||
# remote tokens are below the threshold.
|
||||
if (
|
||||
self.kv_recompute_threshold > 0
|
||||
and count < self.kv_recompute_threshold
|
||||
):
|
||||
logger.debug(
|
||||
"Skipping remote pull for %s: %d remote tokens < threshold %d",
|
||||
request.request_id,
|
||||
count,
|
||||
self.kv_recompute_threshold,
|
||||
)
|
||||
return 0, False
|
||||
return count, True
|
||||
|
||||
# No remote prefill for this request.
|
||||
return 0, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NIXLConnector update_state_after_alloc: "
|
||||
"num_external_tokens=%s, kv_transfer_params=%s",
|
||||
num_external_tokens,
|
||||
params,
|
||||
)
|
||||
|
||||
if not params:
|
||||
return
|
||||
|
||||
if params.get("do_remote_decode") or (
|
||||
params.get("do_remote_prefill") and self.is_bidirectional_kv_xfer_enabled
|
||||
):
|
||||
self._reqs_in_batch.add(request.request_id)
|
||||
if self.use_host_buffer and params.get("do_remote_decode"):
|
||||
# NOTE: when accelerator is not directly supported by Nixl,
|
||||
# prefilled blocks need to be saved to host memory before transfer.
|
||||
self._reqs_need_save[request.request_id] = request
|
||||
elif params.get("do_remote_prefill") or (
|
||||
params.get("do_remote_decode")
|
||||
and self.is_bidirectional_kv_xfer_enabled
|
||||
and not params.get("_remote_blocks_processed")
|
||||
):
|
||||
if params.get("remote_block_ids"):
|
||||
if all(
|
||||
p in params
|
||||
for p in (
|
||||
"remote_engine_id",
|
||||
"remote_request_id",
|
||||
"remote_host",
|
||||
"remote_port",
|
||||
)
|
||||
):
|
||||
# If remote_blocks and num_external_tokens = 0, we have
|
||||
# a full prefix cache hit on the local node. We need to call
|
||||
# send_notif in _read_blocks to free the memory on the remote node.
|
||||
|
||||
unhashed_local_block_ids: BlockIds = (
|
||||
blocks.get_unhashed_block_ids_all_groups()
|
||||
if num_external_tokens > 0
|
||||
else ()
|
||||
)
|
||||
local_block_ids = self.get_exchange_clipped_blocks(
|
||||
unhashed_local_block_ids
|
||||
)
|
||||
# Blocks covered by the local prefix cache, per KV cache group.
|
||||
# Each count fixes where that group's DCP slice starts, which the
|
||||
# worker needs to line up with the remote's slice.
|
||||
local_num_computed_blocks = tuple(
|
||||
sum(
|
||||
block.block_hash is not None and not block.is_null
|
||||
for block in group
|
||||
)
|
||||
for group in blocks.blocks
|
||||
)
|
||||
|
||||
# Get unhashed blocks to pull from remote. Mind that a full prefix
|
||||
# cache hit is indicated with an empty list.
|
||||
self._reqs_need_recv[request.request_id] = (
|
||||
request,
|
||||
local_block_ids,
|
||||
local_num_computed_blocks,
|
||||
# Parked in WAITING_FOR_REMOTE_KVS only when there is
|
||||
# something to pull; a full local hit stays RUNNING.
|
||||
num_external_tokens > 0,
|
||||
)
|
||||
|
||||
else:
|
||||
logger.warning(
|
||||
"Got invalid KVTransferParams: %s. This "
|
||||
"request will not utilize KVTransfer",
|
||||
params,
|
||||
)
|
||||
else:
|
||||
assert num_external_tokens == 0
|
||||
# Only trigger 1 KV transfer per request.
|
||||
params["do_remote_prefill"] = False
|
||||
params["_remote_blocks_processed"] = True
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: "BlockIds",
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Once a request is finished, determine whether request blocks
|
||||
should be freed now or will be sent asynchronously and freed later.
|
||||
"""
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NIXLConnector request_finished(%s), request_status=%s, "
|
||||
"kv_transfer_params=%s",
|
||||
request.request_id,
|
||||
request.status,
|
||||
params,
|
||||
)
|
||||
if not params:
|
||||
return False, None
|
||||
|
||||
is_p_node = bool(params.get("do_remote_decode"))
|
||||
is_d_node = not is_p_node
|
||||
|
||||
# Stop heartbeating for aborted requests that never reached finished_recving:
|
||||
# normal path cleans up in update_connector_output.
|
||||
self._stop_heartbeat(request.request_id)
|
||||
|
||||
if params.get("do_remote_prefill"):
|
||||
# If do_remote_prefill is still True when the request is finished,
|
||||
# update_state_after_alloc must not have been called (the request
|
||||
# must have been aborted before it was scheduled, e.g. via the
|
||||
# abort_immediately path used to clean up KV-transfer requests
|
||||
# rejected at the D-side serving layer).
|
||||
# To avoid stranding the prefill blocks in the prefill instance,
|
||||
# we must add empty block_ids to _reqs_need_recv so that our
|
||||
# worker side will notify and free blocks in the prefill instance.
|
||||
self._reqs_need_recv[request.request_id] = (request, [], (), False)
|
||||
params["do_remote_prefill"] = False
|
||||
return False, None
|
||||
|
||||
if is_d_node and not self.is_bidirectional_kv_xfer_enabled:
|
||||
return False, None
|
||||
|
||||
if request.status not in (
|
||||
RequestStatus.FINISHED_LENGTH_CAPPED,
|
||||
RequestStatus.FINISHED_STOPPED,
|
||||
):
|
||||
# Also include the case of a P/D Prefill request with immediate
|
||||
# block free (eg abort). Stop tracking this request.
|
||||
self._reqs_not_processed.add(request.request_id)
|
||||
# Clear _reqs_need_save if a request is aborted as partial prefill.
|
||||
self._reqs_need_save.pop(request.request_id, None)
|
||||
return False, None
|
||||
|
||||
# TODO: check whether block_ids actually ever be 0. If not we could
|
||||
# remove the conditional below
|
||||
delay_free_blocks = any(len(group) > 0 for group in block_ids)
|
||||
remote_num_tokens = 0
|
||||
blocks_expiry_time = None
|
||||
if delay_free_blocks:
|
||||
# Prefill request on remote. It will be read from D upon completion
|
||||
request_kv_blocks_ttl = self._kv_lease_duration
|
||||
if is_d_node:
|
||||
# For blocks pinned on D, use a simpler timeout for now instead of a
|
||||
# lease mechanism as turn2 request is client-driven.
|
||||
request_kv_blocks_ttl = self.decoder_kv_blocks_ttl
|
||||
logger.debug(
|
||||
"NIXLConnector request_finished(%s) waiting for %d seconds "
|
||||
"before releasing blocks",
|
||||
request.request_id,
|
||||
request_kv_blocks_ttl,
|
||||
)
|
||||
self._reqs_need_send[request.request_id] = (
|
||||
time.perf_counter() + request_kv_blocks_ttl
|
||||
)
|
||||
if is_d_node:
|
||||
# D blocks expiry time exported for the turn-2 readback.
|
||||
blocks_expiry_time = self._reqs_need_send[request.request_id]
|
||||
# NOTE HMA will "mark" empty/null blocks in groups with 0s (eg SWA ones),
|
||||
# trimming down after allocating for the whole sequence length. Empty
|
||||
# blocks are always at the start of the list.
|
||||
# Here we "unpad" blocks to send the actual remote blocks to be read.
|
||||
block_ids = self.get_exchange_clipped_blocks(block_ids)
|
||||
|
||||
remote_num_tokens = request.num_computed_tokens
|
||||
|
||||
return delay_free_blocks, dict(
|
||||
do_remote_prefill=is_p_node,
|
||||
do_remote_decode=is_d_node,
|
||||
remote_block_ids=block_ids,
|
||||
remote_engine_id=self.engine_id,
|
||||
remote_request_id=request.request_id,
|
||||
remote_host=self.side_channel_host,
|
||||
remote_port=self.side_channel_port,
|
||||
tp_size=self.transfer_tp_size,
|
||||
dcp_size=self.vllm_config.parallel_config.decode_context_parallel_size,
|
||||
pp_size=self.vllm_config.parallel_config.pipeline_parallel_size,
|
||||
remote_num_tokens=remote_num_tokens,
|
||||
remote_blocks_expiry_time=blocks_expiry_time,
|
||||
transfer_mode=self._TRANSFER_MODE,
|
||||
)
|
||||
@@ -1,693 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pull-specific (READ) worker-side logic for the NIXL connector."""
|
||||
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
|
||||
NixlBaseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlConnectorMetadata,
|
||||
ReqMeta,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
ReadSpec,
|
||||
_is_attention_spec,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Slack (seconds) subtracted from D's exported block-expiry deadline on the turn-2
|
||||
# readback, absorbing clock-offset error and read latency.
|
||||
_KV_BLOCKS_EXPIRY_SAFETY_MARGIN = 5.0
|
||||
|
||||
|
||||
class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
"""Pull-specific (READ) worker logic."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
engine_id: str,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, engine_id, kv_cache_config)
|
||||
|
||||
def start_load_kv(self, metadata: NixlConnectorMetadata):
|
||||
"""Start loading by triggering non-blocking nixl_xfer.
|
||||
We check for these trnxs to complete in each step().
|
||||
"""
|
||||
for req_id, meta in metadata.reqs_to_recv.items():
|
||||
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
|
||||
meta.local_block_ids, self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
assert meta.remote is not None
|
||||
# Remote block IDs are kept logical here; expanded in
|
||||
# _read_blocks_for_req using the remote engine's phys ratio.
|
||||
remote_engine_id = meta.remote.engine_id
|
||||
logger.debug(
|
||||
"start_load_kv for request %s from remote engine %s. "
|
||||
"Num local_block_ids: %s. Num remote_block_ids: %s. ",
|
||||
req_id,
|
||||
remote_engine_id,
|
||||
len(meta.local_physical_block_ids),
|
||||
len(meta.remote.block_ids),
|
||||
)
|
||||
# Full local hits and aborted cleanup only notify P; no recv is awaited.
|
||||
# On a full local hit:
|
||||
# - Notification failure must not fail the request: its KV is local.
|
||||
# - Receive completion must not be reported: num_external_tokens == 0,
|
||||
# so the scheduler never entered WAITING_FOR_REMOTE_KVS to begin with.
|
||||
# Aborted cleanup requests have already been removed from the scheduler.
|
||||
if meta.awaiting_kvs or any(meta.local_block_ids):
|
||||
self._recving_metadata[req_id] = meta
|
||||
self._read_blocks_for_req(req_id, meta)
|
||||
|
||||
# Start transfers for requests whose handshakes have now finished.
|
||||
while not self._ready_requests.empty():
|
||||
self._read_blocks_for_req(*self._ready_requests.get_nowait())
|
||||
|
||||
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
|
||||
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
|
||||
# has nothing to send. Report the requests as sent right away so
|
||||
# the scheduler-side aggregation (world_size workers, and any
|
||||
# sibling connector inside a MultiConnector) still completes.
|
||||
self._replicated_pcp_done_sending.update(metadata.reqs_to_send)
|
||||
return
|
||||
|
||||
# Keep around the requests that have been part of a batch. This is
|
||||
# needed because async scheduling pushes the misalignment between the
|
||||
# moment in which requests expiration is set (P side) and the moment in
|
||||
# which blocks are read from D. As P can now more easily lag behind D
|
||||
# while processing the next batch, we make sure to only set an
|
||||
# expiration for requests that have not been read from D yet.
|
||||
for req_id in metadata.reqs_in_batch:
|
||||
self._reqs_to_process.add(req_id)
|
||||
|
||||
# Remove all requests that are not to be processed (eg aborted).
|
||||
for req_id in metadata.reqs_not_processed:
|
||||
self._reqs_to_process.discard(req_id)
|
||||
# We should never get an abort after setting an expiry timer
|
||||
assert req_id not in self._reqs_to_send
|
||||
|
||||
# Add to requests that are waiting to be read and track expiration.
|
||||
# Deadlines are stamped with the scheduler process's perf_counter,
|
||||
# which is not comparable to ours when the worker runs in another
|
||||
# process on another node (perf_counter epochs differ by boot time).
|
||||
# Rebase the remaining TTL onto our clock; broadcast latency only
|
||||
# lengthens the lease, which is the safe direction. A cross-node
|
||||
# epoch gap larger than the TTL otherwise expires the lease on
|
||||
# arrival and the blocks are freed before D reads them.
|
||||
now_local = time.perf_counter()
|
||||
for req_id, expiration_time in metadata.reqs_to_send.items():
|
||||
if req_id in self._reqs_to_process:
|
||||
if metadata.scheduler_clock:
|
||||
expiration_time = now_local + (
|
||||
expiration_time - metadata.scheduler_clock
|
||||
)
|
||||
self._reqs_to_send[req_id] = expiration_time
|
||||
|
||||
# Send heartbeats to P-side engines to keep KV blocks alive while
|
||||
# requests sit in the D scheduler WAITING queue.
|
||||
self._send_heartbeats(metadata)
|
||||
|
||||
def _is_turn2_read_expired(self, meta: ReqMeta) -> bool:
|
||||
"""Whether D's cached blocks for this turn-2 readback have (nearly) expired."""
|
||||
assert meta.remote is not None
|
||||
blocks_expiry_time = meta.remote.blocks_expiry_time
|
||||
# Deadline may be absent (router may not forward it) -> read as usual.
|
||||
if blocks_expiry_time is None or not meta.local_physical_block_ids:
|
||||
return False
|
||||
clock_offset = self._engine_clock_offset[meta.remote.engine_id]
|
||||
deadline = blocks_expiry_time - clock_offset
|
||||
return time.perf_counter() + _KV_BLOCKS_EXPIRY_SAFETY_MARGIN >= deadline
|
||||
|
||||
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
|
||||
assert meta.remote is not None and self.transfer_topo is not None
|
||||
engine_id = meta.remote.engine_id
|
||||
if engine_id in self._invalid_remote_engines:
|
||||
self._handle_failed_transfer(
|
||||
req_id, None, self._recv_failures, record_failed_transfer=False
|
||||
)
|
||||
return
|
||||
if engine_id not in self._remote_agents:
|
||||
# Also re-handshakes queued requests that outlived invalid-peer cleanup.
|
||||
self._background_nixl_handshake(req_id, engine_id, meta)
|
||||
return
|
||||
# Update last activity from this remote. Mind that cleanup is done on main
|
||||
# thread (this one), so we don't race on this structure.
|
||||
self._engine_last_active[engine_id] = time.perf_counter()
|
||||
|
||||
if self._bidirectional_kv_xfer_enabled and self._is_turn2_read_expired(meta):
|
||||
logger.warning(
|
||||
"Declining expired remote read for %s from engine %s.",
|
||||
req_id,
|
||||
engine_id,
|
||||
)
|
||||
self.xfer_stats.record_kv_expired_req()
|
||||
# KV expiry is reported separately from transport failures, so only
|
||||
# the state cleanup side of _handle_failed_transfer runs here.
|
||||
self._handle_failed_transfer(
|
||||
req_id, None, self._recv_failures, record_failed_transfer=False
|
||||
)
|
||||
return
|
||||
|
||||
if any(len(group) > 0 for group in meta.local_block_ids):
|
||||
# The scheduler waits for finished_recving from *every* worker.
|
||||
# Under DCP a rank's slice can legitimately come out empty when its
|
||||
# interleaved positions fall past the end of the sequence. _read_blocks
|
||||
# then takes the notify-only path without registering a transfer.
|
||||
# Seed the entry so this rank still reports completion.
|
||||
self._recving_transfers.setdefault(req_id, [])
|
||||
|
||||
plan = self.tp_mappings[engine_id]
|
||||
remote_info = self.transfer_topo.get_engine_info(engine_id)
|
||||
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
|
||||
|
||||
dcp_active = self.dcp_size > 1 or remote_info.remote_dcp_size > 1
|
||||
local_block_ids = meta.local_physical_block_ids
|
||||
remote_region_groups = self.dst_region_group_ids[engine_id]
|
||||
local_region_groups = self.region_group_ids or remote_region_groups
|
||||
if not local_block_ids:
|
||||
# Region expansion cannot index empty groups. Pass empty specs to
|
||||
# _read_blocks so its existing cache-hit notification path runs.
|
||||
read_specs = [
|
||||
ReadSpec(remote_rank=rank, local_block_ids=[], remote_block_ids=[])
|
||||
for rank in plan.all_source_ranks
|
||||
]
|
||||
elif local_region_groups != remote_region_groups:
|
||||
if not self.use_mla or self._has_mamba:
|
||||
raise NotImplementedError(
|
||||
"Different NIXL cache-group layouts are only supported for "
|
||||
"pure MLA models"
|
||||
)
|
||||
if self.block_size != remote_info.remote_block_size:
|
||||
raise NotImplementedError(
|
||||
"Region-mapped NIXL transfers require matching physical block sizes"
|
||||
)
|
||||
remote_physical_block_ids = self._logical_to_kernel_block_ids(
|
||||
meta.remote.block_ids,
|
||||
remote_info.remote_physical_blocks_per_logical,
|
||||
)
|
||||
remote_by_region = self._block_ids_by_region(
|
||||
remote_physical_block_ids, remote_region_groups
|
||||
)
|
||||
local_by_region = self._block_ids_by_region(
|
||||
local_block_ids, local_region_groups
|
||||
)
|
||||
num_computed_blocks = None
|
||||
num_remote_blocks = None
|
||||
if (
|
||||
meta.remote.num_tokens is not None
|
||||
and meta.local_num_computed_blocks
|
||||
and all(group >= 0 for group in local_region_groups)
|
||||
and all(group >= 0 for group in remote_region_groups)
|
||||
):
|
||||
transfer_groups = self.kv_cache_config.transfer_group_ids
|
||||
num_computed_blocks = [
|
||||
meta.local_num_computed_blocks[transfer_groups[group]]
|
||||
* self._physical_blocks_per_logical_kv_block
|
||||
for group in local_region_groups
|
||||
]
|
||||
num_remote_blocks = cdiv(
|
||||
meta.remote.num_tokens, remote_info.remote_block_size
|
||||
)
|
||||
elif any(local_by_region) and (
|
||||
dcp_active
|
||||
or remote_info.remote_physical_blocks_per_logical
|
||||
!= self._physical_blocks_per_logical_kv_block
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Region-mapped pulls with DCP or different logical block sizes "
|
||||
"require remote_num_tokens, per-group prefix counts "
|
||||
"and unshared regions"
|
||||
)
|
||||
read_specs = [
|
||||
ReadSpec(
|
||||
rank,
|
||||
*self._apply_prefix_caching_by_region(
|
||||
local_by_region,
|
||||
remote_by_region,
|
||||
num_computed_blocks=num_computed_blocks,
|
||||
num_remote_blocks=num_remote_blocks,
|
||||
remote_rank=rank,
|
||||
remote_dcp_size=remote_info.remote_dcp_size,
|
||||
),
|
||||
block_ids_by_region=True,
|
||||
)
|
||||
for rank in plan.all_source_ranks
|
||||
]
|
||||
meta.region_blocks_to_zero = [
|
||||
list(blocks[sum(len(spec.local_block_ids[r]) for spec in read_specs) :])
|
||||
for r, blocks in enumerate(local_by_region)
|
||||
]
|
||||
else:
|
||||
remote_logical_block_ids = meta.remote.block_ids
|
||||
meta.remote.block_ids = self._logical_to_kernel_block_ids(
|
||||
remote_logical_block_ids,
|
||||
remote_info.remote_physical_blocks_per_logical,
|
||||
)
|
||||
num_groups = len(meta.local_block_ids)
|
||||
|
||||
def group_ids(block_ids: BlockIds, rank: int) -> list[list[int]]:
|
||||
return [
|
||||
list(block_ids[g]) if rank in plan.source_ranks_per_group[g] else []
|
||||
for g in range(num_groups)
|
||||
]
|
||||
|
||||
read_specs = []
|
||||
for rank in plan.all_source_ranks:
|
||||
if dcp_active:
|
||||
local_ids = group_ids(meta.local_block_ids, rank)
|
||||
remote_ids = group_ids(remote_logical_block_ids, rank)
|
||||
for g in range(num_groups):
|
||||
if not local_ids[g] or not _is_attention_spec(
|
||||
self._group_spec_types[g]
|
||||
):
|
||||
continue
|
||||
local_ids[g], remote_ids[g] = self._apply_dcp_prefix_caching(
|
||||
local_ids[g],
|
||||
remote_ids[g],
|
||||
remote_rank=rank,
|
||||
local_dcp_size=self.dcp_size,
|
||||
local_dcp_rank=self.dcp_rank,
|
||||
remote_dcp_size=remote_info.remote_dcp_size,
|
||||
local_num_computed_blocks=(
|
||||
meta.local_num_computed_blocks[g]
|
||||
),
|
||||
)
|
||||
local_physical_ids = self._logical_to_kernel_block_ids(
|
||||
local_ids, self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
remote_physical_ids = self._logical_to_kernel_block_ids(
|
||||
remote_ids,
|
||||
remote_info.remote_physical_blocks_per_logical,
|
||||
)
|
||||
else:
|
||||
local_physical_ids = group_ids(meta.local_physical_block_ids, rank)
|
||||
remote_physical_ids = group_ids(meta.remote.block_ids, rank)
|
||||
read_specs.append(
|
||||
ReadSpec(
|
||||
remote_rank=rank,
|
||||
local_block_ids=local_physical_ids,
|
||||
remote_block_ids=remote_physical_ids,
|
||||
)
|
||||
)
|
||||
|
||||
# D may have to perform multiple reads from different remote ranks.
|
||||
# Pure MLA reads once because its cache is replicated. Hybrid
|
||||
# MLA+SSM still needs one read per SSM source rank. With DCP, pure
|
||||
# MLA may also read from multiple ranks (disjoint token slices).
|
||||
if self.use_mla and tp_ratio < 0 and not self._has_mamba and not dcp_active:
|
||||
assert len(read_specs) == 1
|
||||
|
||||
for i, spec in enumerate(read_specs):
|
||||
remote_block_size = remote_info.remote_block_size
|
||||
logger.debug(
|
||||
"Remote agent %s available, calling _read_blocks"
|
||||
" on remote rank %s with remote block size %s for req %s",
|
||||
meta.remote.engine_id,
|
||||
spec.remote_rank,
|
||||
remote_block_size,
|
||||
req_id,
|
||||
)
|
||||
# Get side handles.
|
||||
if tp_ratio < 0 and (not self.use_mla or len(read_specs) > 1):
|
||||
# Remote tp_size > local tp_size: we must perform multiple
|
||||
# reads. Get the memory chunk onto which we will write to.
|
||||
split_key = (tp_ratio, remote_block_size)
|
||||
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i]
|
||||
local_dram_handle = (
|
||||
self._dram_src_handles_by_tp_ratio[split_key][i]
|
||||
if self._mixed_mem_types
|
||||
else None
|
||||
)
|
||||
else:
|
||||
# Single read from remote, we write to the whole memory region.
|
||||
# Also handle remote block size different from local block size.
|
||||
local_xfer_side_handle = self.src_xfer_handles_by_block_size[
|
||||
remote_block_size
|
||||
]
|
||||
local_dram_handle = (
|
||||
self._dram_src_handles_by_block_size[remote_block_size]
|
||||
if self._mixed_mem_types
|
||||
else None
|
||||
)
|
||||
|
||||
# Destination handle: remote_engine_id -> remote_rank -> handle.
|
||||
remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][
|
||||
spec.remote_rank
|
||||
]
|
||||
|
||||
# Once a read routes the request to failure reporting, the
|
||||
# scheduler may free and reuse its blocks, so no sibling READs
|
||||
# may be posted (and P must not be notified).
|
||||
if not self._read_blocks(
|
||||
read_spec=spec,
|
||||
request_id=req_id,
|
||||
dst_engine_id=meta.remote.engine_id,
|
||||
remote_request_id=meta.remote.request_id,
|
||||
local_xfer_side_handle=local_xfer_side_handle,
|
||||
local_dram_handle=local_dram_handle,
|
||||
remote_xfer_side_handle=remote_xfer_side_handle,
|
||||
expected_consumers=plan.local_consumers,
|
||||
awaiting_kvs=meta.awaiting_kvs,
|
||||
):
|
||||
return
|
||||
|
||||
if self.use_mla and tp_ratio < 0 and len(read_specs) == 1:
|
||||
# ..but we still need to notify the other remote ranks that we
|
||||
# have the blocks we need so they can update the request state.
|
||||
# Same thing for DCP (tp_size == dcp_size), so the raw tp_ratio already
|
||||
# reflects whether any remote replica is left unchosen.
|
||||
notif_id = f"{meta.remote.request_id}:{plan.local_consumers}".encode()
|
||||
remote_agents = self._remote_agents[meta.remote.engine_id]
|
||||
for rank_to_notify, agent in remote_agents.items():
|
||||
if rank_to_notify != (0, read_specs[0].remote_rank):
|
||||
try:
|
||||
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="notification_failed",
|
||||
msg="Remote rank will not update request state. "
|
||||
"This may indicate network issues.",
|
||||
req_id=req_id,
|
||||
error=e,
|
||||
dst_engine_id=meta.remote.engine_id,
|
||||
remote_rank=rank_to_notify[1],
|
||||
remote_agent_name=agent,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
|
||||
def _read_blocks(
|
||||
self,
|
||||
read_spec: ReadSpec,
|
||||
dst_engine_id: str,
|
||||
request_id: str,
|
||||
remote_request_id: str,
|
||||
local_xfer_side_handle: int,
|
||||
local_dram_handle: int | None,
|
||||
remote_xfer_side_handle: int,
|
||||
expected_consumers: int,
|
||||
awaiting_kvs: bool,
|
||||
) -> bool:
|
||||
"""Post a READ point-to-point xfer request from a single local worker to
|
||||
a single remote worker.
|
||||
|
||||
Returns True when the read was posted (or was unnecessary), False
|
||||
when the request was routed to failure reporting — the caller must
|
||||
not post further transfers for it.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
remote_rank = read_spec.remote_rank
|
||||
local_block_ids = read_spec.local_block_ids
|
||||
remote_block_ids = read_spec.remote_block_ids
|
||||
|
||||
remote_info = self.transfer_topo.get_engine_info(dst_engine_id)
|
||||
block_size_ratio = self.transfer_topo.block_size_ratio(
|
||||
remote_info.remote_block_size
|
||||
)
|
||||
if block_size_ratio > 1:
|
||||
if read_spec.block_ids_by_region:
|
||||
raise NotImplementedError(
|
||||
"Region-mapped NIXL transfers require matching physical block sizes"
|
||||
)
|
||||
local_block_ids, remote_block_ids = (
|
||||
self._map_block_ids_for_block_size_ratio(
|
||||
local_block_ids, remote_block_ids, block_size_ratio
|
||||
)
|
||||
)
|
||||
# NOTE(rob): having the staging blocks be on the READER side is
|
||||
# not going to work well (since we will have to call rearrange tensors).
|
||||
# after we detect the txn is complete (which means we cannot make the
|
||||
# read trxn async easily). If we want to make "READ" happen cleanly,
|
||||
# then we will need to have the staging blocks on the remote side.
|
||||
|
||||
# NOTE(rob): according to nvidia the staging blocks are used to
|
||||
# saturate IB with heterogeneous TP sizes.
|
||||
|
||||
# Number of local workers that will notify this producer worker.
|
||||
# Propagate on notification so dst worker can wait before freeing.
|
||||
notif_id = f"{remote_request_id}:{expected_consumers}".encode()
|
||||
|
||||
# Full prefix cache hit: do not need to read remote blocks,
|
||||
# just notify P worker that we have the blocks we need.
|
||||
if not any(len(group) > 0 for group in local_block_ids):
|
||||
# A full prefix cache hit is indicated with an empty list.
|
||||
agent_name = self._remote_agents[dst_engine_id][(0, remote_rank)]
|
||||
try:
|
||||
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id)
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="notification_failed",
|
||||
msg="P worker blocks will be freed after timeout. "
|
||||
"This may indicate network issues.",
|
||||
req_id=request_id,
|
||||
error=e,
|
||||
dst_engine_id=dst_engine_id,
|
||||
remote_rank=remote_rank,
|
||||
remote_agent_name=agent_name,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
# Report even on notification failure: the KV is already local, and
|
||||
# an unreported parked request would hold its blocks forever.
|
||||
# Notify-only recvs must stay unreported (scheduler asserts).
|
||||
if awaiting_kvs:
|
||||
self._recving_transfers.setdefault(request_id, [])
|
||||
return True
|
||||
|
||||
if not read_spec.block_ids_by_region:
|
||||
assert (
|
||||
len(remote_block_ids)
|
||||
== len(local_block_ids)
|
||||
== len(self.kv_cache_config.transfer_groups)
|
||||
)
|
||||
if not (self.dcp_size > 1 or remote_info.remote_dcp_size > 1):
|
||||
local_block_ids, remote_block_ids = self._apply_prefix_caching(
|
||||
decode_block_ids=local_block_ids,
|
||||
prefill_block_ids=remote_block_ids,
|
||||
decode_physical_per_logical=(
|
||||
self._physical_blocks_per_logical_kv_block
|
||||
),
|
||||
prefill_physical_per_logical=(
|
||||
remote_info.remote_physical_blocks_per_logical
|
||||
),
|
||||
)
|
||||
|
||||
# NOTE (nicolo) With homogeneous TP, each TP worker loads KV from
|
||||
# corresponding rank. With heterogeneous TP, fixing D>P, the D tp
|
||||
# workers will issue xfers to parts of the P worker remote kv caches.
|
||||
|
||||
# Get descs ids.
|
||||
remote_block_descs_ids = self._compute_desc_ids(
|
||||
block_ids=remote_block_ids,
|
||||
dst_num_blocks=self.dst_num_blocks[dst_engine_id],
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical,
|
||||
region_num_blocks=(self.dst_region_num_blocks.get(dst_engine_id) or None),
|
||||
region_group_ids=(
|
||||
list(range(self.num_regions))
|
||||
if read_spec.block_ids_by_region
|
||||
else (self.dst_region_group_ids.get(dst_engine_id) or None)
|
||||
),
|
||||
uses_region_group_mapping=(
|
||||
self.num_regions > 1
|
||||
if read_spec.block_ids_by_region
|
||||
else self.dst_uses_region_group_mapping[dst_engine_id]
|
||||
),
|
||||
)
|
||||
local_block_descs_ids = self._compute_desc_ids(
|
||||
block_ids=local_block_ids,
|
||||
dst_num_blocks=self.dst_num_blocks[self.engine_id],
|
||||
block_size_ratio=block_size_ratio,
|
||||
physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block,
|
||||
region_num_blocks=(self.dst_region_num_blocks.get(self.engine_id) or None),
|
||||
region_group_ids=(
|
||||
list(range(self.num_regions))
|
||||
if read_spec.block_ids_by_region
|
||||
else (self.region_group_ids or None)
|
||||
),
|
||||
uses_region_group_mapping=(
|
||||
self.num_regions > 1
|
||||
if read_spec.block_ids_by_region
|
||||
else self._uses_region_group_mapping
|
||||
),
|
||||
)
|
||||
|
||||
assert len(local_block_descs_ids) == len(remote_block_descs_ids)
|
||||
|
||||
# Prepare transfer with Nixl.
|
||||
handle = None
|
||||
try:
|
||||
if self._mixed_mem_types:
|
||||
self._read_blocks_mixed(
|
||||
request_id=request_id,
|
||||
local_block_size_key=remote_info.remote_block_size,
|
||||
local_device_handle=local_xfer_side_handle,
|
||||
local_dram_handle=local_dram_handle,
|
||||
remote_xfer_side_handle=remote_xfer_side_handle,
|
||||
local_block_descs_ids=local_block_descs_ids,
|
||||
remote_block_descs_ids=remote_block_descs_ids,
|
||||
notif_agent=self._remote_agents[dst_engine_id][(0, remote_rank)],
|
||||
notif_id=notif_id,
|
||||
)
|
||||
return True
|
||||
handle = self.nixl_wrapper.make_prepped_xfer(
|
||||
"READ",
|
||||
local_xfer_side_handle,
|
||||
local_block_descs_ids,
|
||||
remote_xfer_side_handle,
|
||||
remote_block_descs_ids,
|
||||
notif_msg=notif_id,
|
||||
)
|
||||
|
||||
# Begin async xfer.
|
||||
self.nixl_wrapper.transfer(handle)
|
||||
|
||||
# Use handle to check completion in future step().
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
return True
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="transfer_setup_failed",
|
||||
req_id=request_id,
|
||||
msg="Deferring failure reporting until outstanding transfers finish",
|
||||
error=e,
|
||||
dst_engine_id=dst_engine_id,
|
||||
remote_rank=remote_rank,
|
||||
)
|
||||
if not self._handle_failed_transfer(
|
||||
request_id, handle, self._recv_failures
|
||||
):
|
||||
assert handle is not None
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
return False
|
||||
|
||||
def _read_blocks_mixed(
|
||||
self,
|
||||
request_id: str,
|
||||
local_block_size_key: int,
|
||||
local_device_handle: int,
|
||||
local_dram_handle: int | None,
|
||||
remote_xfer_side_handle: int,
|
||||
local_block_descs_ids: np.ndarray,
|
||||
remote_block_descs_ids: np.ndarray,
|
||||
notif_agent: str,
|
||||
notif_id: bytes,
|
||||
) -> None:
|
||||
"""Split a READ across the local DRAM and device descriptor lists."""
|
||||
desc_is_dram = self._desc_is_dram_by_block_size[local_block_size_key]
|
||||
desc_pos = self._desc_pos_by_block_size[local_block_size_key]
|
||||
local_ids = np.asarray(local_block_descs_ids)
|
||||
remote_ids = np.asarray(remote_block_descs_ids)
|
||||
is_dram = desc_is_dram[local_ids]
|
||||
|
||||
if self._skip_dram_xfer:
|
||||
# TP rank 0 reads the DRAM (shared host pool) part for every
|
||||
# local rank; keep only the device part. Register the request
|
||||
# even if nothing is left, so it still completes and notifies.
|
||||
keep = ~is_dram
|
||||
local_ids = local_ids[keep]
|
||||
remote_ids = remote_ids[keep]
|
||||
is_dram = is_dram[keep]
|
||||
self._recving_transfers.setdefault(request_id, [])
|
||||
assert local_dram_handle is not None or not is_dram.any()
|
||||
reads = (
|
||||
(is_dram, local_dram_handle),
|
||||
(~is_dram, local_device_handle),
|
||||
)
|
||||
handles: list[int] = []
|
||||
try:
|
||||
for mask, local_handle in reads:
|
||||
if mask.any():
|
||||
handles.append(
|
||||
self.nixl_wrapper.make_prepped_xfer(
|
||||
"READ",
|
||||
local_handle,
|
||||
desc_pos[local_ids[mask]],
|
||||
remote_xfer_side_handle,
|
||||
remote_ids[mask],
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
for handle in handles:
|
||||
if not self._try_release_xfer_handle(request_id, handle):
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
raise
|
||||
|
||||
self._pending_recv_notifs.setdefault(request_id, []).append(
|
||||
(notif_agent, notif_id)
|
||||
)
|
||||
for i, handle in enumerate(handles):
|
||||
try:
|
||||
self.nixl_wrapper.transfer(handle)
|
||||
except Exception:
|
||||
for unstarted in handles[i:]:
|
||||
if not self._try_release_xfer_handle(request_id, unstarted):
|
||||
self._recving_transfers[request_id].append(unstarted)
|
||||
raise
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
|
||||
def _get_new_notifs(self) -> set[str]:
|
||||
"""Get req_ids which got a remote xfer message. When multiple consumers
|
||||
are reading from the same producer (heterogeneous TP or DCP
|
||||
scenario), wait for all consumers to be done pulling.
|
||||
|
||||
Also handles heartbeat notifications ("HB:req1,req2,...") by
|
||||
extending the lease on the referenced requests.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
notified_req_ids: set[str] = set()
|
||||
for notifs in self.nixl_wrapper.get_new_notifs().values():
|
||||
for notif in notifs:
|
||||
msg = notif.decode("utf-8")
|
||||
|
||||
# Handle heartbeat messages from D-side.
|
||||
if msg.startswith("HB:"):
|
||||
self._handle_heartbeat(msg[3:])
|
||||
continue
|
||||
|
||||
req_id, expected_consumers = msg.rsplit(":", 1)
|
||||
if (
|
||||
req_id not in self._reqs_to_send
|
||||
and req_id not in self._reqs_to_process
|
||||
):
|
||||
self.xfer_stats.record_notification_after_expiry()
|
||||
logger.error(
|
||||
"Potentially invalid KV blocks for "
|
||||
"unrecognized request %s were retrieved by "
|
||||
"a decode worker. They may have expired.",
|
||||
req_id,
|
||||
)
|
||||
continue
|
||||
|
||||
# Every reader of this req_id reports the same count (it's
|
||||
# derived from aggregate topology, not the specific rank),
|
||||
# so repeated notifications never disagree on it.
|
||||
self.expected_consumer_notifications_by_req[req_id] = int(
|
||||
expected_consumers
|
||||
)
|
||||
|
||||
self.consumer_notification_counts_by_req[req_id] += 1
|
||||
# Wait all consumers (D) to be done reading before freeing.
|
||||
if (
|
||||
self.consumer_notification_counts_by_req[req_id]
|
||||
== self.expected_consumer_notifications_by_req[req_id]
|
||||
):
|
||||
notified_req_ids.add(req_id)
|
||||
del self.consumer_notification_counts_by_req[req_id]
|
||||
del self.expected_consumer_notifications_by_req[req_id]
|
||||
self._reqs_to_process.remove(req_id)
|
||||
self._reqs_to_send.pop(req_id, None)
|
||||
return notified_req_ids
|
||||
@@ -1,357 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Push-specific scheduler-side logic for the NIXL connector.
|
||||
|
||||
In push mode, scheduler-side responsibilities are:
|
||||
|
||||
* D side (decode): on ``update_state_after_alloc``, stash registration data
|
||||
(D's identity + locally allocated block IDs) into
|
||||
``_push_pending_registrations``. The D worker drains it from
|
||||
``meta.push_registrations`` next step and sends a NIXL notification to the
|
||||
P worker (no scheduler-level networking).
|
||||
* P side (prefill): on ``request_finished``, stash the finished block IDs
|
||||
into ``_finished_request_blocks`` for the lease, and into
|
||||
``_newly_finished_push_blocks`` so the P worker picks them up via
|
||||
``meta.push_finished_blocks`` and matches against any D registrations
|
||||
it already received via NIXL notifications.
|
||||
* Both sides: ``has_pending_push_work`` keeps the engine main loop stepping
|
||||
while pushes are in flight. ``update_connector_output`` cleans up
|
||||
``_finished_request_blocks`` once the WRITE completes.
|
||||
|
||||
A soft per-registration watchdog on the D scheduler fails requests that have
|
||||
been registered but not fulfilled within a configurable timeout.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
|
||||
NixlBaseConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlConnectorMetadata,
|
||||
ReqId,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
"""Push-specific scheduler logic (WRITE-based KV transfer).
|
||||
|
||||
All P2P communication is deferred to the worker level via NIXL
|
||||
notifications. The scheduler communicates with workers only through
|
||||
the standard ``build_connector_meta`` / ``update_connector_output``
|
||||
hooks.
|
||||
"""
|
||||
|
||||
_TRANSFER_MODE: str = "push"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
engine_id: str,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
super().__init__(vllm_config, engine_id, kv_cache_config)
|
||||
if self.is_bidirectional_kv_xfer_enabled:
|
||||
raise NotImplementedError(
|
||||
"Bidirectional KV transfer is not supported for NIXL push connector."
|
||||
)
|
||||
|
||||
# D-side: registration data to pass to D workers via metadata on
|
||||
# the next ``build_connector_meta`` call.
|
||||
self._push_pending_registrations: dict[ReqId, dict[str, Any]] = {}
|
||||
|
||||
# D-side: track the wall-clock deadline for each registered request
|
||||
# to detect "registered but never fulfilled" failures (e.g. the P
|
||||
# node disappeared after registration). Keyed by D request_id.
|
||||
self._push_registration_deadlines: dict[ReqId, float] = {}
|
||||
|
||||
# P-side: block IDs for finished requests, kept for the lease and
|
||||
# used to drive ``has_pending_push_work``.
|
||||
self._finished_request_blocks: dict[ReqId, BlockIds] = {}
|
||||
# P-side: newly finished blocks to ship to P workers on next step.
|
||||
self._newly_finished_push_blocks: dict[ReqId, BlockIds] = {}
|
||||
|
||||
# Soft watchdog timeout (seconds) for D-side registrations that
|
||||
# never receive a push completion. Defaults to the existing
|
||||
# decoder KV blocks TTL so behaviour matches the lease.
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
self._push_registration_timeout: float = float(
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"push_registration_timeout",
|
||||
self.decoder_kv_blocks_ttl,
|
||||
)
|
||||
)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
) -> tuple[int, bool]:
|
||||
"""In push mode, D doesn't pull — it registers blocks and waits.
|
||||
|
||||
However, we still need to handle the do_remote_prefill case where D
|
||||
needs to know how many tokens will be pushed.
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NixlPushConnector get_num_new_matched_tokens: "
|
||||
"num_computed_tokens=%s, kv_transfer_params=%s",
|
||||
num_computed_tokens,
|
||||
params,
|
||||
)
|
||||
|
||||
if params is not None and params.get("do_remote_prefill"):
|
||||
token_ids = request.prompt_token_ids or []
|
||||
actual = self._get_remote_prefill_token_count(len(token_ids))
|
||||
count = actual - num_computed_tokens
|
||||
if count > 0:
|
||||
return count, True
|
||||
|
||||
return 0, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int
|
||||
):
|
||||
"""In push mode, D stores registration data for the worker to send
|
||||
to P via NIXL notification (deferred to ``build_connector_meta``).
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NixlPushConnector update_state_after_alloc: "
|
||||
"num_external_tokens=%s, kv_transfer_params=%s",
|
||||
num_external_tokens,
|
||||
params,
|
||||
)
|
||||
|
||||
if not params:
|
||||
return
|
||||
|
||||
# P side: track the request as in-batch so the lease accounting
|
||||
# matches what the worker expects on the next step.
|
||||
if params.get("do_remote_decode"):
|
||||
self._reqs_in_batch.add(request.request_id)
|
||||
|
||||
# P side with host-buffer offload: defer save to the worker.
|
||||
if self.use_host_buffer and params.get("do_remote_decode"):
|
||||
self._reqs_need_save[request.request_id] = request
|
||||
return
|
||||
|
||||
# D side: only act on the first call (``do_remote_prefill`` is
|
||||
# unset on re-entry by the marker below).
|
||||
if not params.get("do_remote_prefill"):
|
||||
return
|
||||
|
||||
if num_external_tokens <= 0:
|
||||
# Nothing to receive: full prefix-cache hit on D, no
|
||||
# registration to stage.
|
||||
return
|
||||
|
||||
# First-pass D path: stash registration data the worker will
|
||||
# ship to P on the next ``build_connector_meta`` cycle.
|
||||
logger.debug(
|
||||
"KV PUSH mode: D node storing registration for request %s",
|
||||
request.request_id,
|
||||
)
|
||||
local_block_ids: BlockIds = blocks.get_unhashed_block_ids_all_groups()
|
||||
local_block_ids = self.get_exchange_clipped_blocks(local_block_ids)
|
||||
|
||||
# ``remote_*`` fields are P's coordinates (from D's perspective).
|
||||
# ``decode_*`` fields are D's own info that P needs for the
|
||||
# reverse handshake before WRITE-ing.
|
||||
self._push_pending_registrations[request.request_id] = {
|
||||
"request_id": request.request_id,
|
||||
"decode_engine_id": self.engine_id,
|
||||
"decode_host": self.side_channel_host,
|
||||
"decode_port": self.side_channel_port,
|
||||
"decode_tp_size": (self.vllm_config.parallel_config.tensor_parallel_size),
|
||||
"local_block_ids": local_block_ids,
|
||||
"remote_engine_id": params["remote_engine_id"],
|
||||
"remote_host": params["remote_host"],
|
||||
"remote_port": params["remote_port"],
|
||||
"remote_tp_size": params["tp_size"],
|
||||
"remote_pp_size": params.get("pp_size", 1),
|
||||
}
|
||||
self._push_registration_deadlines[request.request_id] = (
|
||||
time.perf_counter() + self._push_registration_timeout
|
||||
)
|
||||
# In push mode D doesn't know P's blocks; P determines them
|
||||
# from the registration. We still track the request as
|
||||
# needing recv so the engine waits for P's WRITE completion.
|
||||
# ``remote_block_ids`` is also seeded to an empty tuple so the
|
||||
# base scheduler's ``add_new_req_to_recv`` can build the
|
||||
# ReqMeta without a KeyError — the actual remote block IDs are
|
||||
# learned by P over the NIXL handshake at WRITE time.
|
||||
params["remote_block_ids"] = ()
|
||||
self._reqs_need_recv[request.request_id] = (request, local_block_ids, (), False)
|
||||
|
||||
# Mark as processed so a re-entry (e.g. preemption + reschedule)
|
||||
# doesn't re-stage the registration.
|
||||
params["do_remote_prefill"] = False
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: BlockIds,
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Push-mode request_finished: stores blocks for workers."""
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NixlPushConnector request_finished(%s), request_status=%s, "
|
||||
"kv_transfer_params=%s",
|
||||
request.request_id,
|
||||
request.status,
|
||||
params,
|
||||
)
|
||||
if not params:
|
||||
return False, None
|
||||
|
||||
is_p_node = bool(params.get("do_remote_decode"))
|
||||
|
||||
self._stop_heartbeat(request.request_id)
|
||||
# Drop any pending registration deadline; the request either
|
||||
# completed or was cancelled.
|
||||
self._push_registration_deadlines.pop(request.request_id, None)
|
||||
|
||||
if params.get("do_remote_prefill"):
|
||||
# ``do_remote_prefill`` is still set, which means
|
||||
# ``update_state_after_alloc`` never ran (it would have
|
||||
# flipped this flag to False). The request was aborted
|
||||
# before it could be scheduled — e.g. rejected at the D
|
||||
# serving layer via abort_immediately. To keep P from
|
||||
# stranding the prefill blocks, we still register an empty
|
||||
# recv so the worker emits a notif that lets P free them.
|
||||
# Seed remote_block_ids so add_new_req_to_recv won't KeyError.
|
||||
params["remote_block_ids"] = ()
|
||||
self._reqs_need_recv[request.request_id] = (request, [], (), False)
|
||||
params["do_remote_prefill"] = False
|
||||
return False, None
|
||||
|
||||
# Push connector only acts on the P-side terminal path; D-side
|
||||
# finishing without a remote prefill is a no-op.
|
||||
if not is_p_node:
|
||||
return False, None
|
||||
|
||||
if request.status not in (
|
||||
RequestStatus.FINISHED_LENGTH_CAPPED,
|
||||
RequestStatus.FINISHED_STOPPED,
|
||||
):
|
||||
self._reqs_not_processed.add(request.request_id)
|
||||
self._reqs_need_save.pop(request.request_id, None)
|
||||
return False, None
|
||||
|
||||
delay_free_blocks = any(len(group) > 0 for group in block_ids)
|
||||
remote_num_tokens = 0
|
||||
if delay_free_blocks:
|
||||
logger.debug(
|
||||
"NixlPushConnector request_finished(%s) waiting for %d seconds "
|
||||
"before releasing blocks",
|
||||
request.request_id,
|
||||
self._kv_lease_duration,
|
||||
)
|
||||
self._reqs_need_send[request.request_id] = (
|
||||
time.perf_counter() + self._kv_lease_duration
|
||||
)
|
||||
|
||||
block_ids = self.get_exchange_clipped_blocks(block_ids)
|
||||
remote_num_tokens = request.num_computed_tokens
|
||||
|
||||
# Store finished blocks for worker-level matching with D
|
||||
# registrations (via NIXL notifications).
|
||||
self._finished_request_blocks[request.request_id] = block_ids
|
||||
self._newly_finished_push_blocks[request.request_id] = block_ids
|
||||
|
||||
return delay_free_blocks, dict(
|
||||
do_remote_prefill=True,
|
||||
do_remote_decode=False,
|
||||
remote_block_ids=block_ids,
|
||||
remote_engine_id=self.engine_id,
|
||||
remote_request_id=request.request_id,
|
||||
remote_host=self.side_channel_host,
|
||||
remote_port=self.side_channel_port,
|
||||
tp_size=self.transfer_tp_size,
|
||||
dcp_size=self.vllm_config.parallel_config.decode_context_parallel_size,
|
||||
pp_size=self.vllm_config.parallel_config.pipeline_parallel_size,
|
||||
remote_num_tokens=remote_num_tokens,
|
||||
transfer_mode=self._TRANSFER_MODE,
|
||||
)
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
meta = super().build_connector_meta(scheduler_output)
|
||||
assert isinstance(meta, NixlConnectorMetadata)
|
||||
|
||||
# Watchdog: any D-side registration whose deadline has passed without
|
||||
# a corresponding push completion is treated as failed and cleaned up.
|
||||
# The corresponding request is already tracked via _reqs_need_recv;
|
||||
# the engine layer will eventually time it out via the lease, but we
|
||||
# at least drop the stale registration so we don't keep retrying.
|
||||
now = time.perf_counter()
|
||||
# Deadlines are inserted in non-decreasing order (monotonic clock +
|
||||
# constant timeout, armed once per request), and dict insertion order
|
||||
# is preserved across key deletions, so we can stop at the first
|
||||
# not-yet-expired entry instead of scanning the whole dict.
|
||||
expired = []
|
||||
for rid, deadline in self._push_registration_deadlines.items():
|
||||
if deadline > now:
|
||||
break
|
||||
expired.append(rid)
|
||||
for rid in expired:
|
||||
self._push_registration_deadlines.pop(rid, None)
|
||||
# Avoid resending a registration that already timed out.
|
||||
self._push_pending_registrations.pop(rid, None)
|
||||
logger.warning(
|
||||
"NixlPushConnector: registration for request %s timed out "
|
||||
"after %.1fs without a push completion",
|
||||
rid,
|
||||
self._push_registration_timeout,
|
||||
)
|
||||
|
||||
# D side: package pending registrations for D workers to send out.
|
||||
if self._push_pending_registrations:
|
||||
meta.push_registrations = dict(self._push_pending_registrations)
|
||||
self._push_pending_registrations.clear()
|
||||
|
||||
# P side: package newly finished blocks for P workers to match against
|
||||
# any D registrations they have received via NIXL notifications.
|
||||
if self._newly_finished_push_blocks:
|
||||
meta.push_finished_blocks = dict(self._newly_finished_push_blocks)
|
||||
self._newly_finished_push_blocks.clear()
|
||||
|
||||
return meta
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
# Keep the engine main loop alive while we have:
|
||||
# - finished P blocks awaiting WRITE completion, or
|
||||
# - pending D registrations the worker has not yet shipped, or
|
||||
# - newly finished blocks not yet shipped to P workers.
|
||||
return bool(self._finished_request_blocks or self._push_pending_registrations)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Clean up finished request blocks after push completes."""
|
||||
super().update_connector_output(connector_output)
|
||||
for req_id in connector_output.finished_sending or ():
|
||||
self._finished_request_blocks.pop(req_id, None)
|
||||
# On D side, finished_recving means the push completed; clear the
|
||||
# watchdog so we don't trip an expiration on a fulfilled request.
|
||||
for req_id in connector_output.finished_recving or ():
|
||||
self._push_registration_deadlines.pop(req_id, None)
|
||||
@@ -1,827 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Push-specific (WRITE) worker-side logic for the NIXL connector.
|
||||
|
||||
A dedicated ``nixl-push-writer`` thread owns all push-related NIXL ops:
|
||||
calls ``get_new_notifs`` (routing PUSH_REG internally; HB / completion
|
||||
notifs are forwarded to the engine main thread), sends PUSH_REG via
|
||||
``send_notif``, matches D registrations with P finished blocks, and
|
||||
issues WRITE transfers via ``make_prepped_xfer`` / ``transfer``.
|
||||
|
||||
The engine main thread feeds the writer through queues:
|
||||
``_reg_send_inbox`` (D-side regs to send), ``_finished_blocks_inbox``
|
||||
(P-side blocks from metadata) and ``_pending_completion_notifs``
|
||||
(non-PUSH_REG notifs forwarded back for HB / completion accounting).
|
||||
The handshake-completion callback feeds ``_deferred_push_inbox`` with
|
||||
matched pushes whose P→D handshake has finished so the writer can
|
||||
(re-)issue the WRITE without ever blocking on the network.
|
||||
|
||||
Wake model: the writer self-polls every
|
||||
``_PUSH_WRITER_POLL_INTERVAL_MS`` only while it has unmatched
|
||||
``_push_finished_blocks`` (i.e. P-side blocks waiting for a D PUSH_REG
|
||||
notif that has no other wake source). All other progress is
|
||||
event-driven: the engine main thread sets ``_push_writer_wake`` from
|
||||
``start_load_kv`` (when handing it new work) and from ``get_finished``
|
||||
(so each engine step gives the writer a chance to drain NIXL notifs);
|
||||
the handshake-completion callback sets the same event after a deferred
|
||||
PUSH_REG send or a deferred push WRITE has been queued. When a request's
|
||||
lease expires (the base worker reports it via ``done_sending``) or the WRITE completes,
|
||||
``get_finished`` enqueues an eviction onto ``_evict_finished_inbox`` so
|
||||
the writer drops any leftover ``_push_finished_blocks`` /
|
||||
``_pending_d_registrations`` and stops self-polling.
|
||||
"""
|
||||
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import Future
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import msgspec
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
|
||||
NixlBaseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
PUSH_REG_NOTIF_PREFIX,
|
||||
NixlConnectorMetadata,
|
||||
RemoteMeta,
|
||||
ReqId,
|
||||
ReqMeta,
|
||||
TransferHandle,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
ReadSpec,
|
||||
_is_attention_spec,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import get_base_request_id
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Writer-thread poll cadence while there is in-flight push state. When
|
||||
# fully idle, the writer blocks on a wake event signalled by the engine
|
||||
# main thread (start_load_kv / get_finished). Smaller -> lower latency
|
||||
# while active, slightly more CPU.
|
||||
_PUSH_WRITER_POLL_INTERVAL_MS = 1.0
|
||||
|
||||
|
||||
class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
"""Push-specific (WRITE) worker logic. See module docstring."""
|
||||
|
||||
# Distinguishes push from pull in the NIXL compatibility hash.
|
||||
_TRANSFER_MODE: str = "push"
|
||||
|
||||
_supports_pp_hma = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
engine_id: str,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, engine_id, kv_cache_config)
|
||||
|
||||
# Heartbeat handshakes to a PP-sharded producer must be notif-only,
|
||||
# like the PUSH_REG path.
|
||||
self._hb_handshake_notif_only = True
|
||||
|
||||
# Push-specific state.
|
||||
# P-side: outgoing WRITE handles awaiting completion, keyed by
|
||||
# request_id. Mutated by writer (submit) and main thread
|
||||
# (``_pop_done_transfers``); guarded by
|
||||
# ``_sending_transfers_lock``.
|
||||
self._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list)
|
||||
self._send_failures: set[ReqId] = set()
|
||||
self._sending_transfers_lock = threading.Lock()
|
||||
|
||||
# Writer-thread owned matching state.
|
||||
# P-side: finished request blocks received from scheduler metadata
|
||||
# that have not yet been matched with an incoming D registration.
|
||||
self._push_finished_blocks: dict[ReqId, BlockIds] = {}
|
||||
# P-side: D registrations received via NIXL notification that have
|
||||
# not yet been matched with a finished P request.
|
||||
self._pending_d_registrations: dict[ReqId, dict[str, Any]] = {}
|
||||
|
||||
# Cross-thread channels.
|
||||
self._reg_send_inbox: queue.Queue[tuple[str, dict[str, Any]]] = queue.Queue()
|
||||
self._finished_blocks_inbox: queue.Queue[tuple[str, BlockIds]] = queue.Queue()
|
||||
self._pending_completion_notifs: queue.Queue[bytes] = queue.Queue()
|
||||
# Main thread → writer: req_ids whose lease has expired or whose
|
||||
# WRITE has completed. Writer drops them from
|
||||
# ``_push_finished_blocks`` so an unmatched entry doesn't keep the
|
||||
# writer busy-polling forever.
|
||||
self._evict_finished_inbox: queue.Queue[str] = queue.Queue()
|
||||
# Handshakes that have just completed and are ready for the WRITE on wthread
|
||||
self._deferred_push_inbox = queue.Queue[tuple[str, BlockIds, dict[str, Any]]]()
|
||||
|
||||
# Wake signal from engine main thread (start_load_kv / get_finished).
|
||||
# Writer self-polls at _PUSH_WRITER_POLL_INTERVAL_MS while it has
|
||||
# active in-flight state; otherwise it blocks until signalled.
|
||||
self._push_writer_wake = threading.Event()
|
||||
|
||||
self._push_writer_stop = threading.Event()
|
||||
self._push_writer_thread: threading.Thread | None = None
|
||||
|
||||
# --- Lifecycle ----------------------------------------------------- #
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, "torch.Tensor"]):
|
||||
super().register_kv_caches(kv_caches)
|
||||
if self._mixed_mem_types:
|
||||
raise NotImplementedError(
|
||||
"NixlPushConnector does not support mixed-memory KV caches"
|
||||
)
|
||||
if self._push_writer_thread is None:
|
||||
self._push_writer_thread = threading.Thread(
|
||||
target=self._push_writer_loop,
|
||||
daemon=True,
|
||||
name="nixl-push-writer",
|
||||
)
|
||||
self._push_writer_thread.start()
|
||||
logger.info("nixl-push-writer thread started (rank=%d)", self.tp_rank)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._push_writer_stop.set()
|
||||
# Unblock the writer if it's waiting in the no-active-state branch.
|
||||
self._push_writer_wake.set()
|
||||
if self._push_writer_thread is not None:
|
||||
self._push_writer_thread.join(timeout=2)
|
||||
self._push_writer_thread = None
|
||||
with self._sending_transfers_lock:
|
||||
for handles in self._sending_transfers.values():
|
||||
for handle in handles:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self._sending_transfers.clear()
|
||||
self._send_failures.clear()
|
||||
super().shutdown()
|
||||
|
||||
# --- Engine-main-thread entry point -------------------------------- #
|
||||
|
||||
def start_load_kv(self, metadata: NixlConnectorMetadata):
|
||||
"""Pre-process metadata; defer NIXL ops to the writer thread."""
|
||||
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
|
||||
return
|
||||
|
||||
# D-side: track reqs waiting for P to push.
|
||||
for req_id, meta in metadata.reqs_to_recv.items():
|
||||
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
|
||||
meta.local_block_ids, self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
assert meta.remote is not None
|
||||
remote_engine_id = meta.remote.engine_id
|
||||
# Update last activity from this remote (same as pull), but only for
|
||||
# an already-connected engine so we never leave an _engine_last_active
|
||||
# entry without a _remote_agents entry (breaks _cleanup_remote_engine).
|
||||
if remote_engine_id in self._remote_agents:
|
||||
self._engine_last_active[remote_engine_id] = time.perf_counter()
|
||||
|
||||
logger.debug(
|
||||
"start_load_kv (push) for request %s from remote engine %s. "
|
||||
"Num local_block_ids: %s. Num remote_block_ids: %s. ",
|
||||
req_id,
|
||||
remote_engine_id,
|
||||
len(meta.local_physical_block_ids),
|
||||
len(meta.remote.block_ids),
|
||||
)
|
||||
self._recving_metadata[req_id] = meta
|
||||
|
||||
# --- D-side: registrations to send to P via NIXL ---
|
||||
if metadata.push_registrations:
|
||||
for req_id, reg_data in metadata.push_registrations.items():
|
||||
self._reg_send_inbox.put((req_id, reg_data))
|
||||
self._push_writer_wake.set()
|
||||
|
||||
# --- P-side: newly finished blocks awaiting a D registration match ---
|
||||
if metadata.push_finished_blocks:
|
||||
for req_id, block_ids in metadata.push_finished_blocks.items():
|
||||
self._finished_blocks_inbox.put((req_id, block_ids))
|
||||
self._push_writer_wake.set()
|
||||
|
||||
# Batch + lease tracking (same as pull).
|
||||
for req_id in metadata.reqs_in_batch:
|
||||
self._reqs_to_process.add(req_id)
|
||||
for req_id in metadata.reqs_not_processed:
|
||||
self._reqs_to_process.discard(req_id)
|
||||
assert req_id not in self._reqs_to_send
|
||||
# Rebase scheduler-clock deadlines onto this worker's clock — see the
|
||||
# equivalent block in pull_worker.start_load_kv for the rationale.
|
||||
now_local = time.perf_counter()
|
||||
for req_id, expiration_time in metadata.reqs_to_send.items():
|
||||
if req_id in self._reqs_to_process:
|
||||
if metadata.scheduler_clock:
|
||||
expiration_time = now_local + (
|
||||
expiration_time - metadata.scheduler_clock
|
||||
)
|
||||
self._reqs_to_send[req_id] = expiration_time
|
||||
|
||||
# Heartbeats still leave from the main thread (base worker behaviour).
|
||||
self._send_heartbeats(metadata)
|
||||
|
||||
# --- Writer thread ------------------------------------------------- #
|
||||
|
||||
def _push_writer_loop(self) -> None:
|
||||
sleep_s = _PUSH_WRITER_POLL_INTERVAL_MS / 1000.0
|
||||
|
||||
while not self._push_writer_stop.is_set():
|
||||
try:
|
||||
# 1. D registrations to send.
|
||||
while True:
|
||||
try:
|
||||
rid, rd = self._reg_send_inbox.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self._send_registration_to_p(rid, rd)
|
||||
|
||||
# 2. Deferred P→D pushes whose handshake just completed; do xfer now
|
||||
while True:
|
||||
try:
|
||||
rid, blocks, rd = self._deferred_push_inbox.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self._do_start_push_kv(rid, blocks, rd)
|
||||
|
||||
# 3. P-side finished blocks; match against pending regs.
|
||||
while True:
|
||||
try:
|
||||
rid, blocks = self._finished_blocks_inbox.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
matched = self._pop_matching_registration(rid)
|
||||
if matched is not None:
|
||||
self._do_start_push_kv(rid, blocks, matched)
|
||||
else:
|
||||
self._push_finished_blocks[rid] = blocks
|
||||
|
||||
# 3b. Evict finished blocks for requests that have either
|
||||
# completed (WRITE acknowledged) or whose lease expired
|
||||
# without a D registration. Drop pending registrations
|
||||
# for the same reason so we don't leak state.
|
||||
while True:
|
||||
try:
|
||||
rid = self._evict_finished_inbox.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self._push_finished_blocks.pop(rid, None)
|
||||
self._pending_d_registrations.pop(rid, None)
|
||||
|
||||
# 4. NIXL notifs: route PUSH_REG; forward the rest.
|
||||
for notifs in self.nixl_wrapper.get_new_notifs().values():
|
||||
for notif in notifs:
|
||||
if notif.startswith(PUSH_REG_NOTIF_PREFIX):
|
||||
self._handle_push_reg_notif(notif)
|
||||
else:
|
||||
self._pending_completion_notifs.put(notif)
|
||||
except Exception:
|
||||
logger.exception("nixl-push-writer error; continuing")
|
||||
|
||||
# Self-poll only while there is no other wake source: P-side
|
||||
# finished blocks waiting for a D PUSH_REG match. All other
|
||||
# progress is event-driven (see module docstring).
|
||||
if self._push_finished_blocks:
|
||||
self._push_writer_stop.wait(timeout=sleep_s)
|
||||
else:
|
||||
self._push_writer_wake.wait()
|
||||
self._push_writer_wake.clear()
|
||||
|
||||
def _handle_push_reg_notif(self, notif: bytes) -> None:
|
||||
try:
|
||||
reg_data = msgspec.msgpack.decode(notif[len(PUSH_REG_NOTIF_PREFIX) :])
|
||||
except Exception:
|
||||
logger.exception("Failed to decode PUSH_REG notification payload")
|
||||
return
|
||||
rid = reg_data.get("request_id") if isinstance(reg_data, dict) else None
|
||||
if not isinstance(rid, str):
|
||||
logger.warning("PUSH_REG notif missing request_id; dropping")
|
||||
return
|
||||
|
||||
match = self._pop_matching_finished_blocks(rid)
|
||||
if match is not None:
|
||||
fin_id, blocks = match
|
||||
self._do_start_push_kv(fin_id, blocks, reg_data)
|
||||
else:
|
||||
self._pending_d_registrations[rid] = reg_data
|
||||
|
||||
# --- D-side registration send (writer thread) ---------------------- #
|
||||
|
||||
def _send_registration_to_p(
|
||||
self,
|
||||
req_id: str,
|
||||
reg_data: dict[str, Any],
|
||||
) -> None:
|
||||
"""Handshake (if needed) then send PUSH_REG. ``send_notif`` always
|
||||
executes on the writer; the handshake runs on the background executor
|
||||
and the request is re-queued onto ``_reg_send_inbox`` once it
|
||||
completes (at which point ``_ensure_handshake`` returns ``None`` and we
|
||||
send directly)."""
|
||||
remote_pp_size = reg_data.get("remote_pp_size", 1)
|
||||
fut = self._ensure_handshake(
|
||||
reg_data["remote_engine_id"],
|
||||
reg_data["remote_host"],
|
||||
reg_data["remote_port"],
|
||||
reg_data["remote_tp_size"],
|
||||
pp_size=remote_pp_size,
|
||||
# D only ever sends PUSH_REG notifs to P and never reads or writes
|
||||
# P's memory in push mode, so it never needs the transfer
|
||||
# descriptors set up by the full add_remote_agent path.
|
||||
notif_agents_only=True,
|
||||
)
|
||||
if fut is None:
|
||||
self._do_send_reg_notif(req_id, reg_data)
|
||||
return
|
||||
|
||||
def _on_handshake(
|
||||
f: Future[tuple[dict[tuple[int, int], str], float]],
|
||||
rid: str = req_id,
|
||||
rd: dict[str, Any] = reg_data,
|
||||
) -> None:
|
||||
try:
|
||||
f.result()
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="push_reg_handshake_failed", req_id=rid, error=e
|
||||
)
|
||||
self._failed_recv_reqs.put(rid)
|
||||
return
|
||||
# Re-queue for the writer to send now that the handshake is done.
|
||||
self._reg_send_inbox.put((rid, rd))
|
||||
# Wake the writer so it sends the PUSH_REG promptly even if
|
||||
# otherwise parked.
|
||||
self._push_writer_wake.set()
|
||||
|
||||
fut.add_done_callback(_on_handshake)
|
||||
|
||||
def _do_send_reg_notif(self, req_id: str, reg_data: dict[str, Any]) -> None:
|
||||
engine_id = reg_data["remote_engine_id"]
|
||||
notif_msg = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(reg_data)
|
||||
# _remote_agents is mutated on other threads; snapshot under the lock.
|
||||
with self._handshake_lock:
|
||||
agents = dict(self._remote_agents.get(engine_id) or {})
|
||||
if not agents:
|
||||
logger.error(
|
||||
"No remote agents for engine %s; cannot send registration for %s",
|
||||
engine_id,
|
||||
req_id,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
return
|
||||
for rank, agent_name in agents.items():
|
||||
try:
|
||||
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_msg)
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="push_reg_notif_failed",
|
||||
req_id=req_id,
|
||||
error=e,
|
||||
remote_rank=rank,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
# Earlier registrations may still trigger WRITEs into D's blocks.
|
||||
# Keep the receive pending until those writes are finished.
|
||||
logger.debug(
|
||||
"Sent PUSH_REG for %s to engine %s (%dB)", req_id, engine_id, len(notif_msg)
|
||||
)
|
||||
|
||||
# --- Matching helpers --------------------------------------------- #
|
||||
|
||||
def _pop_matching_registration(self, request_id: str) -> dict[str, Any] | None:
|
||||
"""Pop the D-side registration matching *request_id*.
|
||||
|
||||
Exact key first, then a match after stripping the random suffix from
|
||||
both sides. No match leaves the request unmatched (push not started).
|
||||
"""
|
||||
data = self._pending_d_registrations.pop(request_id, None)
|
||||
if data is not None:
|
||||
return data
|
||||
base_id = get_base_request_id(request_id)
|
||||
for reg_id in list(self._pending_d_registrations):
|
||||
if get_base_request_id(reg_id) == base_id:
|
||||
return self._pending_d_registrations.pop(reg_id)
|
||||
return None
|
||||
|
||||
def _pop_matching_finished_blocks(
|
||||
self, request_id: str
|
||||
) -> tuple[str, BlockIds] | None:
|
||||
"""Pop the P-side finished blocks matching *request_id*.
|
||||
|
||||
Same lookup as ``_pop_matching_registration``: exact key, then a
|
||||
match after stripping the random suffix from both sides.
|
||||
"""
|
||||
blocks = self._push_finished_blocks.pop(request_id, None)
|
||||
if blocks is not None:
|
||||
return request_id, blocks
|
||||
base_id = get_base_request_id(request_id)
|
||||
for fin_id in list(self._push_finished_blocks):
|
||||
if get_base_request_id(fin_id) == base_id:
|
||||
return fin_id, self._push_finished_blocks.pop(fin_id)
|
||||
return None
|
||||
|
||||
# --- WRITE transfer logic (writer thread) ------------------------- #
|
||||
|
||||
def _do_start_push_kv(
|
||||
self,
|
||||
request_id: str,
|
||||
local_block_ids: BlockIds,
|
||||
registration_data: dict[str, Any],
|
||||
) -> None:
|
||||
"""Start push-based KV transfer from P worker to D node.
|
||||
|
||||
The P→D handshake runs on the base worker's background executor.
|
||||
If it isn't ready yet we register a completion callback, defer the
|
||||
WRITE, and re-drive this request via ``_deferred_push_inbox`` once
|
||||
the handshake resolves -- so the writer thread never blocks on the
|
||||
network (mirrors ``_send_registration_to_p``).
|
||||
"""
|
||||
if not local_block_ids:
|
||||
logger.warning("No local blocks to push for request %s", request_id)
|
||||
return
|
||||
|
||||
# ``local_block_ids`` are P's logical block IDs; ``remote_block_ids``
|
||||
# (D's, from the PUSH_REG notif) are also logical.
|
||||
decode_engine_id = registration_data["decode_engine_id"]
|
||||
remote_block_ids = registration_data["local_block_ids"]
|
||||
decode_request_id = registration_data["request_id"]
|
||||
|
||||
# Runs on the background executor; defer the WRITE until it's ready.
|
||||
fut = self._ensure_handshake(
|
||||
decode_engine_id,
|
||||
registration_data["decode_host"],
|
||||
registration_data["decode_port"],
|
||||
registration_data["decode_tp_size"],
|
||||
)
|
||||
if fut is not None:
|
||||
|
||||
def _on_handshake(
|
||||
f: Future[tuple[dict[tuple[int, int], str], float]],
|
||||
rid: str = request_id,
|
||||
blocks: BlockIds = local_block_ids,
|
||||
rd: dict[str, Any] = registration_data,
|
||||
) -> None:
|
||||
if (e := f.exception()) is not None:
|
||||
# The engine reclaims the blocks via the TTL so we dont free here
|
||||
self._log_failure(
|
||||
failure_type="push_handshake_failed", req_id=rid, error=e
|
||||
)
|
||||
return
|
||||
self._deferred_push_inbox.put((rid, blocks, rd))
|
||||
self._push_writer_wake.set()
|
||||
|
||||
fut.add_done_callback(_on_handshake)
|
||||
return
|
||||
# Keep the engine alive while it is actively receiving pushes, mirroring
|
||||
# how pull-mode transfers touch _engine_last_active in start_load_kv.
|
||||
self._engine_last_active[decode_engine_id] = time.perf_counter()
|
||||
|
||||
logical_local = self._as_grouped_block_ids(local_block_ids)
|
||||
logical_remote = self._as_grouped_block_ids(remote_block_ids)
|
||||
physical_local = self._logical_to_kernel_block_ids(
|
||||
logical_local, self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
|
||||
push_meta = ReqMeta(
|
||||
local_block_ids=logical_local,
|
||||
local_physical_block_ids=physical_local,
|
||||
tp_size=self.world_size,
|
||||
remote=RemoteMeta(
|
||||
block_ids=logical_remote,
|
||||
host="",
|
||||
port=0,
|
||||
engine_id=decode_engine_id,
|
||||
request_id=decode_request_id,
|
||||
),
|
||||
)
|
||||
|
||||
t0 = time.perf_counter()
|
||||
self._xfer_blocks_for_req(req_id=request_id, meta=push_meta)
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000.0
|
||||
if elapsed_ms > 200.0:
|
||||
logger.warning(
|
||||
"_do_start_push_kv for %s took %.1fms (slow NIXL submission)",
|
||||
request_id,
|
||||
elapsed_ms,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _as_grouped_block_ids(block_ids: BlockIds) -> BlockIds:
|
||||
"""Normalise a sequence of block IDs to a tuple-of-groups shape.
|
||||
|
||||
``BlockIds`` is canonically a tuple of per-group lists, but some
|
||||
registration payloads collapse a single-group case to a flat
|
||||
list. Re-wrap that case so downstream group-aware helpers see a
|
||||
consistent shape."""
|
||||
if block_ids and not isinstance(block_ids[0], (list, tuple)):
|
||||
return (list(block_ids),)
|
||||
return block_ids
|
||||
|
||||
def _xfer_blocks_for_req(self, req_id: str, meta: ReqMeta):
|
||||
"""Issue WRITE transfers to one or more remote TP ranks."""
|
||||
assert meta.remote is not None and self.transfer_topo is not None
|
||||
engine_id = meta.remote.engine_id
|
||||
plan = self.tp_mappings[engine_id]
|
||||
remote_info = self.transfer_topo.get_engine_info(engine_id)
|
||||
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
|
||||
|
||||
# Expand D's logical IDs using the ratio learned during the
|
||||
# NIXL handshake. ``meta`` is freshly built by
|
||||
# ``_do_start_push_kv`` so mutating it here is safe.
|
||||
meta.remote.block_ids = self._logical_to_kernel_block_ids(
|
||||
meta.remote.block_ids,
|
||||
remote_info.remote_physical_blocks_per_logical,
|
||||
)
|
||||
remote_block_ids = meta.remote.block_ids
|
||||
local_block_ids = meta.local_physical_block_ids
|
||||
local_region_groups = self.region_group_ids
|
||||
remote_region_groups = self.dst_region_group_ids[engine_id]
|
||||
groups_differ = local_region_groups != remote_region_groups
|
||||
if groups_differ and not self._transfer_layer_group_ids:
|
||||
raise NotImplementedError(
|
||||
"NixlPushConnector does not support different producer and "
|
||||
"consumer cache-group layouts"
|
||||
)
|
||||
|
||||
# MLA latent is replicated across D's TP ranks: the tp-mapping
|
||||
# collapses it to one rank (fine for reads), but push must WRITE every
|
||||
# D rank or the rest decode stale KV. For hybrid MLA+SSM the sharded
|
||||
# SSM state already targets every covered D rank, so only the
|
||||
# attention groups need widening; pure MLA writes to all handshaked
|
||||
# ranks (only the dst differs per rank).
|
||||
replicate_attn = self.use_mla and tp_ratio < 0
|
||||
if replicate_attn and not self._has_mamba:
|
||||
assert len(plan.all_source_ranks) == 1
|
||||
write_ranks = sorted(self.dst_xfer_side_handles[engine_id])
|
||||
else:
|
||||
write_ranks = list(plan.all_source_ranks)
|
||||
|
||||
num_groups = len(local_block_ids)
|
||||
|
||||
def group_ids(block_ids: BlockIds, rank: int) -> BlockIds:
|
||||
return [
|
||||
list(block_ids[g])
|
||||
if (self._is_csa_linear and tp_ratio < 0)
|
||||
or (replicate_attn and _is_attention_spec(self._group_spec_types[g]))
|
||||
or rank in plan.source_ranks_per_group[g]
|
||||
else []
|
||||
for g in range(num_groups)
|
||||
]
|
||||
|
||||
read_specs = [
|
||||
ReadSpec(
|
||||
remote_rank=rank,
|
||||
local_block_ids=group_ids(local_block_ids, rank),
|
||||
remote_block_ids=group_ids(remote_block_ids, rank),
|
||||
)
|
||||
for rank in write_ranks
|
||||
]
|
||||
|
||||
handles: list[int] = []
|
||||
for i, spec in enumerate(read_specs):
|
||||
remote_block_size = remote_info.remote_block_size
|
||||
logger.debug(
|
||||
"Remote agent %s available, calling _xfer_blocks"
|
||||
" on remote rank %s with remote block size %s for req %s",
|
||||
meta.remote.engine_id,
|
||||
spec.remote_rank,
|
||||
remote_block_size,
|
||||
req_id,
|
||||
)
|
||||
if tp_ratio < 0 and (not self.use_mla or len(plan.all_source_ranks) > 1):
|
||||
# Multiple targets: write each rank its chunk of local memory.
|
||||
# Hybrid MLA+SSM also lands here: its split handles replicate
|
||||
# the attention descriptors and chunk only the SSM state.
|
||||
split_key = (tp_ratio, remote_block_size)
|
||||
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i]
|
||||
else:
|
||||
local_xfer_side_handle = self.src_xfer_handles_by_block_size[
|
||||
remote_block_size
|
||||
]
|
||||
|
||||
remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][
|
||||
spec.remote_rank
|
||||
]
|
||||
|
||||
handle = self._xfer_blocks(
|
||||
read_spec=spec,
|
||||
request_id=req_id,
|
||||
dst_engine_id=meta.remote.engine_id,
|
||||
remote_request_id=meta.remote.request_id,
|
||||
local_xfer_side_handle=local_xfer_side_handle,
|
||||
remote_xfer_side_handle=remote_xfer_side_handle,
|
||||
)
|
||||
if handle is not None:
|
||||
handles.append(handle)
|
||||
|
||||
# Publish all the request's WRITE handles in one locked update: a
|
||||
# partial set would let ``_pop_done_transfers`` finish the request
|
||||
# early, then double-report it as the remaining writes land.
|
||||
if handles:
|
||||
with self._sending_transfers_lock:
|
||||
self._sending_transfers[req_id].extend(handles)
|
||||
|
||||
def _xfer_blocks(
|
||||
self,
|
||||
read_spec: ReadSpec,
|
||||
dst_engine_id: str,
|
||||
request_id: str,
|
||||
remote_request_id: str,
|
||||
local_xfer_side_handle: int,
|
||||
remote_xfer_side_handle: int,
|
||||
) -> int | None:
|
||||
"""Post a WRITE point-to-point xfer request.
|
||||
|
||||
Returns the in-flight transfer handle (so the caller can track all of
|
||||
a request's handles atomically), or ``None`` if nothing was submitted.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
remote_rank = read_spec.remote_rank
|
||||
local_block_ids = read_spec.local_block_ids
|
||||
remote_block_ids = read_spec.remote_block_ids
|
||||
|
||||
remote_info = self.transfer_topo.get_engine_info(dst_engine_id)
|
||||
block_size_ratio = self.transfer_topo.block_size_ratio(
|
||||
remote_info.remote_block_size
|
||||
)
|
||||
if block_size_ratio > 1:
|
||||
local_block_ids, remote_block_ids = (
|
||||
self._map_block_ids_for_block_size_ratio(
|
||||
local_block_ids, remote_block_ids, block_size_ratio
|
||||
)
|
||||
)
|
||||
|
||||
notif_id = f"{remote_request_id}:{self.world_size}".encode()
|
||||
|
||||
if len(local_block_ids) == 0:
|
||||
logger.warning("No blocks to push for request %s", request_id)
|
||||
return None
|
||||
|
||||
# Prefix caching: D allocated only uncached blocks, so on a partial hit it
|
||||
# sends fewer than P's. End-trim P's blocks to that same suffix so we WRITE only
|
||||
# the uncomputed tail into D's slots. Runs on kernel ids, post-expansion.
|
||||
remote_block_ids, local_block_ids = self._apply_prefix_caching(
|
||||
decode_block_ids=remote_block_ids,
|
||||
prefill_block_ids=local_block_ids,
|
||||
decode_physical_per_logical=remote_info.remote_physical_blocks_per_logical,
|
||||
prefill_physical_per_logical=self._physical_blocks_per_logical_kv_block,
|
||||
)
|
||||
|
||||
local_block_ids = list(local_block_ids)
|
||||
remote_block_ids = list(remote_block_ids)
|
||||
assert len(local_block_ids) == len(remote_block_ids), (
|
||||
f"push group-count mismatch for {request_id}: {len(local_block_ids)} "
|
||||
f"local vs {len(remote_block_ids)} remote groups"
|
||||
)
|
||||
for i in range(len(local_block_ids)):
|
||||
assert len(local_block_ids[i]) == len(remote_block_ids[i]), (
|
||||
f"push block-count mismatch for {request_id} group {i}: "
|
||||
f"{len(local_block_ids[i])} local vs "
|
||||
f"{len(remote_block_ids[i])} remote blocks"
|
||||
)
|
||||
|
||||
# Get descs ids.
|
||||
remote_block_descs_ids = self._compute_desc_ids(
|
||||
block_ids=remote_block_ids,
|
||||
dst_num_blocks=self.dst_num_blocks[dst_engine_id],
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical,
|
||||
region_num_blocks=self.dst_region_num_blocks[dst_engine_id],
|
||||
region_group_ids=self.dst_region_group_ids[dst_engine_id],
|
||||
uses_region_group_mapping=self.dst_uses_region_group_mapping[dst_engine_id],
|
||||
)
|
||||
local_block_descs_ids = self._compute_desc_ids(
|
||||
block_ids=local_block_ids,
|
||||
dst_num_blocks=self.dst_num_blocks[self.engine_id],
|
||||
block_size_ratio=block_size_ratio,
|
||||
physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block,
|
||||
region_num_blocks=self.dst_region_num_blocks[self.engine_id],
|
||||
region_group_ids=self.region_group_ids,
|
||||
uses_region_group_mapping=self._uses_region_group_mapping,
|
||||
)
|
||||
|
||||
assert len(local_block_descs_ids) == len(remote_block_descs_ids)
|
||||
|
||||
handle = None
|
||||
try:
|
||||
handle = self.nixl_wrapper.make_prepped_xfer(
|
||||
"WRITE",
|
||||
local_xfer_side_handle,
|
||||
local_block_descs_ids,
|
||||
remote_xfer_side_handle,
|
||||
remote_block_descs_ids,
|
||||
notif_msg=notif_id,
|
||||
)
|
||||
self.nixl_wrapper.transfer(handle)
|
||||
# Caller tracks the handle (atomically with the request's other
|
||||
# writes) so P can free blocks once all of them are done.
|
||||
return handle
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="transfer_setup_failed",
|
||||
req_id=request_id,
|
||||
msg="Push WRITE submission failed; releasing handle",
|
||||
error=e,
|
||||
dst_engine_id=dst_engine_id,
|
||||
remote_rank=remote_rank,
|
||||
)
|
||||
# On the P side this WRITE failure is purely outbound; we
|
||||
# don't have a ``_recving_metadata`` entry to invalidate, so
|
||||
# we just release the handle and let the engine reschedule
|
||||
# via the lease / watchdog.
|
||||
with self._sending_transfers_lock:
|
||||
self._send_failures.add(request_id)
|
||||
if not self._handle_failed_transfer(request_id, handle):
|
||||
return handle
|
||||
return None
|
||||
|
||||
# --- Notification handling on engine main thread ------------------ #
|
||||
|
||||
def _get_new_notifs(self) -> set[str]:
|
||||
"""Drain HB / completion notifs forwarded by the writer thread.
|
||||
|
||||
The writer owns ``nixl_wrapper.get_new_notifs`` for push; PUSH_REG
|
||||
notifs are handled there. P learns of finished pushes from its WRITE
|
||||
handles, so this never reports finished sends.
|
||||
"""
|
||||
while True:
|
||||
try:
|
||||
notif = self._pending_completion_notifs.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
msg = notif.decode("utf-8")
|
||||
if msg.startswith("HB:"):
|
||||
self._handle_heartbeat(msg[3:])
|
||||
continue
|
||||
|
||||
req_id, tp_size = msg.rsplit(":", 1)
|
||||
|
||||
if (meta := self._recving_metadata.get(req_id)) is None:
|
||||
# The lease may have expired before the notif arrived.
|
||||
self.xfer_stats.record_notification_after_expiry()
|
||||
logger.error(
|
||||
"Unrecognized request %s notif (may have expired).", req_id
|
||||
)
|
||||
continue
|
||||
# Consumer waits for one notif per producer rank writing
|
||||
# here: pp_size stages * producers-per-consumer (>1 when
|
||||
# producer TP > consumer TP; tp_size is the producer TP).
|
||||
producers_per_consumer = max(1, int(tp_size) // self.world_size)
|
||||
expected_notifs = meta.pp_size * producers_per_consumer
|
||||
self.consumer_notification_counts_by_req[req_id] += 1
|
||||
if self.consumer_notification_counts_by_req[req_id] < expected_notifs:
|
||||
continue
|
||||
del self.consumer_notification_counts_by_req[req_id]
|
||||
# P drove the transfer (we own no NIXL handle), so
|
||||
# materialise an empty ``_recving_transfers`` entry for
|
||||
# ``_pop_done_transfers`` to report done.
|
||||
self._recving_transfers.setdefault(req_id, [])
|
||||
return set()
|
||||
|
||||
def get_transfer_results(self) -> KVConnectorTransferResults:
|
||||
# Engine main thread asking for completions: also wake the writer
|
||||
# so it gets a chance to drain NIXL notifs (heartbeats, completion
|
||||
# notifs, late PUSH_REGs) even if it had been parked.
|
||||
self._push_writer_wake.set()
|
||||
|
||||
results = super().get_transfer_results()
|
||||
done_sending = results.finished_sending
|
||||
|
||||
# ``_pop_done_transfers`` mutates ``_sending_transfers``; the
|
||||
# writer thread also appends to it, so guard the pop.
|
||||
with self._sending_transfers_lock:
|
||||
done_pushing, failed_pushing = self._pop_done_transfers(
|
||||
self._sending_transfers
|
||||
)
|
||||
# Remember failures until the final sibling WRITE completes.
|
||||
self._send_failures.update(failed_pushing)
|
||||
successful = {
|
||||
req_id
|
||||
for req_id in done_pushing - self._send_failures
|
||||
if req_id in self._reqs_to_send or req_id in self._reqs_to_process
|
||||
}
|
||||
self._send_failures.difference_update(done_pushing | done_sending)
|
||||
# Expired requests were already reported, even if their WRITEs finish later.
|
||||
done_pushing = successful
|
||||
for req_id in done_pushing:
|
||||
self._reqs_to_send.pop(req_id, None)
|
||||
self._reqs_to_process.discard(req_id)
|
||||
done_sending.add(req_id)
|
||||
|
||||
# Tell the writer to drop any state it still holds for any
|
||||
# request that just finished (push completed) or expired
|
||||
# (lease ran out without a D registration ever arriving).
|
||||
for req_id in done_sending:
|
||||
self._evict_finished_inbox.put(req_id)
|
||||
if done_sending:
|
||||
self._push_writer_wake.set()
|
||||
|
||||
return results
|
||||
@@ -1,12 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Backward-compatible re-export of NixlPullConnectorScheduler."""
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
|
||||
# Backward compatibility: NixlConnectorScheduler is the pull-based scheduler.
|
||||
NixlConnectorScheduler = NixlPullConnectorScheduler
|
||||
|
||||
__all__ = ["NixlConnectorScheduler", "NixlPullConnectorScheduler"]
|
||||
@@ -1,328 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stats and Prometheus metrics for the NIXL connector."""
|
||||
|
||||
import copy
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.v1.metrics.utils import create_metric_per_engine
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.nixl_utils import nixlXferTelemetry
|
||||
|
||||
|
||||
@dataclass
|
||||
class NixlKVConnectorStats(KVConnectorStats):
|
||||
"""Container for transfer performance metrics."""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
# Empty container init, no data is passed in.
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
# Must be serializable
|
||||
self.data: dict[str, list[float | int]] = {
|
||||
"transfer_duration": [],
|
||||
"post_duration": [],
|
||||
"bytes_transferred": [],
|
||||
"num_descriptors": [],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
}
|
||||
|
||||
def record_transfer(self, res: "nixlXferTelemetry | None"):
|
||||
# Keep metrics units consistent with rest of the code: time us->s
|
||||
if res is None:
|
||||
return
|
||||
self.data["transfer_duration"].append(res.xferDuration / 1e6)
|
||||
self.data["post_duration"].append(res.postDuration / 1e6)
|
||||
self.data["bytes_transferred"].append(res.totalBytes)
|
||||
self.data["num_descriptors"].append(res.descCount)
|
||||
|
||||
def record_failed_transfer(self):
|
||||
"""Record a failed NIXL transfer operation."""
|
||||
self.data["num_failed_transfers"].append(1)
|
||||
|
||||
def record_failed_notification(self):
|
||||
"""Record a failed NIXL notification (send_notif)."""
|
||||
self.data["num_failed_notifications"].append(1)
|
||||
|
||||
def record_failed_handshake(self):
|
||||
"""Record a failed NIXL handshake."""
|
||||
self.data["num_failed_handshakes"].append(1)
|
||||
|
||||
def record_kv_expired_req(self):
|
||||
"""Record a request that had its KV blocks expire."""
|
||||
self.data["num_kv_expired_reqs"].append(1)
|
||||
|
||||
def record_notification_after_expiry(self):
|
||||
"""Record a completion notification that arrived after its request expired."""
|
||||
self.data["num_notifications_after_expiry"].append(1)
|
||||
|
||||
def clone_and_reset(self) -> "NixlKVConnectorStats":
|
||||
old = copy.copy(self)
|
||||
self.reset()
|
||||
return old
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
# Do not discard metrics update that are entirely failures related.
|
||||
return (
|
||||
self.num_successful_transfers == 0
|
||||
and len(self.data["num_failed_transfers"]) == 0
|
||||
and len(self.data["num_failed_notifications"]) == 0
|
||||
and len(self.data["num_failed_handshakes"]) == 0
|
||||
and len(self.data["num_kv_expired_reqs"]) == 0
|
||||
and len(self.data["num_notifications_after_expiry"]) == 0
|
||||
)
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if not other.is_empty():
|
||||
for k, v in other.data.items():
|
||||
accumulator = self.data[k]
|
||||
assert isinstance(accumulator, list)
|
||||
accumulator.extend(v)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
# Compute compact representative stats suitable for CLI logging.
|
||||
# Failure counts are reported on every interval: transfer, handshake
|
||||
# and notification failures are grouped as sporadic
|
||||
# lower-transport-layer events, while KV expiry is reported separately
|
||||
# as it is an actionable autoscaler signal rather than a transport
|
||||
# issue. Notifications arriving after expiry are surfaced too, as the
|
||||
# reader may have consumed KV blocks that were already reused.
|
||||
failure_counts = {
|
||||
"Num failed transfers": len(self.data["num_failed_transfers"])
|
||||
+ len(self.data["num_failed_handshakes"])
|
||||
+ len(self.data["num_failed_notifications"]),
|
||||
"Num KV expired reqs": len(self.data["num_kv_expired_reqs"]),
|
||||
"Num notifs after expiry": len(self.data["num_notifications_after_expiry"]),
|
||||
}
|
||||
if self.num_successful_transfers == 0:
|
||||
# Timing / throughput stats only cover successful transfers. If
|
||||
# all requests in the interval were unsuccessful, the failure
|
||||
# counts above still surface what happened.
|
||||
return {
|
||||
"Num successful transfers": 0,
|
||||
"Avg xfer time (ms)": 0,
|
||||
"P90 xfer time (ms)": 0,
|
||||
"Avg post time (ms)": 0,
|
||||
"P90 post time (ms)": 0,
|
||||
"Avg MB per transfer": 0,
|
||||
"Throughput (MB/s)": 0,
|
||||
"Avg number of descriptors": 0,
|
||||
**failure_counts,
|
||||
}
|
||||
|
||||
xfer_time = np.asarray(self.data["transfer_duration"])
|
||||
post_time = np.asarray(self.data["post_duration"])
|
||||
# Convert to MB for CLI logging.
|
||||
mb = np.asarray(self.data["bytes_transferred"]) / 2**20
|
||||
descs = np.asarray(self.data["num_descriptors"], dtype=np.uint32)
|
||||
n = len(descs)
|
||||
assert n == self.num_successful_transfers
|
||||
|
||||
total_mb = mb.sum()
|
||||
avg_mb = total_mb / n
|
||||
|
||||
total_time_seconds = xfer_time.sum()
|
||||
throughput_mb_s = total_mb / total_time_seconds
|
||||
|
||||
return {
|
||||
"Num successful transfers": n,
|
||||
"Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
|
||||
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
|
||||
"Avg post time (ms)": round(post_time.mean().item() * 1e3, 3),
|
||||
"P90 post time (ms)": round(np.percentile(post_time, 90).item() * 1e3, 3),
|
||||
"Avg MB per transfer": round(avg_mb.item(), 3),
|
||||
"Throughput (MB/s)": round(throughput_mb_s.item(), 3),
|
||||
"Avg number of descriptors": round(descs.mean().item(), 1),
|
||||
**failure_counts,
|
||||
}
|
||||
|
||||
@property
|
||||
def num_successful_transfers(self) -> int:
|
||||
return len(self.data["transfer_duration"])
|
||||
|
||||
|
||||
class NixlPromMetrics(KVConnectorPromMetrics):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
|
||||
buckets = [
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.025,
|
||||
0.05,
|
||||
0.075,
|
||||
0.1,
|
||||
0.2,
|
||||
0.3,
|
||||
0.5,
|
||||
0.75,
|
||||
1.0,
|
||||
5.0,
|
||||
]
|
||||
nixl_histogram_xfer_time = self._histogram_cls(
|
||||
name="vllm:nixl_xfer_time_seconds",
|
||||
documentation="Histogram of transfer duration for NIXL KV Cache transfers.",
|
||||
buckets=buckets[1:],
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.nixl_histogram_xfer_time = create_metric_per_engine(
|
||||
nixl_histogram_xfer_time, self.per_engine_labelvalues
|
||||
)
|
||||
nixl_histogram_post_time = self._histogram_cls(
|
||||
name="vllm:nixl_post_time_seconds",
|
||||
documentation="Histogram of transfer post time for NIXL KV"
|
||||
" Cache transfers.",
|
||||
buckets=buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.nixl_histogram_post_time = create_metric_per_engine(
|
||||
nixl_histogram_post_time, self.per_engine_labelvalues
|
||||
)
|
||||
# uniform 2kb to 16gb range
|
||||
buckets = [2 ** (10 + i) for i in range(1, 25, 2)]
|
||||
nixl_histogram_bytes_transferred = self._histogram_cls(
|
||||
name="vllm:nixl_bytes_transferred",
|
||||
documentation="Histogram of bytes transferred per NIXL KV Cache transfers.",
|
||||
buckets=buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.nixl_histogram_bytes_transferred = create_metric_per_engine(
|
||||
nixl_histogram_bytes_transferred, self.per_engine_labelvalues
|
||||
)
|
||||
buckets = [
|
||||
10,
|
||||
20,
|
||||
30,
|
||||
50,
|
||||
75,
|
||||
100,
|
||||
200,
|
||||
400,
|
||||
1000,
|
||||
2000,
|
||||
4000,
|
||||
10000,
|
||||
20000,
|
||||
50000,
|
||||
]
|
||||
nixl_histogram_num_descriptors = self._histogram_cls(
|
||||
name="vllm:nixl_num_descriptors",
|
||||
documentation="Histogram of number of descriptors per NIXL"
|
||||
" KV Cache transfers.",
|
||||
buckets=buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.nixl_histogram_num_descriptors = create_metric_per_engine(
|
||||
nixl_histogram_num_descriptors, self.per_engine_labelvalues
|
||||
)
|
||||
counter_nixl_num_failed_transfers = self._counter_cls(
|
||||
name="vllm:nixl_num_failed_transfers",
|
||||
documentation="Number of failed NIXL KV Cache transfers, including"
|
||||
" handshake and notification failures. NOTE: KV expiry is tracked"
|
||||
" separately in vllm:nixl_num_kv_expired_reqs.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_failed_transfers = create_metric_per_engine(
|
||||
counter_nixl_num_failed_transfers, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
counter_nixl_num_failed_notifications = self._counter_cls(
|
||||
name="vllm:nixl_num_failed_notifications",
|
||||
documentation="Number of failed NIXL KV Cache notifications. "
|
||||
"Retained for compatibility; these failures are also included in "
|
||||
"vllm:nixl_num_failed_transfers.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_failed_notifications = create_metric_per_engine(
|
||||
counter_nixl_num_failed_notifications, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
counter_nixl_num_kv_expired_reqs = self._counter_cls(
|
||||
name="vllm:nixl_num_kv_expired_reqs",
|
||||
documentation="Number of requests that had their KV expire. "
|
||||
"NOTE: This metric is tracked on the P instance.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_kv_expired_reqs = create_metric_per_engine(
|
||||
counter_nixl_num_kv_expired_reqs, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
counter_nixl_num_notifications_after_expiry = self._counter_cls(
|
||||
name="vllm:nixl_num_notifications_after_expiry",
|
||||
documentation="Number of completion notifications for requests that "
|
||||
"were no longer tracked, usually because their KV lease expired. The "
|
||||
"KV blocks may have been reused before the transfer finished. Counted "
|
||||
"per notification (one per remote rank), not per request.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_notifications_after_expiry = create_metric_per_engine(
|
||||
counter_nixl_num_notifications_after_expiry, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
for prom_obj, list_item_key in zip(
|
||||
[
|
||||
self.nixl_histogram_xfer_time,
|
||||
self.nixl_histogram_post_time,
|
||||
self.nixl_histogram_bytes_transferred,
|
||||
self.nixl_histogram_num_descriptors,
|
||||
],
|
||||
[
|
||||
"transfer_duration",
|
||||
"post_duration",
|
||||
"bytes_transferred",
|
||||
"num_descriptors",
|
||||
],
|
||||
):
|
||||
for list_item in transfer_stats_data[list_item_key]:
|
||||
prom_obj[engine_idx].observe(list_item)
|
||||
for counter_obj, counter_item_keys in zip(
|
||||
[
|
||||
self.counter_nixl_num_failed_transfers,
|
||||
self.counter_nixl_num_failed_notifications,
|
||||
self.counter_nixl_num_kv_expired_reqs,
|
||||
self.counter_nixl_num_notifications_after_expiry,
|
||||
],
|
||||
[
|
||||
# Transfer, handshake and notification failures are grouped:
|
||||
# all are sporadic lower-transport-layer events. KV expiry is
|
||||
# reported separately as it signals autoscaler behavior, not
|
||||
# transport health.
|
||||
(
|
||||
"num_failed_transfers",
|
||||
"num_failed_handshakes",
|
||||
"num_failed_notifications",
|
||||
),
|
||||
("num_failed_notifications",),
|
||||
("num_kv_expired_reqs",),
|
||||
("num_notifications_after_expiry",),
|
||||
],
|
||||
):
|
||||
for counter_item_key in counter_item_keys:
|
||||
for list_item in transfer_stats_data[counter_item_key]:
|
||||
counter_obj[engine_idx].inc(list_item)
|
||||
@@ -1,161 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TP mapping computation for NIXL KV cache transfers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
BlockIds,
|
||||
TransferTopology,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, MambaSpec
|
||||
|
||||
# ======================================================================
|
||||
# Data structures
|
||||
# ======================================================================
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReadSpec:
|
||||
"""Specification for a single remote block read operation."""
|
||||
|
||||
remote_rank: int
|
||||
local_block_ids: BlockIds
|
||||
remote_block_ids: BlockIds
|
||||
block_ids_by_region: bool = False
|
||||
|
||||
|
||||
def _is_attention_spec(spec_type: type[KVCacheSpec]) -> bool:
|
||||
return issubclass(spec_type, AttentionSpec)
|
||||
|
||||
|
||||
def _is_ssm_spec(spec_type: type[KVCacheSpec]) -> bool:
|
||||
return issubclass(spec_type, MambaSpec)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TPMapping:
|
||||
"""Complete local-to-remote TP mapping for one remote engine.
|
||||
|
||||
Generated once per remote engine during handshake.
|
||||
"""
|
||||
|
||||
# Remote TP ranks that this local rank reads from, per group.
|
||||
# Position = local piece index.
|
||||
source_ranks_per_group: tuple[tuple[int, ...], ...]
|
||||
|
||||
# Superset of all source ranks (union of all groups).
|
||||
all_source_ranks: tuple[int, ...]
|
||||
|
||||
# Maps each source rank to its FA head slot index.
|
||||
rank_to_attention_slot: dict[int, int]
|
||||
|
||||
# FA head offset factor for hetero-TP (D_TP > P_TP).
|
||||
rank_offset_factor: int
|
||||
|
||||
# Local ranks (in aggregate) that read from a given source rank. The producer frees
|
||||
# a request's blocks only once that many notifications have come in.
|
||||
local_consumers: int = 1
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# TP mapping computation
|
||||
# ======================================================================
|
||||
|
||||
|
||||
def compute_tp_mapping(
|
||||
transfer_topology: TransferTopology,
|
||||
remote_tp_size: int,
|
||||
group_spec_types: tuple[type[KVCacheSpec], ...],
|
||||
remote_dcp_size: int = 1,
|
||||
) -> TPMapping:
|
||||
"""Build the complete local-to-remote TP mapping.
|
||||
|
||||
Computes source ranks, head slot assignments, and the rank offset
|
||||
factor in a single pass.
|
||||
|
||||
DCP support is scoped to MLA only, with a side is either fully replicated or fully
|
||||
sharded. DCP-branch reuses the same rank set used at handshake selection.
|
||||
"""
|
||||
tp_rank = transfer_topology.tp_rank
|
||||
tp_size = transfer_topology.tp_size
|
||||
total_num_kv_heads = transfer_topology.total_num_kv_heads
|
||||
# --- Attention source ranks ---
|
||||
if transfer_topology.is_mla or tp_size >= remote_tp_size:
|
||||
if transfer_topology.is_mla and remote_dcp_size > 1:
|
||||
attn_ranks = transfer_topology.dcp_source_ranks(
|
||||
remote_tp_size, remote_dcp_size
|
||||
)
|
||||
else:
|
||||
# D (local TP) > P (remote TP): multiple local ranks read different chunks
|
||||
# from *one* remote rank, corresponding to different kv heads.
|
||||
# For MLA, we only need one remote since cache is duplicated. When
|
||||
# P TP=k*TP k, this will spread mla ranks to read from remote k*tp_rank.
|
||||
attn_ranks = [tp_rank * remote_tp_size // tp_size]
|
||||
else:
|
||||
# P (remote TP) > D (local TP): one local rank
|
||||
# reads from multiple remote ranks.
|
||||
# GQA dedup: when K < remote_tp_size, several remote ranks
|
||||
# hold the same KV head. np.unique keeps only the first
|
||||
# rank per unique head so we don't issue redundant reads.
|
||||
abs_tp = remote_tp_size // tp_size
|
||||
start = tp_rank * abs_tp
|
||||
heads = np.arange(start, start + abs_tp) * total_num_kv_heads // remote_tp_size
|
||||
_, unique_idx = np.unique(heads, return_index=True)
|
||||
attn_ranks = (start + np.sort(unique_idx)).tolist()
|
||||
|
||||
# --- SSM source ranks ---
|
||||
has_ssm = any(_is_ssm_spec(t) for t in group_spec_types)
|
||||
if has_ssm:
|
||||
if tp_size < remote_tp_size:
|
||||
abs_tp = remote_tp_size // tp_size
|
||||
ssm_ranks = list(range(tp_rank * abs_tp, (tp_rank + 1) * abs_tp))
|
||||
else:
|
||||
ssm_ranks = list(attn_ranks)
|
||||
else:
|
||||
ssm_ranks = []
|
||||
|
||||
all_ranks = sorted(set(attn_ranks) | set(ssm_ranks))
|
||||
|
||||
# --- Per-group ordered source ranks ---
|
||||
source_ranks_per_group = tuple(
|
||||
tuple(ssm_ranks) if _is_ssm_spec(t) else tuple(attn_ranks)
|
||||
for t in group_spec_types
|
||||
)
|
||||
|
||||
# --- Attention head slots ---
|
||||
head_to_slot: dict[int, int] = {}
|
||||
for i, r in enumerate(attn_ranks):
|
||||
head_to_slot[r * total_num_kv_heads // remote_tp_size] = i
|
||||
rank_to_attention_slot = {
|
||||
r: head_to_slot.get(r * total_num_kv_heads // remote_tp_size, 0)
|
||||
for r in all_ranks
|
||||
}
|
||||
|
||||
# --- Rank offset factor ---
|
||||
if transfer_topology.is_mla or tp_size <= remote_tp_size:
|
||||
# We don't index into remote for reading, no offset needed.
|
||||
rank_offset_factor = 0
|
||||
elif tp_size > total_num_kv_heads:
|
||||
local_head = tp_rank * total_num_kv_heads // tp_size
|
||||
p_start = attn_ranks[0] * total_num_kv_heads // remote_tp_size
|
||||
rank_offset_factor = local_head - p_start
|
||||
else:
|
||||
# D TP > P TP: we index into remote to read different heads depending on rank.
|
||||
rank_offset_factor = tp_rank % (tp_size // remote_tp_size)
|
||||
|
||||
local_consumers = transfer_topology.dcp_consumer_count(
|
||||
remote_tp_size, remote_dcp_size
|
||||
)
|
||||
|
||||
return TPMapping(
|
||||
source_ranks_per_group=source_ranks_per_group,
|
||||
all_source_ranks=tuple(all_ranks),
|
||||
rank_to_attention_slot=rank_to_attention_slot,
|
||||
rank_offset_factor=rank_offset_factor,
|
||||
local_consumers=local_consumers,
|
||||
)
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared constants, lazy imports and helpers for the NIXL connector."""
|
||||
|
||||
import contextlib
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import zmq
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import make_zmq_socket
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, UniformTypeKVCacheSpecs
|
||||
|
||||
# Supported platforms and types of kv transfer buffer.
|
||||
# {device: tuple of supported kv buffer types}
|
||||
_NIXL_SUPPORTED_DEVICE = {
|
||||
"cuda": (
|
||||
"cuda",
|
||||
"cpu",
|
||||
),
|
||||
"tpu": ("cpu",),
|
||||
"xpu": (
|
||||
"cpu",
|
||||
"xpu",
|
||||
),
|
||||
"cpu": ("cpu",),
|
||||
}
|
||||
# support for oot platform by providing mapping in current_platform
|
||||
_NIXL_SUPPORTED_DEVICE.update(current_platform.get_nixl_supported_devices())
|
||||
|
||||
|
||||
# TODO: merge with vllm.utils.network_utils.zmq_socket_ctx
|
||||
@contextlib.contextmanager
|
||||
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
|
||||
"""Context manager for a ZMQ socket."""
|
||||
if socket_type not in (zmq.ROUTER, zmq.REQ):
|
||||
raise ValueError(f"Unexpected socket type: {socket_type}")
|
||||
|
||||
ctx: zmq.Context | None = None
|
||||
try:
|
||||
ctx = zmq.Context() # type: ignore[attr-defined]
|
||||
yield make_zmq_socket(
|
||||
ctx=ctx, path=addr, socket_type=socket_type, bind=socket_type == zmq.ROUTER
|
||||
)
|
||||
finally:
|
||||
if ctx is not None:
|
||||
ctx.destroy(linger=0)
|
||||
|
||||
|
||||
def get_representative_spec_type(spec: KVCacheSpec) -> type[KVCacheSpec]:
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
# All inner specs are the same type; pick any.
|
||||
inner = next(iter(spec.kv_cache_specs.values()))
|
||||
return type(inner)
|
||||
return type(spec)
|
||||
|
||||
|
||||
# Trailing 8-hex randomization suffix appended by
|
||||
# ``input_processor.assign_request_id`` as ``-{random_uuid():.8}``.
|
||||
_RANDOM_SUFFIX_RE = re.compile(r"-[0-9a-f]{8}$", re.IGNORECASE)
|
||||
|
||||
|
||||
def get_base_request_id(request_id: str) -> str:
|
||||
"""Strip the per-request ``-<8 hex>`` randomization suffix, if present."""
|
||||
return _RANDOM_SUFFIX_RE.sub("", request_id)
|
||||
@@ -1,13 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Backward-compatible re-export of NixlPullConnectorWorker."""
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
|
||||
NixlPullConnectorWorker,
|
||||
)
|
||||
|
||||
# Backward compatibility: NixlConnectorWorker is the pull-based worker.
|
||||
NixlConnectorWorker = NixlPullConnectorWorker
|
||||
|
||||
|
||||
__all__ = ["NixlConnectorWorker", "NixlPullConnectorWorker"]
|
||||
@@ -1,386 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""SimpleCPUOffloadConnector: minimal CPU KV cache offloading."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_events import KVCacheEvent
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.simple_kv_offload.manager import (
|
||||
BoundaryStoreStats,
|
||||
SimpleCPUOffloadScheduler,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metadata import (
|
||||
SimpleCPUOffloadMetadata,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metrics import (
|
||||
SimpleCPUOffloadPromMetrics,
|
||||
SimpleCPUOffloadStats,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.worker import (
|
||||
SimpleCPUOffloadWorker,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Default CPU capacity: 8 GB
|
||||
DEFAULT_CPU_CAPACITY_BYTES = 8 * (1024**3)
|
||||
|
||||
VALID_KV_OFFLOAD_BACKENDS = ("cpu", "disk")
|
||||
# Keys that only apply to the disk backend, warned about under "cpu".
|
||||
_DISK_ONLY_KEYS = (
|
||||
"disk_path",
|
||||
"disk_capacity_bytes",
|
||||
"disk_buffer_slots",
|
||||
"use_page_cache",
|
||||
)
|
||||
|
||||
|
||||
class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""CPU KV cache offloading with custom kernel transfers and BlockPool LRU."""
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
|
||||
# future cache miss, so opt out of the producer-role default.
|
||||
return False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
|
||||
enable_prefix_caching = vllm_config.cache_config.enable_prefix_caching
|
||||
extra_config = self._kv_transfer_config.kv_connector_extra_config or {}
|
||||
|
||||
cpu_capacity_bytes = int(
|
||||
extra_config.get("cpu_bytes_to_use", DEFAULT_CPU_CAPACITY_BYTES)
|
||||
)
|
||||
# cpu_bytes_to_use is server-wide for compatibility;
|
||||
# cpu_bytes_to_use_per_rank overrides for per-rank capacity.
|
||||
world_size = vllm_config.parallel_config.world_size
|
||||
cpu_capacity_per_rank = cpu_capacity_bytes // world_size
|
||||
if "cpu_bytes_to_use_per_rank" in extra_config:
|
||||
explicit = int(extra_config["cpu_bytes_to_use_per_rank"])
|
||||
if explicit != cpu_capacity_per_rank:
|
||||
logger.warning(
|
||||
"cpu_bytes_to_use_per_rank (%.2f GB) != "
|
||||
"cpu_bytes_to_use/world_size (%.2f GB). Using per-rank value.",
|
||||
explicit / (1024**3),
|
||||
cpu_capacity_per_rank / (1024**3),
|
||||
)
|
||||
cpu_capacity_per_rank = explicit
|
||||
|
||||
# When set, num_cpu_blocks overrides automatic pool sizing from byte budgets
|
||||
# (including cpu_bytes_to_use and cpu_bytes_to_use_per_rank), allowing ranks
|
||||
# with different KV layouts (e.g. PP stages) to use the same pool size.
|
||||
# Byte budgets still bound capacity.
|
||||
num_cpu_blocks = extra_config.get("num_cpu_blocks")
|
||||
if num_cpu_blocks is not None and (
|
||||
type(num_cpu_blocks) is not int or num_cpu_blocks <= 0
|
||||
):
|
||||
raise ValueError("num_cpu_blocks must be a positive integer")
|
||||
|
||||
lazy_offload = bool(extra_config.get("lazy_offload", False))
|
||||
|
||||
kv_offload_backend = str(extra_config.get("kv_offload_backend", "cpu"))
|
||||
if kv_offload_backend not in VALID_KV_OFFLOAD_BACKENDS:
|
||||
raise ValueError(
|
||||
f"Unknown kv_offload_backend {kv_offload_backend!r}; "
|
||||
f"expected one of {VALID_KV_OFFLOAD_BACKENDS}"
|
||||
)
|
||||
disk_mode = kv_offload_backend == "disk"
|
||||
self._cache_hit_source = (
|
||||
CacheHitSource.DISK if disk_mode else CacheHitSource.HOST
|
||||
)
|
||||
|
||||
disk_path = extra_config.get("disk_path", None) or None
|
||||
disk_capacity_bytes = int(
|
||||
extra_config.get("disk_capacity_bytes", 100 * (1024**3))
|
||||
)
|
||||
disk_buffer_slots = max(1, int(extra_config.get("disk_buffer_slots", 2)))
|
||||
use_page_cache = bool(extra_config.get("use_page_cache", False))
|
||||
|
||||
if disk_mode:
|
||||
if disk_path is None:
|
||||
raise ValueError(
|
||||
'kv_offload_backend="disk" requires disk_path to be set.'
|
||||
)
|
||||
if disk_capacity_bytes <= 0:
|
||||
raise ValueError(
|
||||
'kv_offload_backend="disk" requires disk_capacity_bytes > 0.'
|
||||
)
|
||||
else:
|
||||
ignored = [k for k in _DISK_ONLY_KEYS if k in extra_config]
|
||||
if ignored:
|
||||
logger.warning(
|
||||
'kv_offload_backend is "cpu", ignoring disk-only config: %s. '
|
||||
'Set kv_offload_backend="disk" to enable the disk backend.',
|
||||
", ".join(ignored),
|
||||
)
|
||||
disk_path = None
|
||||
|
||||
self.scheduler_manager: SimpleCPUOffloadScheduler | None = None
|
||||
self.worker_handler: SimpleCPUOffloadWorker | None = None
|
||||
|
||||
if not enable_prefix_caching:
|
||||
logger.warning(
|
||||
"Detected prefix caching disabled, disabling CPU offload "
|
||||
"since it requires prefix caching."
|
||||
)
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"SimpleCPUOffloadConnector: role=%s, "
|
||||
"per_rank=%.2f GB, world_size=%d, mode=%s, backend=%s, disk=%s",
|
||||
role.name,
|
||||
cpu_capacity_per_rank / (1024**3),
|
||||
world_size,
|
||||
"lazy" if lazy_offload else "eager",
|
||||
kv_offload_backend,
|
||||
disk_path or "none",
|
||||
)
|
||||
if num_cpu_blocks is not None:
|
||||
logger.info(
|
||||
"Using explicit num_cpu_blocks=%d for offload pool sizing "
|
||||
"instead of automatic sizing from byte budgets.",
|
||||
num_cpu_blocks,
|
||||
)
|
||||
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
|
||||
assert kv_cache_config is not None
|
||||
scheduler_block_size, hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
self.scheduler_manager = SimpleCPUOffloadScheduler(
|
||||
vllm_config,
|
||||
kv_cache_config,
|
||||
cpu_capacity_per_rank,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
lazy_offload=lazy_offload,
|
||||
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
|
||||
use_page_cache=use_page_cache if disk_mode else False,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.worker_handler = SimpleCPUOffloadWorker(
|
||||
vllm_config,
|
||||
kv_cache_config,
|
||||
cpu_capacity_per_rank,
|
||||
kv_offload_backend=kv_offload_backend,
|
||||
disk_path=disk_path,
|
||||
disk_capacity_bytes=disk_capacity_bytes,
|
||||
disk_buffer_slots=disk_buffer_slots,
|
||||
use_page_cache=use_page_cache,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
)
|
||||
|
||||
# --- Worker-side methods ---
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
if self.worker_handler is not None:
|
||||
self.worker_handler.register_kv_caches(kv_caches)
|
||||
|
||||
def bind_connector_metadata(
|
||||
self,
|
||||
connector_metadata: KVConnectorMetadata,
|
||||
) -> None:
|
||||
super().bind_connector_metadata(connector_metadata)
|
||||
if self.worker_handler is not None:
|
||||
assert isinstance(connector_metadata, SimpleCPUOffloadMetadata)
|
||||
self.worker_handler.bind_connector_metadata(connector_metadata)
|
||||
|
||||
def clear_connector_metadata(self) -> None:
|
||||
super().clear_connector_metadata()
|
||||
if self.worker_handler is not None:
|
||||
self.worker_handler.clear_connector_metadata()
|
||||
|
||||
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata) -> None:
|
||||
if self.worker_handler is not None:
|
||||
assert isinstance(kv_connector_metadata, SimpleCPUOffloadMetadata)
|
||||
self.worker_handler.handle_preemptions(kv_connector_metadata)
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
if self.worker_handler is not None:
|
||||
self.worker_handler.start_load_kv()
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
pass # Always load asynchronously, issued in start_load_kv()
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: "AttentionMetadata",
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
pass # Always save asynchronously, issued in wait_for_save()
|
||||
|
||||
def wait_for_save(self) -> None:
|
||||
if self.worker_handler is not None:
|
||||
self.worker_handler.wait_for_save()
|
||||
|
||||
def get_finished(
|
||||
self,
|
||||
finished_req_ids: set[str],
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
if self.worker_handler is not None:
|
||||
return self.worker_handler.get_finished(finished_req_ids)
|
||||
return None, None
|
||||
|
||||
def build_connector_worker_meta(self):
|
||||
if self.worker_handler is not None:
|
||||
return self.worker_handler.build_connector_worker_meta()
|
||||
return None
|
||||
|
||||
# --- Scheduler-side methods ---
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None:
|
||||
if self.scheduler_manager is not None:
|
||||
self.scheduler_manager.bind_gpu_block_pool(gpu_block_pool)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
return 0, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: "Request",
|
||||
blocks: "KVCacheBlocks",
|
||||
num_external_tokens: int,
|
||||
) -> None:
|
||||
if self.scheduler_manager is not None:
|
||||
self.scheduler_manager.update_state_after_alloc(
|
||||
request, blocks, num_external_tokens
|
||||
)
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.build_connector_meta(scheduler_output)
|
||||
return SimpleCPUOffloadMetadata()
|
||||
|
||||
def update_connector_output(
|
||||
self,
|
||||
connector_output: KVConnectorOutput,
|
||||
) -> None:
|
||||
if self.scheduler_manager is not None:
|
||||
self.scheduler_manager.update_connector_output(connector_output)
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.request_finished(request, block_ids)
|
||||
return False, None
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.request_finished_all_groups(
|
||||
request, block_ids
|
||||
)
|
||||
return False, None
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.has_pending_stores()
|
||||
return False
|
||||
|
||||
def get_boundary_store_stats(self) -> BoundaryStoreStats | None:
|
||||
"""Return cumulative boundary-handoff store diagnostics."""
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.get_boundary_store_stats()
|
||||
return None
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
# Worker-side callers may invoke this hook on connectors constructed
|
||||
# via __new__ (as in test_worker.py), which lack scheduler_manager.
|
||||
scheduler_manager: SimpleCPUOffloadScheduler | None = getattr(
|
||||
self, "scheduler_manager", None
|
||||
)
|
||||
if scheduler_manager is not None:
|
||||
return scheduler_manager.get_stats()
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
SimpleCPUOffloadStats(data=data)
|
||||
if data is not None
|
||||
else SimpleCPUOffloadStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return SimpleCPUOffloadPromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.take_events()
|
||||
return []
|
||||
|
||||
# NOTE: Workers are not contacted. In-flight transfers drain naturally,
|
||||
# and stale completions are ignored by the guarded
|
||||
# SimpleCPUOffloadScheduler._process_store_event().
|
||||
def reset_cache(self) -> bool | None:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.reset()
|
||||
return None
|
||||
@@ -1,235 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mamba conv-state sub-projection decomposition for NIXL transfer.
|
||||
|
||||
With DS conv state layout (dim, state_len), sub-projections are
|
||||
contiguous in memory. Each D rank reads its slices via separate
|
||||
RDMA transfers — no P-side permutation needed.
|
||||
|
||||
Supported model types:
|
||||
- Mamba1: conv = [x], temporal = (intermediate_size, state_size)
|
||||
- Mamba2: conv = [x, B, C], temporal = (num_heads, head_dim)
|
||||
- GDN (Gated Delta Net): conv = [Q, K, V] (dim(Q)==dim(K)),
|
||||
temporal = (num_v_heads, v_dim, k_dim)
|
||||
"""
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MambaConvSplitInfo:
|
||||
"""Per-rank byte sizes of the conv sub-projections.
|
||||
|
||||
Used by both P and D sides for NIXL descriptor registration.
|
||||
All fields are LOCAL to this engine's TP (already divided by TP size).
|
||||
|
||||
DS memory layout within one page (contiguous):
|
||||
Mamba1: |---- x ----| (single sub-projection, no decomposition)
|
||||
Mamba2: |-- x --|- B -|- C -| (B == C)
|
||||
GDN: |- Q -|- K -|-- V --| (dim(Q)==dim(K), V may differ)
|
||||
"""
|
||||
|
||||
conv_rows: int # conv_kernel - 1 (typically 3)
|
||||
# Per-rank column counts per sub-projection:
|
||||
# 1 entry for Mamba1, 3 for Mamba2/GDN.
|
||||
local_proj_dims: tuple[int, ...]
|
||||
conv_dtype_size: int # bytes per element (e.g. 2 for float16)
|
||||
ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes)
|
||||
|
||||
@property
|
||||
def local_conv_dim(self) -> int:
|
||||
"""Total conv columns per rank."""
|
||||
return sum(self.local_proj_dims)
|
||||
|
||||
@property
|
||||
def proj_bytes(self) -> tuple[int, ...]:
|
||||
"""Byte sizes of the sub-projections for one rank."""
|
||||
row_bytes = self.conv_rows * self.conv_dtype_size
|
||||
return tuple(d * row_bytes for d in self.local_proj_dims)
|
||||
|
||||
@property
|
||||
def local_conv_offsets(self) -> list[tuple[int, int]]:
|
||||
"""(byte_offset, byte_size) of each sub-projection within this
|
||||
engine's page.
|
||||
|
||||
Used by both P and D for local descriptor registration.
|
||||
"""
|
||||
offsets: list[tuple[int, int]] = []
|
||||
offset = 0
|
||||
for size in self.proj_bytes:
|
||||
offsets.append((offset, size))
|
||||
offset += size
|
||||
return offsets
|
||||
|
||||
def remote_conv_offsets(
|
||||
self, local_rank_offset: int, tp_ratio: int
|
||||
) -> list[tuple[int, int]]:
|
||||
"""(byte_offset, byte_size) of this D rank's sub-projection slices
|
||||
within one P page.
|
||||
|
||||
Used by D side only, during remote descriptor registration.
|
||||
|
||||
Args:
|
||||
local_rank_offset: which slice this D rank reads.
|
||||
tp_ratio: signed TP ratio.
|
||||
>= 1: D_TP >= P_TP — P page is larger, D reads its slice.
|
||||
< 0: P_TP > D_TP — P pages are smaller, D reads entire
|
||||
P page. Local dims are scaled down by |tp_ratio|
|
||||
to get P-sized offsets.
|
||||
|
||||
"""
|
||||
offsets: list[tuple[int, int]] = []
|
||||
if tp_ratio >= 1:
|
||||
remote_base = 0
|
||||
for size in self.proj_bytes:
|
||||
offsets.append((remote_base + local_rank_offset * size, size))
|
||||
remote_base += size * tp_ratio
|
||||
else:
|
||||
# NOTE (ZhanqiuHu): tp_ratio < 0 means P_TP > D_TP, so P pages
|
||||
# are smaller than D's. Local dims are D-sized, but we need
|
||||
# P-sized offsets. Scale down by |tp_ratio|.
|
||||
abs_ratio = -tp_ratio
|
||||
remote_base = 0
|
||||
for size in self.proj_bytes:
|
||||
remote_size = size // abs_ratio
|
||||
offsets.append((remote_base, remote_size))
|
||||
remote_base += remote_size
|
||||
return offsets
|
||||
|
||||
|
||||
def derive_mamba_conv_split(
|
||||
mamba_spec: MambaSpec,
|
||||
local_tp: int,
|
||||
) -> MambaConvSplitInfo:
|
||||
"""Derive per-rank sub-projection byte sizes from a MambaSpec.
|
||||
|
||||
Called once at init on both P and D. Decomposes the conv dimension
|
||||
into its sub-projection parts based on the model type.
|
||||
|
||||
Args:
|
||||
mamba_spec: MambaSpec whose shapes are:
|
||||
shapes[0] = conv state: (conv_dim_local, conv_rows) in DS layout.
|
||||
shapes[1] = temporal state (model-specific shape).
|
||||
local_tp: this engine's tensor-parallel size.
|
||||
|
||||
Returns:
|
||||
MambaConvSplitInfo with per-rank sub-projection dims, conv_rows,
|
||||
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
|
||||
|
||||
"""
|
||||
_supported = (
|
||||
MambaAttentionBackendEnum.MAMBA1,
|
||||
MambaAttentionBackendEnum.MAMBA2,
|
||||
MambaAttentionBackendEnum.GDN_ATTN,
|
||||
)
|
||||
if mamba_spec.mamba_type not in _supported:
|
||||
raise NotImplementedError(
|
||||
f"Conv transfer only supports Mamba1, Mamba2 and GDN models, "
|
||||
f"got mamba_type={mamba_spec.mamba_type!r}."
|
||||
)
|
||||
|
||||
conv_shape = mamba_spec.shapes[0]
|
||||
assert len(conv_shape) == 2, f"Expected 2D conv state shape, got {conv_shape}"
|
||||
|
||||
# NOTE (ZhanqiuHu): 3-read requires DS layout, which is already asserted
|
||||
# in nixl worker __init__. Use it directly instead of heuristic detection.
|
||||
assert is_conv_state_dim_first(), "3-read requires DS conv state layout"
|
||||
local_conv_dim = conv_shape[0] # DS: (conv_dim_local, conv_rows)
|
||||
conv_rows = conv_shape[1]
|
||||
|
||||
conv_dtype_size = torch.tensor(
|
||||
[],
|
||||
dtype=mamba_spec.dtypes[0], # type: ignore[misc]
|
||||
).element_size()
|
||||
ssm_dtype_size = torch.tensor(
|
||||
[],
|
||||
dtype=mamba_spec.dtypes[1], # type: ignore[misc]
|
||||
).element_size()
|
||||
conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size
|
||||
ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size
|
||||
|
||||
local_proj_dims: tuple[int, ...]
|
||||
if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA1:
|
||||
# Mamba1 conv state holds only x (no B/C), so it's a single
|
||||
# contiguous TP shard with no sub-projection decomposition.
|
||||
temporal_shape = mamba_spec.shapes[1]
|
||||
assert temporal_shape[0] == local_conv_dim, (
|
||||
f"Mamba1 temporal state dim ({temporal_shape[0]}) doesn't match "
|
||||
f"conv dim ({local_conv_dim}); both should be "
|
||||
f"intermediate_size/TP."
|
||||
)
|
||||
local_proj_dims = (local_conv_dim,)
|
||||
elif mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2:
|
||||
# NOTE (ZhanqiuHu): intermediate_size (= global x dim) is not stored
|
||||
# in MambaSpec, so we reconstruct it from the SSM temporal state shape:
|
||||
# shapes[1] = (local_num_heads, head_dim), already divided by TP.
|
||||
head_dim = mamba_spec.shapes[1][1]
|
||||
local_num_heads = mamba_spec.shapes[1][0]
|
||||
intermediate_size = local_num_heads * local_tp * head_dim
|
||||
|
||||
# NOTE (ZhanqiuHu): global conv dim = intermediate_size + 2 * groups_ss,
|
||||
# where groups_ss is the B (= C) dimension. B and C are always the same
|
||||
# size, so we recover groups_ss from the remainder after subtracting x.
|
||||
remainder = local_conv_dim * local_tp - intermediate_size
|
||||
assert remainder > 0 and remainder % 2 == 0, (
|
||||
f"Conv dim ({local_conv_dim}*tp={local_tp}) doesn't decompose "
|
||||
f"into intermediate_size={intermediate_size} + 2*groups_ss. "
|
||||
f"remainder={remainder}"
|
||||
)
|
||||
groups_ss = remainder // 2
|
||||
|
||||
# Divide by TP to get per-rank column counts.
|
||||
x_local = intermediate_size // local_tp
|
||||
b_local = groups_ss // local_tp
|
||||
local_proj_dims = (x_local, b_local, b_local)
|
||||
elif mamba_spec.mamba_type == MambaAttentionBackendEnum.GDN_ATTN:
|
||||
# GDN: conv = [Q, K, V] where dim(Q) == dim(K) == key_dim.
|
||||
# conv_dim = key_dim*2 + value_dim (all global, divided by TP).
|
||||
# Temporal state shape is (num_v_heads/TP, head_v_dim, head_k_dim).
|
||||
temporal_shape = mamba_spec.shapes[1]
|
||||
num_v_heads_local = temporal_shape[0]
|
||||
head_v_dim = temporal_shape[1]
|
||||
value_dim_local = num_v_heads_local * head_v_dim
|
||||
|
||||
remainder = local_conv_dim - value_dim_local
|
||||
assert remainder > 0 and remainder % 2 == 0, (
|
||||
f"GDN conv dim ({local_conv_dim}) doesn't decompose into "
|
||||
f"2*key_dim_local + value_dim_local={value_dim_local}. "
|
||||
f"remainder={remainder}"
|
||||
)
|
||||
key_dim_local = remainder // 2
|
||||
local_proj_dims = (key_dim_local, key_dim_local, value_dim_local)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Conv split not supported for mamba_type={mamba_spec.mamba_type!r}"
|
||||
)
|
||||
|
||||
return MambaConvSplitInfo(
|
||||
conv_rows=conv_rows,
|
||||
local_proj_dims=local_proj_dims,
|
||||
conv_dtype_size=conv_dtype_size,
|
||||
ssm_sizes=(conv_state_bytes, ssm_state_bytes),
|
||||
)
|
||||
|
||||
|
||||
def compute_physical_blocks_per_logical(
|
||||
ssm_sizes: tuple[int, ...], block_len: int
|
||||
) -> int:
|
||||
"""Derive _physical_blocks_per_logical_kv_block from remote metadata.
|
||||
|
||||
The remote engine's ratio is not sent directly in the handshake, so we
|
||||
reconstruct it: total mamba state per logical block / block_len.
|
||||
|
||||
Args:
|
||||
ssm_sizes: (conv_state_bytes, ssm_state_bytes) from NixlAgentMetadata.
|
||||
block_len: the engine's block_len in bytes (from block_lens[0]).
|
||||
|
||||
"""
|
||||
return math.ceil((ssm_sizes[0] + ssm_sizes[1]) / block_len)
|
||||
Reference in New Issue
Block a user