[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers

- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs,
  moriio, hisparse, example and hidden-states connectors
- Reduce weight_transfer to the base API used by model loading
- Remove kv_offload tiering, simple_kv_offload, ec cpu/example,
  snapshot command, engine snapshot server
- Remove transformers model fallback, dsv4 tokenizers, dead platforms
  cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn
  ops, dead layers (short_conv, fused_embed_norm, indexer_topk,
  relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar
- Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant
- Rebuild parser hubs around Qwen3 adapters only (36 dead adapters)
- Remove dead tests with the code they covered

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 02:19:03 +07:00
co-authored by Hinata
parent 54a396586e
commit 2b544b7ba0
353 changed files with 29 additions and 121269 deletions
@@ -150,48 +150,12 @@ class KVConnectorFactory:
# The registration should not be done in each individual file, as we want to
# only load the files corresponding to the current connector.
KVConnectorFactory.register_connector(
"ExampleHiddenStatesConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.example_hidden_states_connector",
"ExampleHiddenStatesConnector",
)
KVConnectorFactory.register_connector(
"NixlConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
"NixlConnector",
)
KVConnectorFactory.register_connector(
"NixlPullConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
"NixlPullConnector",
)
KVConnectorFactory.register_connector(
"NixlPushConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
"NixlPushConnector",
)
KVConnectorFactory.register_connector(
"MultiConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.multi_connector",
"MultiConnector",
)
KVConnectorFactory.register_connector(
"HiSparseConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector",
"HiSparseConnector",
)
KVConnectorFactory.register_connector(
"MoRIIOConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector",
"MoRIIOConnector",
)
KVConnectorFactory.register_connector(
"OffloadingConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector",
@@ -203,14 +167,3 @@ KVConnectorFactory.register_connector(
"vllm.distributed.kv_transfer.kv_connector.v1.decode_bench_connector",
"DecodeBenchConnector",
)
KVConnectorFactory.register_connector(
"SimpleCPUOffloadConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector",
"SimpleCPUOffloadConnector",
)
KVConnectorFactory.register_connector(
"HF3FSKVConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_connector",
"HF3FSKVConnector",
)
@@ -1,302 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import multiprocessing
import os
import threading
from functools import wraps
from pathlib import Path
import torch
import torch.utils.cpp_extension
from torch.utils.cpp_extension import load
root = Path(__file__).parent.resolve()
cuda_include_path = os.path.join(torch.utils.cpp_extension.CUDA_HOME, "include")
hf3fs_utils = load(
name="hf3fs_utils",
sources=[f"{root}/utils/hf3fs_utils.cpp"],
extra_include_paths=[cuda_include_path],
)
logger = logging.getLogger(__name__)
HF3FS_AVAILABLE = True
try:
from hf3fs_fuse.io import (
deregister_fd,
extract_mount_point,
make_ioring,
make_iovec,
register_fd,
)
except ImportError:
HF3FS_AVAILABLE = False
def rsynchronized():
def _decorator(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
with self.rlock:
return func(self, *args, **kwargs)
return wrapper
return _decorator
def wsynchronized():
def _decorator(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
with self.wlock:
return func(self, *args, **kwargs)
return wrapper
return _decorator
class Hf3fsClient:
def __init__(self, path: str, size: int, bytes_per_page: int, entries: int):
"""Initialize the HF3FS client with hf3fs_fuse.
Args:
path: Path to the file used for storage
size: Total size of the storage file in bytes
bytes_per_page: Size of each page in bytes
entries: Maximum number of concurrent operations
"""
if not HF3FS_AVAILABLE:
raise ImportError(
"hf3fs_fuse.io is not available. Please install the hf3fs_fuse package."
)
self.path = path
self.size = size
self.bytes_per_page = bytes_per_page
self.entries = entries
self._closed = False
self.file = None
self.shm_r = None
self.shm_w = None
self.ior_r = None
self.ior_w = None
self.iov_r = None
self.iov_w = None
try:
# Create the file if it doesn't exist and set its size
self.file = os.open(self.path, os.O_RDWR | os.O_CREAT)
os.ftruncate(self.file, size)
register_fd(self.file)
self.hf3fs_mount_point = extract_mount_point(path)
self.bs = self.bytes_per_page
self.shm_r = multiprocessing.shared_memory.SharedMemory(
size=self.bs * self.entries, create=True
)
self.shm_w = multiprocessing.shared_memory.SharedMemory(
size=self.bs * self.entries, create=True
)
self.shm_r_tensor = torch.frombuffer(self.shm_r.buf, dtype=torch.uint8)
self.shm_w_tensor = torch.frombuffer(self.shm_w.buf, dtype=torch.uint8)
numel = self.bs * self.entries
self.r_pinned = torch.empty(
numel,
dtype=torch.uint8,
device="cpu",
pin_memory=True,
)
self.w_pinned = torch.empty(
numel,
dtype=torch.uint8,
device="cpu",
pin_memory=True,
)
self.numa = -1
self.ior_r = make_ioring(
self.hf3fs_mount_point,
self.entries,
for_read=True,
timeout=1,
numa=self.numa,
)
self.ior_w = make_ioring(
self.hf3fs_mount_point,
self.entries,
for_read=False,
timeout=1,
numa=self.numa,
)
self.iov_r = make_iovec(self.shm_r, self.hf3fs_mount_point)
self.iov_w = make_iovec(self.shm_w, self.hf3fs_mount_point)
self.shm_r.unlink()
self.shm_w.unlink()
self.rlock = threading.RLock()
self.wlock = threading.RLock()
self.stream = torch.cuda.Stream()
self.stream_ptr_int = self.stream.cuda_stream
except Exception:
self._release_resources()
raise
logger.debug(
"Initialized HF3FS client with file: %s, size: %s bytes", path, size
)
def _release_resources(self) -> None:
"""Release all acquired resources safely."""
# iov must be released before ioring and shm
for attr in ("iov_r", "iov_w", "ior_r", "ior_w"):
obj = getattr(self, attr, None)
if obj is not None:
del obj
setattr(self, attr, None)
for attr in ("shm_r", "shm_w"):
shm = getattr(self, attr, None)
if shm is not None:
try:
shm.close()
except Exception as e:
logger.warning("Failed to close %s: %s", attr, e)
setattr(self, attr, None)
if self.file is not None:
try:
deregister_fd(self.file)
except Exception as e:
logger.warning("deregister_fd failed: %s", e)
try:
os.close(self.file)
except OSError as e:
logger.warning("os.close failed: %s", e)
self.file = None
@rsynchronized()
def batch_read(self, offsets: list[int], tensors: list[torch.Tensor]) -> list[int]:
"""Read data from the file at specified offsets into tensors.
Args:
offsets: List of byte offsets to read from
tensors: List of tensors to read data into
Returns:
List of operation results (0 for success, non-zero for error)
"""
self.check(offsets, tensors)
assert self.ior_r is not None
assert self.iov_r is not None
# prepare
current = 0
for offset, tensor in zip(offsets, tensors):
size = tensor.numel() * tensor.itemsize
self.ior_r.prepare(
self.iov_r[current : current + size], True, self.file, offset
)
current += size
# submit
ionum = len(offsets)
resv = self.ior_r.submit().wait(min_results=ionum)
# results
with torch.cuda.stream(self.stream):
hf3fs_utils.read_shm(
self.shm_r_tensor, self.r_pinned, tensors, self.stream_ptr_int
)
results = [res.result for res in resv]
return results
@wsynchronized()
def batch_write(
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
) -> list[int]:
"""Write data from tensors to the file at specified offsets.
Args:
offsets: List of byte offsets to write to
tensors: List of tensors containing data to write
event: CUDA event to synchronize on before reading the tensors
Returns:
List of operation results (0 for success, non-zero for error)
"""
self.check(offsets, tensors)
assert self.ior_w is not None
assert self.iov_w is not None
# prepare
with torch.cuda.stream(self.stream):
self.stream.wait_event(event)
hf3fs_utils.write_shm(
tensors, self.shm_w_tensor, self.w_pinned, self.stream_ptr_int
)
current = 0
for offset, tensor in zip(offsets, tensors):
size = tensor.numel() * tensor.itemsize
self.ior_w.prepare(
self.iov_w[current : current + size], False, self.file, offset
)
current += size
# submit
ionum = len(offsets)
resv = self.ior_w.submit().wait(min_results=ionum)
# results
results = [res.result for res in resv]
return results
def check(self, offsets: list[int], tensors: list[torch.Tensor]) -> None:
sizes = [t.numel() * t.itemsize for t in tensors]
if any(
[
len(offsets) > self.entries,
len(offsets) != len(sizes),
any(
offset < 0 or offset + size > self.size
for offset, size in zip(offsets, sizes)
),
any(size > self.bytes_per_page for size in sizes),
]
):
self.close()
raise ValueError("Hf3fsClient.check Failed")
def get_size(self) -> int:
"""Get the total size of the storage file.
Returns:
Size of the file in bytes
"""
return self.size
def close(self) -> None:
"""Close the client and clean up resources."""
if self._closed:
return
self._closed = True
self._release_resources()
def flush(self) -> None:
"""Flush any pending writes to disk."""
if not self._closed and self.file is not None:
os.fsync(self.file)
File diff suppressed because it is too large Load Diff
@@ -1,532 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HF3FS Metadata Server with key-based organization."""
import argparse
import logging
import threading
from abc import ABC, abstractmethod
from dataclasses import dataclass
try:
import orjson
HAS_ORJSON = True
except ImportError:
import json as orjson # type: ignore
HAS_ORJSON = False
import requests
from fastapi import FastAPI, HTTPException, Request, Response
from fastapi.responses import ORJSONResponse
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)
@dataclass
class RankFileMetadata:
"""Manages file page allocation for a single rank."""
rank_id: int
num_pages: int
free_pages: list[int]
def allocate_pages(self, num_pages: int) -> list[int]:
"""Allocate specified number of free pages."""
if len(self.free_pages) < num_pages:
return []
allocated = self.free_pages[:num_pages]
self.free_pages = self.free_pages[num_pages:]
return allocated
def release_pages(self, page_indices: list[int]) -> None:
"""Release pages back to free pool."""
for page_idx in page_indices:
if page_idx not in self.free_pages:
self.free_pages.append(page_idx)
def get_free_page_count(self) -> int:
"""Get current number of free pages."""
return len(self.free_pages)
@dataclass
class KeyMetadata:
"""Manages metadata for a single key across multiple ranks."""
key: str
rank_to_page: dict[int, int] # rank -> allocated page index
tp_world_size: int
def add_rank_page(self, rank: int, page_index: int) -> None:
"""Add page allocation for a specific rank."""
self.rank_to_page[rank] = page_index
def get_all_pages(self) -> list[tuple[int, int]]:
"""Get all (rank, page) pairs for this key."""
return [(rank, page) for rank, page in self.rank_to_page.items()]
def get_rank_page(self, rank: int) -> int | None:
"""Get page index for a specific rank."""
return self.rank_to_page.get(rank)
def is_complete(self) -> bool:
"""Check if all ranks in the TP world have allocated pages."""
return len(self.rank_to_page) == self.tp_world_size
class GlobalMetadataState:
"""Manages global metadata state across all ranks and keys."""
def __init__(self):
self.global_lock = threading.RLock()
self.rank_metadata: dict[int, RankFileMetadata] = {}
self.key_metadata: dict[str, KeyMetadata] = {}
def clear(self) -> None:
"""Clear all metadata state."""
with self.global_lock:
self.rank_metadata.clear()
self.key_metadata.clear()
logger.info("Cleared all metadata state")
def initialize_rank(self, rank: int, num_pages: int) -> None:
"""Initialize a new rank with specified number of pages."""
with self.global_lock:
if rank not in self.rank_metadata:
self.rank_metadata[rank] = RankFileMetadata(
rank, num_pages, list(range(num_pages))
)
logger.info("Initialized rank %s with %s pages", rank, num_pages)
def allocate_pages_for_keys(
self, rank: int, keys: list[tuple[str, str]]
) -> dict[str, int]:
"""Allocate one page for each key on the specified rank.
Args:
rank: Rank ID to allocate pages on
keys: List of keys to allocate pages for
Returns:
Dictionary mapping key -> allocated page index
"""
with self.global_lock:
if rank not in self.rank_metadata:
raise ValueError(f"Rank {rank} not initialized")
# Batch allocate pages for all keys
num_pages_needed = len(keys)
allocated_pages = self.rank_metadata[rank].allocate_pages(num_pages_needed)
if len(allocated_pages) < num_pages_needed:
logger.warning(
"Rank %s only allocated %s pages for %s keys",
rank,
len(allocated_pages),
num_pages_needed,
)
allocation_results = {}
for i, (key, prefix_key) in enumerate(keys):
if key in self.key_metadata:
key_meta = self.key_metadata[key]
if key_meta.is_complete() and rank in key_meta.rank_to_page:
# key is already fully written, reuse the existing page
# and release the allocated pages back to the free pool.
if i < len(allocated_pages):
self.rank_metadata[rank].release_pages([allocated_pages[i]])
allocation_results[key] = key_meta.rank_to_page[rank]
continue
if i < len(allocated_pages):
allocation_results[key] = allocated_pages[i]
else:
allocation_results[key] = -1 # No pages available
return allocation_results
def confirm_write_for_keys(
self,
rank: int,
key_confirmations: list[tuple[str, int]],
pages_to_release: list[int] | None = None,
) -> None:
"""Confirm write operations for keys and update metadata.
Args:
rank: Rank ID that confirmed the writes
key_confirmations: List of (key, page_index) tuples
pages_to_release: List of page indices to release back to free pool
"""
with self.global_lock:
# Confirm successful writes
for key, page_index in key_confirmations:
if key not in self.key_metadata:
# Need to determine tp_world_size from rank_metadata
tp_world_size = len(self.rank_metadata)
self.key_metadata[key] = KeyMetadata(key, {}, tp_world_size)
# Add confirmed page to key metadata
self.key_metadata[key].add_rank_page(rank, page_index)
# Release specified pages back to free pool
if pages_to_release:
self.rank_metadata[rank].release_pages(pages_to_release)
logger.debug(
"Released %s pages on rank %s: %s",
len(pages_to_release),
rank,
pages_to_release,
)
def batch_key_exists(self, keys: list[str]) -> list[bool]:
"""Check if keys exist in metadata and all ranks have confirmed writes.
Args:
keys: List of keys to check
Returns:
List of boolean values indicating key existence and completion
"""
with self.global_lock:
results = []
for key in keys:
if key not in self.key_metadata:
results.append(False)
else:
# Check if all ranks in the TP world have confirmed writes
key_meta = self.key_metadata[key]
results.append(key_meta.is_complete())
return results
def get_key_locations(self, rank: int, keys: list[str]) -> list[int | None]:
"""Get page indices for keys on a specific rank.
Args:
rank: Rank ID to query
keys: List of keys to look up
Returns:
List of page indices in the same order as input keys (None if key not found)
"""
with self.global_lock:
if rank not in self.rank_metadata:
raise ValueError(f"Rank {rank} not initialized")
results = []
for key in keys:
if key in self.key_metadata:
key_meta = self.key_metadata[key]
if key_meta.is_complete():
page_index = key_meta.get_rank_page(rank)
else:
page_index = None
results.append(page_index)
else:
results.append(None)
return results
class Hf3fsMetadataServer:
"""HF3FS Metadata Server with improved key-based organization."""
def __init__(self, persistence_path: str | None = None, save_interval: int = 60):
self.state = GlobalMetadataState()
if HAS_ORJSON:
self.app = FastAPI(default_response_class=ORJSONResponse)
else:
self.app = FastAPI()
self._setup_routes()
async def _read_json(self, request: Request) -> dict:
"""Parse request JSON using orjson if available."""
body = await request.body()
return orjson.loads(body)
def _json_response(self, content: dict):
"""Return ORJSONResponse when available to bypass jsonable_encoder."""
if HAS_ORJSON:
return ORJSONResponse(content)
else:
return content
def _setup_routes(self):
"""Setup FastAPI routes for new API design."""
self.app.post("/rank/{rank}/initialize")(self.initialize_rank)
self.app.post("/keys/batch_allocate")(self.batch_allocate_pages_for_keys)
self.app.post("/keys/confirm_write")(self.confirm_write_for_keys)
self.app.post("/keys/batch_exists")(self.batch_key_exists)
self.app.post("/keys/get_locations")(self.get_key_locations)
self.app.post("/clear")(self.clear)
async def initialize_rank(self, rank: int, request: Request):
"""Initialize a rank with specified number of pages."""
data = await self._read_json(request)
role = data.get("role", "worker")
num_pages = data.get("num_pages", 0)
if role == "scheduler":
return self._json_response(
{"message": "Scheduler role does not require initialization"}
)
if role == "worker" and num_pages > 0:
self.state.initialize_rank(rank, num_pages)
return self._json_response(
{"message": f"Rank {rank} initialized with {num_pages} pages"}
)
else:
raise HTTPException(
status_code=400, detail="Invalid initialization parameters"
)
async def batch_allocate_pages_for_keys(self, request: Request):
"""Allocate one page for each key on a specific rank."""
data = await self._read_json(request)
rank = data.get("rank")
keys = data.get("keys", [])
# Validate input format
if rank is None or not isinstance(keys, list):
raise HTTPException(
status_code=400, detail="Invalid request format: need 'rank' and 'keys'"
)
try:
# Perform allocation
results = self.state.allocate_pages_for_keys(rank, keys)
# Convert results to response format
response = {"rank": rank, "results": list(results.items())}
return self._json_response(response)
except Exception as e:
raise HTTPException(
status_code=500, detail=f"Allocation failed: {str(e)}"
) from e
async def confirm_write_for_keys(self, request: Request):
"""Confirm write operations for keys."""
data = await self._read_json(request)
rank = data.get("rank")
confirmations = data.get("confirmations", [])
pages_to_release = data.get("pages_to_release", [])
# Validate input format
if rank is None or not isinstance(confirmations, list):
raise HTTPException(
status_code=400,
detail="Invalid request format: need 'rank' and 'confirmations'",
)
try:
self.state.confirm_write_for_keys(rank, confirmations, pages_to_release)
return Response(status_code=204)
except Exception as e:
logger.error("Confirm write for keys failed: %s", e)
raise HTTPException(
status_code=500, detail=f"Confirmation failed: {str(e)}"
) from e
async def batch_key_exists(self, request: Request):
"""Check if multiple keys exist in metadata."""
data = await self._read_json(request)
keys = data.get("keys", [])
if not isinstance(keys, list):
raise HTTPException(status_code=400, detail="Invalid keys format")
try:
exists_results = self.state.batch_key_exists(keys)
return self._json_response({"exists": exists_results})
except Exception as e:
raise HTTPException(
status_code=500, detail=f"Key existence check failed: {str(e)}"
) from e
async def get_key_locations(self, request: Request):
"""Get page indices for keys on a specific rank."""
data = await self._read_json(request)
rank = data.get("rank")
keys = data.get("keys", [])
# Validate input format
if rank is None or not isinstance(keys, list):
raise HTTPException(
status_code=400, detail="Invalid request format: need 'rank' and 'keys'"
)
try:
# Get key locations
locations = self.state.get_key_locations(rank, keys)
return self._json_response({"locations": locations})
except Exception as e:
raise HTTPException(
status_code=500, detail=f"Failed to get key locations: {str(e)}"
) from e
async def clear(self, request: Request):
"""Clear the metadata server."""
self.state.clear()
return Response(status_code=204)
def run(self, host: str = "0.0.0.0", port: int = 18000):
"""Run the metadata server."""
import uvicorn
logger.info("Starting improved metadata server on http://%s:%s", host, port)
uvicorn.run(self.app, host=host, port=port)
# --- Client implementation ---
class Hf3fsMetadataInterface(ABC):
"""Interface for HF3FS metadata operations."""
@abstractmethod
def initialize(self, rank: int, num_pages: int = 0, role: str = "worker") -> None:
"""Initialize the metadata service with specified number of pages."""
pass
@abstractmethod
def allocate_pages_for_keys(
self, rank: int, keys: list[tuple[str, str]]
) -> list[tuple[str, int]]:
"""Allocate one page for each key on the specified rank."""
pass
@abstractmethod
def confirm_write_for_keys(
self,
rank: int,
key_confirmations: list[tuple[str, int]],
pages_to_release: list[int] | None = None,
) -> None:
"""Confirm write operations for keys and optionally release pages."""
pass
@abstractmethod
def batch_key_exists(self, keys: list[str]) -> list[bool]:
"""Check if keys exist and are complete across all ranks."""
pass
@abstractmethod
def get_key_locations(self, rank: int, keys: list[str]) -> list[int]:
"""Get page indices for keys on a specific rank."""
pass
class Hf3fsGlobalMetadataClient(Hf3fsMetadataInterface):
"""Global HTTP metadata client for HF3FS."""
def __init__(self, base_url: str = "http://localhost:18000", max_retries: int = 3):
self.base_url = base_url.rstrip("/")
self._session = requests.Session()
retry_strategy = Retry(
total=max_retries,
backoff_factor=0.3,
status_forcelist=[500, 502, 503, 504],
allowed_methods=["GET", "POST"],
)
adapter = HTTPAdapter(max_retries=retry_strategy)
self._session.mount("http://", adapter)
def _post(self, endpoint: str, json_data: dict) -> dict:
"""Make POST request to metadata server."""
try:
url = f"{self.base_url}/{endpoint}"
headers = {"Content-Type": "application/json"}
if HAS_ORJSON:
payload = orjson.dumps(json_data)
else:
import json
payload = json.dumps(json_data).encode("utf-8")
response = self._session.post(url, data=payload, headers=headers)
response.raise_for_status()
if response.status_code == 204 or not response.content:
return {}
if HAS_ORJSON:
return orjson.loads(response.content)
else:
return response.json()
except requests.exceptions.RequestException as e:
logger.error("Failed to POST to %s after retries: %s", endpoint, e)
raise RuntimeError(f"Failed to connect to metadata server: {e}") from e
def initialize(self, rank: int, num_pages: int = 0, role: str = "worker") -> None:
"""Initialize a rank with specified number of pages."""
self._post(f"rank/{rank}/initialize", {"num_pages": num_pages, "role": role})
def allocate_pages_for_keys(
self, rank: int, keys: list[tuple[str, str]]
) -> list[tuple[str, int]]:
"""Allocate pages for keys on the specified rank."""
response = self._post("keys/batch_allocate", {"rank": rank, "keys": keys})
# Convert response to expected format
return response.get("results", {})
def confirm_write_for_keys(
self,
rank: int,
key_confirmations: list[tuple[str, int]],
pages_to_release: list[int] | None = None,
) -> None:
"""Confirm write operations for keys and optionally release pages."""
payload = {
"rank": rank,
"confirmations": key_confirmations,
"pages_to_release": pages_to_release or [],
}
self._post("keys/confirm_write", payload)
def batch_key_exists(self, keys: list[str]) -> list[bool]:
"""Check if keys exist and are complete across all ranks."""
response = self._post("keys/batch_exists", {"keys": keys})
return response.get("exists", [])
def get_key_locations(self, rank: int, keys: list[str]) -> list[int]:
"""Get page indices for keys on a specific rank."""
response = self._post("keys/get_locations", {"rank": rank, "keys": keys})
return response.get("locations", [])
def run_metadata_server(
host: str = "0.0.0.0",
port: int = 18000,
):
"""Run the improved HF3FS metadata server."""
server = Hf3fsMetadataServer()
server.run(host=host, port=port)
# --- Main Execution ---
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Improved HF3FS Metadata Server")
parser.add_argument(
"--host", type=str, default="0.0.0.0", help="Host to bind the server to."
)
parser.add_argument(
"--port", type=int, default=18000, help="Port to run the server on."
)
args = parser.parse_args()
run_metadata_server(args.host, args.port)
@@ -1,139 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from dataclasses import dataclass, field
from typing import Optional
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.v1.request import Request
class AtomicCounter:
"""Thread-safe atomic counter for round-robin operations."""
def __init__(self, n: int):
assert n > 0, "Counter size must be positive"
self._n = n
self._value = 0
self._lock = threading.Lock()
def next(self) -> int:
"""Get next value in round-robin fashion."""
with self._lock:
current = self._value
self._value = (current + 1) % self._n
return current
@dataclass
class LoadBlockInfo:
"""Operation for loading blocks from external storage."""
num_computed_blocks: int
num_blocks_to_load: int
need_fetch_block_ids: list[int]
@dataclass
class SaveBlockInfo:
"""Operation for saving blocks to external storage."""
skip_leading_blocks: int
@dataclass
class RequestSchedulingState:
"""Unified request scheduling state management."""
request_id: str
request: Request | None = None
# Token and block tracking
token_ids: list[int] = field(default_factory=list)
allocated_block_ids: list[int] = field(default_factory=list)
num_saved_blocks: int = 0
# Load operation info
load_op: LoadBlockInfo | None = None
# Scheduling phase
phase: str = "NEW" # NEW -> WAITING_TO_LOAD -> ACTIVE -> FINISHED
def needs_loading(self) -> bool:
"""Check if request needs loading."""
return self.load_op is not None and self.load_op.num_blocks_to_load > 0
def is_ready_to_load(self) -> bool:
"""Check if request is ready for loading."""
return self.phase == "WAITING_TO_LOAD" and self.needs_loading()
def update_tokens_and_blocks(self, new_token_ids: list[int], new_block_ids) -> None:
"""Update with new tokens and blocks."""
if new_token_ids:
self.token_ids.extend(new_token_ids)
if new_block_ids is not None:
normalized_block_ids = self._normalize_block_ids(new_block_ids)
self.allocated_block_ids.extend(normalized_block_ids)
def _normalize_block_ids(self, block_ids) -> list[int]:
"""Normalize block_ids to list format."""
if not block_ids:
return []
if isinstance(block_ids, tuple):
return block_ids[0] if block_ids else []
if isinstance(block_ids, list):
return block_ids
return []
@dataclass
class HF3FSRequestMetadata:
"""Metadata for a single request in HF3FS connector."""
request_id: str
token_ids: list[int]
block_ids: list[int]
load_block_op: LoadBlockInfo | None = None
save_block_op: SaveBlockInfo | None = None
@staticmethod
def from_scheduling_state(
state: "RequestSchedulingState",
block_size: int,
load_op: LoadBlockInfo | None = None,
skip_leading_blocks: int | None = None,
) -> Optional["HF3FSRequestMetadata"]:
"""Create request metadata from scheduling state."""
token_count = len(state.token_ids)
total_blocks = token_count // block_size
skip_blocks = (
state.num_saved_blocks
if skip_leading_blocks is None
else skip_leading_blocks
)
new_blocks_to_save = total_blocks - state.num_saved_blocks
if new_blocks_to_save <= 0 and load_op is None:
return None
state.num_saved_blocks = total_blocks
return HF3FSRequestMetadata(
request_id=state.request_id,
token_ids=state.token_ids,
block_ids=state.allocated_block_ids,
load_block_op=load_op,
save_block_op=SaveBlockInfo(skip_leading_blocks=skip_blocks),
)
class HF3FSConnectorMetadata(KVConnectorMetadata):
"""Container for HF3FS connector metadata."""
def __init__(self):
self.requests: list[HF3FSRequestMetadata] = []
def add_request(self, request_metadata: HF3FSRequestMetadata) -> None:
"""Add request to metadata."""
self.requests.append(request_metadata)
@@ -1,262 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import async_tensor_h2d
@triton.jit
def kv_cache_scatter_kernel(
kv_cache_ptrs_ptr,
source_ptr,
token_indices_ptr,
num_tokens_in_block,
total_token_in_kvcache,
num_layers,
tokens_per_block,
block_stride,
head_stride,
token_stride,
content_stride,
num_heads: tl.constexpr,
content_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
layer_idx = tl.program_id(0)
token_pos = tl.program_id(1)
if layer_idx >= num_layers or token_pos >= num_tokens_in_block:
return
token_idx = tl.load(token_indices_ptr + token_pos)
kv_cache_ptr = tl.cast(tl.load(kv_cache_ptrs_ptr + layer_idx), source_ptr.dtype)
if token_idx >= total_token_in_kvcache:
return
block_idx = token_idx // tokens_per_block
state_idx = token_idx % tokens_per_block
hidden_size = num_heads * content_size
for i in range(0, hidden_size, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < hidden_size
head_idx = offset // content_size
content_idx = offset % content_size
source_offset = (
(layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos
) * content_size + content_idx
target_offset = (
block_idx * block_stride
+ head_idx * head_stride
+ state_idx * token_stride
+ content_idx * content_stride
)
value = tl.load(source_ptr + source_offset, mask=mask)
tl.store(kv_cache_ptr + target_offset, value, mask=mask)
@triton.jit
def kv_cache_gather_kernel(
kv_cache_ptrs_ptr,
dst_ptr,
token_indices_ptr,
num_tokens_in_block,
total_token_in_kvcache,
num_layers,
tokens_per_block,
block_stride,
head_stride,
token_stride,
content_stride,
num_heads: tl.constexpr,
content_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
layer_idx = tl.program_id(0)
token_pos = tl.program_id(1)
if layer_idx >= num_layers or token_pos >= num_tokens_in_block:
return
token_idx = tl.load(token_indices_ptr + token_pos)
kv_cache_ptr = tl.cast(tl.load(kv_cache_ptrs_ptr + layer_idx), dst_ptr.dtype)
if token_idx >= total_token_in_kvcache:
return
block_idx = token_idx // tokens_per_block
state_idx = token_idx % tokens_per_block
hidden_size = num_heads * content_size
for i in range(0, hidden_size, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < hidden_size
head_idx = offset // content_size
content_idx = offset % content_size
kvcache_offset = (
block_idx * block_stride
+ head_idx * head_stride
+ state_idx * token_stride
+ content_idx * content_stride
)
dst_offset = (
(layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos
) * content_size + content_idx
value = tl.load(kv_cache_ptr + kvcache_offset, mask=mask)
tl.store(dst_ptr + dst_offset, value, mask=mask)
def scatter_kv_caches(
kv_caches_ptrs: torch.Tensor,
total_token_in_kvcache: int,
src_tensor: torch.Tensor,
token_indices: list[int],
tokens_per_block: int,
num_heads: int,
content_size: int,
kv_cache_strides: tuple[int, ...],
) -> None:
"""Scatter KV cache data from source tensor to KV cache storage.
Args:
kv_caches_ptrs: Tensor of KV cache pointers (one per layer)
total_token_in_kvcache: Total number of tokens in KV cache
src_tensor: Source ``[L, H, N, C]`` tensor containing data to scatter
token_indices: List of token positions to update
tokens_per_block: Number of stored states in each cache block
num_heads: Size of the H axis
content_size: Size of the C axis
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
"""
num_layers = len(kv_caches_ptrs)
num_tokens_in_block = len(token_indices)
assert src_tensor.shape == (
num_layers,
num_heads,
num_tokens_in_block,
content_size,
)
assert len(kv_cache_strides) == 4
device = src_tensor.device
token_indices_tensor = async_tensor_h2d(
token_indices, device=device, dtype=torch.int32
)
grid = (num_layers, num_tokens_in_block)
BLOCK_SIZE = 128
kv_cache_scatter_kernel[grid](
kv_caches_ptrs,
src_tensor,
token_indices_tensor,
num_tokens_in_block,
total_token_in_kvcache,
num_layers,
tokens_per_block,
*kv_cache_strides,
num_heads=num_heads,
content_size=content_size,
BLOCK_SIZE=BLOCK_SIZE,
)
def gather_kv_caches(
kv_caches_ptrs: torch.Tensor,
total_token_in_kvcache: int,
dst_tensor: torch.Tensor,
token_indices: list[int],
tokens_per_block: int,
num_heads: int,
content_size: int,
kv_cache_strides: tuple[int, ...],
) -> None:
"""Gather KV cache data from KV cache storage to destination tensor.
Args:
kv_caches_ptrs: Tensor of KV cache pointers (one per layer)
total_token_in_kvcache: Total number of tokens in KV cache
dst_tensor: Destination ``[L, H, N, C]`` tensor
token_indices: List of token positions to gather
tokens_per_block: Number of stored states in each cache block
num_heads: Size of the H axis
content_size: Size of the C axis
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
"""
num_layers = kv_caches_ptrs.shape[0]
num_tokens_in_block = len(token_indices)
assert dst_tensor.shape == (
num_layers,
num_heads,
num_tokens_in_block,
content_size,
)
assert len(kv_cache_strides) == 4
device = dst_tensor.device
token_indices_tensor = async_tensor_h2d(
token_indices, device=device, dtype=torch.int32
)
grid = (num_layers, num_tokens_in_block)
BLOCK_SIZE = 128
kv_cache_gather_kernel[grid](
kv_caches_ptrs,
dst_tensor,
token_indices_tensor,
num_tokens_in_block,
total_token_in_kvcache,
num_layers,
tokens_per_block,
*kv_cache_strides,
num_heads=num_heads,
content_size=content_size,
BLOCK_SIZE=BLOCK_SIZE,
)
class CopyBufferAllocator:
"""Memory pool for tensor buffers to avoid frequent allocation/deallocation."""
def __init__(
self, device: torch.device, dtype: torch.dtype, shape: list, max_count: int
):
self._shape = shape
self._max_count = max_count
self._device = device
self._free_buffers = [
torch.empty(shape, dtype=dtype, device=device) for _ in range(max_count)
]
self._inuse_count = 0
def alloc_buffer(self, count: int) -> list[torch.Tensor] | None:
"""Allocate buffers from the pool."""
if count == 0:
return []
if self._inuse_count + count <= self._max_count:
self._inuse_count += count
result = self._free_buffers[-count:]
del self._free_buffers[-count:]
return result
return None
def free_buffer(self, buffers: list[torch.Tensor]) -> None:
"""Return buffers to the pool."""
if not buffers:
return
if self._inuse_count >= len(buffers):
self._inuse_count -= len(buffers)
self._free_buffers.extend(buffers)
else:
raise RuntimeError("Attempted to free more buffers than allocated")
logger = init_logger(__name__)
@@ -1,133 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import os
import torch
logger = logging.getLogger(__name__)
HF3FS_AVAILABLE = True
class Hf3fsClient:
"""Mock HF3FS client using file backend for debugging and testing."""
def __init__(self, path: str, size: int, bytes_per_page: int, entries: int):
self._size = size
self._bytes_per_page = bytes_per_page
self._entries = entries
self._file_path = path
self._ensure_file_exists()
logger.debug("Initialized mock HF3FS client: %s (%d bytes)", path, size)
def _ensure_file_exists(self) -> None:
"""Create file if it doesn't exist."""
if not os.path.exists(self._file_path):
with open(self._file_path, "w+b") as f:
f.truncate(self._size)
def batch_read(self, offsets: list[int], tensors: list[torch.Tensor]) -> list[int]:
"""Read data from file at specified offsets into tensors."""
results = []
try:
with open(self._file_path, "rb") as f:
for offset, tensor in zip(offsets, tensors):
num_bytes = tensor.numel() * tensor.element_size()
if offset < 0 or offset + num_bytes > self._size:
results.append(-1)
continue
f.seek(offset)
buffer_data = f.read(num_bytes)
if len(buffer_data) == num_bytes == self._bytes_per_page:
tensor_data = self._convert_buffer_to_tensor(
buffer_data, tensor.dtype
)
tensor.copy_(
tensor_data.reshape(tensor.shape).to(tensor.device)
)
results.append(self._bytes_per_page)
else:
logger.error(
"Read size mismatch: got %d, expected %d",
len(buffer_data),
num_bytes,
)
results.append(-1)
except Exception as e:
logger.error("Batch read error: %s", e)
results.extend([-1] * (len(offsets) - len(results)))
return results
def _convert_buffer_to_tensor(
self, buffer_data: bytes, dtype: torch.dtype
) -> torch.Tensor:
"""Convert buffer data to tensor with proper dtype handling."""
if dtype == torch.bfloat16:
tensor_data = torch.frombuffer(buffer_data, dtype=torch.uint16)
return tensor_data.view(dtype=torch.bfloat16)
else:
return torch.frombuffer(buffer_data, dtype=dtype)
def batch_write(
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
) -> list[int]:
"""Write data from tensors to file at specified offsets."""
results = []
try:
torch.cuda.current_stream().wait_event(event)
# Convert tensors to bytes
data_bytes_list = [self._tensor_to_bytes(tensor) for tensor in tensors]
# Write to file
with open(self._file_path, "r+b") as f:
for offset, data_bytes in zip(offsets, data_bytes_list):
if offset < 0 or offset + len(data_bytes) > self._size:
results.append(-1)
continue
f.seek(offset)
bytes_written = f.write(data_bytes)
if bytes_written == len(data_bytes) == self._bytes_per_page:
results.append(self._bytes_per_page)
else:
logger.error(
"Write size mismatch: wrote %d, expected %d",
bytes_written,
self._bytes_per_page,
)
results.append(-1)
except Exception as e:
logger.error("Batch write error: %s", e)
results.extend([-1] * (len(offsets) - len(results)))
return results
def _tensor_to_bytes(self, tensor: torch.Tensor) -> bytes:
"""Convert tensor to bytes with proper dtype handling."""
cpu_tensor = tensor.cpu()
if cpu_tensor.dtype == torch.bfloat16:
return cpu_tensor.view(dtype=torch.uint16).numpy().tobytes()
else:
return cpu_tensor.numpy().tobytes()
def get_size(self) -> int:
"""Get the total size of the storage file."""
return self._size
def close(self) -> None:
"""Close the client (no-op for file backend)."""
pass
def flush(self) -> None:
"""Flush any pending writes (no-op for file backend)."""
pass
@@ -1,57 +0,0 @@
#include <cuda_runtime.h>
#include <torch/extension.h>
#include <cstring>
#include <vector>
void read_shm(const torch::Tensor& shm, const torch::Tensor& pin,
std::vector<torch::Tensor> dst, uint64_t stream_ptr) {
py::gil_scoped_release release;
cudaStream_t stream = reinterpret_cast<cudaStream_t>(stream_ptr);
// Copy from shared memory to pinned memory
char* shm_ptr = static_cast<char*>(shm.data_ptr());
char* src_ptr = static_cast<char*>(pin.data_ptr());
std::memcpy(src_ptr, shm_ptr, shm.numel() * shm.element_size());
// Copy from pinned memory to GPU tensors
size_t current = 0;
for (size_t i = 0; i < dst.size(); ++i) {
auto& t = dst[i];
size_t t_bytes = t.numel() * t.element_size();
char* dst_ptr = static_cast<char*>(t.data_ptr());
cudaMemcpyAsync(dst_ptr, src_ptr + current, t_bytes, cudaMemcpyHostToDevice,
stream);
current += t_bytes;
}
cudaStreamSynchronize(stream);
}
void write_shm(const std::vector<torch::Tensor> src, torch::Tensor& shm,
const torch::Tensor& pin, uint64_t stream_ptr) {
py::gil_scoped_release release;
cudaStream_t stream = reinterpret_cast<cudaStream_t>(stream_ptr);
// Copy from GPU tensors to pinned memory
char* dst_ptr = static_cast<char*>(pin.data_ptr());
size_t current = 0;
for (size_t i = 0; i < src.size(); ++i) {
auto& t = src[i];
size_t t_bytes = t.numel() * t.element_size();
char* src_ptr = static_cast<char*>(t.data_ptr());
cudaMemcpyAsync(dst_ptr + current, src_ptr, t_bytes, cudaMemcpyDeviceToHost,
stream);
current += t_bytes;
}
cudaStreamSynchronize(stream);
// Copy from pinned memory to shared memory
char* shm_ptr = static_cast<char*>(shm.data_ptr());
std::memcpy(shm_ptr, dst_ptr, shm.numel() * shm.element_size());
}
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
m.def("read_shm", &read_shm, "Read tensors from shared memory");
m.def("write_shm", &write_shm, "Write tensors to shared memory");
}
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,408 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Connector boundary for HiSparse scheduler and worker state."""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorWorkerMetadata,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparseKVConnectorStats,
HiSparsePromMetrics,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.hisparse.layout import get_hisparse_steady_state_concurrency
from vllm.v1.hisparse.types import (
SparseKVOffloadCommand,
SparseKVResidencyUpdate,
SparseKVRowMirror,
)
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.forward_context import ForwardContext
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
from vllm.v1.hisparse.coordinator import HiSparseCoordinator
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
@dataclass
class HiSparseConnectorMetadata(KVConnectorMetadata):
command: SparseKVOffloadCommand | None
host_block_copies: tuple[KVCacheBlockCopy, ...]
source_block_ids: tuple[int, ...]
row_mirrors: dict[str, tuple[SparseKVRowMirror, ...]]
all_context_pages_resident: bool
residency_updates: dict[str, SparseKVResidencyUpdate]
@dataclass
class HiSparseConnectorWorkerMetadata(KVConnectorWorkerMetadata):
enqueued_transfer_counts: dict[int, int]
completed_transfer_counts: dict[int, int]
completed_host_copy_dst_ids: tuple[int, ...] = ()
def aggregate(self, other: KVConnectorWorkerMetadata) -> KVConnectorWorkerMetadata:
assert isinstance(other, HiSparseConnectorWorkerMetadata)
def add_counts(first: dict[int, int], second: dict[int, int]) -> dict[int, int]:
combined = first.copy()
for transfer_id, count in second.items():
combined[transfer_id] = combined.get(transfer_id, 0) + count
return combined
return HiSparseConnectorWorkerMetadata(
enqueued_transfer_counts=add_counts(
self.enqueued_transfer_counts, other.enqueued_transfer_counts
),
completed_transfer_counts=add_counts(
self.completed_transfer_counts, other.completed_transfer_counts
),
# Every worker runs the same copies, so a union is the ack.
completed_host_copy_dst_ids=tuple(
{*self.completed_host_copy_dst_ids, *other.completed_host_copy_dst_ids}
),
)
class HiSparseConnectorScheduler:
def __init__(
self,
*,
async_speculative: bool,
draft_kv_lookahead: int = 0,
) -> None:
self.coordinator: HiSparseCoordinator | None = None
self.async_speculative = async_speculative
self.draft_kv_lookahead = draft_kv_lookahead
self.requests: dict[str, Request] = {}
def bind_coordinator(self, coordinator: HiSparseCoordinator) -> None:
assert self.coordinator is None
self.coordinator = coordinator
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
if self.coordinator is None:
return None
host_pool = self.coordinator.get_host_block_pool()
assert host_pool is not None
stats = HiSparseKVConnectorStats()
stats.record_host_usage(
host_pool.get_usage(), len(self.coordinator.pending_spills)
)
if host_pool.metrics_collector is not None:
stats.record_host_evictions(host_pool.metrics_collector.drain_events())
return stats
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> HiSparseConnectorMetadata:
assert self.coordinator is not None
# HiSparse rebinds worker state every step, so the load must start
# before the forward rather than being deferred to post-forward.
scheduler_output.has_sync_kv_loads = True
num_computed_tokens = {
request.req_id: request.num_computed_tokens
for request in scheduler_output.scheduled_new_reqs
}
num_computed_tokens.update(
zip(
scheduler_output.scheduled_cached_reqs.req_ids,
scheduler_output.scheduled_cached_reqs.num_computed_tokens,
)
)
scheduled_requests = tuple(
(
request_id,
num_computed_tokens[request_id],
scheduled_count,
)
for request_id, scheduled_count in (
scheduler_output.num_scheduled_tokens.items()
)
)
# Runs first: it plans this step's transfers and residency changes.
self.coordinator.advance_scheduled(
(
request_id,
min(start + count, self.requests[request_id].num_tokens),
)
for request_id, start, count in scheduled_requests
)
command = self.coordinator.build_offload_command()
host_block_copies = self.coordinator.take_host_block_copies()
source_group_id = self.coordinator.host_group_id
assert source_group_id is not None
source_block_ids = [
block_id
for request in scheduler_output.scheduled_new_reqs
for block_id in request.block_ids[source_group_id]
]
for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids:
if new_block_ids is not None:
source_block_ids.extend(new_block_ids[source_group_id])
row_mirrors = {}
for request_id, scheduled_start, scheduled_count in scheduled_requests:
mirror_start = scheduled_start
if self.async_speculative:
mirror_start = max(
0,
scheduled_start - self.requests[request_id].num_output_placeholders,
)
row_mirrors[request_id] = self.coordinator.build_row_mirrors(
(
(
request_id,
mirror_start,
scheduled_count
+ scheduled_start
- mirror_start
+ self.draft_kv_lookahead,
),
),
)
return HiSparseConnectorMetadata(
command,
host_block_copies,
tuple(source_block_ids),
row_mirrors,
self.coordinator.all_context_pages_resident(scheduled_requests),
self.coordinator.take_residency_updates(
scheduler_output.num_scheduled_tokens
),
)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
assert self.coordinator is not None
for request_id in connector_output.finished_recving or ():
self.coordinator.finish_host_import(
request_id, failed=request_id in connector_output.failed_recving
)
metadata = connector_output.kv_connector_worker_meta
if metadata is None:
return
assert isinstance(metadata, HiSparseConnectorWorkerMetadata)
self.coordinator.release_completed_host_copies(
metadata.completed_host_copy_dst_ids
)
self.coordinator.update_spills(
metadata.enqueued_transfer_counts,
metadata.completed_transfer_counts,
)
class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
"""Join the scheduler coordinator to the worker's transfer engine."""
@classmethod
def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str:
return "BLHNC"
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig,
) -> None:
super().__init__(vllm_config, role, kv_cache_config)
if kv_cache_config.hisparse_host_num_blocks is None:
raise ValueError("HiSparseConnector requires a HiSparse host pool")
self.connector_scheduler: HiSparseConnectorScheduler | None = None
self.connector_worker: HiSparseConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
speculative_config = vllm_config.speculative_config
self.connector_scheduler = HiSparseConnectorScheduler(
async_speculative=bool(
vllm_config.scheduler_config.async_scheduling
and speculative_config is not None
),
draft_kv_lookahead=vllm_config.num_lookahead_tokens,
)
max_model_len = vllm_config.model_config.max_model_len
steady_concurrency = get_hisparse_steady_state_concurrency(
vllm_config, kv_cache_config
)
logger.info_once(
"HiSparse steady-state maximum concurrency for %s tokens per "
"request: %.2fx (running requests reading from host).",
f"{max_model_len:,}",
steady_concurrency,
)
elif role == KVConnectorRole.WORKER:
self.connector_worker = HiSparseConnectorWorker(
vllm_config, kv_cache_config
)
else:
raise ValueError(f"Unsupported KV connector role: {role}")
def bind_kv_cache_manager(self, kv_cache_manager: KVCacheManager) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.bind_coordinator(
get_hisparse_coordinator(kv_cache_manager)
)
@property
def requires_kv_delivery(self) -> bool:
return False
def has_pending_push_work(self) -> bool:
assert self.connector_scheduler is not None
assert self.connector_scheduler.coordinator is not None
return self.connector_scheduler.coordinator.has_pending_work()
def has_pending_block_frees(self) -> bool:
assert self.connector_scheduler is not None
assert self.connector_scheduler.coordinator is not None
return self.connector_scheduler.coordinator.has_pending_reclamation()
def finish_forward(self) -> None:
assert self.connector_worker is not None
self.connector_worker.finish_forward()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def reset_capture_state(self) -> None:
assert self.connector_worker is not None
self.connector_worker.reset_hot_state()
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is not None:
return self.connector_worker.get_kv_connector_stats()
if self.connector_scheduler is not None:
return self.connector_scheduler.get_kv_connector_stats()
return None
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
HiSparseKVConnectorStats(data=data)
if data is not None
else HiSparseKVConnectorStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return HiSparsePromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, HiSparseConnectorMetadata)
request_state_indices = kwargs.get("request_state_indices")
assert request_state_indices is None or isinstance(
request_state_indices, torch.Tensor
)
request_ids = kwargs.get("request_ids")
assert request_ids is None or isinstance(request_ids, list)
attn_metadata = kwargs.get("attn_metadata")
assert attn_metadata is None or isinstance(attn_metadata, dict)
self.connector_worker.start_step(
metadata,
request_state_indices,
request_ids,
num_tokens=int(kwargs.get("num_tokens") or 0),
)
self.connector_worker.prepare_forward(attn_metadata)
def wait_for_layer_load(self, layer_name: str) -> None:
return
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
return
def wait_for_save(self) -> None:
return
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
assert self.connector_worker is not None
enqueued, completed = self.connector_worker.take_transfer_updates()
host_copies = self.connector_worker.take_completed_host_copies()
if not enqueued and not completed and not host_copies:
return None
return HiSparseConnectorWorkerMetadata(
enqueued_transfer_counts={transfer_id: 1 for transfer_id in enqueued},
completed_transfer_counts={transfer_id: 1 for transfer_id in completed},
completed_host_copy_dst_ids=tuple(host_copies),
)
def shutdown(self) -> None:
if self.connector_worker is not None:
self.connector_worker.shutdown()
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int | None, bool]:
return 0, False
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.requests[request.request_id] = request
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
def request_finished_all_groups(
self,
request: Request,
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
self.connector_scheduler.requests.pop(request.request_id, None)
return False, None
@@ -1,214 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stats and Prometheus metrics for the HiSparse connector."""
from dataclasses import dataclass
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.v1.metrics.buckets import histogram_buckets
from vllm.v1.metrics.stats import KVCacheEvictionEvent
from vllm.v1.metrics.utils import create_metric_per_engine
_HISPARSE_LEVEL_KEYS = frozenset({"host_cache_usage_perc", "pending_page_transfers"})
@dataclass
class HiSparseKVConnectorStats(KVConnectorStats):
"""Container for HiSparse hot-buffer residency metrics.
Each list entry is the delta recorded over one device counter snapshot
interval, so a list can hold multiple snapshots per logging interval.
The host-tier gauges are level values, not deltas: aggregation keeps
only the most recent observation.
"""
def __post_init__(self):
if not self.data:
# Empty container init, no data is passed in.
self.reset()
def reset(self):
# Must be serializable
self.data: dict[str, list[int | float]] = {
"cache_hits": [],
"cache_misses": [],
"host_to_device_bytes": [],
"host_cache_usage_perc": [],
"pending_page_transfers": [],
"host_block_lifetime_seconds": [],
"host_block_idle_before_evict_seconds": [],
"host_block_reuse_gap_seconds": [],
}
def record_snapshot(self, hits: int, misses: int, host_to_device_bytes: int):
self.data["cache_hits"].append(hits)
self.data["cache_misses"].append(misses)
self.data["host_to_device_bytes"].append(host_to_device_bytes)
def record_host_usage(self, usage: float, pending_page_transfers: int) -> None:
self.data["host_cache_usage_perc"].append(usage)
self.data["pending_page_transfers"].append(pending_page_transfers)
def record_host_evictions(self, events: list[KVCacheEvictionEvent]) -> None:
for event in events:
self.data["host_block_lifetime_seconds"].append(event.lifetime_seconds)
self.data["host_block_idle_before_evict_seconds"].append(event.idle_seconds)
self.data["host_block_reuse_gap_seconds"].extend(event.reuse_gaps_seconds)
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if not other.is_empty():
for k, v in other.data.items():
accumulator = self.data[k]
assert isinstance(accumulator, list)
if k in _HISPARSE_LEVEL_KEYS:
accumulator.extend(v[-1:])
else:
accumulator.extend(v)
return self
def reduce(self) -> dict[str, int | float]:
# Compute compact representative stats suitable for CLI logging.
reduced: dict[str, int | float] = {
"HiSparse hot-buffer hits": sum(self.data["cache_hits"]),
"HiSparse hot-buffer misses": sum(self.data["cache_misses"]),
"HiSparse host-to-device bytes": sum(self.data["host_to_device_bytes"]),
}
if self.data["host_cache_usage_perc"]:
usage = self.data["host_cache_usage_perc"][-1]
reduced["HiSparse host KV cache usage %"] = round(100 * usage, 1)
return reduced
def is_empty(self) -> bool:
return all(len(values) == 0 for values in self.data.values())
class HiSparsePromMetrics(KVConnectorPromMetrics):
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
counter_cache_hits = self._counter_cls(
name="vllm:hisparse_cache_hits",
documentation="Number of HiSparse device hot-buffer hits.",
labelnames=labelnames,
)
counter_cache_misses = self._counter_cls(
name="vllm:hisparse_cache_misses",
documentation="Number of HiSparse device hot-buffer misses.",
labelnames=labelnames,
)
counter_host_to_device_bytes = self._counter_cls(
name="vllm:hisparse_host_to_device_bytes",
documentation=(
"Bytes transferred from host KV storage to HiSparse hot buffers."
),
labelnames=labelnames,
)
self.hisparse_counters: dict[str, Any] = {
"cache_hits": create_metric_per_engine(
counter_cache_hits, self.per_engine_labelvalues
),
"cache_misses": create_metric_per_engine(
counter_cache_misses, self.per_engine_labelvalues
),
"host_to_device_bytes": create_metric_per_engine(
counter_host_to_device_bytes, self.per_engine_labelvalues
),
}
gauge_host_cache_usage = self._gauge_cls(
name="vllm:hisparse_host_cache_usage_perc",
documentation=(
"HiSparse host KV-cache usage. 1 means 100 percent usage. "
"Evictable cached blocks count as free."
),
multiprocess_mode="mostrecent",
labelnames=labelnames,
)
gauge_pending_page_transfers = self._gauge_cls(
name="vllm:hisparse_pending_page_transfers",
documentation=(
"HiSparse page transfers (write-backs and restores) not yet completed."
),
multiprocess_mode="mostrecent",
labelnames=labelnames,
)
self.hisparse_gauges: dict[str, Any] = {
"host_cache_usage_perc": create_metric_per_engine(
gauge_host_cache_usage, self.per_engine_labelvalues
),
"pending_page_transfers": create_metric_per_engine(
gauge_pending_page_transfers, self.per_engine_labelvalues
),
}
self.hisparse_histograms: dict[str, Any] = {}
if vllm_config.observability_config.kv_cache_metrics:
residency_buckets = histogram_buckets(
"kv_cache_residency",
overrides=vllm_config.observability_config.custom_histogram_buckets,
)
histogram_host_block_lifetime = self._histogram_cls(
name="vllm:hisparse_host_block_lifetime_seconds",
documentation=(
"Histogram of HiSparse host KV block lifetime from allocation "
"to eviction. Sampled metrics (controlled by "
"--kv-cache-metrics-sample)."
),
buckets=residency_buckets,
labelnames=labelnames,
)
histogram_host_block_idle_before_evict = self._histogram_cls(
name="vllm:hisparse_host_block_idle_before_evict_seconds",
documentation=(
"Histogram of HiSparse host KV block idle time before eviction. "
"Sampled metrics (controlled by --kv-cache-metrics-sample)."
),
buckets=residency_buckets,
labelnames=labelnames,
)
histogram_host_block_reuse_gap = self._histogram_cls(
name="vllm:hisparse_host_block_reuse_gap_seconds",
documentation=(
"Histogram of time gaps between consecutive HiSparse host KV "
"block accesses. Sampled metrics (controlled by "
"--kv-cache-metrics-sample)."
),
buckets=residency_buckets,
labelnames=labelnames,
)
self.hisparse_histograms = {
"host_block_lifetime_seconds": create_metric_per_engine(
histogram_host_block_lifetime, self.per_engine_labelvalues
),
"host_block_idle_before_evict_seconds": create_metric_per_engine(
histogram_host_block_idle_before_evict, self.per_engine_labelvalues
),
"host_block_reuse_gap_seconds": create_metric_per_engine(
histogram_host_block_reuse_gap, self.per_engine_labelvalues
),
}
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
for name, counter in self.hisparse_counters.items():
for value in transfer_stats_data.get(name, []):
counter[engine_idx].inc(value)
for name, gauge in self.hisparse_gauges.items():
values = transfer_stats_data.get(name, [])
if values:
gauge[engine_idx].set(values[-1])
for name, histogram in self.hisparse_histograms.items():
for value in transfer_stats_data.get(name, []):
histogram[engine_idx].observe(value)
@@ -1,978 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HiSparse worker-side host/hot state and data movement."""
from __future__ import annotations
from collections import deque
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any, NamedTuple
import numpy as np
import torch
from vllm import _custom_ops as ops
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparseKVConnectorStats,
)
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tp_group,
)
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import current_stream
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
from vllm.v1.hisparse.runtime import (
HiSparseCacheHandle,
drop_hisparse_host_pool,
release_pinned_state,
update_hisparse_residency,
)
from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror
from vllm.v1.kv_cache_interface import (
HiSparseHotSpec,
KVCacheConfig,
)
from vllm.v1.worker.utils import copy_kv_cache_blocks_inplace
if TYPE_CHECKING:
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnectorMetadata,
)
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
class _DMADescriptors(NamedTuple):
src: torch.Tensor
dst: torch.Tensor
sizes: torch.Tensor
src_np: np.ndarray
dst_np: np.ndarray
sizes_np: np.ndarray
@dataclass
class _SlotMappingStaging:
stream: torch.Stream
event: torch.Event
slots: torch.Tensor
candidates: tuple[SparseKVRowMirror, ...] = ()
num_tokens: int = 0
source_index: int = 0
def _allocate_dma_descriptors(size: int) -> _DMADescriptors:
src, dst, sizes = (torch.empty(size, dtype=torch.int64) for _ in range(3))
return _DMADescriptors(
src,
dst,
sizes,
src.numpy(),
dst.numpy(),
sizes.numpy(),
)
_METRICS_INTERVAL = 2000
def _get_hisparse_cache(
forward_context: dict[str, Any], layer_name: str
) -> HiSparseCacheHandle:
attention_layer = forward_context[layer_name]
hisparse_cache = attention_layer.hisparse_cache
assert hisparse_cache is not None
return hisparse_cache
def _flatten_row_mirrors(
row_mirrors: Mapping[str, tuple[SparseKVRowMirror, ...]],
request_ids: Sequence[str] | None,
) -> tuple[SparseKVRowMirror, ...]:
ordered_ids = row_mirrors if request_ids is None else request_ids
return tuple(
mirror
for request_id in ordered_ids
for mirror in row_mirrors.get(request_id, ())
)
def _select_written_row_mirrors(
candidates: tuple[SparseKVRowMirror, ...],
source_slots: np.ndarray,
source_index: int,
) -> tuple[SparseKVRowMirror, ...]:
"""Select and coalesce GPU-written rows from a scheduler-owned envelope."""
source_slots = source_slots[source_slots >= 0]
if source_slots.size == 0 or not candidates:
return ()
starts = np.asarray([mirror.source_starts for mirror in candidates], dtype=np.int64)
counts = np.fromiter((mirror.num_rows for mirror in candidates), dtype=np.int64)
destinations = np.fromiter(
[mirror.destination_start for mirror in candidates], dtype=np.int64
)
if source_index >= starts.shape[1]:
raise RuntimeError("HiSparse row-mirror source index is out of range.")
source_starts = starts[:, source_index]
order = np.argsort(source_starts)
matches = np.searchsorted(source_starts[order], source_slots, side="right") - 1
candidate_ids = order[np.maximum(matches, 0)]
offsets = source_slots - source_starts[candidate_ids]
valid = (matches >= 0) & (offsets < counts[candidate_ids])
if not np.all(valid):
# A page the scheduler could not map has no host destination, so its
# rows are simply not mirrorable this step.
source_slots = source_slots[valid]
candidate_ids = candidate_ids[valid]
offsets = offsets[valid]
if source_slots.size == 0:
return ()
sources = starts[candidate_ids] + offsets[:, None]
destinations = destinations[candidate_ids] + offsets
boundaries = (
np.flatnonzero(
(candidate_ids[1:] != candidate_ids[:-1])
| (np.diff(destinations) != 1)
| np.any(np.diff(sources, axis=0) != 1, axis=1)
)
+ 1
)
run_starts = np.concatenate(([0], boundaries))
run_ends = np.concatenate((boundaries, [destinations.size]))
return tuple(
SparseKVRowMirror(
source_starts=tuple(int(value) for value in sources[start]),
destination_start=int(destinations[start]),
num_rows=int(end - start),
)
for start, end in zip(run_starts, run_ends, strict=True)
)
def _is_hisparse_host_writer(
shared_host_region: SharedOffloadRegion | None,
) -> bool:
return shared_host_region is None or get_tensor_model_parallel_rank() == 0
def _create_hisparse_host_events(
shared_host_region: SharedOffloadRegion | None,
is_host_writer: bool,
device: torch.device,
) -> tuple[torch.Event, torch.Event]:
if shared_host_region is None:
return torch.Event(), torch.Event()
events: tuple[torch.Event, torch.Event] | None = None
ipc_handles = None
if is_host_writer:
events = (
torch.cuda.Event(interprocess=True),
torch.cuda.Event(interprocess=True),
)
stream = current_stream()
for event in events:
event.record(stream)
ipc_handles = tuple(event.ipc_handle() for event in events)
ipc_handles = get_tp_group().broadcast_object(ipc_handles, src=0)
if events is None:
events = (
torch.cuda.Event.from_ipc_handle(device, ipc_handles[0]),
torch.cuda.Event.from_ipc_handle(device, ipc_handles[1]),
)
return events
class HiSparseConnectorWorker:
"""Own HiSparse host/hot state and execute its worker-side transfers."""
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> None:
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
self._initialized = False
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
forward_context = self.vllm_config.compilation_config.static_forward_context
cache_handles: list[HiSparseCacheHandle] = []
cache_layer_names: list[str] = []
for group in self.kv_cache_config.kv_cache_groups:
if not isinstance(group.kv_cache_spec, HiSparseHotSpec):
continue
for cache_name in group.layer_names:
assert cache_name.endswith(HISPARSE_HOT_SUFFIX)
layer_name = cache_name[: -len(HISPARSE_HOT_SUFFIX)]
cache_handles.append(_get_hisparse_cache(forward_context, layer_name))
cache_layer_names.append(layer_name)
if not cache_handles:
raise RuntimeError("HiSparse connector found no hot-cache handles.")
hot_backings: dict[int, torch.Tensor] = {}
registered_host_pools: dict[int, torch.Tensor] = {}
shared_host_regions: dict[int, SharedOffloadRegion] = {}
for cache in cache_handles:
hot_backing = cache.runtime.hot_backing
hot_backings[hot_backing.untyped_storage().data_ptr()] = hot_backing
registered_pool = cache.runtime.registered_host_pool
registered_host_pools[registered_pool.data_ptr()] = registered_pool
if (region := cache.runtime.shared_host_region) is not None:
shared_host_regions[id(region)] = region
if len(hot_backings) != 1:
raise RuntimeError("HiSparse hot tensors must share one GPU backing.")
hot_backing = next(iter(hot_backings.values()))
if len(shared_host_regions) > 1:
raise RuntimeError("HiSparse caches must share one host region.")
shared_host_region = next(iter(shared_host_regions.values()), None)
pinned_host_pools = (
[]
if shared_host_region is not None
else list(registered_host_pools.values())
)
is_host_writer = _is_hisparse_host_writer(shared_host_region)
resident = cache_handles[0].view
assert resident is not None
host_num_blocks = self.kv_cache_config.hisparse_host_num_blocks
assert host_num_blocks is not None
try:
self.initialize(
cache_handles,
cache_layer_names,
hot_backing,
self.vllm_config.scheduler_config.max_num_seqs,
host_num_blocks,
hot_backing.device,
pinned_host_pools,
shared_host_region=shared_host_region,
is_host_writer=is_host_writer,
)
except Exception:
release_pinned_state(
[cache.runtime for cache in cache_handles],
pinned_host_pools,
shared_host_region,
)
drop_hisparse_host_pool(self.vllm_config)
raise
def initialize(
self,
cache_handles: list[HiSparseCacheHandle],
cache_layer_names: list[str],
hot_backing: torch.Tensor,
max_num_reqs: int,
host_num_blocks: int,
device: torch.device,
pinned_host_pools: list[torch.Tensor],
*,
shared_host_region: SharedOffloadRegion | None = None,
is_host_writer: bool = True,
) -> None:
if self._initialized:
raise RuntimeError("HiSparse connector worker is already initialized.")
resident = cache_handles[0].view
assert resident is not None
self.kernel_block_size = resident.block_size
self.host_num_blocks = host_num_blocks
self.pinned_host_pools = pinned_host_pools
self.shared_host_region = shared_host_region
self.is_host_writer = is_host_writer
self.dma_stream = (
torch.cuda.Stream(device=device) if self.is_host_writer else None
)
self._slot_mapping_staging = None
if self.is_host_writer:
max_mirror_rows = (
self.vllm_config.scheduler_config.max_num_batched_tokens
+ max_num_reqs * (self.vllm_config.num_lookahead_tokens + 1)
)
self._slot_mapping_staging = _SlotMappingStaging(
stream=torch.Stream(device=device),
event=torch.Event(),
slots=torch.empty(
max_mirror_rows,
dtype=torch.int64,
pin_memory=True,
),
)
self._dma_free_descriptors: list[_DMADescriptors] = []
self._pending_dma_descriptors: deque[tuple[torch.Event, _DMADescriptors]] = (
deque()
)
self._dma_submitted = False
self._per_layer_mirrored: set[int] = set()
self._submitted_mirror_layers: set[int] = set()
self._layer_ready_events = tuple(torch.Event() for _ in cache_handles)
self._forward_ready_event = torch.Event()
self._draft_layers = tuple(
index for index, cache in enumerate(cache_handles) if cache.draft_layer
)
self._draft_mirror_pending = False
self._set_row_mirrors(())
self.cache_layer_names = cache_layer_names
self._group_leaders = tuple(
(layer_name, cache)
for layer_name, cache in zip(cache_layer_names, cache_handles, strict=True)
if cache.runtime.is_group_leader
)
self.host_write_events = _create_hisparse_host_events(
shared_host_region, is_host_writer, device
)
self.host_write_event = self.host_write_events[1]
self._next_host_write_event = 0
self.cache_handles = cache_handles
self.leader_runtimes = [cache.runtime for _, cache in self._group_leaders]
request_state_indices = {
indices.data_ptr(): indices
for cache in cache_handles
if (indices := cache.runtime.request_state_indices) is not None
}
if len(request_state_indices) != 1:
raise RuntimeError(
"HiSparse runtimes must share one request-state mapping."
)
self.request_state_indices = next(iter(request_state_indices.values()))
if self.request_state_indices.numel() != max_num_reqs:
raise RuntimeError(
"HiSparse request-state mapping does not match max_num_seqs."
)
self.hot_backing = hot_backing
self.residency: torch.Tensor | None = next(
(cache.residency for cache in cache_handles if cache.residency is not None),
None,
)
self._pending_invalid_block_ids: list[int] = []
# Destination block ids of host copies this worker has run.
self._completed_host_copy_dst_ids: list[int] = []
self._post_forward_transfers: list[SparseKVPageTransfer] = []
self._enqueued_transfer_ids: list[int] = []
self._pending_transfer_events: deque[tuple[torch.Event, tuple[int, ...]]] = (
deque()
)
self._metrics_calls = 0
self._metrics_event = torch.Event()
self._metrics_pending = False
self._init_dma()
if self.is_host_writer:
for layer_index, handle in enumerate(cache_handles):
# Draft layers are mirrored once in the next start_step instead.
if not handle.draft_layer:
handle.submit_layer_mirror = partial(
self._enqueue_layer_mirror, layer_index
)
self._initialized = True
def set_request_state_indices(self, indices: torch.Tensor) -> None:
if indices.numel() > self.request_state_indices.numel():
raise ValueError(
"HiSparse request-state mapping exceeds max_num_seqs: "
f"{indices.numel()} > {self.request_state_indices.numel()}."
)
if torch.cuda.is_current_stream_capturing():
return
# Attention indexes persistent request state by input-batch row. Refresh
# that indirection after every batch compaction or reorder.
self.request_state_indices.fill_(-1)
self.request_state_indices[: indices.numel()].copy_(indices)
if self._pending_invalid_block_ids:
self.invalidate_blocks(self._pending_invalid_block_ids, indices)
self._pending_invalid_block_ids.clear()
def _init_dma(self) -> None:
host_caches = tuple(cache.runtime.host_cache for cache in self.cache_handles)
resident_caches = []
for cache in self.cache_handles:
assert cache.view is not None and cache.slot_mapping is not None
resident_caches.append(cache.view.cache)
self.host_caches = host_caches
self.resident_caches = tuple(resident_caches)
for resident_cache, host_cache in zip(resident_caches, host_caches):
row_bytes = resident_cache.shape[-1] * resident_cache.element_size()
if (
resident_cache.ndim != 3
or resident_cache.shape[1] != self.kernel_block_size
or resident_cache.stride(1) * resident_cache.element_size() != row_bytes
):
raise RuntimeError("HiSparse DMA requires contiguous resident rows.")
if (
host_cache.ndim != 2
or not host_cache.is_contiguous()
or host_cache.shape[1] * host_cache.element_size() != row_bytes
):
raise RuntimeError("HiSparse DMA requires contiguous host rows.")
def start_step(
self,
metadata: HiSparseConnectorMetadata,
request_state_indices: torch.Tensor | None,
request_ids: list[str] | None = None,
num_tokens: int = 0,
) -> None:
self._stage_row_mirror_mapping(num_tokens)
self._finish_previous_step()
previous_host_write_event = self.host_write_event
self.host_write_event = self.host_write_events[self._next_host_write_event]
self._next_host_write_event ^= 1
current_stream().wait_event(previous_host_write_event)
if self.shared_host_region is not None and not num_tokens:
# Without a forward, no TP collective keeps rank 0 from recording
# a shared event again before every rank has issued its wait.
get_tp_group().barrier()
self._release_completed_dma_descriptors()
self._dma_submitted = False
mirrors = _flatten_row_mirrors(metadata.row_mirrors, request_ids)
if self._slot_mapping_staging is not None:
self._slot_mapping_staging.candidates = mirrors
self._set_row_mirrors(mirrors)
self._clear_forward_mirror_state()
for handle in self.cache_handles:
handle.all_context_pages_resident = metadata.all_context_pages_resident
self._copy_host_blocks(metadata.host_block_copies, previous_host_write_event)
transfers = (
metadata.command.page_transfers if metadata.command is not None else []
)
self._restore_pages([transfer for transfer in transfers if transfer.restore])
self._post_forward_transfers = [
transfer
for transfer in transfers
if not transfer.restore and transfer.after_forward
]
self._submit_transfers(
[
transfer
for transfer in transfers
if not transfer.restore and not transfer.after_forward
]
)
self._pending_invalid_block_ids.extend(metadata.source_block_ids)
if request_state_indices is not None:
self.set_request_state_indices(request_state_indices)
if metadata.residency_updates:
assert self.residency is not None
assert request_state_indices is not None and request_ids is not None
update_hisparse_residency(
self.residency,
metadata.residency_updates,
request_ids,
request_state_indices,
)
def _clear_forward_mirror_state(self) -> None:
self._per_layer_mirrored.clear()
self._submitted_mirror_layers.clear()
for handle in self.cache_handles:
handle.decode_batch = False
handle.host_mirror_required = False
handle.num_actual_tokens = 0
handle.num_decode_tokens = 0
handle.req_id_per_token = None
def prepare_forward(self, attn_metadata: Mapping[str, Any] | None) -> None:
if attn_metadata is None:
return
for layer_name, handle in self._group_leaders:
metadata = attn_metadata.get(layer_name)
if metadata is not None:
handle.prepare_group_for_batch(metadata)
def _stage_row_mirror_mapping(self, num_tokens: int) -> None:
"""Snapshot the rows this forward will write, off the compute stream.
The resident slot mapping is a persistent view bound at registration
time, so the rows are already staged by the time the forward launches.
"""
state = self._slot_mapping_staging
if state is None or not num_tokens:
return
handle = self.cache_handles[0]
slots = handle.slot_mapping
assert slots is not None
source_index = handle.runtime.resident_source_index
start = state.num_tokens
end = start + num_tokens
if end > state.slots.shape[0]:
raise ValueError(
"HiSparse row mapping exceeds staging capacity: "
f"{end} > {state.slots.shape[0]}."
)
main_stream = current_stream()
state.stream.wait_stream(main_stream)
with torch.cuda.stream(state.stream):
state.slots[start:end].copy_(
slots[:num_tokens],
non_blocking=True,
)
state.event.record(state.stream)
state.num_tokens = end
state.source_index = source_index
def _resolve_row_mirrors(self, state: _SlotMappingStaging) -> None:
"""Compute the final row mirrors from the staged GPU slot mapping."""
self._set_row_mirrors(
_select_written_row_mirrors(
state.candidates,
state.slots[: state.num_tokens].numpy(),
state.source_index,
)
)
state.num_tokens = 0
def _copy_host_blocks(
self,
host_block_copies: Sequence[KVCacheBlockCopy],
previous_host_write_event: torch.Event,
) -> None:
if not host_block_copies:
return
self._completed_host_copy_dst_ids.extend(
copy.dst_block_id for copy in host_block_copies
)
if self.shared_host_region is None or get_tensor_model_parallel_rank() == 0:
if self.host_caches:
previous_host_write_event.synchronize()
copy_kv_cache_blocks_inplace(
self.host_caches,
self.host_num_blocks,
host_block_copies,
)
if self.shared_host_region is not None:
get_tp_group().barrier()
def invalidate_blocks(
self, block_ids: list[int], request_state_indices: torch.Tensor
) -> None:
"""Invalidate recycled host slots in this worker's leader runtimes."""
if not block_ids:
return
device = self.cache_handles[0].runtime.device
staging = torch.tensor(block_ids, dtype=torch.int32, pin_memory=True)
blocks = staging.to(device, dtype=torch.int32, non_blocking=True)
offsets = torch.arange(self.kernel_block_size, dtype=torch.int32, device=device)
slots = (blocks[:, None] * self.kernel_block_size + offsets[None, :]).flatten()
sorted_slots = torch.sort(slots).values
state_indices = request_state_indices.to(device=device, dtype=torch.long)
for runtime in self.leader_runtimes:
runtime.invalidate_sorted_slots(sorted_slots, state_indices)
def reset_hot_state(self) -> None:
for runtime in self.leader_runtimes:
runtime.reset_hot_state()
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
stats = None
if self._metrics_pending and self._metrics_event.query():
stats = HiSparseKVConnectorStats()
for runtime in self.leader_runtimes:
group = runtime.index_group
hits, misses = group.swap_stats_host.tolist()
if hits or misses:
stats.record_snapshot(hits, misses, misses * group.stats_row_bytes)
self._metrics_pending = False
if stats.is_empty():
stats = None
self._metrics_calls += 1
if (
self._metrics_calls % _METRICS_INTERVAL == 0
and not self._metrics_pending
and not torch.cuda.is_current_stream_capturing()
):
compute_stream = current_stream()
for runtime in self.leader_runtimes:
group = runtime.index_group
compute_stream.wait_stream(group.copy_stream)
group.swap_stats_host.copy_(group.swap_stats, non_blocking=True)
group.swap_stats.zero_()
group.copy_stream.wait_stream(compute_stream)
self._metrics_event.record()
self._metrics_pending = True
return stats
def _release_completed_dma_descriptors(self) -> None:
pending = self._pending_dma_descriptors
while pending and pending[0][0].query():
_, descriptors = pending.popleft()
self._dma_free_descriptors.append(descriptors)
def _acquire_dma_descriptors(self, size: int) -> _DMADescriptors:
for index, descriptors in enumerate(self._dma_free_descriptors):
if descriptors.src.numel() >= size:
return self._dma_free_descriptors.pop(index)
return _allocate_dma_descriptors(size)
def _submit_dma_descriptors(
self,
descriptors: _DMADescriptors,
descriptor_count: int,
transfer_ids: tuple[int, ...] = (),
ready_event: torch.Event | None = None,
) -> None:
stream = self.dma_stream
assert stream is not None
if ready_event is None:
stream.wait_stream(current_stream())
else:
stream.wait_event(ready_event)
completion_event = torch.Event()
with torch.cuda.stream(stream):
ops.swap_blocks_batch(
descriptors.src[:descriptor_count],
descriptors.dst[:descriptor_count],
descriptors.sizes[:descriptor_count],
)
self.host_write_event.record(stream)
completion_event.record(stream)
self._pending_dma_descriptors.append((completion_event, descriptors))
self._dma_submitted = True
if transfer_ids:
self._pending_transfer_events.append((completion_event, transfer_ids))
self._enqueued_transfer_ids.extend(transfer_ids)
def _set_row_mirrors(self, mirrors: tuple[SparseKVRowMirror, ...]) -> None:
self._row_mirrors = mirrors
self._row_mirror_destination_starts = np.fromiter(
(mirror.destination_start for mirror in mirrors),
dtype=np.int64,
count=len(mirrors),
)
self._row_mirror_counts = np.fromiter(
(mirror.num_rows for mirror in mirrors),
dtype=np.int64,
count=len(mirrors),
)
if mirrors:
self._row_mirror_source_starts = np.asarray(
[mirror.source_starts for mirror in mirrors], dtype=np.int64
)
if self._row_mirror_source_starts.ndim != 2:
raise RuntimeError("HiSparse DMA source mappings must be rectangular.")
else:
self._row_mirror_source_starts = np.empty((0, 0), dtype=np.int64)
self._row_mirror_num_rows = int(self._row_mirror_counts.sum())
def _enqueue_row_dma(
self, layer_indices: Sequence[int], ready_event: torch.Event | None = None
) -> None:
if (
not layer_indices
or not self._row_mirror_num_rows
or not self.is_host_writer
):
return
mirrors = self._row_mirrors
num_layers = len(layer_indices)
descriptor_count = len(mirrors) * num_layers
descriptors = self._acquire_dma_descriptors(descriptor_count)
destination_starts = self._row_mirror_destination_starts
row_counts = self._row_mirror_counts
for descriptor_offset, layer_index in enumerate(layer_indices):
cache = self.cache_handles[layer_index]
source_index = cache.runtime.resident_source_index
if source_index >= self._row_mirror_source_starts.shape[1]:
raise RuntimeError("HiSparse row DMA source index is out of range.")
source_rows = self._row_mirror_source_starts[:, source_index]
source = self.resident_caches[layer_index]
destination = self.host_caches[layer_index]
row_bytes = source.shape[-1] * source.element_size()
if (
source.stride(1) * source.element_size() != row_bytes
or destination.shape[1] * destination.element_size() != row_bytes
):
raise RuntimeError("HiSparse row DMA requires contiguous rows.")
source_blocks, source_row_offsets = np.divmod(
source_rows, self.kernel_block_size
)
source_out_of_range = (
np.any(source_blocks < 0)
or np.any(source_blocks >= source.shape[0])
or np.any(source_row_offsets + row_counts > self.kernel_block_size)
)
if source_out_of_range:
raise RuntimeError("HiSparse row DMA source is out of range.")
if np.any(destination_starts < 0) or np.any(
destination_starts + row_counts > destination.shape[0]
):
raise RuntimeError("HiSparse row DMA index is out of range.")
descriptor_slice = slice(descriptor_offset, descriptor_count, num_layers)
descriptors.src_np[descriptor_slice] = (
source.data_ptr()
+ source_blocks * source.stride(0) * source.element_size()
+ source_row_offsets * row_bytes
)
descriptors.dst_np[descriptor_slice] = (
destination.data_ptr() + destination_starts * row_bytes
)
descriptors.sizes_np[descriptor_slice] = row_counts * row_bytes
self._submit_dma_descriptors(
descriptors, descriptor_count, ready_event=ready_event
)
def _enqueue_layer_mirror(self, layer_index: int) -> None:
handle = self.cache_handles[layer_index]
if not handle.host_mirror_required:
return
if layer_index in self._per_layer_mirrored:
raise RuntimeError(f"HiSparse layer {layer_index} mirrored twice.")
state = self._slot_mapping_staging
if state is not None and state.num_tokens:
# The staging stream waited on the compute stream before the
# forward launched, so this only drains pre-forward work that is
# already queued ahead of the layer kernels: no GPU bubble.
state.event.synchronize()
self._resolve_row_mirrors(state)
self._per_layer_mirrored.add(layer_index)
next_layer = layer_index + 1
if (
next_layer < len(self.cache_handles)
and self.cache_handles[next_layer].runtime.resident_source_index
== handle.runtime.resident_source_index
):
return
ready_event = self._layer_ready_events[layer_index]
ready_event.record()
pending_layers = tuple(
sorted(self._per_layer_mirrored - self._submitted_mirror_layers)
)
self._enqueue_row_dma(
pending_layers,
ready_event=ready_event,
)
self._submitted_mirror_layers.update(pending_layers)
def _record_transfer_completion(
self,
transfers: list[SparseKVPageTransfer],
*,
stream: torch.Stream | None = None,
) -> None:
if not transfers:
return
if stream is None:
if not self.is_host_writer:
return
stream = self.dma_stream
assert stream is not None
completion_event = torch.Event()
completion_event.record(stream)
transfer_ids = tuple(transfer.transfer_id for transfer in transfers)
self._pending_transfer_events.append((completion_event, transfer_ids))
self._enqueued_transfer_ids.extend(transfer_ids)
def _restore_pages(self, transfers: list[SparseKVPageTransfer]) -> None:
"""Restore imported tails on every rank before its forward or graph replay."""
if not transfers:
return
for layer_index, cache in enumerate(self.cache_handles):
source_index = cache.runtime.resident_source_index
source = self.host_caches[layer_index]
destination = self.resident_caches[layer_index]
for transfer in transfers:
host_start = transfer.host_block_id * self.kernel_block_size
resident_block = transfer.resident_block_ids[source_index]
# Each import pays one H2D page copy and enqueue per layer/rank
# instead of tail prefill on D. Restore the prompt rows before
# decode appends to this page; unused rows do not extend the
# logical sequence.
# cudaHostRegister pins this pool outside PyTorch's allocator,
# so the sync checker cannot recognize it via is_pinned().
with gpu_sync_allowed():
destination[resident_block].copy_(
source[host_start : host_start + self.kernel_block_size],
non_blocking=True,
)
self._record_transfer_completion(transfers, stream=current_stream())
def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
if self.cache_handles[0].runtime.eager_host_mirror:
self._record_transfer_completion(transfers)
else:
self._enqueue_transfers(transfers)
def _enqueue_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
if not transfers or not self.is_host_writer:
return
num_layers = len(self.cache_handles)
descriptor_count = len(transfers) * num_layers
descriptors = self._acquire_dma_descriptors(descriptor_count)
destination_rows = np.fromiter(
(transfer.host_block_id * self.kernel_block_size for transfer in transfers),
dtype=np.int64,
count=len(transfers),
)
source_blocks_by_transfer = np.asarray(
[transfer.resident_block_ids for transfer in transfers], dtype=np.int64
)
if source_blocks_by_transfer.ndim != 2:
raise RuntimeError(
"HiSparse spill DMA source mappings must be rectangular."
)
for layer_index, cache in enumerate(self.cache_handles):
source_index = cache.runtime.resident_source_index
if source_index >= source_blocks_by_transfer.shape[1]:
raise RuntimeError("HiSparse spill DMA source index is out of range.")
source_blocks = source_blocks_by_transfer[:, source_index]
source = self.resident_caches[layer_index]
destination = self.host_caches[layer_index]
if np.any(source_blocks < 0) or np.any(source_blocks >= source.shape[0]):
raise RuntimeError("HiSparse spill DMA source is out of range.")
if np.any(destination_rows < 0) or np.any(
destination_rows + self.kernel_block_size > destination.shape[0]
):
raise RuntimeError("HiSparse spill DMA destination is out of range.")
row_bytes = source.shape[-1] * source.element_size()
descriptor_slice = slice(layer_index, descriptor_count, num_layers)
descriptors.src_np[descriptor_slice] = (
source.data_ptr()
+ source_blocks * source.stride(0) * source.element_size()
)
descriptors.dst_np[descriptor_slice] = (
destination.data_ptr() + destination_rows * row_bytes
)
descriptors.sizes_np[descriptor_slice] = self.kernel_block_size * row_bytes
self._submit_dma_descriptors(
descriptors,
descriptor_count,
transfer_ids=tuple(transfer.transfer_id for transfer in transfers),
)
def _enqueue_host_mirror(
self,
ready_event: torch.Event | None = None,
) -> None:
active = [
(index, handle)
for index, handle in enumerate(self.cache_handles)
if handle.num_actual_tokens != 0
]
if not active:
return
def mirror_key(handle: HiSparseCacheHandle) -> tuple:
slots = handle.mirror_slot_mapping
return (
handle.num_actual_tokens,
handle.num_decode_tokens,
handle.decode_batch,
handle.host_mirror_required,
handle.runtime.eager_host_mirror,
None if slots is None else slots.data_ptr(),
)
keys = {mirror_key(handle) for _, handle in active}
if len(keys) > 1:
raise RuntimeError(
f"HiSparse cache layers disagree on mirror metadata: {list(keys)}."
)
cache = active[0][1]
if not cache.host_mirror_required:
return
dst_slots = cache.mirror_slot_mapping
if dst_slots is None:
raise RuntimeError("HiSparse host mirror has no source slot mapping.")
num_rows = min(cache.num_actual_tokens, dst_slots.numel())
if num_rows == 0:
return
if self.is_host_writer:
# The drafter has not written its layers' rows yet; the next
# start_step mirrors them after it runs.
self._draft_mirror_pending = bool(self._draft_layers)
expected_layers = {index for index, _ in active}.difference(
self._draft_layers
)
if self._per_layer_mirrored and self._per_layer_mirrored != expected_layers:
raise RuntimeError(
"HiSparse per-layer DMA did not mirror every active layer: "
f"expected {sorted(expected_layers)}, got "
f"{sorted(self._per_layer_mirrored)}."
)
pending_layers = tuple(
sorted(expected_layers - self._submitted_mirror_layers)
)
if pending_layers:
self._enqueue_row_dma(pending_layers, ready_event=ready_event)
self._submitted_mirror_layers.update(pending_layers)
assert cache.req_id_per_token is not None
for _, handle in active:
if handle.runtime.is_group_leader:
handle.runtime.invalidate_written_slots(
dst_slots[:num_rows],
cache.req_id_per_token[:num_rows],
)
def _finish_mirror_phase(self, ready_event: torch.Event | None = None) -> None:
state = self._slot_mapping_staging
if state is not None and state.num_tokens:
# Measured free: blocking here costs no throughput (26.7 vs 27.6
# gen tok/s) and lets every step mirror exactly the written rows.
state.event.synchronize()
self._resolve_row_mirrors(state)
self._enqueue_host_mirror(ready_event)
self._clear_forward_mirror_state()
def finish_forward(self) -> None:
compute_stream = current_stream()
self._forward_ready_event.record()
self._finish_mirror_phase(self._forward_ready_event)
if self.is_host_writer:
if self._dma_submitted:
compute_stream.wait_event(self.host_write_event)
self._dma_submitted = False
else:
self.host_write_event.record(compute_stream)
self._release_completed_dma_descriptors()
def _finish_previous_step(self) -> None:
"""Mirror the previous step's draft-layer rows, then hand its pages over.
Runs before this step's work, so after the previous step's drafter.
Post-forward transfers are ordered behind the draft mirror, so a page is
never reported clean, and its resident block never released, before
its draft-layer rows reach the host. Both land in the previous step's
host write event, which this step waits on before its forward and its
host copies, since the scheduler may already have reused the
finished requests' blocks they read and write.
"""
if self._draft_mirror_pending:
self._draft_mirror_pending = False
self._enqueue_row_dma(self._draft_layers)
transfers = self._post_forward_transfers
self._post_forward_transfers = []
self._submit_transfers(transfers)
def take_completed_host_copies(self) -> list[int]:
"""Drain host copies this worker has enqueued for this step."""
completed = self._completed_host_copy_dst_ids
self._completed_host_copy_dst_ids = []
return completed
def take_transfer_updates(self) -> tuple[list[int], list[int]]:
enqueued = self._enqueued_transfer_ids
self._enqueued_transfer_ids = []
completed: list[int] = []
while self._pending_transfer_events:
event, transfer_ids = self._pending_transfer_events[0]
if not event.query():
break
self._pending_transfer_events.popleft()
completed.extend(transfer_ids)
return enqueued, completed
def shutdown(self) -> None:
if not self._initialized:
return
if self._slot_mapping_staging is not None:
self._slot_mapping_staging.stream.synchronize()
if self.dma_stream is not None:
self.dma_stream.synchronize()
release_pinned_state(
[cache.runtime for cache in self.cache_handles],
self.pinned_host_pools,
self.shared_host_region,
)
drop_hisparse_host_pool(self.vllm_config)
self._initialized = False
@@ -1,656 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import math
import os
import threading
import time
from collections.abc import Iterator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, NamedTuple
import msgspec
import regex as re
import torch
import zmq
from vllm.config import KVTransferConfig, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
)
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.utils.network_utils import (
get_ip,
get_open_port,
make_zmq_socket,
)
if TYPE_CHECKING:
pass
from dataclasses import field
from enum import Enum
logger = init_logger(__name__)
Transfer = tuple[int, float]
EngineId = str
ReqId = str
TransferId = str
TransferOffsetsKey = tuple[str, tuple[int, ...], tuple[int, ...], torch.dtype]
def _positive_finite_timeout(name: str, value: Any) -> float:
timeout = float(value)
if not math.isfinite(timeout) or timeout <= 0:
raise ValueError(f"{name} must be finite and greater than zero")
return timeout
class MoRIIOTransferAck(NamedTuple):
transfer_id: TransferId
consumer_tp_size: int = 1
@dataclass
class WriteTask:
request_id: ReqId
transfer_id: TransferId
dst_engine_id: str
local_block_ids: BlockIds
remote_block_ids_hint: BlockIds | None
layer_name: str
event: torch.cuda.Event
remote_notify_port: int
remote_ip: str
remote_tp_size: int | None = None
multi_pod_hosts: list[str] = field(default_factory=list)
remote_dp_size_local: int = 0
remote_dp_size: int = 1
enqueue_time: float = field(default_factory=time.perf_counter)
retried: int = 0
@dataclass
class LayerTransferPlan:
"""Plan for transferring a single layer."""
request_id: ReqId
transfer_id: TransferId
layer_name: str
sess_idx: int
transfer_local_offsets: list[int]
transfer_remote_offsets: list[int]
transfer_sizes: list[int]
use_batch: bool = True
@dataclass
class RemoteAllocInfo:
"""Information about remote block allocation."""
block_ids: BlockIds | None
writes_done: int = 0
writes_expected: int | None = None
decode_dp_rank: int = 0
completion_request_id: str | None = None
completion_notify_port: int | None = None
completion_remote_ip: str | None = None
completion_notified: bool = False
transfer_statuses: list[Any] = field(default_factory=list)
transfer_offsets: dict[
TransferOffsetsKey,
tuple[list[int], list[int], list[int]],
] = field(default_factory=dict)
class ROLE(Enum):
PRODUCER = "producer"
CONSUMER = "consumer"
NOTINIT = "notinit"
class MoRIIOAgentMetadata(
msgspec.Struct,
omit_defaults=True, # type: ignore[call-arg]
# required for @cached_property.d
dict=True,
):
engine_id: str
agent_metadata: bytes
kv_caches_base_addr: list[int]
num_blocks: int
block_len: int
attn_backend_name: str
class RoleManager:
"""Manages role state across the connector."""
_instance: "RoleManager | None" = None
_lock = threading.Lock()
def __init__(self) -> None:
self._role: ROLE = ROLE.NOTINIT
@classmethod
def get_instance(cls) -> "RoleManager":
if cls._instance is None:
with cls._lock:
if cls._instance is None:
cls._instance = cls()
return cls._instance
def set_role(self, role: ROLE) -> None:
"""Set the current role."""
with self._lock:
self._role = role
def get_role(self) -> ROLE:
"""Get the current role."""
return self._role
def set_role(role: ROLE):
"""Set the global role."""
RoleManager.get_instance().set_role(role)
def get_role() -> ROLE:
"""Get the global role."""
return RoleManager.get_instance().get_role()
class MoRIIOMode(Enum):
READ = "read"
WRITE = "write"
class TransferBatchState(Enum):
"""Verdict for a group of transfer statuses that belong to one request.
A request's KV transfer is spread over one status per layer (two for a KDA
layer), so a single status never decides the request: PENDING means at
least one is still in flight and none has failed, FAILED means at least
one failed, DONE means all succeeded.
"""
DONE = "done"
FAILED = "failed"
PENDING = "pending"
class MoRIIOError(Exception):
"""Base exception for MoRIIO operations."""
pass
class HandshakeError(MoRIIOError):
"""Exception raised when handshake fails."""
pass
class TransferError(MoRIIOError):
"""Exception raised when transfer fails."""
pass
def get_moriio_mode(kv_transfer_config: KVTransferConfig) -> MoRIIOMode:
read_mode = str(
kv_transfer_config.kv_connector_extra_config.get("read_mode", "false")
).lower().strip() in ("true", "1")
logger.debug("MoRIIO Connector read_mode: %s", read_mode)
if read_mode:
return MoRIIOMode.READ
else:
return MoRIIOMode.WRITE
def get_port_offset(dp_rank: int, tp_rank: int, tp_size: int = 1) -> int:
return (dp_rank) * tp_size + tp_rank
def fold_local_rank(global_dp_rank: int, dp_size_local: int) -> int:
"""Fold a global DP rank into its pod-local rank [0, dp_size_local).
``dp_size_local == 0`` is the external-DP sentinel (local size unknown):
return the rank unchanged, since a global DP rank is always < the global
DP size so no folding is needed and the modulo is skipped (never divides
by zero).
"""
return global_dp_rank % dp_size_local if dp_size_local else global_dp_rank
def pod_index(global_dp_rank: int, dp_size_local: int) -> int:
"""Pod index (0-based) a global DP rank lives on for Wide-EP multi-pod.
``dp_size_local == 0`` (external-DP sentinel) collapses to a single pod.
"""
return global_dp_rank // dp_size_local if dp_size_local else 0
def resolve_host_ip(extra_config: dict) -> str:
"""The IP this MoRIIO process advertises for KV transfer.
Honors an explicit ``host_ip`` in ``kv_connector_extra_config`` before
falling back to ``get_ip()``. An external router/orchestrator can set it to
the node's routable address; this is required under frameworks (e.g. Ray)
where ``get_ip()`` resolves to an unroutable public IP and ``VLLM_HOST_IP``
cannot be propagated to the worker processes that bind the transfer engine.
"""
return extra_config.get("host_ip") or get_ip()
_DEPRECATED_ENV_VARS: dict[str, str] = {
"VLLM_MORIIO_CONNECTOR_READ_MODE": "read_mode",
"VLLM_MORIIO_QP_PER_TRANSFER": "qp_per_transfer",
"VLLM_MORIIO_POST_BATCH_SIZE": "post_batch_size",
"VLLM_MORIIO_NUM_WORKERS": "num_workers",
}
def _warn_deprecated_env_vars() -> None:
for env_var, new_key in _DEPRECATED_ENV_VARS.items():
if env_var in os.environ:
logger.warning_once(
"The environment variable %s is deprecated and ignored. "
"Set %r inside kv_transfer_config.kv_connector_extra_config "
"instead.",
env_var,
new_key,
)
@dataclass
class MoRIIOConfig:
local_ip: str
local_kv_port: int
proxy_ip: str
local_ping_port: int
proxy_ping_port: int
http_port: int
handshake_port: int
notify_port: int
tp_rank: int
dp_rank: int
dp_size: int
tp_size: int
transfer_timeout: float
defer_timeout: float
recv_abort_timeout: float
read_mode: bool = False
qp_per_transfer: int = 1
post_batch_size: int = -1
num_workers: int = 1
backend: str = "rdma"
@classmethod
def from_vllm_config(cls, vllm_config: VllmConfig) -> "MoRIIOConfig":
# Port Configuration:
# local_ping_port -> Outgoing heartbeat to proxy
# proxy_ping_port -> Remote proxy's heartbeat ingress port
# http_port -> Instance's HTTP service endpoint
# local_kv_port -> service port for mori engine
# notify_port -> For synchronizing stages between prefill and decode
# handshake_port -> For initial handshake between mori engine
# Optional tuning knobs
# read_mode -> If true, run the connector in READ mode (consumer
# pulls KV from producer) instead of the default
# WRITE mode.
# transfer_timeout -> Timeout for waiting_for_transfer_complete before
# raising TransferError (sec).
# defer_timeout -> Timeout for deferred WRITE allocation or a missing
# READ release notification (sec).
# recv_abort_timeout -> Timeout before an in-flight recv whose RDMA
# completion never arrived is aborted (sec).
# Knobs for RDMA transfers, ignored if on xgmi backend
# qp_per_transfer -> Number of RDMA Queue Pairs per KV transfer.
# post_batch_size -> Batch size for posting transfer work requests
# (-1 lets the MoRI backend choose).
# num_workers -> Number of background worker threads the MoRI
# engine uses for transfer processing.
# TODO : merge notify_port and handshake_port to simplify port management
# supports non-contiguous ports
assert vllm_config.kv_transfer_config is not None, (
"kv_transfer_config must be set for MoRIIOConnector"
)
_warn_deprecated_env_vars()
kv_transfer_config = vllm_config.kv_transfer_config
extra_config = kv_transfer_config.kv_connector_extra_config
tp_rank = get_tensor_model_parallel_rank()
# Fold the global data_parallel_rank back to [0, dp_size_local) for
# per-node port allocation (handles the external-DP sentinel).
pc = vllm_config.parallel_config
dp_rank = fold_local_rank(pc.data_parallel_rank, pc.data_parallel_size_local)
base_notify_port = int(extra_config["notify_port"])
tp_size = get_tensor_model_parallel_world_size()
port_offset = get_port_offset(dp_rank, tp_rank)
backend = str(extra_config.get("backend", "rdma")).lower()
if backend not in ("rdma", "xgmi"):
raise ValueError(
f"Invalid MoRIIO backend {backend!r} in kv_connector_extra_config; "
"must be one of 'rdma' or 'xgmi'."
)
transfer_timeout = _positive_finite_timeout(
"kv_connector_extra_config.transfer_timeout",
extra_config.get(
"transfer_timeout", MoRIIOConstants.DEFAULT_TRANSFER_TIMEOUT
),
)
defer_timeout = _positive_finite_timeout(
"kv_connector_extra_config.defer_timeout",
extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT),
)
legacy_recv_abort_timeout = os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S")
if legacy_recv_abort_timeout is not None:
logger.warning_once(
"The environment variable VLLM_MORIIO_TRANSFER_TIMEOUT_S is "
"deprecated. Set 'recv_abort_timeout' inside "
"kv_transfer_config.kv_connector_extra_config instead."
)
recv_abort_timeout = _positive_finite_timeout(
"kv_connector_extra_config.recv_abort_timeout",
extra_config.get(
"recv_abort_timeout",
legacy_recv_abort_timeout or MoRIIOConstants.DEFAULT_RECV_ABORT_TIMEOUT,
),
)
return cls(
local_ip=resolve_host_ip(extra_config),
local_kv_port=get_open_port(),
proxy_ip=extra_config.get("proxy_ip", ""),
local_ping_port=get_open_port(),
proxy_ping_port=int(extra_config.get("proxy_ping_port", 0)),
http_port=int(extra_config["http_port"]),
handshake_port=int(extra_config["handshake_port"]),
notify_port=base_notify_port + port_offset,
tp_rank=tp_rank,
dp_rank=dp_rank,
# Advertise the GLOBAL DP world size to the routing proxy so it can
# address every rank across pods; dp_size_local above is only for
# per-pod port folding, not for routing.
dp_size=pc.data_parallel_size,
tp_size=tp_size,
read_mode=get_moriio_mode(kv_transfer_config) == MoRIIOMode.READ,
qp_per_transfer=int(extra_config.get("qp_per_transfer", 1)),
post_batch_size=int(extra_config.get("post_batch_size", -1)),
num_workers=int(extra_config.get("num_workers", 1)),
backend=backend,
transfer_timeout=transfer_timeout,
defer_timeout=defer_timeout,
recv_abort_timeout=recv_abort_timeout,
)
class MoRIIOConstants:
"""Constants for MoRIIO connector."""
# ZMQ message types
GET_META_MSG = b"get_meta_msg"
POP_DONE_RECV = b"pop_done_recv"
OVER = b"OVER"
COMPLETION_PREFIX = "cmpl"
TRANSFER_PREFIX = "tx"
PING_INTERVAL = 3
MAX_PING_RETRIES = 100
DEFAULT_HANDSHAKE_PORT = "6301"
DEFAULT_NOTIFY_PORT = "61005"
VLLM_MORI_READ_ABORT_REQUEST_TIMEOUT = 3600
# Timeout (seconds) for waiting_for_transfer_complete before raising TransferError.
# Overridable via kv_connector_extra_config["transfer_timeout"].
DEFAULT_TRANSFER_TIMEOUT = 30.0
# Timeout (seconds) for deferred WRITE allocation or a missing READ release
# notification.
# Overridable via kv_connector_extra_config["defer_timeout"].
DEFAULT_DEFER_TIMEOUT = 60.0
# Timeout (seconds) before an in-flight recv whose RDMA completion was lost
# is aborted, so the decode worker does not hang on it forever.
# Overridable via kv_connector_extra_config["recv_abort_timeout"].
DEFAULT_RECV_ABORT_TIMEOUT = 120.0
# The router embeds both zmq_addresses in the request_id:
# "___prefill_addr_{zmq}___decode_addr_{zmq}_{32-hex-uuid}"
# MoRIIO zmq_address format: "host:IP,handshake:PORT,notify:PORT"
#
# This lets each connector side parse the peer's connection info without
# requiring the router to pass it explicitly in kv_transfer_params.
_PREFILL_ZMQ_RE = re.compile(r"___prefill_addr_(.+?)___decode_addr_")
# vLLM wraps the router's X-Request-Id as "cmpl-<id>-<seq>-<hex>" so there may
# be a trailing "-<seq>-<hex>" suffix after the 32-char UUID. Allow it.
_DECODE_ZMQ_RE = re.compile(r"___decode_addr_(.+)_[0-9a-f]{32}(?:-.*)?$")
def parse_moriio_zmq_address(
zmq_address: str,
) -> tuple[str, int, int]:
"""Parse the MoRI-IO zmq address into its components.
Parses ``"host:IP,handshake:PORT,notify:PORT"`` into
(host, handshake_port, notify_port).
Each key-value pair is split on the *first* colon so that IPv6 addresses
(e.g. ``host:::1``) are handled correctly. Raises ``ValueError`` if any
of ``host``, ``handshake``, or ``notify`` keys are absent or if the port
values are non-numeric.
"""
parts: dict[str, str] = {}
for segment in zmq_address.split(","):
key, _, val = segment.partition(":")
parts[key.strip()] = val.strip()
try:
host = parts["host"]
handshake_port = int(parts["handshake"])
notify_port = int(parts["notify"])
except (KeyError, ValueError) as e:
raise ValueError(
f"Malformed zmq_address {zmq_address!r}: expected "
f"'host:IP,handshake:PORT,notify:PORT' format"
) from e
return host, handshake_port, notify_port
def get_peer_zmq_from_request_id(request_id: str, is_producer: bool) -> str | None:
"""Extract the *peer's* zmq_address from the vLLM router request_id.
The producer (prefill) needs the decode's address; the consumer (decode)
needs the prefill's address.
Returns ``None`` when the request_id does not encode peer info. The
llm-d routing sidecar (``llm-d-inference-scheduler``) does not embed
addresses in ``request_id``; instead it passes ``remote_host``,
``remote_handshake_port`` and ``remote_notify_port`` explicitly in
``kv_transfer_params``. Callers must handle the ``None`` return by
falling back to those fields. See ``add_new_req`` for the canonical
fallback path.
"""
if is_producer:
m = _DECODE_ZMQ_RE.search(request_id)
else:
m = _PREFILL_ZMQ_RE.search(request_id)
if m is None:
return None
return m.group(1)
@dataclass
class ReqMeta:
"""Metadata for a single request."""
transfer_id: TransferId
local_block_ids: BlockIds
remote_block_ids: BlockIds
remote_host: str
remote_port: int
remote_handshake_port: int
remote_notify_port: int
remote_engine_id: str
tp_size: int
remote_dp_size: int
# Prefill DP rank that owns this request's KV (forwarded by the proxy). The
# read must target this rank's memory registration; the default 0 preserves
# the symmetric single-DP behaviour.
remote_dp_rank: int = 0
# Multi-pod: list of remote pod IPs indexed by pod_idx.
multi_pod_hosts: list[str] = field(default_factory=list)
# Per-pod DP size; 0 means fallback to remote_dp_size.
remote_dp_size_local: int = 0
class MoRIIOConnectorMetadata(KVConnectorMetadata):
def __init__(self):
self.reqs_to_recv: dict[ReqId, ReqMeta] = {}
self.reqs_to_save: dict[ReqId, ReqMeta] = {}
self.reqs_to_send: dict[ReqId, float] = {}
self.transfer_id_to_request_id: dict[TransferId, ReqId] = {}
def __repr__(self):
return (
f"MoRIIOConnectorMetadata: reqs_to_recv={self.reqs_to_recv}, "
f"reqs_to_save={self.reqs_to_save}, "
f"reqs_to_send={self.reqs_to_send}, "
f"transfer_id_to_request_id={self.transfer_id_to_request_id}"
)
def add_new_req(
self,
request_id: ReqId,
local_block_ids: BlockIds,
kv_transfer_params: dict[str, Any],
write_mode=False,
):
"""Ingest a peer's ``kv_transfer_params`` into a typed ``ReqMeta``.
This is the single place peer info enters the connector. The
``kv_transfer_params`` contract (as produced by the llm-d sidecar /
vLLM router, or echoed by the prefill leg's ``request_finished``):
Required (always):
* ``transfer_id`` -- stable id shared by both legs.
* ``remote_engine_id`` -- peer engine id for the handshake table.
* ``remote_block_ids`` -- peer block ids (may be [] in WRITE mode,
where decode allocates its own blocks).
Peer address -- ONE of the following two must resolve:
* embedded in ``request_id`` (vLLM-router PD id form), OR
* explicit ``remote_host`` + ``remote_handshake_port`` +
``remote_notify_port`` (llm-d sidecar / returnable path).
If neither resolves we raise -- there is no safe default host/port.
Optional (defaulted):
* ``tp_size`` (default 1) -- peer TP size.
* ``remote_dp_size`` (default 1) -- peer GLOBAL DP size.
* ``remote_dp_size_local`` (default = ``remote_dp_size``) -- per-pod
DP size for Wide-EP multi-pod port/host
folding; 0/absent means single-pod.
* ``remote_hosts`` (default [remote_host]) -- per-pod IP list
indexed by ``pod_idx = rank // dp_local``.
Routing keys consumed elsewhere (NOT here): ``remote_dp_rank`` /
``remote_dp_rank_override`` gate the decode->prefill notify target in
MoRIIOConnectorScheduler; they are router-authoritative and never
self-derived (see that class's request routing contract).
"""
transfer_id = kv_transfer_params["transfer_id"]
# Try request_id embedded address first, fallback to explicit params.
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode)
if peer_zmq is not None:
remote_host, remote_handshake_port, remote_notify_port = (
parse_moriio_zmq_address(peer_zmq)
)
else:
try:
remote_host = kv_transfer_params["remote_host"]
if not remote_host:
raise ValueError(
f"request_id {request_id!r} does not embed a peer "
f"zmq_address and kv_transfer_params['remote_host'] is "
f"empty; cannot route MoRI-IO transfer"
)
remote_handshake_port = int(kv_transfer_params["remote_handshake_port"])
remote_notify_port = int(kv_transfer_params["remote_notify_port"])
except (KeyError, TypeError, ValueError) as e:
raise ValueError(
f"request_id {request_id!r} does not embed a peer "
f"zmq_address and kv_transfer_params is missing one or "
f"more sidecar-fallback keys (need remote_host, "
f"remote_handshake_port, remote_notify_port): {e}"
) from e
# Multi-pod: use multi_pod_hosts list or fallback to single host.
_pod_hosts = kv_transfer_params.get("remote_hosts") or [remote_host]
if not isinstance(_pod_hosts, list):
_pod_hosts = [_pod_hosts]
_pod_hosts = [str(h) for h in _pod_hosts]
_remote_dp_size_local = int(
kv_transfer_params.get(
"remote_dp_size_local",
kv_transfer_params.get("remote_dp_size", 1),
)
)
_req = ReqMeta(
transfer_id=transfer_id,
local_block_ids=local_block_ids,
remote_block_ids=kv_transfer_params["remote_block_ids"],
remote_engine_id=kv_transfer_params["remote_engine_id"],
remote_host=remote_host,
remote_port=int(remote_handshake_port),
remote_handshake_port=int(remote_handshake_port),
remote_notify_port=int(remote_notify_port),
# Remote peer TP degree (used as remote_tp_size downstream). The
# decode side must know the prefiller's TP degree to map producer
# ranks onto the correct decode rank (and vice versa). The old plain
# "tp_size" key (#46332) is absent on WRITE producer requests and
# defaulted to 1, collapsing every producer rank onto decode rank 0
# and corrupting the transfer; prefer "remote_tp_size". 0 == unknown
# (treated as homogeneous downstream).
tp_size=int(
kv_transfer_params.get("remote_tp_size")
or kv_transfer_params.get("tp_size")
or 0
),
remote_dp_size=kv_transfer_params.get("remote_dp_size", 1),
remote_dp_rank=kv_transfer_params.get("remote_dp_rank", 0),
multi_pod_hosts=_pod_hosts,
remote_dp_size_local=_remote_dp_size_local,
)
if write_mode:
self.reqs_to_save[request_id] = _req
else:
self.reqs_to_recv[request_id] = _req
@contextlib.contextmanager
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
"""Context manager for a ZMQ socket."""
if socket_type not in (zmq.ROUTER, zmq.REQ, zmq.DEALER):
raise ValueError(f"Unexpected socket type: {socket_type}")
ctx: zmq.Context | None = None
try:
ctx = zmq.Context() # type: ignore[attr-defined]
yield make_zmq_socket(
ctx=ctx, path=addr, socket_type=socket_type, bind=socket_type == zmq.ROUTER
)
finally:
if ctx is not None:
ctx.destroy(linger=0)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,96 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Keep discovery heartbeats independent of the model worker's GIL."""
import json
import logging
import select
import subprocess
import sys
from typing import Any
import msgpack
import zmq
logger = logging.getLogger(__name__)
class MoRIIOHeartbeat:
def __init__(
self,
endpoint: str,
payload: dict[str, Any],
interval: float,
max_retries: int = 100,
):
self._process = subprocess.Popen(
[sys.executable, __file__, endpoint, str(interval), str(max_retries)],
stdin=subprocess.PIPE,
text=True,
)
assert self._process.stdin is not None
try:
self._process.stdin.write(json.dumps(payload) + "\n")
self._process.stdin.flush()
except BaseException:
self.shutdown()
raise
def check_health(self) -> None:
returncode = self._process.poll()
if returncode is not None:
raise RuntimeError(
f"MoRIIO discovery heartbeat exited with code {returncode}"
)
def shutdown(self) -> None:
assert self._process.stdin is not None
try:
if not self._process.stdin.closed:
self._process.stdin.close()
except BrokenPipeError:
pass
finally:
try:
self._process.wait(timeout=5)
except subprocess.TimeoutExpired:
self._process.terminate()
try:
self._process.wait(timeout=1)
except subprocess.TimeoutExpired:
self._process.kill()
self._process.wait(timeout=1)
def main() -> None:
endpoint, interval_arg, max_retries_arg = sys.argv[1:]
initial = sys.stdin.readline()
if not initial:
return
payload = msgpack.dumps(json.loads(initial))
interval = float(interval_arg)
max_retries = int(max_retries_arg)
failures = 0
with zmq.Context() as context, context.socket(zmq.DEALER) as sock:
sock.setsockopt(zmq.LINGER, 0)
sock.setsockopt(zmq.SNDTIMEO, 1000)
sock.connect(endpoint)
while not select.select([sys.stdin], [], [], 0)[0]:
try:
sock.send(payload)
failures = 0
except zmq.Again:
failures = 0
logger.warning("MoRIIO discovery heartbeat send failed; retrying")
except zmq.ZMQError:
failures += 1
logger.exception("MoRIIO discovery heartbeat send failed")
if failures >= max_retries:
raise
# EOF also stops the child when the worker exits without cleanup.
if select.select([sys.stdin], [], [], interval)[0]:
break
if __name__ == "__main__":
main()
@@ -1,615 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Mapping
from typing import NamedTuple, cast
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import (
MambaConvSplitInfo,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheConfig,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
UniformTypeKVCacheSpecs,
)
class LayerTransferGeometry(NamedTuple):
num_blocks: int
block_size: int
block_len: int
slot_size_bytes: int
block_stride: int
local_kv_stride: int | None
remote_kv_stride: int | None
transfers_per_block: int
regions_per_block: int
split_kv_regions: bool
class MambaTransferGeometry(NamedTuple):
"""Geometry of a hybrid (mamba/KDA) layer's recurrent state.
A KDA layer's packed page is exposed as ``conv_state`` and ``ssm_state``
views. Each is slot-strided, so a slot's bytes live at
``slot * slot_stride * element_size`` and span ``slot_bytes``. The two
tensors are registered as two separate MoRIIO regions per layer.
"""
conv_slot_stride: int
ssm_slot_stride: int
ssm_slot_bytes: int
conv_element_size: int
ssm_element_size: int
conv_region_len: int
ssm_region_len: int
def get_mamba_transfer_geometry(
conv: torch.Tensor,
ssm: torch.Tensor,
) -> MambaTransferGeometry:
"""Derive the slot-strided geometry of a KDA layer's conv + ssm state.
``conv``/``ssm`` are zero-copy views over the layer's packed page. Slot byte
offset is ``slot * slot_stride * element_size`` and each
slot spans ``subtensor[0].numel() * element_size`` bytes; the whole tensor
is registered as one region (``*_region_len`` bytes).
"""
if conv.ndim == 0 or ssm.ndim == 0 or conv.shape[0] == 0 or ssm.shape[0] == 0:
raise ValueError("KDA conv/ssm caches must contain at least one slot")
if conv.shape[0] != ssm.shape[0]:
raise ValueError(
f"KDA conv/ssm slot count mismatch: {conv.shape[0]} != {ssm.shape[0]}"
)
conv_slot_bytes = conv[0].numel() * conv.element_size()
ssm_slot_bytes = ssm[0].numel() * ssm.element_size()
conv_available_bytes = (
conv.untyped_storage().nbytes() - conv.storage_offset() * conv.element_size()
)
ssm_available_bytes = (
ssm.untyped_storage().nbytes() - ssm.storage_offset() * ssm.element_size()
)
if conv.shape[0]:
conv_required_bytes = (conv.shape[0] - 1) * conv.stride(
0
) * conv.element_size() + conv_slot_bytes
ssm_required_bytes = (ssm.shape[0] - 1) * ssm.stride(
0
) * ssm.element_size() + ssm_slot_bytes
if conv_required_bytes > conv_available_bytes:
raise ValueError("KDA conv slots exceed the registered storage extent")
if ssm_required_bytes > ssm_available_bytes:
raise ValueError("KDA ssm slots exceed the registered storage extent")
return MambaTransferGeometry(
conv_slot_stride=conv.stride(0),
ssm_slot_stride=ssm.stride(0),
ssm_slot_bytes=ssm_slot_bytes,
conv_element_size=conv.element_size(),
ssm_element_size=ssm.element_size(),
# Registered region must span the full slot-strided extent: stride(0)
# may exceed the per-slot element count, so numel() would under-size the
# region and truncate the highest-slot transfers. shape[0]*stride(0)
# covers every byte a ``slot * slot_stride * elem`` offset can address.
# Cap at the bytes remaining from each view's storage offset. A packed
# conv+ssm page places ssm at a non-zero intra-buffer offset,
# so do not register past the end of the shared backing allocation.
conv_region_len=min(
conv.shape[0] * conv.stride(0) * conv.element_size(),
conv_available_bytes,
),
ssm_region_len=min(
ssm.shape[0] * ssm.stride(0) * ssm.element_size(),
ssm_available_bytes,
),
)
def build_layer_to_spec(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheSpec]:
layer_to_spec: dict[str, KVCacheSpec] = {}
for group in kv_cache_config.kv_cache_groups:
group_spec = group.kv_cache_spec
if isinstance(group_spec, UniformTypeKVCacheSpecs):
layer_to_spec.update(
{
layer_name: group_spec.kv_cache_specs[layer_name]
for layer_name in group.layer_names
}
)
else:
layer_to_spec.update(
{layer_name: group_spec for layer_name in group.layer_names}
)
return layer_to_spec
def is_mla_cache_layer(
layer_to_spec: Mapping[str, KVCacheSpec], layer_name: str
) -> bool:
try:
spec = layer_to_spec[layer_name]
except KeyError as e:
raise ValueError(f"Missing KV cache spec for layer {layer_name}") from e
return isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec))
def _spec_dim_matches(value: int, expected: int | None) -> bool:
return expected is None or value == expected
def _kernel_layout_matches(
spec: KVCacheSpec, kernel_block_size: int, num_kv_heads: int, head_dim: int
) -> bool:
if kernel_block_size <= 0 or spec.block_size % kernel_block_size != 0:
return False
return _spec_dim_matches(
num_kv_heads, getattr(spec, "num_kv_heads", None)
) and _spec_dim_matches(head_dim, getattr(spec, "head_size", None))
def _select_kernel_block_layout(
layer_name: str, shape: torch.Size, spec: KVCacheSpec
) -> tuple[int, int, int]:
axis2_matches = _kernel_layout_matches(spec, shape[2], shape[3], shape[4])
axis3_matches = _kernel_layout_matches(spec, shape[3], shape[2], shape[4])
if axis2_matches and axis3_matches and shape[2] != shape[3]:
raise ValueError(
f"Ambiguous MoRIIO kernel-block K/V cache shape for layer "
f"{layer_name}: {tuple(shape)}"
)
if axis2_matches:
return shape[2], shape[3], shape[4]
if axis3_matches:
return shape[3], shape[2], shape[4]
raise ValueError(
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
f"{tuple(shape)} does not contain block size {spec.block_size}"
)
def get_layer_transfer_geometry(
layer_name: str,
kv_cache: torch.Tensor,
layer_to_spec: Mapping[str, KVCacheSpec],
remote_num_blocks: int | None = None,
) -> LayerTransferGeometry:
spec = layer_to_spec[layer_name]
shape = kv_cache.shape
stride = kv_cache.stride()
element_size = kv_cache.element_size()
is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name)
if not is_mla_cache and len(shape) == 5 and shape[0] == 2:
_, num_blocks = shape[:2]
kernel_blocks_per_block = 1
if shape[2] == spec.block_size:
block_size, num_kv_heads, head_dim = shape[2:]
elif shape[3] == spec.block_size:
num_kv_heads, block_size, head_dim = shape[2:]
else:
kernel_num_blocks = num_blocks
kernel_block_size, num_kv_heads, head_dim = _select_kernel_block_layout(
layer_name, shape, spec
)
kernel_blocks_per_block = spec.block_size // kernel_block_size
if kernel_num_blocks % kernel_blocks_per_block != 0:
raise ValueError(
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, "
f"not divisible by {kernel_blocks_per_block}"
)
num_blocks = kernel_num_blocks // kernel_blocks_per_block
block_size = spec.block_size
slot_size_bytes = num_kv_heads * head_dim * element_size
block_len = block_size * slot_size_bytes
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[1] * kernel_blocks_per_block,
local_kv_stride=stride[0],
remote_kv_stride=(
stride[1] * kernel_blocks_per_block * (remote_num_blocks or num_blocks)
),
transfers_per_block=2,
regions_per_block=1,
split_kv_regions=True,
)
if not is_mla_cache and len(shape) == 5 and shape[1] == 2:
num_blocks = shape[0]
if shape[2] == spec.block_size:
block_size, num_kv_heads, head_dim = shape[2:]
slot_size_bytes = num_kv_heads * head_dim * element_size
block_len = block_size * slot_size_bytes
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[0],
local_kv_stride=stride[1],
remote_kv_stride=stride[1],
transfers_per_block=2,
regions_per_block=2,
split_kv_regions=False,
)
elif shape[3] == spec.block_size:
num_kv_heads, block_size, head_dim = shape[2:]
else:
kernel_num_blocks = num_blocks
kernel_block_size, _, _ = _select_kernel_block_layout(
layer_name, shape, spec
)
kernel_blocks_per_block = spec.block_size // kernel_block_size
if kernel_num_blocks % kernel_blocks_per_block != 0:
raise ValueError(
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, "
f"not divisible by {kernel_blocks_per_block}"
)
num_blocks = kernel_num_blocks // kernel_blocks_per_block
block_size = spec.block_size
block_stride = stride[0] * kernel_blocks_per_block
block_len = block_stride * element_size
slot_size_bytes = block_len // block_size
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=block_stride,
local_kv_stride=None,
remote_kv_stride=None,
transfers_per_block=1,
regions_per_block=1,
split_kv_regions=False,
)
slot_size_bytes = num_kv_heads * head_dim * element_size
block_len = block_size * slot_size_bytes
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[0],
local_kv_stride=stride[1],
remote_kv_stride=stride[1],
transfers_per_block=2,
regions_per_block=2,
split_kv_regions=False,
)
if (
isinstance(spec, AttentionSpec)
and len(shape) == 4
and shape[1] == spec.num_heads
and shape[2] == spec.num_states
and shape[3] * element_size == spec.state_content_size_bytes
):
# Standardized per-layer [B, H, N, C] view (MLA is just H == 1).
num_blocks, num_heads, num_states, content_dim = shape
slot_size_bytes = num_heads * content_dim * element_size
block_len = num_states * slot_size_bytes
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=spec.block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[0],
local_kv_stride=None,
remote_kv_stride=None,
transfers_per_block=1,
regions_per_block=1,
split_kv_regions=False,
)
cache_kind = "MLA" if is_mla_cache else "K/V"
raise ValueError(
f"Unsupported MoRIIO {cache_kind} cache shape for layer "
f"{layer_name}: {tuple(shape)}"
)
def kda_conv_ssm(
kv_cache: torch.Tensor,
spec: MambaSpec,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Return zero-copy conv and SSM views over a packed two-state page."""
from math import prod
from vllm.utils.torch_utils import get_dtype_size
if len(spec.shapes) != 2 or len(spec.dtypes) != 2:
raise ValueError(
"MoRIIO KDA transfer requires exactly two states (conv and SSM)"
)
if kv_cache.element_size() != 1:
raise ValueError("MoRIIO KDA packed cache must use a byte-sized dtype")
if kv_cache.ndim != 4 or kv_cache.shape[1:3] != (1, 1) or kv_cache.stride(3) != 1:
raise ValueError(
"MoRIIO KDA packed cache must have shape [B, 1, 1, C] with "
"a contiguous byte dimension"
)
# Mirror MambaBase.bind_kv_cache: blocks may be strided within a shared
# multi-layer allocation, while every individual byte page is contiguous.
pages = kv_cache.squeeze(dim=(1, 2))
states: list[torch.Tensor] = []
offset = 0
for shp, dt in zip(spec.shapes, spec.dtypes, strict=True):
nbytes = prod(shp) * get_dtype_size(dt)
if offset + nbytes > pages.shape[1]:
raise ValueError(
"MoRIIO KDA state layout exceeds the packed cache page size"
)
state = pages[:, offset : offset + nbytes].view(dt)
states.append(state.view(-1, *shp))
offset += nbytes
return states[0], states[1]
def iter_layer_registration_regions(
layer_name: str,
kv_cache: torch.Tensor,
layer_to_spec: Mapping[str, KVCacheSpec],
) -> list[tuple[torch.Tensor, int]]:
spec = layer_to_spec[layer_name]
if isinstance(spec, MambaSpec):
# KDA layer: register the conv and ssm tensors as two whole-tensor
# regions (the conv 3-subprojection split is expressed as transfer
# offset triples, not extra registrations).
conv, ssm = kda_conv_ssm(kv_cache, spec)
geom = get_mamba_transfer_geometry(conv, ssm)
return [(conv, geom.conv_region_len), (ssm, geom.ssm_region_len)]
geometry = cast(
LayerTransferGeometry,
get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec),
)
region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len
if geometry.regions_per_block == 1:
# Padded/interleaved pages stride past block_len: register the strided
# span, ending at the last block's bytes so it stays in-allocation.
block_stride_bytes = geometry.block_stride * kv_cache.element_size()
region_len = max(
region_len,
(geometry.num_blocks - 1) * block_stride_bytes + geometry.block_len,
)
if geometry.split_kv_regions:
return [(cache, region_len) for cache in kv_cache]
return [(kv_cache, region_len)]
def merge_contiguous_offsets(
offsets_local: list[int],
offsets_remote: list[int],
sizes: list[int],
) -> tuple[list[int], list[int], list[int]]:
if not offsets_local:
return [], [], []
if not (len(offsets_local) == len(offsets_remote) == len(sizes)):
raise ValueError("Input list lengths mismatch")
rows = sorted(zip(offsets_local, offsets_remote, sizes), key=lambda row: row[0])
merged: list[list[int]] = []
for local, remote, size in rows:
if (
merged
and local == merged[-1][0] + merged[-1][2]
and remote == merged[-1][1] + merged[-1][2]
):
merged[-1][2] += size
else:
merged.append([local, remote, size])
return (
[row[0] for row in merged],
[row[1] for row in merged],
[row[2] for row in merged],
)
def compute_block_transfer_offsets(
layer_name: str,
kv_cache: torch.Tensor,
layer_to_spec: Mapping[str, KVCacheSpec],
local_block_ids: list[int],
remote_block_ids: list[int],
remote_num_blocks: int,
merge_fn: Callable[
[list[int], list[int], list[int]], tuple[list[int], list[int], list[int]]
] = merge_contiguous_offsets,
) -> tuple[list[int], list[int], list[int]]:
# A shorter/empty local list transfers only what decode allocated (READ-mode
# partial or full-prefix-hit); a longer local list is a genuine bug.
if len(local_block_ids) > len(remote_block_ids):
raise ValueError(
"local_block_ids longer than remote_block_ids: "
f"{len(local_block_ids)} > {len(remote_block_ids)}"
)
geometry = cast(
LayerTransferGeometry,
get_layer_transfer_geometry(
layer_name, kv_cache, layer_to_spec, remote_num_blocks
),
)
invalid_local = next(
(
block_id
for block_id in local_block_ids
if not 0 <= block_id < geometry.num_blocks
),
None,
)
if invalid_local is not None:
raise ValueError(
f"local block id {invalid_local} is outside [0, {geometry.num_blocks})"
)
invalid_remote = next(
(
block_id
for block_id in remote_block_ids
if not 0 <= block_id < remote_num_blocks
),
None,
)
if invalid_remote is not None:
raise ValueError(
f"remote block id {invalid_remote} is outside [0, {remote_num_blocks})"
)
element_size = kv_cache.element_size()
transfer_size_byte = geometry.block_len
per_block = geometry.transfers_per_block
total = len(local_block_ids) * per_block
offset_local = [0] * total
offset_remote = [0] * total
sizes = [transfer_size_byte] * total
w = 0
for lb, rb in zip(local_block_ids, remote_block_ids):
offset_local[w] = element_size * (lb * geometry.block_stride)
offset_remote[w] = element_size * (rb * geometry.block_stride)
w += 1
if per_block == 2:
assert geometry.local_kv_stride is not None
assert geometry.remote_kv_stride is not None
offset_local[w] = element_size * (
geometry.local_kv_stride + lb * geometry.block_stride
)
offset_remote[w] = element_size * (
geometry.remote_kv_stride + rb * geometry.block_stride
)
w += 1
return merge_fn(offset_local, offset_remote, sizes)
class MambaOffsetTemplate(NamedTuple):
"""Slot-independent conv+ssm offset decomposition for a KDA layer.
Homogeneous-TP geometry (conv/ssm slot strides, conv sub-projection
offsets, ssm per-slot size) is identical across requests and across the
homogeneous GDN layers, so it is computed once and reused; only the
per-request slot bases vary (see ``apply_mamba_offset_template``).
"""
num_slots: int
conv_slot_stride_bytes: int
ssm_slot_stride_bytes: int
ssm_read_bytes: int
conv_subprojs: tuple[tuple[int, int, int], ...]
def build_mamba_offset_template(
conv: torch.Tensor,
ssm: torch.Tensor,
split_info: MambaConvSplitInfo,
tp_ratio: int,
) -> MambaOffsetTemplate:
"""Precompute the slot-independent conv+ssm offset decomposition.
Only homogeneous TP (``tp_ratio == 1``) is supported; heterogeneous TP is
gated with ``NotImplementedError`` (see the design doc). The returned
template is a pure function of the layer geometry and ``split_info``, so it
can be cached and reused across requests without recomputation.
"""
if tp_ratio != 1:
# Heterogeneous-TP recurrent-state transfer needs the remote page's
# slot stride (which differs from the local page) and, for
# P_TP > D_TP, a multi-rank gather of conv/ssm slices. Both are
# follow-ups. The mamba path bypasses the attention KV-head validator,
# so fail loudly here instead of silently transferring wrong bytes.
raise NotImplementedError(
"heterogeneous-TP mamba/KDA transfer is not supported yet "
f"(tp_ratio={tp_ratio}); use equal prefill/decode TP for "
"hybrid models"
)
geom = get_mamba_transfer_geometry(conv, ssm)
conv_local_offsets = split_info.local_conv_offsets
conv_remote_offsets = split_info.remote_conv_offsets(0, 1)
ssm_read_bytes = geom.ssm_slot_bytes
conv_subprojs = tuple(
(loff, roff, rsz)
for (loff, _lsz), (roff, rsz) in zip(conv_local_offsets, conv_remote_offsets)
)
return MambaOffsetTemplate(
num_slots=conv.shape[0],
conv_slot_stride_bytes=geom.conv_slot_stride * geom.conv_element_size,
ssm_slot_stride_bytes=geom.ssm_slot_stride * geom.ssm_element_size,
ssm_read_bytes=ssm_read_bytes,
conv_subprojs=conv_subprojs,
)
def apply_mamba_offset_template(
template: MambaOffsetTemplate,
local_slots: list[int],
remote_slots: list[int],
) -> tuple[list[int], list[int], list[int]]:
"""Apply per-request slot bases to a cached ``MambaOffsetTemplate``.
Conv sub-projection entries (all slots) come first, followed by one ssm
entry per slot.
"""
if len(local_slots) != len(remote_slots):
raise ValueError(
"local and remote mamba slot counts must match: "
f"{len(local_slots)} != {len(remote_slots)}"
)
invalid_local = next(
(slot for slot in local_slots if not 0 <= slot < template.num_slots), None
)
if invalid_local is not None:
raise ValueError(
f"local mamba slot {invalid_local} is outside [0, {template.num_slots})"
)
invalid_remote = next((slot for slot in remote_slots if slot < 0), None)
if invalid_remote is not None:
raise ValueError(f"remote mamba slot {invalid_remote} must be non-negative")
# The peer's KDA slot count is not part of MoRIIOAgentMetadata yet. Do not
# compare remote slot IDs against the local pool size: two otherwise
# compatible engines may reserve different numbers of slots.
local_offs: list[int] = []
remote_offs: list[int] = []
sizes: list[int] = []
# Conv sub-projections (region 0), all slots first.
conv_slot_stride_bytes = template.conv_slot_stride_bytes
for ls, rs in zip(local_slots, remote_slots):
lbase = ls * conv_slot_stride_bytes
rbase = rs * conv_slot_stride_bytes
for loff, roff, rsz in template.conv_subprojs:
local_offs.append(lbase + loff)
remote_offs.append(rbase + roff)
sizes.append(rsz)
# SSM temporal state (region 1), one entry per slot.
ssm_slot_stride_bytes = template.ssm_slot_stride_bytes
ssm_read_bytes = template.ssm_read_bytes
for ls, rs in zip(local_slots, remote_slots):
local_offs.append(ls * ssm_slot_stride_bytes)
remote_offs.append(rs * ssm_slot_stride_bytes)
sizes.append(ssm_read_bytes)
return local_offs, remote_offs, sizes
def compute_mamba_conv_split_count(
local_slots: list[int],
split_info: MambaConvSplitInfo,
) -> int:
"""Number of leading conv entries in a KDA layer transfer plan.
Entries ``[:count]`` are conv sub-projections (conv region/session);
entries ``[count:]`` are the per-slot ssm state (ssm region/session).
"""
return len(local_slots) * len(split_info.local_conv_offsets)
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NIXL KV-cache transfer connector (disaggregated prefill / decode)."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
NixlBaseConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
NixlBaseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector import (
NixlBaseConnector,
NixlConnector,
NixlPullConnector,
NixlPushConnector,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlAgentMetadata,
NixlConnectorMetadata,
NixlHandshakePayload,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
NixlPullConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import (
NixlPushConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
NixlPushConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import (
NixlConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
NixlKVConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
__all__ = [
"NixlAgentMetadata",
"NixlBaseConnector",
"NixlBaseConnectorScheduler",
"NixlBaseConnectorWorker",
"NixlConnector",
"NixlConnectorMetadata",
"NixlConnectorScheduler",
"NixlConnectorWorker",
"NixlHandshakePayload",
"NixlKVConnectorStats",
"NixlPullConnector",
"NixlPullConnectorScheduler",
"NixlPullConnectorWorker",
"NixlPushConnector",
"NixlPushConnectorScheduler",
"NixlPushConnectorWorker",
]
@@ -1,549 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Base scheduler-side logic for the NIXL connector."""
import threading
import time
from typing import TYPE_CHECKING, Any
import msgspec
import zmq
from vllm import envs
from vllm.distributed.kv_transfer.kv_connector.utils import (
BlockIds,
EngineId,
clip_ssm_state_blocks,
yield_req_data,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorHandshakeMetadata,
KVConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
GET_META_MSG,
HeartbeatInfo,
NixlConnectorMetadata,
NixlHandshakePayload,
ReqId,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import zmq_ctx
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.network_utils import make_zmq_path
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
MambaSpec,
SlidingWindowSpec,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
class NixlBaseConnectorScheduler:
"""Base implementation of Scheduler side methods shared by pull and push."""
# Emitted in kv_transfer_params so an external router can distinguish a
# pull (READ) producer from a push (WRITE) one. Overridden by the push
# scheduler.
_TRANSFER_MODE: str = "pull"
def __init__(
self,
vllm_config: "VllmConfig",
engine_id: str,
kv_cache_config: "KVCacheConfig",
):
self.vllm_config = vllm_config
parallel_config = vllm_config.parallel_config
# TP1 PCP+DCP exposes its DCP shards as transfer ranks.
self.transfer_tp_size = max(
parallel_config.tensor_parallel_size,
parallel_config.decode_context_parallel_size,
)
self.block_size = vllm_config.cache_config.block_size
self.engine_id: EngineId = engine_id
self.kv_cache_config = kv_cache_config
self.side_channel_host = envs.VLLM_NIXL_SIDE_CHANNEL_HOST
self.side_channel_port = (
envs.VLLM_NIXL_SIDE_CHANNEL_PORT
+ vllm_config.parallel_config.data_parallel_index
)
assert vllm_config.kv_transfer_config is not None
self._kv_lease_duration: int = (
vllm_config.kv_transfer_config.get_from_extra_config(
"kv_lease_duration", 30
)
)
# NOTE (NickLucche): For now we use a hardcoded value for a simpler interface.
self._heartbeat_interval = self._kv_lease_duration // 6
if current_platform.device_type == "cpu":
self.use_host_buffer = False
else:
self.use_host_buffer = (
vllm_config.kv_transfer_config.kv_buffer_device == "cpu"
)
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
# Also handle unlikely SW-only model case instead of checking num_groups>1.
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.transfer_groups
)
)
self._has_mamba = kv_cache_config.has_mamba_layers
self._bounded_replay = any(
g.kv_cache_spec.prefix_replay_tokens > 0
for g in kv_cache_config.kv_cache_groups
)
logger.info("Initializing NIXL Scheduler %s", engine_id)
if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager:
logger.info("Hybrid Memory Allocator is enabled with NIXL")
# Background thread for handling new handshake requests.
self._nixl_handshake_listener_t: threading.Thread | None = None
self._stop_event = threading.Event()
# Requests that need to start recv/send.
# New requests are added by update_state_after_alloc in
# the scheduler. Used to make metadata passed to Worker.
self._reqs_need_recv: dict[
ReqId, tuple[Request, BlockIds, tuple[int, ...], bool]
] = {}
self._reqs_need_save: dict[ReqId, Request] = {}
# Reqs to send and their expiration time
self._reqs_need_send: dict[ReqId, float] = {}
self._reqs_in_batch: set[ReqId] = set()
# Reqs to remove from processed set because they're not to send after
# remote prefill or aborted.
self._reqs_not_processed: set[ReqId] = set()
# Heartbeat tracking: requests needing periodic lease-renewal heartbeats to
# remote P-side, stored as ready-to-send HeartbeatInfo grouped by remote engine
self._heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {}
# Reverse lookup: local req_id -> (engine_id, remote_req_id) for O(1) removal
self._heartbeat_req_engine: dict[ReqId, tuple[EngineId, ReqId]] = {}
self._last_heartbeat_time: float = 0.0
# Gather Sliding Window sizes for each kv cache group (if any) in number of
# blocks per KV cache group. This is used to clip the local attention window.
sw_sizes_tokens: list[tuple[int, int]] = [
(g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size)
if isinstance(g.kv_cache_spec, SlidingWindowSpec)
else (0, self.block_size)
for g in kv_cache_config.transfer_groups
]
# cdiv(n_tokens, block_size) gives blocks/window; add 1 to conservatively
# account for boundary overlap eg window isn't fully aligned with blocks.
self.blocks_per_sw = [
cdiv(n_tokens, block_size) + 1 if n_tokens else 0
for n_tokens, block_size in sw_sizes_tokens
]
# Trailing scratch slots that mamba managers co-allocate per request
# for speculative decoding; None for non-SSM groups.
self._ssm_spec_blocks = [
g.kv_cache_spec.num_speculative_blocks
if isinstance(g.kv_cache_spec, MambaSpec)
else None
for g in kv_cache_config.transfer_groups
]
# Threshold to decide whether to compute kv cache locally
# or pull from a remote node: minimum number of remote
# tokens to amortize the xfer latencies
self.kv_recompute_threshold: int = int(
vllm_config.kv_transfer_config.get_from_extra_config(
"kv_recompute_threshold", 64
)
)
# Bi-directional KV transfer feature supports KV block
# transfers from D node to P node
self.is_bidirectional_kv_xfer_enabled = (
vllm_config.kv_transfer_config.get_from_extra_config(
"bidirectional_kv_xfer", False
)
)
self.decoder_kv_blocks_ttl = (
vllm_config.kv_transfer_config.get_from_extra_config(
"decoder_kv_blocks_ttl", 480
)
)
if self.is_bidirectional_kv_xfer_enabled and self.kv_recompute_threshold > 0:
logger.info(
"Bidirectional KV transfer is enabled and the kv "
"recompute threshold is set to %d tokens."
"KV blocks on D are released after a TTL of %d seconds.",
self.kv_recompute_threshold,
self.decoder_kv_blocks_ttl,
)
def shutdown(self):
self._stop_event.set()
if self._nixl_handshake_listener_t is not None:
self._nixl_handshake_listener_t.join()
self._nixl_handshake_listener_t = None
def on_new_request(self, request: "Request") -> None:
"""Track a request that may need heartbeats."""
params = request.kv_transfer_params
if params is not None and params.get("do_remote_decode"):
self._truncate_request_for_prefill(request)
# NOTE (NickLucche) This excludes request meant for P, ie heartbeats are
# effectively disabled for Bidirectional KV transfer.
if params is None or not params.get("do_remote_prefill"):
return
# Only track if all required remote fields are present.
remote_engine_id = params.get("remote_engine_id")
remote_request_id = params.get("remote_request_id")
host = params.get("remote_host")
port = params.get("remote_port")
tp_size = params.get("tp_size")
dcp_size = params.get("dcp_size", 1)
pp_size = params.get("pp_size", 1)
if (
remote_engine_id is None
or remote_request_id is None
or host is None
or port is None
or tp_size is None
):
return
if remote_engine_id not in self._heartbeat_by_engine:
self._heartbeat_by_engine[remote_engine_id] = HeartbeatInfo(
req_ids=set(),
host=host,
port=port,
tp_size=tp_size,
dcp_size=dcp_size,
pp_size=pp_size,
)
self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id)
self._heartbeat_req_engine[request.request_id] = (
remote_engine_id,
remote_request_id,
)
def _stop_heartbeat(self, req_id: ReqId) -> None:
"""Remove *req_id* from heartbeat tracking (if tracked)."""
if key := self._heartbeat_req_engine.pop(req_id, None):
engine_id, remote_id = key
if info := self._heartbeat_by_engine.get(engine_id):
info.req_ids.discard(remote_id)
if not info.req_ids:
# Clean up empty engines so we don't leak a key when remote dies.
del self._heartbeat_by_engine[engine_id]
def get_exchange_clipped_blocks(
self, block_ids: BlockIds, clip_ssm: bool = True
) -> BlockIds:
"""Clip a request's block lists down to the transferable blocks.
Sliding-window groups keep only the in-window tail: the KV cache
manager allocates blocks for the entire sequence length and cleans up
out-of-window blocks only prior to the `request_finished_all_groups`
hook.
SSM groups keep only their state-bearing slot: the trailing
speculative scratch slots and everything before the running state
(null placeholders and the previous step's superseded state) go.
Use this at every block-id exchange point. Pass ``clip_ssm=False``
for per-step partial lists (host-buffer save), where the SSM strip
does not apply.
"""
if len(block_ids) == 0:
# No blocks to clip, e.g. a full prefix cache hit.
return block_ids
block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids)
if not self._is_hma_required:
return block_ids
# NOTE (NickLucche) This logic is currently handled at the connector level
# because offloading connectors might want to receive the whole sequence even
# for SWA groups. We will abstract this logic once the interface is more stable
assert len(block_ids) == len(self.blocks_per_sw), (
"Number of KV cache groups must match"
)
clipped = []
for i, blocks in enumerate(block_ids):
if n_sw := self.blocks_per_sw[i]:
blocks = blocks[-n_sw:]
elif (
clip_ssm
and blocks
and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None
):
blocks = clip_ssm_state_blocks(blocks, n_spec_blocks)
clipped.append(blocks)
return tuple(clipped)
def set_xfer_handshake_metadata(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""Set the KV connector handshake metadata for this connector.
Args:
metadata (dict): the handshake metadata to set.
"""
encoded_data: dict[tuple[int, int], bytes] = {}
encoder = msgspec.msgpack.Encoder()
for (pp_rank, tp_rank), rank_metadata in metadata.items():
if not isinstance(rank_metadata, NixlHandshakePayload):
raise ValueError(
"NixlConnectorScheduler expects NixlHandshakePayload for "
"handshake metadata."
)
encoded_data[(pp_rank, tp_rank)] = encoder.encode(rank_metadata)
logger.debug(
"PP rank %d, TP rank %d: encoded NixlHandshakePayload size: %s bytes",
pp_rank,
tp_rank,
str(len(encoded_data[(pp_rank, tp_rank)])),
)
# Only start the listener when we have metadata to serve.
if self._nixl_handshake_listener_t is None:
ready_event = threading.Event()
self._nixl_handshake_listener_t = threading.Thread(
target=self._nixl_handshake_listener,
args=(
encoded_data,
ready_event,
self._stop_event,
self.side_channel_host,
self.side_channel_port,
),
daemon=True,
name="nixl_handshake_listener",
)
self._nixl_handshake_listener_t.start()
ready_event.wait() # Wait for listener ZMQ socket to be ready.
@staticmethod
def _nixl_handshake_listener(
encoded_data: dict[tuple[int, int], Any],
ready_event: threading.Event,
stop_event: threading.Event,
host: str,
port: int,
):
"""Background thread for getting new NIXL handshakes."""
# NOTE(rob): this is a simple implementation. We will move
# to a better approach via HTTP endpoint soon.
# Listen for new requests for metadata.
path = make_zmq_path("tcp", host, port)
logger.debug("Starting listening on path: %s", path)
with zmq_ctx(zmq.ROUTER, path) as sock:
sock.setsockopt(zmq.RCVTIMEO, 1000)
ready_event.set()
while True:
try:
identity, _, msg = sock.recv_multipart()
except zmq.Again:
if stop_event.is_set():
break
continue
# Decode (GET_META_MSG, pp_rank, tp_rank).
msg, target_pp_rank, target_tp_rank = msgspec.msgpack.decode(msg)
logger.debug(
"Received message for pp rank %s, tp rank %s",
target_pp_rank,
target_tp_rank,
)
if msg != GET_META_MSG:
logger.warning("Connection listener got unexpected message %s", msg)
# Echo our perf_counter so P can estimate the clock offset.
# perf_counter is only comparable within a process, so this
# listener must run in the same process that stamps the block
# expiry deadline (`_reqs_need_send`).
ts = msgspec.msgpack.encode(time.perf_counter())
sock.send_multipart(
(identity, b"", encoded_data[(target_pp_rank, target_tp_rank)], ts)
)
def _prefill_backoff(self) -> int:
"""Trailing prompt tokens the prefiller must not compute; the decoder
recomputes them locally.
Mamba needs h(N-1) so the decoder can derive h(N) itself. DSV41's sliding
window under bounded replay is not replayed after a P/D load, so it must
be laid out for the token the decoder recomputes. Multi-module
MTP needs to keep its whole lookahead window off of the prefiller, which
would otherwise embed the unverified drafts in the MTP layer's KV cache. The
decoder would never rebuild them, because the update is sized by the rejection
count, which is zero for the first decode.
"""
return max(
1 if self._has_mamba or self._bounded_replay else 0,
self.vllm_config.num_prefill_lookahead_tokens - 1,
)
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
"""D-side only. The number of prompt tokens to load from the prefiller.
Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder
will recompute locally."""
backoff = self._prefill_backoff()
if backoff and num_prompt_tokens > backoff:
return num_prompt_tokens - backoff
return num_prompt_tokens
def _truncate_request_for_prefill(self, request: "Request") -> None:
"""P-side only: drop the trailing ``_prefill_backoff()`` prompt tokens
so the prefiller stops short of what the decoder recomputes locally.
For Mamba that is the single token needed to yield h(N-1); for
multi-module MTP it is the drafter's whole lookahead window.
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
request is preempted and rescheduled."""
backoff = self._prefill_backoff()
params = request.kv_transfer_params
if (
backoff
and params is not None
# Guard against repeated truncation after preemption/reschedule.
and not params.get("_p_side_truncated")
and request.num_prompt_tokens > backoff
# A request that skips reading the prefix cache (prompt logprobs)
# loads nothing on the decoder, which recomputes its whole prompt.
and not request.get_skip_reading_prefix_cache()
):
if request.prompt_token_ids is not None:
del request.prompt_token_ids[-backoff:]
elif request.prompt_embeds is not None:
request.prompt_embeds = request.prompt_embeds[:-backoff]
else:
return
del request._all_token_ids[-backoff:]
request.num_prompt_tokens -= backoff
request.max_tokens = 1
params["_p_side_truncated"] = True
def _build_save_meta(
self,
meta: NixlConnectorMetadata,
scheduler_output: SchedulerOutput,
) -> None:
# only called when use_host_buffer is True to build the save metadata
# NOTE: For the prefill side, there might be a chance that an early added
# request is a chunked prefill, so we need to check if new blocks are added
for req_id, new_block_id_groups, _ in yield_req_data(scheduler_output):
req_to_save = self._reqs_need_save.get(req_id)
if req_to_save is None or new_block_id_groups is None:
continue
req = req_to_save
assert req.kv_transfer_params is not None
clipped_block_id_groups = self.get_exchange_clipped_blocks(
new_block_id_groups, clip_ssm=False
)
meta.add_new_req_to_save(
request_id=req_id,
local_block_ids=clipped_block_id_groups,
kv_transfer_params=req.kv_transfer_params,
)
assert scheduler_output.num_scheduled_tokens is not None
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
is_partial = (
req.num_computed_tokens + num_scheduled_tokens
) < req.num_prompt_tokens
if not is_partial:
# For non-partial prefills, once new req_meta is scheduled, it
# can be removed from _reqs_need_save.
# For partial prefill case, we will retain the request in
# _reqs_need_save until all blocks are scheduled with req_meta.
# Therefore, only pop if `not is_partial`.
self._reqs_need_save.pop(req_id)
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
meta = NixlConnectorMetadata()
# Loop through scheduled reqs and convert to ReqMeta.
for req_id, (
req,
block_ids,
cached,
awaiting_kvs,
) in self._reqs_need_recv.items():
assert req.kv_transfer_params is not None
meta.add_new_req_to_recv(
request_id=req_id,
local_block_ids=block_ids,
kv_transfer_params=req.kv_transfer_params,
local_num_computed_blocks=cached,
awaiting_kvs=awaiting_kvs,
)
if self.use_host_buffer:
self._build_save_meta(meta, scheduler_output)
meta.reqs_to_send = self._reqs_need_send
# Clock reference for reqs_to_send: deadlines above are in this
# process's perf_counter domain; workers (possibly on other nodes,
# where perf_counter has a different epoch) rebase against this.
meta.scheduler_clock = time.perf_counter()
meta.reqs_in_batch = self._reqs_in_batch
meta.reqs_not_processed = self._reqs_not_processed
# Package heartbeats, throttled by heartbeat_interval.
if self._heartbeat_by_engine:
now = time.perf_counter()
if now - self._last_heartbeat_time >= self._heartbeat_interval:
self._last_heartbeat_time = now
meta.heartbeat_by_engine = self._heartbeat_by_engine
# Clear the list once workers start the transfers
self._reqs_need_recv.clear()
self._reqs_in_batch = set()
self._reqs_not_processed = set()
self._reqs_need_send = {}
return meta
def update_connector_output(self, connector_output: "KVConnectorOutput") -> None:
"""Stop heartbeating for requests whose KV transfer completed."""
for req_id in connector_output.finished_recving or ():
self._stop_heartbeat(req_id)
def has_pending_push_work(self) -> bool:
return False
############################################################
# Abstract methods that subclasses must implement
############################################################
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int, bool]:
raise NotImplementedError
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
raise NotImplementedError
def request_finished(
self,
request: "Request",
block_ids: BlockIds,
) -> tuple[bool, dict[str, Any] | None]:
raise NotImplementedError
File diff suppressed because it is too large Load Diff
@@ -1,426 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NIXL connector facades.
This module hosts the thin facade classes that vLLM's KV-connector layer
instantiates. Almost all the real work lives in the per-mode scheduler
and worker classes; the connector classes here only forward calls.
* :class:`NixlBaseConnector` – common logic shared by pull and push.
* :class:`NixlPullConnector` – pull-based (READ) KV transfer.
* :class:`NixlPushConnector` – push-based (WRITE) KV transfer.
* ``NixlConnector`` – backward-compatible alias for :class:`NixlPullConnector`.
"""
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import KVTransferConfig, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import (
EngineId,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
CopyBlocksOp,
KVConnectorBase_V1,
KVConnectorHandshakeMetadata,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
NixlPullConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import (
NixlPushConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
NixlPushConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
NixlKVConnectorStats,
NixlPromMetrics,
)
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
NixlBaseConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
NixlBaseConnectorWorker,
)
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
"""Base connector with common logic shared by pull and push modes."""
_cache_hit_source = CacheHitSource.P2P
@property
def supports_divergent_local_hybrid_hits(self) -> bool:
return True
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, role, kv_cache_config)
assert vllm_config.kv_transfer_config is not None
assert vllm_config.kv_transfer_config.engine_id is not None
if vllm_config.kv_transfer_config.kv_role == "kv_both":
logger.warning_once(
"Using kv_role='kv_both' with NixlConnector is deprecated "
"and will be removed in a future release. Please set "
"kv_role='kv_producer' for prefill instances and "
"kv_role='kv_consumer' for decode instances. "
)
parallel_config = vllm_config.parallel_config
pcp_size = parallel_config.prefill_context_parallel_size
kv_role = vllm_config.kv_transfer_config.kv_role
if pcp_size > 1 and kv_role in ("kv_consumer", "kv_both"):
raise NotImplementedError(
"NixlConnector PCP currently supports kv_producer only. "
"Consumers and kv_both require "
"prefill_context_parallel_size=1."
)
dcp_size = parallel_config.decode_context_parallel_size
if (
pcp_size > 1
and dcp_size > 1
and (parallel_config.tensor_parallel_size != 1 or dcp_size != pcp_size)
):
raise NotImplementedError(
"NixlConnector PCP+DCP requires TP1 with DCP spanning "
"the full PCP group."
)
# TODO: Support PCP with bidirectional KV transfer by tracking separate
# send and receive completion counts.
if pcp_size > 1 and vllm_config.kv_transfer_config.get_from_extra_config(
"bidirectional_kv_xfer", False
):
raise NotImplementedError(
"NixlConnector PCP producers do not support bidirectional KV transfer."
)
self.kv_cache_config = kv_cache_config
self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id
self.kv_transfer_config = vllm_config.kv_transfer_config
# Subclasses must set self.connector_scheduler and self.connector_worker
self.connector_scheduler: NixlBaseConnectorScheduler | None = None
self.connector_worker: NixlBaseConnectorWorker | None = None
############################################################
# Class Methods
############################################################
@classmethod
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
return False
@classmethod
def get_required_kvcache_layout(cls, vllm_config: VllmConfig):
if vllm_config.model_config is None:
logger.warning_once(
"Unable to detect current VLLM config. "
"Fallback to default kv cache layout."
)
return None
use_mla = vllm_config.model_config.use_mla
if use_mla:
# return None when we have mla
# as the layout should not matter in that case,
# which fallback to the default behavior.
return None
logger.info_once(
"NixlConnector setting KV cache layout to LBHNC for "
"better xfer performance."
)
return "LBHNC"
############################################################
# Scheduler Side Methods
############################################################
def get_finished_count(self) -> int | None:
return None
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
return self._kv_cache_config.transfer_group_ids
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
)
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
assert self.connector_scheduler is not None
return self.connector_scheduler.update_state_after_alloc(
request, blocks, num_external_tokens
)
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def on_new_request(self, request: "Request") -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.on_new_request(request)
def update_connector_output(self, connector_output: KVConnectorOutput):
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
def request_finished(
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, (block_ids,))
def request_finished_all_groups(
self,
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
def set_xfer_handshake_metadata_pp_aware(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""Set handshake metadata keyed by (pp_rank, tp_rank) so the side
channel can serve every PP stage's agent metadata.
Args:
metadata (dict): the handshake metadata to set.
"""
assert self.connector_scheduler is not None
self.connector_scheduler.set_xfer_handshake_metadata(metadata)
############################################################
# Worker Side Methods
############################################################
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
assert self.connector_worker is not None
self.connector_worker.set_host_xfer_buffer_ops(copy_operation)
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
"""Get the finished recving and sending requests."""
assert self.connector_worker is not None
return self.connector_worker.get_finished()
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
assert self.connector_worker is not None
results = self.connector_worker.get_transfer_results()
if (
self.kv_transfer_config.kv_role == "kv_producer"
and self.connector_worker.pcp_rank > 0
and not self.connector_worker.pcp_dcp_sharded
):
results.finished_sending.clear()
return results
def get_block_ids_with_load_errors(self) -> set[int]:
"""Get block IDs that failed to load via NIXL."""
assert self.connector_worker is not None
return self.connector_worker.get_block_ids_with_load_errors()
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is None:
return None
return self.connector_worker.get_kv_connector_stats()
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
NixlKVConnectorStats(data=data)
if data is not None
else NixlKVConnectorStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return NixlPromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
def wait_for_layer_load(self, layer_name: str) -> None:
"""NixlConnector does not do layerwise saving."""
pass
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs,
) -> None:
"""NixlConnector does not save explicitly."""
pass
def wait_for_save(self):
assert self.connector_worker is not None
assert isinstance(self._connector_metadata, NixlConnectorMetadata)
if self.connector_worker.use_host_buffer and self.connector_worker.copy_blocks:
self.connector_worker.save_kv_to_host(self._connector_metadata)
def has_pending_push_work(self) -> bool:
if self.connector_scheduler is not None:
return self.connector_scheduler.has_pending_push_work()
return False
def shutdown(self):
if self.connector_worker is not None:
self.connector_worker.shutdown()
if self.connector_scheduler is not None:
self.connector_scheduler.shutdown()
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
"""Get the KVConnector handshake metadata for this connector.
This metadata is used for out-of-band connector handshake
between P/D workers.
Returns:
KVConnectorHandshakeMetadata: the handshake metadata.
None if no handshake metadata is available.
"""
assert self.connector_worker is not None
if (
self.kv_transfer_config.kv_role == "kv_producer"
and self.connector_worker.pcp_rank > 0
and not self.connector_worker.pcp_dcp_sharded
):
return None
return self.connector_worker.xfer_handshake_metadata
class NixlPullConnector(NixlBaseConnector):
"""Pull-based (READ) NIXL KV transfer connector."""
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, role, kv_cache_config)
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = NixlPullConnectorScheduler(
vllm_config, self.engine_id, kv_cache_config
)
self.connector_worker = None
elif role == KVConnectorRole.WORKER:
self.connector_scheduler = None
self.connector_worker = NixlPullConnectorWorker(
vllm_config, self.engine_id, kv_cache_config
)
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
assert self.connector_worker is not None
assert isinstance(self.connector_worker, NixlPullConnectorWorker)
assert isinstance(self._connector_metadata, NixlConnectorMetadata)
self.connector_worker.start_load_kv(self._connector_metadata)
class NixlPushConnector(NixlBaseConnector):
"""Push-based (WRITE) NIXL KV transfer connector."""
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, role, kv_cache_config)
if vllm_config.parallel_config.decode_context_parallel_size > 1:
raise ValueError(
"NixlPushConnector does not support decode_context_parallel_size > 1."
)
self.connector_scheduler: NixlPushConnectorScheduler | None = None
self.connector_worker: NixlPushConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = NixlPushConnectorScheduler(
vllm_config, self.engine_id, kv_cache_config
)
elif role == KVConnectorRole.WORKER:
self.connector_worker = NixlPushConnectorWorker(
vllm_config, self.engine_id, kv_cache_config
)
else:
raise ValueError(f"Unsupported KVConnectorRole: {role}")
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
"""Drive push processing on the worker.
The worker enqueues registrations / finished blocks for the
background ``nixl-push-writer`` thread; the writer issues the
WRITE transfers and polls NIXL notifs without further
engine-thread involvement.
"""
assert self.connector_worker is not None
assert isinstance(self._connector_metadata, NixlConnectorMetadata)
self.connector_worker.start_load_kv(self._connector_metadata)
# Backward compatibility: NixlConnector is the pull-based connector.
NixlConnector = NixlPullConnector
__all__ = [
"NixlBaseConnector",
"NixlConnector",
"NixlPullConnector",
"NixlPushConnector",
]
@@ -1,340 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Metadata dataclasses and helpers for the NIXL connector."""
from dataclasses import dataclass, field
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds, EngineId
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorHandshakeMetadata,
KVConnectorMetadata,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
TransferHandle = int
ReqId = str
GET_META_MSG = b"get_meta_msg"
# Push-mode (WRITE-based) registration notification.
# Sent worker-to-worker over NIXL: D worker -> P worker, encoded as
# PUSH_REG_NOTIF_PREFIX + msgpack(registration_data).
PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
#
# NIXL Connector Version
#
# Increment this version whenever there is an incompatible change to:
# - NixlAgentMetadata schema
# - kv_transfer_params schema or semantics
# - NIXL transfer protocol or wire format
# - KV cache memory layout or block organization
# - Any other change that breaks P/D interoperability
#
# Version History:
# 1: Initial version with compatibility checking
# 2: Add remote_request_id to kv_transfer_params
# 3: Add physical_blocks_per_logical_kv_block to NixlAgentMetadata
# 4: Add KV block lease renewal through heartbeats
# 5: Add remote_blocks_expiry_time to kv_transfer_params + handshake
# clock-sync timestamp
# 6: Validate EAGLE/MTP speculative configuration compatibility
# 7: Include NIXL transfer mode (push vs pull) in the compatibility hash
# 8: Add dcp_size and pcp_size to NixlAgentMetadata
# 9: Add block_strides
# 10: Add dense virtual transfer pages for compressed MLA caches
# 11: Add per-region transfer geometry and memory types to NixlAgentMetadata
# 12: Add per-region member names for PP push
# 13: Add packed-member layouts and order-independent packed-push backend hashes
# 14: Add Qwen3.8-Flash-Next PLE page length - #59997
#
NIXL_CONNECTOR_VERSION: int = 14
@dataclass
class NixlAgentMetadata:
engine_id: str
agent_metadata: bytes
kv_caches_base_addr: list[int]
device_id: int
num_blocks: int
block_lens: list[int]
block_strides: list[int]
kv_cache_layout: str
block_size: int
ssm_sizes: tuple[int, int]
attn_backend_name: str
physical_blocks_per_logical_kv_block: int
region_num_blocks: list[int] | None = None
region_group_ids: list[int] | None = None
region_names: list[str] | None = None
region_mem_types: list[str] | None = None
dcp_size: int = 1
pcp_size: int = 1
# Page length of the PLE short-conv state (None when the model has none).
ple_block_len: int | None = None
# Layer names sharing each advertised region, in region order.
region_members: list[list[str]] = field(default_factory=list)
# Packed member -> (byte offset in its region's block, bytes per page).
packed_member_layouts: dict[str, tuple[int, int]] = field(default_factory=dict)
@dataclass
class NixlHandshakePayload(KVConnectorHandshakeMetadata):
"""Wrapper for NIXL handshake sent over the wire.
Enables two-phase decoding for graceful compatibility checking:
1. Decode NixlHandshakePayload to get compatibility_hash
2. Compute local hash and compare
3. Only if hashes match, decode agent_metadata_bytes
This prevents decoder errors when NixlAgentMetadata schema is
incompatible, allowing graceful failure with clear error message.
"""
compatibility_hash: str
agent_metadata_bytes: bytes # NixlAgentMetadata encoded
def _get_speculative_compatibility_factors(
vllm_config: VllmConfig,
) -> dict[str, Any] | None:
"""Return NIXL compatibility factors for hidden-state-based speculators."""
speculative_config = vllm_config.speculative_config
if speculative_config is None or not speculative_config.use_eagle():
return None
draft_model_config = speculative_config.draft_model_config
assert draft_model_config is not None
auxiliary_layer_ids = getattr(
draft_model_config.hf_config,
"eagle_aux_hidden_state_layer_ids",
None,
)
# kv_cache_dtype is a user override that defaults to None, meaning "inherit
# the target's --kv-cache-dtype". Resolve it to the effective value so an
# explicit setting on one side and inheritance on the other (same effective
# dtype) don't spuriously mismatch.
kv_cache_dtype = (
speculative_config.kv_cache_dtype or vllm_config.cache_config.cache_dtype
)
# Note: the draft attention_backend is intentionally not hashed. Its only
# transfer-relevant effect is the KV block layout/size, which is validated
# per region at runtime in _validate_remote_agent_handshake. The connector
# only sees the raw override here (usually None = auto-select), never the
# resolved backend, so hashing it would cause false mismatches without
# catching anything the runtime layout check misses.
return {
"method": speculative_config.method,
"model": draft_model_config.model,
"revision": draft_model_config.revision,
"code_revision": draft_model_config.code_revision,
"parallel_drafting": speculative_config.parallel_drafting,
"kv_cache_dtype": str(kv_cache_dtype),
"auxiliary_layer_ids": (
tuple(auxiliary_layer_ids) if auxiliary_layer_ids is not None else None
),
}
def compute_nixl_compatibility_hash(
vllm_config: VllmConfig,
attn_backend_name: str,
transfer_mode: str = "pull",
) -> str:
"""Compute compatibility hash for NIXL KV transfer.
Hash only the factors that affect whether two NIXL instances can
successfully transfer KV cache data.
Factors included:
- vLLM version and NIXL connector version
- Model architecture (name, dtype, KV heads, layers)
- KV cache format (dtype, sliding window)
- Attention backend
- EAGLE/MTP configuration that affects transferred state
- Transfer mode (push vs pull)
The transfer mode is included because the push (WRITE) and pull (READ)
connectors use incompatible transfer protocols; a push connector and a
pull connector must never complete a handshake with each other.
Note: Factors like tensor_parallel_size, block_size, and kv_cache_layout
are validated at runtime in _validate_remote_agent_handshake and are not
included in this hash to support heterogeneous deployments.
Note - the set of factors are likely to evolve significantly over
time to be more or less permissive.
Returns:
SHA-256 hex digest
"""
from vllm import __version__ as vllm_version
from vllm.config.utils import hash_factors
model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
is_hma_enabled = not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
factors = {
# Version compatibility
"vllm_version": vllm_version,
"nixl_connector_version": NIXL_CONNECTOR_VERSION,
# Model architecture - affects KV cache shape
"model": model_config.model,
"dtype": str(model_config.dtype),
"num_kv_heads": model_config.get_total_num_kv_heads(),
"head_size": model_config.get_head_size(),
"num_hidden_layers": model_config.get_total_num_hidden_layers(),
# Attention backend and KV cache dtype affect memory layout
"attn_backend_name": attn_backend_name,
"cache_dtype": str(cache_config.cache_dtype),
"is_hma_enabled": is_hma_enabled,
"speculative_config": _get_speculative_compatibility_factors(vllm_config),
# push (WRITE) and pull (READ) connectors are protocol-incompatible
"transfer_mode": transfer_mode,
}
compat_hash = hash_factors(factors)
logger.debug(
"NIXL compatibility hash: %s (model=%s, dtype=%s, num_kv_heads=%d, "
"cache_dtype=%s, attn_backend=%s)",
compat_hash,
factors["model"],
factors["dtype"],
factors["num_kv_heads"],
factors["cache_dtype"],
attn_backend_name,
)
return compat_hash
@dataclass
class HeartbeatInfo:
"""Heartbeat data for a single remote engine, sent from D worker to P."""
req_ids: set[ReqId]
host: str
port: int
tp_size: int
dcp_size: int = 1
pp_size: int = 1
@dataclass
class RemoteMeta:
block_ids: BlockIds
host: str
port: int
engine_id: str
request_id: str
blocks_expiry_time: float | None = None
num_tokens: int | None = None
@dataclass
class ReqMeta:
local_block_ids: BlockIds
# To be used when logical block size does not match the kernel block size
local_physical_block_ids: BlockIds
tp_size: int
dcp_size: int = 1
# Per-KV-cache-group logical blocks this rank already holds, i.e. its
# prefix-cache hit. Fixes where this rank's DCP slice starts relative to
# the remote's; kept per-group since hybrid models (e.g. SWA+FA) can have
# different cache-hit counts per group.
local_num_computed_blocks: tuple[int, ...] = ()
remote: RemoteMeta | None = None
# Remote block size, discovered during NIXL handshake (push mode).
remote_block_size: int | None = None
# Remote producer pipeline-parallel size (push mode, D side).
pp_size: int = 1
# True only when the scheduler parked the request in WAITING_FOR_REMOTE_KVS
# and expects it in finished_recving; notify-only recvs must not be reported.
awaiting_kvs: bool = False
# Worker-only, per-region physical pages to zero after a successful pull.
# None selects group-based completion; empty lists mean no zeroing.
region_blocks_to_zero: BlockIds | None = None
class NixlConnectorMetadata(KVConnectorMetadata):
def __init__(self):
self.reqs_to_recv: dict[ReqId, ReqMeta] = {}
self.reqs_to_save: dict[ReqId, ReqMeta] = {}
self.reqs_to_send: dict[ReqId, float] = {}
# The scheduler process's time.perf_counter() when this metadata was
# built. reqs_to_send deadlines are stamped with the scheduler's
# clock, which is NOT comparable across processes (perf_counter is
# process/boot-local): workers must rebase the remaining TTL onto
# their own clock via this reference. 0.0 = unset (legacy metadata).
self.scheduler_clock: float = 0.0
self.reqs_in_batch: set[ReqId] = set()
self.reqs_not_processed: set[ReqId] = set()
# Heartbeat data grouped by remote engine, sent by D worker to P.
self.heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {}
# Push mode (D side): registration data the D worker should send to
# P workers via NIXL notification on this step.
self.push_registrations: dict[ReqId, dict[str, Any]] = {}
# Push mode (P side): newly finished request blocks to be matched
# against pending D registrations on the P worker.
self.push_finished_blocks: dict[ReqId, BlockIds] = {}
def _add_new_req(
self,
local_block_ids: BlockIds,
kv_transfer_params: dict[str, Any],
local_num_computed_blocks: tuple[int, ...] = (),
awaiting_kvs: bool = False,
) -> ReqMeta:
return ReqMeta(
local_block_ids=local_block_ids,
local_physical_block_ids=local_block_ids,
# P workers don't need to receive these from proxy here.
tp_size=kv_transfer_params.get("tp_size", 1),
dcp_size=kv_transfer_params.get("dcp_size", 1),
remote_block_size=kv_transfer_params.get("remote_block_size"),
pp_size=kv_transfer_params.get("pp_size", 1),
local_num_computed_blocks=local_num_computed_blocks,
awaiting_kvs=awaiting_kvs,
)
def add_new_req_to_save(
self,
request_id: ReqId,
local_block_ids: BlockIds,
kv_transfer_params: dict[str, Any],
):
self.reqs_to_save[request_id] = self._add_new_req(
local_block_ids, kv_transfer_params
)
def add_new_req_to_recv(
self,
request_id: ReqId,
local_block_ids: BlockIds,
kv_transfer_params: dict[str, Any],
local_num_computed_blocks: tuple[int, ...] = (),
awaiting_kvs: bool = False,
):
req = self._add_new_req(
local_block_ids,
kv_transfer_params,
local_num_computed_blocks,
awaiting_kvs,
)
req.remote = RemoteMeta(
block_ids=kv_transfer_params["remote_block_ids"],
engine_id=kv_transfer_params["remote_engine_id"],
request_id=kv_transfer_params["remote_request_id"],
host=kv_transfer_params["remote_host"],
port=kv_transfer_params["remote_port"],
blocks_expiry_time=kv_transfer_params.get("remote_blocks_expiry_time"),
num_tokens=kv_transfer_params.get("remote_num_tokens"),
)
self.reqs_to_recv[request_id] = req
@@ -1,292 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pull-specific scheduler-side logic for the NIXL connector."""
import time
from typing import TYPE_CHECKING, Any
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
NixlBaseConnectorScheduler,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
"""Pull-specific scheduler logic (READ-based KV transfer)."""
def __init__(
self,
vllm_config: "VllmConfig",
engine_id: str,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, engine_id, kv_cache_config)
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int, bool]:
"""For remote prefill, pull all prompt blocks from remote
asynchronously relative to engine execution.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally
computed tokens for this request
Returns:
* the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
* true if the external KV cache tokens will be loaded
asynchronously (between scheduler steps).
"""
params = request.kv_transfer_params
logger.debug(
"NIXLConnector get_num_new_matched_tokens: "
"num_computed_tokens=%s, kv_transfer_params=%s",
num_computed_tokens,
params,
)
if params is not None and params.get("do_remote_prefill"):
# Remote prefill: get all prompt blocks from remote.
token_ids = request.prompt_token_ids or []
actual = self._get_remote_prefill_token_count(len(token_ids))
count = actual - num_computed_tokens
if count > 0:
return count, True
if (
params is not None
and params.get("do_remote_decode")
and params.get("remote_block_ids")
and all(
p in params
for p in (
"remote_engine_id",
"remote_request_id",
"remote_host",
"remote_port",
)
)
):
# Decode node has kv blocks for part of prefill request, so, provide them
# as an external token count to scheduler.
# The tokens will be loaded if not already present
# in the prefill node local cache
remote_num_tokens = params.get("remote_num_tokens") or 0
count = (
min(remote_num_tokens, request.num_prompt_tokens) - num_computed_tokens
)
if count > 0:
# Check kv_recompute_threshold: skip pull if
# remote tokens are below the threshold.
if (
self.kv_recompute_threshold > 0
and count < self.kv_recompute_threshold
):
logger.debug(
"Skipping remote pull for %s: %d remote tokens < threshold %d",
request.request_id,
count,
self.kv_recompute_threshold,
)
return 0, False
return count, True
# No remote prefill for this request.
return 0, False
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
params = request.kv_transfer_params
logger.debug(
"NIXLConnector update_state_after_alloc: "
"num_external_tokens=%s, kv_transfer_params=%s",
num_external_tokens,
params,
)
if not params:
return
if params.get("do_remote_decode") or (
params.get("do_remote_prefill") and self.is_bidirectional_kv_xfer_enabled
):
self._reqs_in_batch.add(request.request_id)
if self.use_host_buffer and params.get("do_remote_decode"):
# NOTE: when accelerator is not directly supported by Nixl,
# prefilled blocks need to be saved to host memory before transfer.
self._reqs_need_save[request.request_id] = request
elif params.get("do_remote_prefill") or (
params.get("do_remote_decode")
and self.is_bidirectional_kv_xfer_enabled
and not params.get("_remote_blocks_processed")
):
if params.get("remote_block_ids"):
if all(
p in params
for p in (
"remote_engine_id",
"remote_request_id",
"remote_host",
"remote_port",
)
):
# If remote_blocks and num_external_tokens = 0, we have
# a full prefix cache hit on the local node. We need to call
# send_notif in _read_blocks to free the memory on the remote node.
unhashed_local_block_ids: BlockIds = (
blocks.get_unhashed_block_ids_all_groups()
if num_external_tokens > 0
else ()
)
local_block_ids = self.get_exchange_clipped_blocks(
unhashed_local_block_ids
)
# Blocks covered by the local prefix cache, per KV cache group.
# Each count fixes where that group's DCP slice starts, which the
# worker needs to line up with the remote's slice.
local_num_computed_blocks = tuple(
sum(
block.block_hash is not None and not block.is_null
for block in group
)
for group in blocks.blocks
)
# Get unhashed blocks to pull from remote. Mind that a full prefix
# cache hit is indicated with an empty list.
self._reqs_need_recv[request.request_id] = (
request,
local_block_ids,
local_num_computed_blocks,
# Parked in WAITING_FOR_REMOTE_KVS only when there is
# something to pull; a full local hit stays RUNNING.
num_external_tokens > 0,
)
else:
logger.warning(
"Got invalid KVTransferParams: %s. This "
"request will not utilize KVTransfer",
params,
)
else:
assert num_external_tokens == 0
# Only trigger 1 KV transfer per request.
params["do_remote_prefill"] = False
params["_remote_blocks_processed"] = True
def request_finished(
self,
request: "Request",
block_ids: "BlockIds",
) -> tuple[bool, dict[str, Any] | None]:
"""Once a request is finished, determine whether request blocks
should be freed now or will be sent asynchronously and freed later.
"""
from vllm.v1.request import RequestStatus
params = request.kv_transfer_params
logger.debug(
"NIXLConnector request_finished(%s), request_status=%s, "
"kv_transfer_params=%s",
request.request_id,
request.status,
params,
)
if not params:
return False, None
is_p_node = bool(params.get("do_remote_decode"))
is_d_node = not is_p_node
# Stop heartbeating for aborted requests that never reached finished_recving:
# normal path cleans up in update_connector_output.
self._stop_heartbeat(request.request_id)
if params.get("do_remote_prefill"):
# If do_remote_prefill is still True when the request is finished,
# update_state_after_alloc must not have been called (the request
# must have been aborted before it was scheduled, e.g. via the
# abort_immediately path used to clean up KV-transfer requests
# rejected at the D-side serving layer).
# To avoid stranding the prefill blocks in the prefill instance,
# we must add empty block_ids to _reqs_need_recv so that our
# worker side will notify and free blocks in the prefill instance.
self._reqs_need_recv[request.request_id] = (request, [], (), False)
params["do_remote_prefill"] = False
return False, None
if is_d_node and not self.is_bidirectional_kv_xfer_enabled:
return False, None
if request.status not in (
RequestStatus.FINISHED_LENGTH_CAPPED,
RequestStatus.FINISHED_STOPPED,
):
# Also include the case of a P/D Prefill request with immediate
# block free (eg abort). Stop tracking this request.
self._reqs_not_processed.add(request.request_id)
# Clear _reqs_need_save if a request is aborted as partial prefill.
self._reqs_need_save.pop(request.request_id, None)
return False, None
# TODO: check whether block_ids actually ever be 0. If not we could
# remove the conditional below
delay_free_blocks = any(len(group) > 0 for group in block_ids)
remote_num_tokens = 0
blocks_expiry_time = None
if delay_free_blocks:
# Prefill request on remote. It will be read from D upon completion
request_kv_blocks_ttl = self._kv_lease_duration
if is_d_node:
# For blocks pinned on D, use a simpler timeout for now instead of a
# lease mechanism as turn2 request is client-driven.
request_kv_blocks_ttl = self.decoder_kv_blocks_ttl
logger.debug(
"NIXLConnector request_finished(%s) waiting for %d seconds "
"before releasing blocks",
request.request_id,
request_kv_blocks_ttl,
)
self._reqs_need_send[request.request_id] = (
time.perf_counter() + request_kv_blocks_ttl
)
if is_d_node:
# D blocks expiry time exported for the turn-2 readback.
blocks_expiry_time = self._reqs_need_send[request.request_id]
# NOTE HMA will "mark" empty/null blocks in groups with 0s (eg SWA ones),
# trimming down after allocating for the whole sequence length. Empty
# blocks are always at the start of the list.
# Here we "unpad" blocks to send the actual remote blocks to be read.
block_ids = self.get_exchange_clipped_blocks(block_ids)
remote_num_tokens = request.num_computed_tokens
return delay_free_blocks, dict(
do_remote_prefill=is_p_node,
do_remote_decode=is_d_node,
remote_block_ids=block_ids,
remote_engine_id=self.engine_id,
remote_request_id=request.request_id,
remote_host=self.side_channel_host,
remote_port=self.side_channel_port,
tp_size=self.transfer_tp_size,
dcp_size=self.vllm_config.parallel_config.decode_context_parallel_size,
pp_size=self.vllm_config.parallel_config.pipeline_parallel_size,
remote_num_tokens=remote_num_tokens,
remote_blocks_expiry_time=blocks_expiry_time,
transfer_mode=self._TRANSFER_MODE,
)
@@ -1,693 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pull-specific (READ) worker-side logic for the NIXL connector."""
import time
from typing import TYPE_CHECKING
import numpy as np
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
NixlBaseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlConnectorMetadata,
ReqMeta,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
ReadSpec,
_is_attention_spec,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_cache_interface import KVCacheConfig
logger = init_logger(__name__)
# Slack (seconds) subtracted from D's exported block-expiry deadline on the turn-2
# readback, absorbing clock-offset error and read latency.
_KV_BLOCKS_EXPIRY_SAFETY_MARGIN = 5.0
class NixlPullConnectorWorker(NixlBaseConnectorWorker):
"""Pull-specific (READ) worker logic."""
def __init__(
self,
vllm_config: "VllmConfig",
engine_id: str,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, engine_id, kv_cache_config)
def start_load_kv(self, metadata: NixlConnectorMetadata):
"""Start loading by triggering non-blocking nixl_xfer.
We check for these trnxs to complete in each step().
"""
for req_id, meta in metadata.reqs_to_recv.items():
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
meta.local_block_ids, self._physical_blocks_per_logical_kv_block
)
assert meta.remote is not None
# Remote block IDs are kept logical here; expanded in
# _read_blocks_for_req using the remote engine's phys ratio.
remote_engine_id = meta.remote.engine_id
logger.debug(
"start_load_kv for request %s from remote engine %s. "
"Num local_block_ids: %s. Num remote_block_ids: %s. ",
req_id,
remote_engine_id,
len(meta.local_physical_block_ids),
len(meta.remote.block_ids),
)
# Full local hits and aborted cleanup only notify P; no recv is awaited.
# On a full local hit:
# - Notification failure must not fail the request: its KV is local.
# - Receive completion must not be reported: num_external_tokens == 0,
# so the scheduler never entered WAITING_FOR_REMOTE_KVS to begin with.
# Aborted cleanup requests have already been removed from the scheduler.
if meta.awaiting_kvs or any(meta.local_block_ids):
self._recving_metadata[req_id] = meta
self._read_blocks_for_req(req_id, meta)
# Start transfers for requests whose handshakes have now finished.
while not self._ready_requests.empty():
self._read_blocks_for_req(*self._ready_requests.get_nowait())
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
# has nothing to send. Report the requests as sent right away so
# the scheduler-side aggregation (world_size workers, and any
# sibling connector inside a MultiConnector) still completes.
self._replicated_pcp_done_sending.update(metadata.reqs_to_send)
return
# Keep around the requests that have been part of a batch. This is
# needed because async scheduling pushes the misalignment between the
# moment in which requests expiration is set (P side) and the moment in
# which blocks are read from D. As P can now more easily lag behind D
# while processing the next batch, we make sure to only set an
# expiration for requests that have not been read from D yet.
for req_id in metadata.reqs_in_batch:
self._reqs_to_process.add(req_id)
# Remove all requests that are not to be processed (eg aborted).
for req_id in metadata.reqs_not_processed:
self._reqs_to_process.discard(req_id)
# We should never get an abort after setting an expiry timer
assert req_id not in self._reqs_to_send
# Add to requests that are waiting to be read and track expiration.
# Deadlines are stamped with the scheduler process's perf_counter,
# which is not comparable to ours when the worker runs in another
# process on another node (perf_counter epochs differ by boot time).
# Rebase the remaining TTL onto our clock; broadcast latency only
# lengthens the lease, which is the safe direction. A cross-node
# epoch gap larger than the TTL otherwise expires the lease on
# arrival and the blocks are freed before D reads them.
now_local = time.perf_counter()
for req_id, expiration_time in metadata.reqs_to_send.items():
if req_id in self._reqs_to_process:
if metadata.scheduler_clock:
expiration_time = now_local + (
expiration_time - metadata.scheduler_clock
)
self._reqs_to_send[req_id] = expiration_time
# Send heartbeats to P-side engines to keep KV blocks alive while
# requests sit in the D scheduler WAITING queue.
self._send_heartbeats(metadata)
def _is_turn2_read_expired(self, meta: ReqMeta) -> bool:
"""Whether D's cached blocks for this turn-2 readback have (nearly) expired."""
assert meta.remote is not None
blocks_expiry_time = meta.remote.blocks_expiry_time
# Deadline may be absent (router may not forward it) -> read as usual.
if blocks_expiry_time is None or not meta.local_physical_block_ids:
return False
clock_offset = self._engine_clock_offset[meta.remote.engine_id]
deadline = blocks_expiry_time - clock_offset
return time.perf_counter() + _KV_BLOCKS_EXPIRY_SAFETY_MARGIN >= deadline
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
assert meta.remote is not None and self.transfer_topo is not None
engine_id = meta.remote.engine_id
if engine_id in self._invalid_remote_engines:
self._handle_failed_transfer(
req_id, None, self._recv_failures, record_failed_transfer=False
)
return
if engine_id not in self._remote_agents:
# Also re-handshakes queued requests that outlived invalid-peer cleanup.
self._background_nixl_handshake(req_id, engine_id, meta)
return
# Update last activity from this remote. Mind that cleanup is done on main
# thread (this one), so we don't race on this structure.
self._engine_last_active[engine_id] = time.perf_counter()
if self._bidirectional_kv_xfer_enabled and self._is_turn2_read_expired(meta):
logger.warning(
"Declining expired remote read for %s from engine %s.",
req_id,
engine_id,
)
self.xfer_stats.record_kv_expired_req()
# KV expiry is reported separately from transport failures, so only
# the state cleanup side of _handle_failed_transfer runs here.
self._handle_failed_transfer(
req_id, None, self._recv_failures, record_failed_transfer=False
)
return
if any(len(group) > 0 for group in meta.local_block_ids):
# The scheduler waits for finished_recving from *every* worker.
# Under DCP a rank's slice can legitimately come out empty when its
# interleaved positions fall past the end of the sequence. _read_blocks
# then takes the notify-only path without registering a transfer.
# Seed the entry so this rank still reports completion.
self._recving_transfers.setdefault(req_id, [])
plan = self.tp_mappings[engine_id]
remote_info = self.transfer_topo.get_engine_info(engine_id)
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
dcp_active = self.dcp_size > 1 or remote_info.remote_dcp_size > 1
local_block_ids = meta.local_physical_block_ids
remote_region_groups = self.dst_region_group_ids[engine_id]
local_region_groups = self.region_group_ids or remote_region_groups
if not local_block_ids:
# Region expansion cannot index empty groups. Pass empty specs to
# _read_blocks so its existing cache-hit notification path runs.
read_specs = [
ReadSpec(remote_rank=rank, local_block_ids=[], remote_block_ids=[])
for rank in plan.all_source_ranks
]
elif local_region_groups != remote_region_groups:
if not self.use_mla or self._has_mamba:
raise NotImplementedError(
"Different NIXL cache-group layouts are only supported for "
"pure MLA models"
)
if self.block_size != remote_info.remote_block_size:
raise NotImplementedError(
"Region-mapped NIXL transfers require matching physical block sizes"
)
remote_physical_block_ids = self._logical_to_kernel_block_ids(
meta.remote.block_ids,
remote_info.remote_physical_blocks_per_logical,
)
remote_by_region = self._block_ids_by_region(
remote_physical_block_ids, remote_region_groups
)
local_by_region = self._block_ids_by_region(
local_block_ids, local_region_groups
)
num_computed_blocks = None
num_remote_blocks = None
if (
meta.remote.num_tokens is not None
and meta.local_num_computed_blocks
and all(group >= 0 for group in local_region_groups)
and all(group >= 0 for group in remote_region_groups)
):
transfer_groups = self.kv_cache_config.transfer_group_ids
num_computed_blocks = [
meta.local_num_computed_blocks[transfer_groups[group]]
* self._physical_blocks_per_logical_kv_block
for group in local_region_groups
]
num_remote_blocks = cdiv(
meta.remote.num_tokens, remote_info.remote_block_size
)
elif any(local_by_region) and (
dcp_active
or remote_info.remote_physical_blocks_per_logical
!= self._physical_blocks_per_logical_kv_block
):
raise NotImplementedError(
"Region-mapped pulls with DCP or different logical block sizes "
"require remote_num_tokens, per-group prefix counts "
"and unshared regions"
)
read_specs = [
ReadSpec(
rank,
*self._apply_prefix_caching_by_region(
local_by_region,
remote_by_region,
num_computed_blocks=num_computed_blocks,
num_remote_blocks=num_remote_blocks,
remote_rank=rank,
remote_dcp_size=remote_info.remote_dcp_size,
),
block_ids_by_region=True,
)
for rank in plan.all_source_ranks
]
meta.region_blocks_to_zero = [
list(blocks[sum(len(spec.local_block_ids[r]) for spec in read_specs) :])
for r, blocks in enumerate(local_by_region)
]
else:
remote_logical_block_ids = meta.remote.block_ids
meta.remote.block_ids = self._logical_to_kernel_block_ids(
remote_logical_block_ids,
remote_info.remote_physical_blocks_per_logical,
)
num_groups = len(meta.local_block_ids)
def group_ids(block_ids: BlockIds, rank: int) -> list[list[int]]:
return [
list(block_ids[g]) if rank in plan.source_ranks_per_group[g] else []
for g in range(num_groups)
]
read_specs = []
for rank in plan.all_source_ranks:
if dcp_active:
local_ids = group_ids(meta.local_block_ids, rank)
remote_ids = group_ids(remote_logical_block_ids, rank)
for g in range(num_groups):
if not local_ids[g] or not _is_attention_spec(
self._group_spec_types[g]
):
continue
local_ids[g], remote_ids[g] = self._apply_dcp_prefix_caching(
local_ids[g],
remote_ids[g],
remote_rank=rank,
local_dcp_size=self.dcp_size,
local_dcp_rank=self.dcp_rank,
remote_dcp_size=remote_info.remote_dcp_size,
local_num_computed_blocks=(
meta.local_num_computed_blocks[g]
),
)
local_physical_ids = self._logical_to_kernel_block_ids(
local_ids, self._physical_blocks_per_logical_kv_block
)
remote_physical_ids = self._logical_to_kernel_block_ids(
remote_ids,
remote_info.remote_physical_blocks_per_logical,
)
else:
local_physical_ids = group_ids(meta.local_physical_block_ids, rank)
remote_physical_ids = group_ids(meta.remote.block_ids, rank)
read_specs.append(
ReadSpec(
remote_rank=rank,
local_block_ids=local_physical_ids,
remote_block_ids=remote_physical_ids,
)
)
# D may have to perform multiple reads from different remote ranks.
# Pure MLA reads once because its cache is replicated. Hybrid
# MLA+SSM still needs one read per SSM source rank. With DCP, pure
# MLA may also read from multiple ranks (disjoint token slices).
if self.use_mla and tp_ratio < 0 and not self._has_mamba and not dcp_active:
assert len(read_specs) == 1
for i, spec in enumerate(read_specs):
remote_block_size = remote_info.remote_block_size
logger.debug(
"Remote agent %s available, calling _read_blocks"
" on remote rank %s with remote block size %s for req %s",
meta.remote.engine_id,
spec.remote_rank,
remote_block_size,
req_id,
)
# Get side handles.
if tp_ratio < 0 and (not self.use_mla or len(read_specs) > 1):
# Remote tp_size > local tp_size: we must perform multiple
# reads. Get the memory chunk onto which we will write to.
split_key = (tp_ratio, remote_block_size)
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i]
local_dram_handle = (
self._dram_src_handles_by_tp_ratio[split_key][i]
if self._mixed_mem_types
else None
)
else:
# Single read from remote, we write to the whole memory region.
# Also handle remote block size different from local block size.
local_xfer_side_handle = self.src_xfer_handles_by_block_size[
remote_block_size
]
local_dram_handle = (
self._dram_src_handles_by_block_size[remote_block_size]
if self._mixed_mem_types
else None
)
# Destination handle: remote_engine_id -> remote_rank -> handle.
remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][
spec.remote_rank
]
# Once a read routes the request to failure reporting, the
# scheduler may free and reuse its blocks, so no sibling READs
# may be posted (and P must not be notified).
if not self._read_blocks(
read_spec=spec,
request_id=req_id,
dst_engine_id=meta.remote.engine_id,
remote_request_id=meta.remote.request_id,
local_xfer_side_handle=local_xfer_side_handle,
local_dram_handle=local_dram_handle,
remote_xfer_side_handle=remote_xfer_side_handle,
expected_consumers=plan.local_consumers,
awaiting_kvs=meta.awaiting_kvs,
):
return
if self.use_mla and tp_ratio < 0 and len(read_specs) == 1:
# ..but we still need to notify the other remote ranks that we
# have the blocks we need so they can update the request state.
# Same thing for DCP (tp_size == dcp_size), so the raw tp_ratio already
# reflects whether any remote replica is left unchosen.
notif_id = f"{meta.remote.request_id}:{plan.local_consumers}".encode()
remote_agents = self._remote_agents[meta.remote.engine_id]
for rank_to_notify, agent in remote_agents.items():
if rank_to_notify != (0, read_specs[0].remote_rank):
try:
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
except Exception as e:
self._log_failure(
failure_type="notification_failed",
msg="Remote rank will not update request state. "
"This may indicate network issues.",
req_id=req_id,
error=e,
dst_engine_id=meta.remote.engine_id,
remote_rank=rank_to_notify[1],
remote_agent_name=agent,
)
self.xfer_stats.record_failed_notification()
def _read_blocks(
self,
read_spec: ReadSpec,
dst_engine_id: str,
request_id: str,
remote_request_id: str,
local_xfer_side_handle: int,
local_dram_handle: int | None,
remote_xfer_side_handle: int,
expected_consumers: int,
awaiting_kvs: bool,
) -> bool:
"""Post a READ point-to-point xfer request from a single local worker to
a single remote worker.
Returns True when the read was posted (or was unnecessary), False
when the request was routed to failure reporting — the caller must
not post further transfers for it.
"""
assert self.transfer_topo is not None
remote_rank = read_spec.remote_rank
local_block_ids = read_spec.local_block_ids
remote_block_ids = read_spec.remote_block_ids
remote_info = self.transfer_topo.get_engine_info(dst_engine_id)
block_size_ratio = self.transfer_topo.block_size_ratio(
remote_info.remote_block_size
)
if block_size_ratio > 1:
if read_spec.block_ids_by_region:
raise NotImplementedError(
"Region-mapped NIXL transfers require matching physical block sizes"
)
local_block_ids, remote_block_ids = (
self._map_block_ids_for_block_size_ratio(
local_block_ids, remote_block_ids, block_size_ratio
)
)
# NOTE(rob): having the staging blocks be on the READER side is
# not going to work well (since we will have to call rearrange tensors).
# after we detect the txn is complete (which means we cannot make the
# read trxn async easily). If we want to make "READ" happen cleanly,
# then we will need to have the staging blocks on the remote side.
# NOTE(rob): according to nvidia the staging blocks are used to
# saturate IB with heterogeneous TP sizes.
# Number of local workers that will notify this producer worker.
# Propagate on notification so dst worker can wait before freeing.
notif_id = f"{remote_request_id}:{expected_consumers}".encode()
# Full prefix cache hit: do not need to read remote blocks,
# just notify P worker that we have the blocks we need.
if not any(len(group) > 0 for group in local_block_ids):
# A full prefix cache hit is indicated with an empty list.
agent_name = self._remote_agents[dst_engine_id][(0, remote_rank)]
try:
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id)
except Exception as e:
self._log_failure(
failure_type="notification_failed",
msg="P worker blocks will be freed after timeout. "
"This may indicate network issues.",
req_id=request_id,
error=e,
dst_engine_id=dst_engine_id,
remote_rank=remote_rank,
remote_agent_name=agent_name,
)
self.xfer_stats.record_failed_notification()
# Report even on notification failure: the KV is already local, and
# an unreported parked request would hold its blocks forever.
# Notify-only recvs must stay unreported (scheduler asserts).
if awaiting_kvs:
self._recving_transfers.setdefault(request_id, [])
return True
if not read_spec.block_ids_by_region:
assert (
len(remote_block_ids)
== len(local_block_ids)
== len(self.kv_cache_config.transfer_groups)
)
if not (self.dcp_size > 1 or remote_info.remote_dcp_size > 1):
local_block_ids, remote_block_ids = self._apply_prefix_caching(
decode_block_ids=local_block_ids,
prefill_block_ids=remote_block_ids,
decode_physical_per_logical=(
self._physical_blocks_per_logical_kv_block
),
prefill_physical_per_logical=(
remote_info.remote_physical_blocks_per_logical
),
)
# NOTE (nicolo) With homogeneous TP, each TP worker loads KV from
# corresponding rank. With heterogeneous TP, fixing D>P, the D tp
# workers will issue xfers to parts of the P worker remote kv caches.
# Get descs ids.
remote_block_descs_ids = self._compute_desc_ids(
block_ids=remote_block_ids,
dst_num_blocks=self.dst_num_blocks[dst_engine_id],
block_size_ratio=None,
physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical,
region_num_blocks=(self.dst_region_num_blocks.get(dst_engine_id) or None),
region_group_ids=(
list(range(self.num_regions))
if read_spec.block_ids_by_region
else (self.dst_region_group_ids.get(dst_engine_id) or None)
),
uses_region_group_mapping=(
self.num_regions > 1
if read_spec.block_ids_by_region
else self.dst_uses_region_group_mapping[dst_engine_id]
),
)
local_block_descs_ids = self._compute_desc_ids(
block_ids=local_block_ids,
dst_num_blocks=self.dst_num_blocks[self.engine_id],
block_size_ratio=block_size_ratio,
physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block,
region_num_blocks=(self.dst_region_num_blocks.get(self.engine_id) or None),
region_group_ids=(
list(range(self.num_regions))
if read_spec.block_ids_by_region
else (self.region_group_ids or None)
),
uses_region_group_mapping=(
self.num_regions > 1
if read_spec.block_ids_by_region
else self._uses_region_group_mapping
),
)
assert len(local_block_descs_ids) == len(remote_block_descs_ids)
# Prepare transfer with Nixl.
handle = None
try:
if self._mixed_mem_types:
self._read_blocks_mixed(
request_id=request_id,
local_block_size_key=remote_info.remote_block_size,
local_device_handle=local_xfer_side_handle,
local_dram_handle=local_dram_handle,
remote_xfer_side_handle=remote_xfer_side_handle,
local_block_descs_ids=local_block_descs_ids,
remote_block_descs_ids=remote_block_descs_ids,
notif_agent=self._remote_agents[dst_engine_id][(0, remote_rank)],
notif_id=notif_id,
)
return True
handle = self.nixl_wrapper.make_prepped_xfer(
"READ",
local_xfer_side_handle,
local_block_descs_ids,
remote_xfer_side_handle,
remote_block_descs_ids,
notif_msg=notif_id,
)
# Begin async xfer.
self.nixl_wrapper.transfer(handle)
# Use handle to check completion in future step().
self._recving_transfers[request_id].append(handle)
return True
except Exception as e:
self._log_failure(
failure_type="transfer_setup_failed",
req_id=request_id,
msg="Deferring failure reporting until outstanding transfers finish",
error=e,
dst_engine_id=dst_engine_id,
remote_rank=remote_rank,
)
if not self._handle_failed_transfer(
request_id, handle, self._recv_failures
):
assert handle is not None
self._recving_transfers[request_id].append(handle)
return False
def _read_blocks_mixed(
self,
request_id: str,
local_block_size_key: int,
local_device_handle: int,
local_dram_handle: int | None,
remote_xfer_side_handle: int,
local_block_descs_ids: np.ndarray,
remote_block_descs_ids: np.ndarray,
notif_agent: str,
notif_id: bytes,
) -> None:
"""Split a READ across the local DRAM and device descriptor lists."""
desc_is_dram = self._desc_is_dram_by_block_size[local_block_size_key]
desc_pos = self._desc_pos_by_block_size[local_block_size_key]
local_ids = np.asarray(local_block_descs_ids)
remote_ids = np.asarray(remote_block_descs_ids)
is_dram = desc_is_dram[local_ids]
if self._skip_dram_xfer:
# TP rank 0 reads the DRAM (shared host pool) part for every
# local rank; keep only the device part. Register the request
# even if nothing is left, so it still completes and notifies.
keep = ~is_dram
local_ids = local_ids[keep]
remote_ids = remote_ids[keep]
is_dram = is_dram[keep]
self._recving_transfers.setdefault(request_id, [])
assert local_dram_handle is not None or not is_dram.any()
reads = (
(is_dram, local_dram_handle),
(~is_dram, local_device_handle),
)
handles: list[int] = []
try:
for mask, local_handle in reads:
if mask.any():
handles.append(
self.nixl_wrapper.make_prepped_xfer(
"READ",
local_handle,
desc_pos[local_ids[mask]],
remote_xfer_side_handle,
remote_ids[mask],
)
)
except Exception:
for handle in handles:
if not self._try_release_xfer_handle(request_id, handle):
self._recving_transfers[request_id].append(handle)
raise
self._pending_recv_notifs.setdefault(request_id, []).append(
(notif_agent, notif_id)
)
for i, handle in enumerate(handles):
try:
self.nixl_wrapper.transfer(handle)
except Exception:
for unstarted in handles[i:]:
if not self._try_release_xfer_handle(request_id, unstarted):
self._recving_transfers[request_id].append(unstarted)
raise
self._recving_transfers[request_id].append(handle)
def _get_new_notifs(self) -> set[str]:
"""Get req_ids which got a remote xfer message. When multiple consumers
are reading from the same producer (heterogeneous TP or DCP
scenario), wait for all consumers to be done pulling.
Also handles heartbeat notifications ("HB:req1,req2,...") by
extending the lease on the referenced requests.
"""
assert self.transfer_topo is not None
notified_req_ids: set[str] = set()
for notifs in self.nixl_wrapper.get_new_notifs().values():
for notif in notifs:
msg = notif.decode("utf-8")
# Handle heartbeat messages from D-side.
if msg.startswith("HB:"):
self._handle_heartbeat(msg[3:])
continue
req_id, expected_consumers = msg.rsplit(":", 1)
if (
req_id not in self._reqs_to_send
and req_id not in self._reqs_to_process
):
self.xfer_stats.record_notification_after_expiry()
logger.error(
"Potentially invalid KV blocks for "
"unrecognized request %s were retrieved by "
"a decode worker. They may have expired.",
req_id,
)
continue
# Every reader of this req_id reports the same count (it's
# derived from aggregate topology, not the specific rank),
# so repeated notifications never disagree on it.
self.expected_consumer_notifications_by_req[req_id] = int(
expected_consumers
)
self.consumer_notification_counts_by_req[req_id] += 1
# Wait all consumers (D) to be done reading before freeing.
if (
self.consumer_notification_counts_by_req[req_id]
== self.expected_consumer_notifications_by_req[req_id]
):
notified_req_ids.add(req_id)
del self.consumer_notification_counts_by_req[req_id]
del self.expected_consumer_notifications_by_req[req_id]
self._reqs_to_process.remove(req_id)
self._reqs_to_send.pop(req_id, None)
return notified_req_ids
@@ -1,357 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Push-specific scheduler-side logic for the NIXL connector.
In push mode, scheduler-side responsibilities are:
* D side (decode): on ``update_state_after_alloc``, stash registration data
(D's identity + locally allocated block IDs) into
``_push_pending_registrations``. The D worker drains it from
``meta.push_registrations`` next step and sends a NIXL notification to the
P worker (no scheduler-level networking).
* P side (prefill): on ``request_finished``, stash the finished block IDs
into ``_finished_request_blocks`` for the lease, and into
``_newly_finished_push_blocks`` so the P worker picks them up via
``meta.push_finished_blocks`` and matches against any D registrations
it already received via NIXL notifications.
* Both sides: ``has_pending_push_work`` keeps the engine main loop stepping
while pushes are in flight. ``update_connector_output`` cleans up
``_finished_request_blocks`` once the WRITE completes.
A soft per-registration watchdog on the D scheduler fails requests that have
been registered but not fulfilled within a configurable timeout.
"""
from __future__ import annotations
import time
from typing import TYPE_CHECKING, Any
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
NixlBaseConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlConnectorMetadata,
ReqId,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
"""Push-specific scheduler logic (WRITE-based KV transfer).
All P2P communication is deferred to the worker level via NIXL
notifications. The scheduler communicates with workers only through
the standard ``build_connector_meta`` / ``update_connector_output``
hooks.
"""
_TRANSFER_MODE: str = "push"
def __init__(
self,
vllm_config: VllmConfig,
engine_id: str,
kv_cache_config: KVCacheConfig,
):
super().__init__(vllm_config, engine_id, kv_cache_config)
if self.is_bidirectional_kv_xfer_enabled:
raise NotImplementedError(
"Bidirectional KV transfer is not supported for NIXL push connector."
)
# D-side: registration data to pass to D workers via metadata on
# the next ``build_connector_meta`` call.
self._push_pending_registrations: dict[ReqId, dict[str, Any]] = {}
# D-side: track the wall-clock deadline for each registered request
# to detect "registered but never fulfilled" failures (e.g. the P
# node disappeared after registration). Keyed by D request_id.
self._push_registration_deadlines: dict[ReqId, float] = {}
# P-side: block IDs for finished requests, kept for the lease and
# used to drive ``has_pending_push_work``.
self._finished_request_blocks: dict[ReqId, BlockIds] = {}
# P-side: newly finished blocks to ship to P workers on next step.
self._newly_finished_push_blocks: dict[ReqId, BlockIds] = {}
# Soft watchdog timeout (seconds) for D-side registrations that
# never receive a push completion. Defaults to the existing
# decoder KV blocks TTL so behaviour matches the lease.
assert vllm_config.kv_transfer_config is not None
self._push_registration_timeout: float = float(
vllm_config.kv_transfer_config.get_from_extra_config(
"push_registration_timeout",
self.decoder_kv_blocks_ttl,
)
)
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int, bool]:
"""In push mode, D doesn't pull — it registers blocks and waits.
However, we still need to handle the do_remote_prefill case where D
needs to know how many tokens will be pushed.
"""
params = request.kv_transfer_params
logger.debug(
"NixlPushConnector get_num_new_matched_tokens: "
"num_computed_tokens=%s, kv_transfer_params=%s",
num_computed_tokens,
params,
)
if params is not None and params.get("do_remote_prefill"):
token_ids = request.prompt_token_ids or []
actual = self._get_remote_prefill_token_count(len(token_ids))
count = actual - num_computed_tokens
if count > 0:
return count, True
return 0, False
def update_state_after_alloc(
self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int
):
"""In push mode, D stores registration data for the worker to send
to P via NIXL notification (deferred to ``build_connector_meta``).
"""
params = request.kv_transfer_params
logger.debug(
"NixlPushConnector update_state_after_alloc: "
"num_external_tokens=%s, kv_transfer_params=%s",
num_external_tokens,
params,
)
if not params:
return
# P side: track the request as in-batch so the lease accounting
# matches what the worker expects on the next step.
if params.get("do_remote_decode"):
self._reqs_in_batch.add(request.request_id)
# P side with host-buffer offload: defer save to the worker.
if self.use_host_buffer and params.get("do_remote_decode"):
self._reqs_need_save[request.request_id] = request
return
# D side: only act on the first call (``do_remote_prefill`` is
# unset on re-entry by the marker below).
if not params.get("do_remote_prefill"):
return
if num_external_tokens <= 0:
# Nothing to receive: full prefix-cache hit on D, no
# registration to stage.
return
# First-pass D path: stash registration data the worker will
# ship to P on the next ``build_connector_meta`` cycle.
logger.debug(
"KV PUSH mode: D node storing registration for request %s",
request.request_id,
)
local_block_ids: BlockIds = blocks.get_unhashed_block_ids_all_groups()
local_block_ids = self.get_exchange_clipped_blocks(local_block_ids)
# ``remote_*`` fields are P's coordinates (from D's perspective).
# ``decode_*`` fields are D's own info that P needs for the
# reverse handshake before WRITE-ing.
self._push_pending_registrations[request.request_id] = {
"request_id": request.request_id,
"decode_engine_id": self.engine_id,
"decode_host": self.side_channel_host,
"decode_port": self.side_channel_port,
"decode_tp_size": (self.vllm_config.parallel_config.tensor_parallel_size),
"local_block_ids": local_block_ids,
"remote_engine_id": params["remote_engine_id"],
"remote_host": params["remote_host"],
"remote_port": params["remote_port"],
"remote_tp_size": params["tp_size"],
"remote_pp_size": params.get("pp_size", 1),
}
self._push_registration_deadlines[request.request_id] = (
time.perf_counter() + self._push_registration_timeout
)
# In push mode D doesn't know P's blocks; P determines them
# from the registration. We still track the request as
# needing recv so the engine waits for P's WRITE completion.
# ``remote_block_ids`` is also seeded to an empty tuple so the
# base scheduler's ``add_new_req_to_recv`` can build the
# ReqMeta without a KeyError — the actual remote block IDs are
# learned by P over the NIXL handshake at WRITE time.
params["remote_block_ids"] = ()
self._reqs_need_recv[request.request_id] = (request, local_block_ids, (), False)
# Mark as processed so a re-entry (e.g. preemption + reschedule)
# doesn't re-stage the registration.
params["do_remote_prefill"] = False
def request_finished(
self,
request: Request,
block_ids: BlockIds,
) -> tuple[bool, dict[str, Any] | None]:
"""Push-mode request_finished: stores blocks for workers."""
from vllm.v1.request import RequestStatus
params = request.kv_transfer_params
logger.debug(
"NixlPushConnector request_finished(%s), request_status=%s, "
"kv_transfer_params=%s",
request.request_id,
request.status,
params,
)
if not params:
return False, None
is_p_node = bool(params.get("do_remote_decode"))
self._stop_heartbeat(request.request_id)
# Drop any pending registration deadline; the request either
# completed or was cancelled.
self._push_registration_deadlines.pop(request.request_id, None)
if params.get("do_remote_prefill"):
# ``do_remote_prefill`` is still set, which means
# ``update_state_after_alloc`` never ran (it would have
# flipped this flag to False). The request was aborted
# before it could be scheduled — e.g. rejected at the D
# serving layer via abort_immediately. To keep P from
# stranding the prefill blocks, we still register an empty
# recv so the worker emits a notif that lets P free them.
# Seed remote_block_ids so add_new_req_to_recv won't KeyError.
params["remote_block_ids"] = ()
self._reqs_need_recv[request.request_id] = (request, [], (), False)
params["do_remote_prefill"] = False
return False, None
# Push connector only acts on the P-side terminal path; D-side
# finishing without a remote prefill is a no-op.
if not is_p_node:
return False, None
if request.status not in (
RequestStatus.FINISHED_LENGTH_CAPPED,
RequestStatus.FINISHED_STOPPED,
):
self._reqs_not_processed.add(request.request_id)
self._reqs_need_save.pop(request.request_id, None)
return False, None
delay_free_blocks = any(len(group) > 0 for group in block_ids)
remote_num_tokens = 0
if delay_free_blocks:
logger.debug(
"NixlPushConnector request_finished(%s) waiting for %d seconds "
"before releasing blocks",
request.request_id,
self._kv_lease_duration,
)
self._reqs_need_send[request.request_id] = (
time.perf_counter() + self._kv_lease_duration
)
block_ids = self.get_exchange_clipped_blocks(block_ids)
remote_num_tokens = request.num_computed_tokens
# Store finished blocks for worker-level matching with D
# registrations (via NIXL notifications).
self._finished_request_blocks[request.request_id] = block_ids
self._newly_finished_push_blocks[request.request_id] = block_ids
return delay_free_blocks, dict(
do_remote_prefill=True,
do_remote_decode=False,
remote_block_ids=block_ids,
remote_engine_id=self.engine_id,
remote_request_id=request.request_id,
remote_host=self.side_channel_host,
remote_port=self.side_channel_port,
tp_size=self.transfer_tp_size,
dcp_size=self.vllm_config.parallel_config.decode_context_parallel_size,
pp_size=self.vllm_config.parallel_config.pipeline_parallel_size,
remote_num_tokens=remote_num_tokens,
transfer_mode=self._TRANSFER_MODE,
)
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
meta = super().build_connector_meta(scheduler_output)
assert isinstance(meta, NixlConnectorMetadata)
# Watchdog: any D-side registration whose deadline has passed without
# a corresponding push completion is treated as failed and cleaned up.
# The corresponding request is already tracked via _reqs_need_recv;
# the engine layer will eventually time it out via the lease, but we
# at least drop the stale registration so we don't keep retrying.
now = time.perf_counter()
# Deadlines are inserted in non-decreasing order (monotonic clock +
# constant timeout, armed once per request), and dict insertion order
# is preserved across key deletions, so we can stop at the first
# not-yet-expired entry instead of scanning the whole dict.
expired = []
for rid, deadline in self._push_registration_deadlines.items():
if deadline > now:
break
expired.append(rid)
for rid in expired:
self._push_registration_deadlines.pop(rid, None)
# Avoid resending a registration that already timed out.
self._push_pending_registrations.pop(rid, None)
logger.warning(
"NixlPushConnector: registration for request %s timed out "
"after %.1fs without a push completion",
rid,
self._push_registration_timeout,
)
# D side: package pending registrations for D workers to send out.
if self._push_pending_registrations:
meta.push_registrations = dict(self._push_pending_registrations)
self._push_pending_registrations.clear()
# P side: package newly finished blocks for P workers to match against
# any D registrations they have received via NIXL notifications.
if self._newly_finished_push_blocks:
meta.push_finished_blocks = dict(self._newly_finished_push_blocks)
self._newly_finished_push_blocks.clear()
return meta
def has_pending_push_work(self) -> bool:
# Keep the engine main loop alive while we have:
# - finished P blocks awaiting WRITE completion, or
# - pending D registrations the worker has not yet shipped, or
# - newly finished blocks not yet shipped to P workers.
return bool(self._finished_request_blocks or self._push_pending_registrations)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Clean up finished request blocks after push completes."""
super().update_connector_output(connector_output)
for req_id in connector_output.finished_sending or ():
self._finished_request_blocks.pop(req_id, None)
# On D side, finished_recving means the push completed; clear the
# watchdog so we don't trip an expiration on a fulfilled request.
for req_id in connector_output.finished_recving or ():
self._push_registration_deadlines.pop(req_id, None)
@@ -1,827 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Push-specific (WRITE) worker-side logic for the NIXL connector.
A dedicated ``nixl-push-writer`` thread owns all push-related NIXL ops:
calls ``get_new_notifs`` (routing PUSH_REG internally; HB / completion
notifs are forwarded to the engine main thread), sends PUSH_REG via
``send_notif``, matches D registrations with P finished blocks, and
issues WRITE transfers via ``make_prepped_xfer`` / ``transfer``.
The engine main thread feeds the writer through queues:
``_reg_send_inbox`` (D-side regs to send), ``_finished_blocks_inbox``
(P-side blocks from metadata) and ``_pending_completion_notifs``
(non-PUSH_REG notifs forwarded back for HB / completion accounting).
The handshake-completion callback feeds ``_deferred_push_inbox`` with
matched pushes whose P→D handshake has finished so the writer can
(re-)issue the WRITE without ever blocking on the network.
Wake model: the writer self-polls every
``_PUSH_WRITER_POLL_INTERVAL_MS`` only while it has unmatched
``_push_finished_blocks`` (i.e. P-side blocks waiting for a D PUSH_REG
notif that has no other wake source). All other progress is
event-driven: the engine main thread sets ``_push_writer_wake`` from
``start_load_kv`` (when handing it new work) and from ``get_finished``
(so each engine step gives the writer a chance to drain NIXL notifs);
the handshake-completion callback sets the same event after a deferred
PUSH_REG send or a deferred push WRITE has been queued. When a request's
lease expires (the base worker reports it via ``done_sending``) or the WRITE completes,
``get_finished`` enqueues an eviction onto ``_evict_finished_inbox`` so
the writer drops any leftover ``_push_finished_blocks`` /
``_pending_d_registrations`` and stops self-polling.
"""
import queue
import threading
import time
from collections import defaultdict
from concurrent.futures import Future
from typing import TYPE_CHECKING, Any
import msgspec
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorTransferResults,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
NixlBaseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
PUSH_REG_NOTIF_PREFIX,
NixlConnectorMetadata,
RemoteMeta,
ReqId,
ReqMeta,
TransferHandle,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
ReadSpec,
_is_attention_spec,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import get_base_request_id
from vllm.logger import init_logger
if TYPE_CHECKING:
import torch
from vllm.config import VllmConfig
from vllm.v1.kv_cache_interface import KVCacheConfig
logger = init_logger(__name__)
# Writer-thread poll cadence while there is in-flight push state. When
# fully idle, the writer blocks on a wake event signalled by the engine
# main thread (start_load_kv / get_finished). Smaller -> lower latency
# while active, slightly more CPU.
_PUSH_WRITER_POLL_INTERVAL_MS = 1.0
class NixlPushConnectorWorker(NixlBaseConnectorWorker):
"""Push-specific (WRITE) worker logic. See module docstring."""
# Distinguishes push from pull in the NIXL compatibility hash.
_TRANSFER_MODE: str = "push"
_supports_pp_hma = True
def __init__(
self,
vllm_config: "VllmConfig",
engine_id: str,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, engine_id, kv_cache_config)
# Heartbeat handshakes to a PP-sharded producer must be notif-only,
# like the PUSH_REG path.
self._hb_handshake_notif_only = True
# Push-specific state.
# P-side: outgoing WRITE handles awaiting completion, keyed by
# request_id. Mutated by writer (submit) and main thread
# (``_pop_done_transfers``); guarded by
# ``_sending_transfers_lock``.
self._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list)
self._send_failures: set[ReqId] = set()
self._sending_transfers_lock = threading.Lock()
# Writer-thread owned matching state.
# P-side: finished request blocks received from scheduler metadata
# that have not yet been matched with an incoming D registration.
self._push_finished_blocks: dict[ReqId, BlockIds] = {}
# P-side: D registrations received via NIXL notification that have
# not yet been matched with a finished P request.
self._pending_d_registrations: dict[ReqId, dict[str, Any]] = {}
# Cross-thread channels.
self._reg_send_inbox: queue.Queue[tuple[str, dict[str, Any]]] = queue.Queue()
self._finished_blocks_inbox: queue.Queue[tuple[str, BlockIds]] = queue.Queue()
self._pending_completion_notifs: queue.Queue[bytes] = queue.Queue()
# Main thread → writer: req_ids whose lease has expired or whose
# WRITE has completed. Writer drops them from
# ``_push_finished_blocks`` so an unmatched entry doesn't keep the
# writer busy-polling forever.
self._evict_finished_inbox: queue.Queue[str] = queue.Queue()
# Handshakes that have just completed and are ready for the WRITE on wthread
self._deferred_push_inbox = queue.Queue[tuple[str, BlockIds, dict[str, Any]]]()
# Wake signal from engine main thread (start_load_kv / get_finished).
# Writer self-polls at _PUSH_WRITER_POLL_INTERVAL_MS while it has
# active in-flight state; otherwise it blocks until signalled.
self._push_writer_wake = threading.Event()
self._push_writer_stop = threading.Event()
self._push_writer_thread: threading.Thread | None = None
# --- Lifecycle ----------------------------------------------------- #
def register_kv_caches(self, kv_caches: dict[str, "torch.Tensor"]):
super().register_kv_caches(kv_caches)
if self._mixed_mem_types:
raise NotImplementedError(
"NixlPushConnector does not support mixed-memory KV caches"
)
if self._push_writer_thread is None:
self._push_writer_thread = threading.Thread(
target=self._push_writer_loop,
daemon=True,
name="nixl-push-writer",
)
self._push_writer_thread.start()
logger.info("nixl-push-writer thread started (rank=%d)", self.tp_rank)
def shutdown(self) -> None:
self._push_writer_stop.set()
# Unblock the writer if it's waiting in the no-active-state branch.
self._push_writer_wake.set()
if self._push_writer_thread is not None:
self._push_writer_thread.join(timeout=2)
self._push_writer_thread = None
with self._sending_transfers_lock:
for handles in self._sending_transfers.values():
for handle in handles:
self.nixl_wrapper.release_xfer_handle(handle)
self._sending_transfers.clear()
self._send_failures.clear()
super().shutdown()
# --- Engine-main-thread entry point -------------------------------- #
def start_load_kv(self, metadata: NixlConnectorMetadata):
"""Pre-process metadata; defer NIXL ops to the writer thread."""
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
return
# D-side: track reqs waiting for P to push.
for req_id, meta in metadata.reqs_to_recv.items():
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
meta.local_block_ids, self._physical_blocks_per_logical_kv_block
)
assert meta.remote is not None
remote_engine_id = meta.remote.engine_id
# Update last activity from this remote (same as pull), but only for
# an already-connected engine so we never leave an _engine_last_active
# entry without a _remote_agents entry (breaks _cleanup_remote_engine).
if remote_engine_id in self._remote_agents:
self._engine_last_active[remote_engine_id] = time.perf_counter()
logger.debug(
"start_load_kv (push) for request %s from remote engine %s. "
"Num local_block_ids: %s. Num remote_block_ids: %s. ",
req_id,
remote_engine_id,
len(meta.local_physical_block_ids),
len(meta.remote.block_ids),
)
self._recving_metadata[req_id] = meta
# --- D-side: registrations to send to P via NIXL ---
if metadata.push_registrations:
for req_id, reg_data in metadata.push_registrations.items():
self._reg_send_inbox.put((req_id, reg_data))
self._push_writer_wake.set()
# --- P-side: newly finished blocks awaiting a D registration match ---
if metadata.push_finished_blocks:
for req_id, block_ids in metadata.push_finished_blocks.items():
self._finished_blocks_inbox.put((req_id, block_ids))
self._push_writer_wake.set()
# Batch + lease tracking (same as pull).
for req_id in metadata.reqs_in_batch:
self._reqs_to_process.add(req_id)
for req_id in metadata.reqs_not_processed:
self._reqs_to_process.discard(req_id)
assert req_id not in self._reqs_to_send
# Rebase scheduler-clock deadlines onto this worker's clock — see the
# equivalent block in pull_worker.start_load_kv for the rationale.
now_local = time.perf_counter()
for req_id, expiration_time in metadata.reqs_to_send.items():
if req_id in self._reqs_to_process:
if metadata.scheduler_clock:
expiration_time = now_local + (
expiration_time - metadata.scheduler_clock
)
self._reqs_to_send[req_id] = expiration_time
# Heartbeats still leave from the main thread (base worker behaviour).
self._send_heartbeats(metadata)
# --- Writer thread ------------------------------------------------- #
def _push_writer_loop(self) -> None:
sleep_s = _PUSH_WRITER_POLL_INTERVAL_MS / 1000.0
while not self._push_writer_stop.is_set():
try:
# 1. D registrations to send.
while True:
try:
rid, rd = self._reg_send_inbox.get_nowait()
except queue.Empty:
break
self._send_registration_to_p(rid, rd)
# 2. Deferred P→D pushes whose handshake just completed; do xfer now
while True:
try:
rid, blocks, rd = self._deferred_push_inbox.get_nowait()
except queue.Empty:
break
self._do_start_push_kv(rid, blocks, rd)
# 3. P-side finished blocks; match against pending regs.
while True:
try:
rid, blocks = self._finished_blocks_inbox.get_nowait()
except queue.Empty:
break
matched = self._pop_matching_registration(rid)
if matched is not None:
self._do_start_push_kv(rid, blocks, matched)
else:
self._push_finished_blocks[rid] = blocks
# 3b. Evict finished blocks for requests that have either
# completed (WRITE acknowledged) or whose lease expired
# without a D registration. Drop pending registrations
# for the same reason so we don't leak state.
while True:
try:
rid = self._evict_finished_inbox.get_nowait()
except queue.Empty:
break
self._push_finished_blocks.pop(rid, None)
self._pending_d_registrations.pop(rid, None)
# 4. NIXL notifs: route PUSH_REG; forward the rest.
for notifs in self.nixl_wrapper.get_new_notifs().values():
for notif in notifs:
if notif.startswith(PUSH_REG_NOTIF_PREFIX):
self._handle_push_reg_notif(notif)
else:
self._pending_completion_notifs.put(notif)
except Exception:
logger.exception("nixl-push-writer error; continuing")
# Self-poll only while there is no other wake source: P-side
# finished blocks waiting for a D PUSH_REG match. All other
# progress is event-driven (see module docstring).
if self._push_finished_blocks:
self._push_writer_stop.wait(timeout=sleep_s)
else:
self._push_writer_wake.wait()
self._push_writer_wake.clear()
def _handle_push_reg_notif(self, notif: bytes) -> None:
try:
reg_data = msgspec.msgpack.decode(notif[len(PUSH_REG_NOTIF_PREFIX) :])
except Exception:
logger.exception("Failed to decode PUSH_REG notification payload")
return
rid = reg_data.get("request_id") if isinstance(reg_data, dict) else None
if not isinstance(rid, str):
logger.warning("PUSH_REG notif missing request_id; dropping")
return
match = self._pop_matching_finished_blocks(rid)
if match is not None:
fin_id, blocks = match
self._do_start_push_kv(fin_id, blocks, reg_data)
else:
self._pending_d_registrations[rid] = reg_data
# --- D-side registration send (writer thread) ---------------------- #
def _send_registration_to_p(
self,
req_id: str,
reg_data: dict[str, Any],
) -> None:
"""Handshake (if needed) then send PUSH_REG. ``send_notif`` always
executes on the writer; the handshake runs on the background executor
and the request is re-queued onto ``_reg_send_inbox`` once it
completes (at which point ``_ensure_handshake`` returns ``None`` and we
send directly)."""
remote_pp_size = reg_data.get("remote_pp_size", 1)
fut = self._ensure_handshake(
reg_data["remote_engine_id"],
reg_data["remote_host"],
reg_data["remote_port"],
reg_data["remote_tp_size"],
pp_size=remote_pp_size,
# D only ever sends PUSH_REG notifs to P and never reads or writes
# P's memory in push mode, so it never needs the transfer
# descriptors set up by the full add_remote_agent path.
notif_agents_only=True,
)
if fut is None:
self._do_send_reg_notif(req_id, reg_data)
return
def _on_handshake(
f: Future[tuple[dict[tuple[int, int], str], float]],
rid: str = req_id,
rd: dict[str, Any] = reg_data,
) -> None:
try:
f.result()
except Exception as e:
self._log_failure(
failure_type="push_reg_handshake_failed", req_id=rid, error=e
)
self._failed_recv_reqs.put(rid)
return
# Re-queue for the writer to send now that the handshake is done.
self._reg_send_inbox.put((rid, rd))
# Wake the writer so it sends the PUSH_REG promptly even if
# otherwise parked.
self._push_writer_wake.set()
fut.add_done_callback(_on_handshake)
def _do_send_reg_notif(self, req_id: str, reg_data: dict[str, Any]) -> None:
engine_id = reg_data["remote_engine_id"]
notif_msg = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(reg_data)
# _remote_agents is mutated on other threads; snapshot under the lock.
with self._handshake_lock:
agents = dict(self._remote_agents.get(engine_id) or {})
if not agents:
logger.error(
"No remote agents for engine %s; cannot send registration for %s",
engine_id,
req_id,
)
self.xfer_stats.record_failed_notification()
self._failed_recv_reqs.put(req_id)
return
for rank, agent_name in agents.items():
try:
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_msg)
except Exception as e:
self._log_failure(
failure_type="push_reg_notif_failed",
req_id=req_id,
error=e,
remote_rank=rank,
)
self.xfer_stats.record_failed_notification()
# Earlier registrations may still trigger WRITEs into D's blocks.
# Keep the receive pending until those writes are finished.
logger.debug(
"Sent PUSH_REG for %s to engine %s (%dB)", req_id, engine_id, len(notif_msg)
)
# --- Matching helpers --------------------------------------------- #
def _pop_matching_registration(self, request_id: str) -> dict[str, Any] | None:
"""Pop the D-side registration matching *request_id*.
Exact key first, then a match after stripping the random suffix from
both sides. No match leaves the request unmatched (push not started).
"""
data = self._pending_d_registrations.pop(request_id, None)
if data is not None:
return data
base_id = get_base_request_id(request_id)
for reg_id in list(self._pending_d_registrations):
if get_base_request_id(reg_id) == base_id:
return self._pending_d_registrations.pop(reg_id)
return None
def _pop_matching_finished_blocks(
self, request_id: str
) -> tuple[str, BlockIds] | None:
"""Pop the P-side finished blocks matching *request_id*.
Same lookup as ``_pop_matching_registration``: exact key, then a
match after stripping the random suffix from both sides.
"""
blocks = self._push_finished_blocks.pop(request_id, None)
if blocks is not None:
return request_id, blocks
base_id = get_base_request_id(request_id)
for fin_id in list(self._push_finished_blocks):
if get_base_request_id(fin_id) == base_id:
return fin_id, self._push_finished_blocks.pop(fin_id)
return None
# --- WRITE transfer logic (writer thread) ------------------------- #
def _do_start_push_kv(
self,
request_id: str,
local_block_ids: BlockIds,
registration_data: dict[str, Any],
) -> None:
"""Start push-based KV transfer from P worker to D node.
The P→D handshake runs on the base worker's background executor.
If it isn't ready yet we register a completion callback, defer the
WRITE, and re-drive this request via ``_deferred_push_inbox`` once
the handshake resolves -- so the writer thread never blocks on the
network (mirrors ``_send_registration_to_p``).
"""
if not local_block_ids:
logger.warning("No local blocks to push for request %s", request_id)
return
# ``local_block_ids`` are P's logical block IDs; ``remote_block_ids``
# (D's, from the PUSH_REG notif) are also logical.
decode_engine_id = registration_data["decode_engine_id"]
remote_block_ids = registration_data["local_block_ids"]
decode_request_id = registration_data["request_id"]
# Runs on the background executor; defer the WRITE until it's ready.
fut = self._ensure_handshake(
decode_engine_id,
registration_data["decode_host"],
registration_data["decode_port"],
registration_data["decode_tp_size"],
)
if fut is not None:
def _on_handshake(
f: Future[tuple[dict[tuple[int, int], str], float]],
rid: str = request_id,
blocks: BlockIds = local_block_ids,
rd: dict[str, Any] = registration_data,
) -> None:
if (e := f.exception()) is not None:
# The engine reclaims the blocks via the TTL so we dont free here
self._log_failure(
failure_type="push_handshake_failed", req_id=rid, error=e
)
return
self._deferred_push_inbox.put((rid, blocks, rd))
self._push_writer_wake.set()
fut.add_done_callback(_on_handshake)
return
# Keep the engine alive while it is actively receiving pushes, mirroring
# how pull-mode transfers touch _engine_last_active in start_load_kv.
self._engine_last_active[decode_engine_id] = time.perf_counter()
logical_local = self._as_grouped_block_ids(local_block_ids)
logical_remote = self._as_grouped_block_ids(remote_block_ids)
physical_local = self._logical_to_kernel_block_ids(
logical_local, self._physical_blocks_per_logical_kv_block
)
push_meta = ReqMeta(
local_block_ids=logical_local,
local_physical_block_ids=physical_local,
tp_size=self.world_size,
remote=RemoteMeta(
block_ids=logical_remote,
host="",
port=0,
engine_id=decode_engine_id,
request_id=decode_request_id,
),
)
t0 = time.perf_counter()
self._xfer_blocks_for_req(req_id=request_id, meta=push_meta)
elapsed_ms = (time.perf_counter() - t0) * 1000.0
if elapsed_ms > 200.0:
logger.warning(
"_do_start_push_kv for %s took %.1fms (slow NIXL submission)",
request_id,
elapsed_ms,
)
@staticmethod
def _as_grouped_block_ids(block_ids: BlockIds) -> BlockIds:
"""Normalise a sequence of block IDs to a tuple-of-groups shape.
``BlockIds`` is canonically a tuple of per-group lists, but some
registration payloads collapse a single-group case to a flat
list. Re-wrap that case so downstream group-aware helpers see a
consistent shape."""
if block_ids and not isinstance(block_ids[0], (list, tuple)):
return (list(block_ids),)
return block_ids
def _xfer_blocks_for_req(self, req_id: str, meta: ReqMeta):
"""Issue WRITE transfers to one or more remote TP ranks."""
assert meta.remote is not None and self.transfer_topo is not None
engine_id = meta.remote.engine_id
plan = self.tp_mappings[engine_id]
remote_info = self.transfer_topo.get_engine_info(engine_id)
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
# Expand D's logical IDs using the ratio learned during the
# NIXL handshake. ``meta`` is freshly built by
# ``_do_start_push_kv`` so mutating it here is safe.
meta.remote.block_ids = self._logical_to_kernel_block_ids(
meta.remote.block_ids,
remote_info.remote_physical_blocks_per_logical,
)
remote_block_ids = meta.remote.block_ids
local_block_ids = meta.local_physical_block_ids
local_region_groups = self.region_group_ids
remote_region_groups = self.dst_region_group_ids[engine_id]
groups_differ = local_region_groups != remote_region_groups
if groups_differ and not self._transfer_layer_group_ids:
raise NotImplementedError(
"NixlPushConnector does not support different producer and "
"consumer cache-group layouts"
)
# MLA latent is replicated across D's TP ranks: the tp-mapping
# collapses it to one rank (fine for reads), but push must WRITE every
# D rank or the rest decode stale KV. For hybrid MLA+SSM the sharded
# SSM state already targets every covered D rank, so only the
# attention groups need widening; pure MLA writes to all handshaked
# ranks (only the dst differs per rank).
replicate_attn = self.use_mla and tp_ratio < 0
if replicate_attn and not self._has_mamba:
assert len(plan.all_source_ranks) == 1
write_ranks = sorted(self.dst_xfer_side_handles[engine_id])
else:
write_ranks = list(plan.all_source_ranks)
num_groups = len(local_block_ids)
def group_ids(block_ids: BlockIds, rank: int) -> BlockIds:
return [
list(block_ids[g])
if (self._is_csa_linear and tp_ratio < 0)
or (replicate_attn and _is_attention_spec(self._group_spec_types[g]))
or rank in plan.source_ranks_per_group[g]
else []
for g in range(num_groups)
]
read_specs = [
ReadSpec(
remote_rank=rank,
local_block_ids=group_ids(local_block_ids, rank),
remote_block_ids=group_ids(remote_block_ids, rank),
)
for rank in write_ranks
]
handles: list[int] = []
for i, spec in enumerate(read_specs):
remote_block_size = remote_info.remote_block_size
logger.debug(
"Remote agent %s available, calling _xfer_blocks"
" on remote rank %s with remote block size %s for req %s",
meta.remote.engine_id,
spec.remote_rank,
remote_block_size,
req_id,
)
if tp_ratio < 0 and (not self.use_mla or len(plan.all_source_ranks) > 1):
# Multiple targets: write each rank its chunk of local memory.
# Hybrid MLA+SSM also lands here: its split handles replicate
# the attention descriptors and chunk only the SSM state.
split_key = (tp_ratio, remote_block_size)
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i]
else:
local_xfer_side_handle = self.src_xfer_handles_by_block_size[
remote_block_size
]
remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][
spec.remote_rank
]
handle = self._xfer_blocks(
read_spec=spec,
request_id=req_id,
dst_engine_id=meta.remote.engine_id,
remote_request_id=meta.remote.request_id,
local_xfer_side_handle=local_xfer_side_handle,
remote_xfer_side_handle=remote_xfer_side_handle,
)
if handle is not None:
handles.append(handle)
# Publish all the request's WRITE handles in one locked update: a
# partial set would let ``_pop_done_transfers`` finish the request
# early, then double-report it as the remaining writes land.
if handles:
with self._sending_transfers_lock:
self._sending_transfers[req_id].extend(handles)
def _xfer_blocks(
self,
read_spec: ReadSpec,
dst_engine_id: str,
request_id: str,
remote_request_id: str,
local_xfer_side_handle: int,
remote_xfer_side_handle: int,
) -> int | None:
"""Post a WRITE point-to-point xfer request.
Returns the in-flight transfer handle (so the caller can track all of
a request's handles atomically), or ``None`` if nothing was submitted.
"""
assert self.transfer_topo is not None
remote_rank = read_spec.remote_rank
local_block_ids = read_spec.local_block_ids
remote_block_ids = read_spec.remote_block_ids
remote_info = self.transfer_topo.get_engine_info(dst_engine_id)
block_size_ratio = self.transfer_topo.block_size_ratio(
remote_info.remote_block_size
)
if block_size_ratio > 1:
local_block_ids, remote_block_ids = (
self._map_block_ids_for_block_size_ratio(
local_block_ids, remote_block_ids, block_size_ratio
)
)
notif_id = f"{remote_request_id}:{self.world_size}".encode()
if len(local_block_ids) == 0:
logger.warning("No blocks to push for request %s", request_id)
return None
# Prefix caching: D allocated only uncached blocks, so on a partial hit it
# sends fewer than P's. End-trim P's blocks to that same suffix so we WRITE only
# the uncomputed tail into D's slots. Runs on kernel ids, post-expansion.
remote_block_ids, local_block_ids = self._apply_prefix_caching(
decode_block_ids=remote_block_ids,
prefill_block_ids=local_block_ids,
decode_physical_per_logical=remote_info.remote_physical_blocks_per_logical,
prefill_physical_per_logical=self._physical_blocks_per_logical_kv_block,
)
local_block_ids = list(local_block_ids)
remote_block_ids = list(remote_block_ids)
assert len(local_block_ids) == len(remote_block_ids), (
f"push group-count mismatch for {request_id}: {len(local_block_ids)} "
f"local vs {len(remote_block_ids)} remote groups"
)
for i in range(len(local_block_ids)):
assert len(local_block_ids[i]) == len(remote_block_ids[i]), (
f"push block-count mismatch for {request_id} group {i}: "
f"{len(local_block_ids[i])} local vs "
f"{len(remote_block_ids[i])} remote blocks"
)
# Get descs ids.
remote_block_descs_ids = self._compute_desc_ids(
block_ids=remote_block_ids,
dst_num_blocks=self.dst_num_blocks[dst_engine_id],
block_size_ratio=None,
physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical,
region_num_blocks=self.dst_region_num_blocks[dst_engine_id],
region_group_ids=self.dst_region_group_ids[dst_engine_id],
uses_region_group_mapping=self.dst_uses_region_group_mapping[dst_engine_id],
)
local_block_descs_ids = self._compute_desc_ids(
block_ids=local_block_ids,
dst_num_blocks=self.dst_num_blocks[self.engine_id],
block_size_ratio=block_size_ratio,
physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block,
region_num_blocks=self.dst_region_num_blocks[self.engine_id],
region_group_ids=self.region_group_ids,
uses_region_group_mapping=self._uses_region_group_mapping,
)
assert len(local_block_descs_ids) == len(remote_block_descs_ids)
handle = None
try:
handle = self.nixl_wrapper.make_prepped_xfer(
"WRITE",
local_xfer_side_handle,
local_block_descs_ids,
remote_xfer_side_handle,
remote_block_descs_ids,
notif_msg=notif_id,
)
self.nixl_wrapper.transfer(handle)
# Caller tracks the handle (atomically with the request's other
# writes) so P can free blocks once all of them are done.
return handle
except Exception as e:
self._log_failure(
failure_type="transfer_setup_failed",
req_id=request_id,
msg="Push WRITE submission failed; releasing handle",
error=e,
dst_engine_id=dst_engine_id,
remote_rank=remote_rank,
)
# On the P side this WRITE failure is purely outbound; we
# don't have a ``_recving_metadata`` entry to invalidate, so
# we just release the handle and let the engine reschedule
# via the lease / watchdog.
with self._sending_transfers_lock:
self._send_failures.add(request_id)
if not self._handle_failed_transfer(request_id, handle):
return handle
return None
# --- Notification handling on engine main thread ------------------ #
def _get_new_notifs(self) -> set[str]:
"""Drain HB / completion notifs forwarded by the writer thread.
The writer owns ``nixl_wrapper.get_new_notifs`` for push; PUSH_REG
notifs are handled there. P learns of finished pushes from its WRITE
handles, so this never reports finished sends.
"""
while True:
try:
notif = self._pending_completion_notifs.get_nowait()
except queue.Empty:
break
msg = notif.decode("utf-8")
if msg.startswith("HB:"):
self._handle_heartbeat(msg[3:])
continue
req_id, tp_size = msg.rsplit(":", 1)
if (meta := self._recving_metadata.get(req_id)) is None:
# The lease may have expired before the notif arrived.
self.xfer_stats.record_notification_after_expiry()
logger.error(
"Unrecognized request %s notif (may have expired).", req_id
)
continue
# Consumer waits for one notif per producer rank writing
# here: pp_size stages * producers-per-consumer (>1 when
# producer TP > consumer TP; tp_size is the producer TP).
producers_per_consumer = max(1, int(tp_size) // self.world_size)
expected_notifs = meta.pp_size * producers_per_consumer
self.consumer_notification_counts_by_req[req_id] += 1
if self.consumer_notification_counts_by_req[req_id] < expected_notifs:
continue
del self.consumer_notification_counts_by_req[req_id]
# P drove the transfer (we own no NIXL handle), so
# materialise an empty ``_recving_transfers`` entry for
# ``_pop_done_transfers`` to report done.
self._recving_transfers.setdefault(req_id, [])
return set()
def get_transfer_results(self) -> KVConnectorTransferResults:
# Engine main thread asking for completions: also wake the writer
# so it gets a chance to drain NIXL notifs (heartbeats, completion
# notifs, late PUSH_REGs) even if it had been parked.
self._push_writer_wake.set()
results = super().get_transfer_results()
done_sending = results.finished_sending
# ``_pop_done_transfers`` mutates ``_sending_transfers``; the
# writer thread also appends to it, so guard the pop.
with self._sending_transfers_lock:
done_pushing, failed_pushing = self._pop_done_transfers(
self._sending_transfers
)
# Remember failures until the final sibling WRITE completes.
self._send_failures.update(failed_pushing)
successful = {
req_id
for req_id in done_pushing - self._send_failures
if req_id in self._reqs_to_send or req_id in self._reqs_to_process
}
self._send_failures.difference_update(done_pushing | done_sending)
# Expired requests were already reported, even if their WRITEs finish later.
done_pushing = successful
for req_id in done_pushing:
self._reqs_to_send.pop(req_id, None)
self._reqs_to_process.discard(req_id)
done_sending.add(req_id)
# Tell the writer to drop any state it still holds for any
# request that just finished (push completed) or expired
# (lease ran out without a D registration ever arriving).
for req_id in done_sending:
self._evict_finished_inbox.put(req_id)
if done_sending:
self._push_writer_wake.set()
return results
@@ -1,12 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backward-compatible re-export of NixlPullConnectorScheduler."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
# Backward compatibility: NixlConnectorScheduler is the pull-based scheduler.
NixlConnectorScheduler = NixlPullConnectorScheduler
__all__ = ["NixlConnectorScheduler", "NixlPullConnectorScheduler"]
@@ -1,328 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stats and Prometheus metrics for the NIXL connector."""
import copy
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import numpy as np
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.v1.metrics.utils import create_metric_per_engine
if TYPE_CHECKING:
from vllm.distributed.nixl_utils import nixlXferTelemetry
@dataclass
class NixlKVConnectorStats(KVConnectorStats):
"""Container for transfer performance metrics."""
def __post_init__(self):
if not self.data:
# Empty container init, no data is passed in.
self.reset()
def reset(self):
# Must be serializable
self.data: dict[str, list[float | int]] = {
"transfer_duration": [],
"post_duration": [],
"bytes_transferred": [],
"num_descriptors": [],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
"num_notifications_after_expiry": [],
}
def record_transfer(self, res: "nixlXferTelemetry | None"):
# Keep metrics units consistent with rest of the code: time us->s
if res is None:
return
self.data["transfer_duration"].append(res.xferDuration / 1e6)
self.data["post_duration"].append(res.postDuration / 1e6)
self.data["bytes_transferred"].append(res.totalBytes)
self.data["num_descriptors"].append(res.descCount)
def record_failed_transfer(self):
"""Record a failed NIXL transfer operation."""
self.data["num_failed_transfers"].append(1)
def record_failed_notification(self):
"""Record a failed NIXL notification (send_notif)."""
self.data["num_failed_notifications"].append(1)
def record_failed_handshake(self):
"""Record a failed NIXL handshake."""
self.data["num_failed_handshakes"].append(1)
def record_kv_expired_req(self):
"""Record a request that had its KV blocks expire."""
self.data["num_kv_expired_reqs"].append(1)
def record_notification_after_expiry(self):
"""Record a completion notification that arrived after its request expired."""
self.data["num_notifications_after_expiry"].append(1)
def clone_and_reset(self) -> "NixlKVConnectorStats":
old = copy.copy(self)
self.reset()
return old
def is_empty(self) -> bool:
# Do not discard metrics update that are entirely failures related.
return (
self.num_successful_transfers == 0
and len(self.data["num_failed_transfers"]) == 0
and len(self.data["num_failed_notifications"]) == 0
and len(self.data["num_failed_handshakes"]) == 0
and len(self.data["num_kv_expired_reqs"]) == 0
and len(self.data["num_notifications_after_expiry"]) == 0
)
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if not other.is_empty():
for k, v in other.data.items():
accumulator = self.data[k]
assert isinstance(accumulator, list)
accumulator.extend(v)
return self
def reduce(self) -> dict[str, int | float]:
# Compute compact representative stats suitable for CLI logging.
# Failure counts are reported on every interval: transfer, handshake
# and notification failures are grouped as sporadic
# lower-transport-layer events, while KV expiry is reported separately
# as it is an actionable autoscaler signal rather than a transport
# issue. Notifications arriving after expiry are surfaced too, as the
# reader may have consumed KV blocks that were already reused.
failure_counts = {
"Num failed transfers": len(self.data["num_failed_transfers"])
+ len(self.data["num_failed_handshakes"])
+ len(self.data["num_failed_notifications"]),
"Num KV expired reqs": len(self.data["num_kv_expired_reqs"]),
"Num notifs after expiry": len(self.data["num_notifications_after_expiry"]),
}
if self.num_successful_transfers == 0:
# Timing / throughput stats only cover successful transfers. If
# all requests in the interval were unsuccessful, the failure
# counts above still surface what happened.
return {
"Num successful transfers": 0,
"Avg xfer time (ms)": 0,
"P90 xfer time (ms)": 0,
"Avg post time (ms)": 0,
"P90 post time (ms)": 0,
"Avg MB per transfer": 0,
"Throughput (MB/s)": 0,
"Avg number of descriptors": 0,
**failure_counts,
}
xfer_time = np.asarray(self.data["transfer_duration"])
post_time = np.asarray(self.data["post_duration"])
# Convert to MB for CLI logging.
mb = np.asarray(self.data["bytes_transferred"]) / 2**20
descs = np.asarray(self.data["num_descriptors"], dtype=np.uint32)
n = len(descs)
assert n == self.num_successful_transfers
total_mb = mb.sum()
avg_mb = total_mb / n
total_time_seconds = xfer_time.sum()
throughput_mb_s = total_mb / total_time_seconds
return {
"Num successful transfers": n,
"Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
"Avg post time (ms)": round(post_time.mean().item() * 1e3, 3),
"P90 post time (ms)": round(np.percentile(post_time, 90).item() * 1e3, 3),
"Avg MB per transfer": round(avg_mb.item(), 3),
"Throughput (MB/s)": round(throughput_mb_s.item(), 3),
"Avg number of descriptors": round(descs.mean().item(), 1),
**failure_counts,
}
@property
def num_successful_transfers(self) -> int:
return len(self.data["transfer_duration"])
class NixlPromMetrics(KVConnectorPromMetrics):
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
buckets = [
0.001,
0.005,
0.01,
0.025,
0.05,
0.075,
0.1,
0.2,
0.3,
0.5,
0.75,
1.0,
5.0,
]
nixl_histogram_xfer_time = self._histogram_cls(
name="vllm:nixl_xfer_time_seconds",
documentation="Histogram of transfer duration for NIXL KV Cache transfers.",
buckets=buckets[1:],
labelnames=labelnames,
)
self.nixl_histogram_xfer_time = create_metric_per_engine(
nixl_histogram_xfer_time, self.per_engine_labelvalues
)
nixl_histogram_post_time = self._histogram_cls(
name="vllm:nixl_post_time_seconds",
documentation="Histogram of transfer post time for NIXL KV"
" Cache transfers.",
buckets=buckets,
labelnames=labelnames,
)
self.nixl_histogram_post_time = create_metric_per_engine(
nixl_histogram_post_time, self.per_engine_labelvalues
)
# uniform 2kb to 16gb range
buckets = [2 ** (10 + i) for i in range(1, 25, 2)]
nixl_histogram_bytes_transferred = self._histogram_cls(
name="vllm:nixl_bytes_transferred",
documentation="Histogram of bytes transferred per NIXL KV Cache transfers.",
buckets=buckets,
labelnames=labelnames,
)
self.nixl_histogram_bytes_transferred = create_metric_per_engine(
nixl_histogram_bytes_transferred, self.per_engine_labelvalues
)
buckets = [
10,
20,
30,
50,
75,
100,
200,
400,
1000,
2000,
4000,
10000,
20000,
50000,
]
nixl_histogram_num_descriptors = self._histogram_cls(
name="vllm:nixl_num_descriptors",
documentation="Histogram of number of descriptors per NIXL"
" KV Cache transfers.",
buckets=buckets,
labelnames=labelnames,
)
self.nixl_histogram_num_descriptors = create_metric_per_engine(
nixl_histogram_num_descriptors, self.per_engine_labelvalues
)
counter_nixl_num_failed_transfers = self._counter_cls(
name="vllm:nixl_num_failed_transfers",
documentation="Number of failed NIXL KV Cache transfers, including"
" handshake and notification failures. NOTE: KV expiry is tracked"
" separately in vllm:nixl_num_kv_expired_reqs.",
labelnames=labelnames,
)
self.counter_nixl_num_failed_transfers = create_metric_per_engine(
counter_nixl_num_failed_transfers, self.per_engine_labelvalues
)
counter_nixl_num_failed_notifications = self._counter_cls(
name="vllm:nixl_num_failed_notifications",
documentation="Number of failed NIXL KV Cache notifications. "
"Retained for compatibility; these failures are also included in "
"vllm:nixl_num_failed_transfers.",
labelnames=labelnames,
)
self.counter_nixl_num_failed_notifications = create_metric_per_engine(
counter_nixl_num_failed_notifications, self.per_engine_labelvalues
)
counter_nixl_num_kv_expired_reqs = self._counter_cls(
name="vllm:nixl_num_kv_expired_reqs",
documentation="Number of requests that had their KV expire. "
"NOTE: This metric is tracked on the P instance.",
labelnames=labelnames,
)
self.counter_nixl_num_kv_expired_reqs = create_metric_per_engine(
counter_nixl_num_kv_expired_reqs, self.per_engine_labelvalues
)
counter_nixl_num_notifications_after_expiry = self._counter_cls(
name="vllm:nixl_num_notifications_after_expiry",
documentation="Number of completion notifications for requests that "
"were no longer tracked, usually because their KV lease expired. The "
"KV blocks may have been reused before the transfer finished. Counted "
"per notification (one per remote rank), not per request.",
labelnames=labelnames,
)
self.counter_nixl_num_notifications_after_expiry = create_metric_per_engine(
counter_nixl_num_notifications_after_expiry, self.per_engine_labelvalues
)
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
for prom_obj, list_item_key in zip(
[
self.nixl_histogram_xfer_time,
self.nixl_histogram_post_time,
self.nixl_histogram_bytes_transferred,
self.nixl_histogram_num_descriptors,
],
[
"transfer_duration",
"post_duration",
"bytes_transferred",
"num_descriptors",
],
):
for list_item in transfer_stats_data[list_item_key]:
prom_obj[engine_idx].observe(list_item)
for counter_obj, counter_item_keys in zip(
[
self.counter_nixl_num_failed_transfers,
self.counter_nixl_num_failed_notifications,
self.counter_nixl_num_kv_expired_reqs,
self.counter_nixl_num_notifications_after_expiry,
],
[
# Transfer, handshake and notification failures are grouped:
# all are sporadic lower-transport-layer events. KV expiry is
# reported separately as it signals autoscaler behavior, not
# transport health.
(
"num_failed_transfers",
"num_failed_handshakes",
"num_failed_notifications",
),
("num_failed_notifications",),
("num_kv_expired_reqs",),
("num_notifications_after_expiry",),
],
):
for counter_item_key in counter_item_keys:
for list_item in transfer_stats_data[counter_item_key]:
counter_obj[engine_idx].inc(list_item)
@@ -1,161 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TP mapping computation for NIXL KV cache transfers."""
from __future__ import annotations
from dataclasses import dataclass
import numpy as np
from vllm.distributed.kv_transfer.kv_connector.utils import (
BlockIds,
TransferTopology,
)
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, MambaSpec
# ======================================================================
# Data structures
# ======================================================================
@dataclass(frozen=True)
class ReadSpec:
"""Specification for a single remote block read operation."""
remote_rank: int
local_block_ids: BlockIds
remote_block_ids: BlockIds
block_ids_by_region: bool = False
def _is_attention_spec(spec_type: type[KVCacheSpec]) -> bool:
return issubclass(spec_type, AttentionSpec)
def _is_ssm_spec(spec_type: type[KVCacheSpec]) -> bool:
return issubclass(spec_type, MambaSpec)
@dataclass(frozen=True)
class TPMapping:
"""Complete local-to-remote TP mapping for one remote engine.
Generated once per remote engine during handshake.
"""
# Remote TP ranks that this local rank reads from, per group.
# Position = local piece index.
source_ranks_per_group: tuple[tuple[int, ...], ...]
# Superset of all source ranks (union of all groups).
all_source_ranks: tuple[int, ...]
# Maps each source rank to its FA head slot index.
rank_to_attention_slot: dict[int, int]
# FA head offset factor for hetero-TP (D_TP > P_TP).
rank_offset_factor: int
# Local ranks (in aggregate) that read from a given source rank. The producer frees
# a request's blocks only once that many notifications have come in.
local_consumers: int = 1
# ======================================================================
# TP mapping computation
# ======================================================================
def compute_tp_mapping(
transfer_topology: TransferTopology,
remote_tp_size: int,
group_spec_types: tuple[type[KVCacheSpec], ...],
remote_dcp_size: int = 1,
) -> TPMapping:
"""Build the complete local-to-remote TP mapping.
Computes source ranks, head slot assignments, and the rank offset
factor in a single pass.
DCP support is scoped to MLA only, with a side is either fully replicated or fully
sharded. DCP-branch reuses the same rank set used at handshake selection.
"""
tp_rank = transfer_topology.tp_rank
tp_size = transfer_topology.tp_size
total_num_kv_heads = transfer_topology.total_num_kv_heads
# --- Attention source ranks ---
if transfer_topology.is_mla or tp_size >= remote_tp_size:
if transfer_topology.is_mla and remote_dcp_size > 1:
attn_ranks = transfer_topology.dcp_source_ranks(
remote_tp_size, remote_dcp_size
)
else:
# D (local TP) > P (remote TP): multiple local ranks read different chunks
# from *one* remote rank, corresponding to different kv heads.
# For MLA, we only need one remote since cache is duplicated. When
# P TP=k*TP k, this will spread mla ranks to read from remote k*tp_rank.
attn_ranks = [tp_rank * remote_tp_size // tp_size]
else:
# P (remote TP) > D (local TP): one local rank
# reads from multiple remote ranks.
# GQA dedup: when K < remote_tp_size, several remote ranks
# hold the same KV head. np.unique keeps only the first
# rank per unique head so we don't issue redundant reads.
abs_tp = remote_tp_size // tp_size
start = tp_rank * abs_tp
heads = np.arange(start, start + abs_tp) * total_num_kv_heads // remote_tp_size
_, unique_idx = np.unique(heads, return_index=True)
attn_ranks = (start + np.sort(unique_idx)).tolist()
# --- SSM source ranks ---
has_ssm = any(_is_ssm_spec(t) for t in group_spec_types)
if has_ssm:
if tp_size < remote_tp_size:
abs_tp = remote_tp_size // tp_size
ssm_ranks = list(range(tp_rank * abs_tp, (tp_rank + 1) * abs_tp))
else:
ssm_ranks = list(attn_ranks)
else:
ssm_ranks = []
all_ranks = sorted(set(attn_ranks) | set(ssm_ranks))
# --- Per-group ordered source ranks ---
source_ranks_per_group = tuple(
tuple(ssm_ranks) if _is_ssm_spec(t) else tuple(attn_ranks)
for t in group_spec_types
)
# --- Attention head slots ---
head_to_slot: dict[int, int] = {}
for i, r in enumerate(attn_ranks):
head_to_slot[r * total_num_kv_heads // remote_tp_size] = i
rank_to_attention_slot = {
r: head_to_slot.get(r * total_num_kv_heads // remote_tp_size, 0)
for r in all_ranks
}
# --- Rank offset factor ---
if transfer_topology.is_mla or tp_size <= remote_tp_size:
# We don't index into remote for reading, no offset needed.
rank_offset_factor = 0
elif tp_size > total_num_kv_heads:
local_head = tp_rank * total_num_kv_heads // tp_size
p_start = attn_ranks[0] * total_num_kv_heads // remote_tp_size
rank_offset_factor = local_head - p_start
else:
# D TP > P TP: we index into remote to read different heads depending on rank.
rank_offset_factor = tp_rank % (tp_size // remote_tp_size)
local_consumers = transfer_topology.dcp_consumer_count(
remote_tp_size, remote_dcp_size
)
return TPMapping(
source_ranks_per_group=source_ranks_per_group,
all_source_ranks=tuple(all_ranks),
rank_to_attention_slot=rank_to_attention_slot,
rank_offset_factor=rank_offset_factor,
local_consumers=local_consumers,
)
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared constants, lazy imports and helpers for the NIXL connector."""
import contextlib
from collections.abc import Iterator
from typing import Any
import regex as re
import zmq
from vllm.platforms import current_platform
from vllm.utils.network_utils import make_zmq_socket
from vllm.v1.kv_cache_interface import KVCacheSpec, UniformTypeKVCacheSpecs
# Supported platforms and types of kv transfer buffer.
# {device: tuple of supported kv buffer types}
_NIXL_SUPPORTED_DEVICE = {
"cuda": (
"cuda",
"cpu",
),
"tpu": ("cpu",),
"xpu": (
"cpu",
"xpu",
),
"cpu": ("cpu",),
}
# support for oot platform by providing mapping in current_platform
_NIXL_SUPPORTED_DEVICE.update(current_platform.get_nixl_supported_devices())
# TODO: merge with vllm.utils.network_utils.zmq_socket_ctx
@contextlib.contextmanager
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
"""Context manager for a ZMQ socket."""
if socket_type not in (zmq.ROUTER, zmq.REQ):
raise ValueError(f"Unexpected socket type: {socket_type}")
ctx: zmq.Context | None = None
try:
ctx = zmq.Context() # type: ignore[attr-defined]
yield make_zmq_socket(
ctx=ctx, path=addr, socket_type=socket_type, bind=socket_type == zmq.ROUTER
)
finally:
if ctx is not None:
ctx.destroy(linger=0)
def get_representative_spec_type(spec: KVCacheSpec) -> type[KVCacheSpec]:
if isinstance(spec, UniformTypeKVCacheSpecs):
# All inner specs are the same type; pick any.
inner = next(iter(spec.kv_cache_specs.values()))
return type(inner)
return type(spec)
# Trailing 8-hex randomization suffix appended by
# ``input_processor.assign_request_id`` as ``-{random_uuid():.8}``.
_RANDOM_SUFFIX_RE = re.compile(r"-[0-9a-f]{8}$", re.IGNORECASE)
def get_base_request_id(request_id: str) -> str:
"""Strip the per-request ``-<8 hex>`` randomization suffix, if present."""
return _RANDOM_SUFFIX_RE.sub("", request_id)
@@ -1,13 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backward-compatible re-export of NixlPullConnectorWorker."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
NixlPullConnectorWorker,
)
# Backward compatibility: NixlConnectorWorker is the pull-based worker.
NixlConnectorWorker = NixlPullConnectorWorker
__all__ = ["NixlConnectorWorker", "NixlPullConnectorWorker"]
@@ -1,386 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SimpleCPUOffloadConnector: minimal CPU KV cache offloading."""
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_events import KVCacheEvent
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.simple_kv_offload.manager import (
BoundaryStoreStats,
SimpleCPUOffloadScheduler,
)
from vllm.v1.simple_kv_offload.metadata import (
SimpleCPUOffloadMetadata,
)
from vllm.v1.simple_kv_offload.metrics import (
SimpleCPUOffloadPromMetrics,
SimpleCPUOffloadStats,
)
from vllm.v1.simple_kv_offload.worker import (
SimpleCPUOffloadWorker,
)
if TYPE_CHECKING:
from vllm.forward_context import ForwardContext
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
# Default CPU capacity: 8 GB
DEFAULT_CPU_CAPACITY_BYTES = 8 * (1024**3)
VALID_KV_OFFLOAD_BACKENDS = ("cpu", "disk")
# Keys that only apply to the disk backend, warned about under "cpu".
_DISK_ONLY_KEYS = (
"disk_path",
"disk_capacity_bytes",
"disk_buffer_slots",
"use_page_cache",
)
class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
"""CPU KV cache offloading with custom kernel transfers and BlockPool LRU."""
@property
def requires_kv_delivery(self) -> bool:
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
# future cache miss, so opt out of the producer-role default.
return False
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, role, kv_cache_config)
enable_prefix_caching = vllm_config.cache_config.enable_prefix_caching
extra_config = self._kv_transfer_config.kv_connector_extra_config or {}
cpu_capacity_bytes = int(
extra_config.get("cpu_bytes_to_use", DEFAULT_CPU_CAPACITY_BYTES)
)
# cpu_bytes_to_use is server-wide for compatibility;
# cpu_bytes_to_use_per_rank overrides for per-rank capacity.
world_size = vllm_config.parallel_config.world_size
cpu_capacity_per_rank = cpu_capacity_bytes // world_size
if "cpu_bytes_to_use_per_rank" in extra_config:
explicit = int(extra_config["cpu_bytes_to_use_per_rank"])
if explicit != cpu_capacity_per_rank:
logger.warning(
"cpu_bytes_to_use_per_rank (%.2f GB) != "
"cpu_bytes_to_use/world_size (%.2f GB). Using per-rank value.",
explicit / (1024**3),
cpu_capacity_per_rank / (1024**3),
)
cpu_capacity_per_rank = explicit
# When set, num_cpu_blocks overrides automatic pool sizing from byte budgets
# (including cpu_bytes_to_use and cpu_bytes_to_use_per_rank), allowing ranks
# with different KV layouts (e.g. PP stages) to use the same pool size.
# Byte budgets still bound capacity.
num_cpu_blocks = extra_config.get("num_cpu_blocks")
if num_cpu_blocks is not None and (
type(num_cpu_blocks) is not int or num_cpu_blocks <= 0
):
raise ValueError("num_cpu_blocks must be a positive integer")
lazy_offload = bool(extra_config.get("lazy_offload", False))
kv_offload_backend = str(extra_config.get("kv_offload_backend", "cpu"))
if kv_offload_backend not in VALID_KV_OFFLOAD_BACKENDS:
raise ValueError(
f"Unknown kv_offload_backend {kv_offload_backend!r}; "
f"expected one of {VALID_KV_OFFLOAD_BACKENDS}"
)
disk_mode = kv_offload_backend == "disk"
self._cache_hit_source = (
CacheHitSource.DISK if disk_mode else CacheHitSource.HOST
)
disk_path = extra_config.get("disk_path", None) or None
disk_capacity_bytes = int(
extra_config.get("disk_capacity_bytes", 100 * (1024**3))
)
disk_buffer_slots = max(1, int(extra_config.get("disk_buffer_slots", 2)))
use_page_cache = bool(extra_config.get("use_page_cache", False))
if disk_mode:
if disk_path is None:
raise ValueError(
'kv_offload_backend="disk" requires disk_path to be set.'
)
if disk_capacity_bytes <= 0:
raise ValueError(
'kv_offload_backend="disk" requires disk_capacity_bytes > 0.'
)
else:
ignored = [k for k in _DISK_ONLY_KEYS if k in extra_config]
if ignored:
logger.warning(
'kv_offload_backend is "cpu", ignoring disk-only config: %s. '
'Set kv_offload_backend="disk" to enable the disk backend.',
", ".join(ignored),
)
disk_path = None
self.scheduler_manager: SimpleCPUOffloadScheduler | None = None
self.worker_handler: SimpleCPUOffloadWorker | None = None
if not enable_prefix_caching:
logger.warning(
"Detected prefix caching disabled, disabling CPU offload "
"since it requires prefix caching."
)
return
logger.info(
"SimpleCPUOffloadConnector: role=%s, "
"per_rank=%.2f GB, world_size=%d, mode=%s, backend=%s, disk=%s",
role.name,
cpu_capacity_per_rank / (1024**3),
world_size,
"lazy" if lazy_offload else "eager",
kv_offload_backend,
disk_path or "none",
)
if num_cpu_blocks is not None:
logger.info(
"Using explicit num_cpu_blocks=%d for offload pool sizing "
"instead of automatic sizing from byte budgets.",
num_cpu_blocks,
)
if role == KVConnectorRole.SCHEDULER:
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
assert kv_cache_config is not None
scheduler_block_size, hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
self.scheduler_manager = SimpleCPUOffloadScheduler(
vllm_config,
kv_cache_config,
cpu_capacity_per_rank,
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
lazy_offload=lazy_offload,
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
use_page_cache=use_page_cache if disk_mode else False,
num_cpu_blocks=num_cpu_blocks,
)
elif role == KVConnectorRole.WORKER:
self.worker_handler = SimpleCPUOffloadWorker(
vllm_config,
kv_cache_config,
cpu_capacity_per_rank,
kv_offload_backend=kv_offload_backend,
disk_path=disk_path,
disk_capacity_bytes=disk_capacity_bytes,
disk_buffer_slots=disk_buffer_slots,
use_page_cache=use_page_cache,
num_cpu_blocks=num_cpu_blocks,
)
# --- Worker-side methods ---
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
if self.worker_handler is not None:
self.worker_handler.register_kv_caches(kv_caches)
def bind_connector_metadata(
self,
connector_metadata: KVConnectorMetadata,
) -> None:
super().bind_connector_metadata(connector_metadata)
if self.worker_handler is not None:
assert isinstance(connector_metadata, SimpleCPUOffloadMetadata)
self.worker_handler.bind_connector_metadata(connector_metadata)
def clear_connector_metadata(self) -> None:
super().clear_connector_metadata()
if self.worker_handler is not None:
self.worker_handler.clear_connector_metadata()
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata) -> None:
if self.worker_handler is not None:
assert isinstance(kv_connector_metadata, SimpleCPUOffloadMetadata)
self.worker_handler.handle_preemptions(kv_connector_metadata)
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
if self.worker_handler is not None:
self.worker_handler.start_load_kv()
def wait_for_layer_load(self, layer_name: str) -> None:
pass # Always load asynchronously, issued in start_load_kv()
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: "AttentionMetadata",
**kwargs: Any,
) -> None:
pass # Always save asynchronously, issued in wait_for_save()
def wait_for_save(self) -> None:
if self.worker_handler is not None:
self.worker_handler.wait_for_save()
def get_finished(
self,
finished_req_ids: set[str],
) -> tuple[set[str] | None, set[str] | None]:
if self.worker_handler is not None:
return self.worker_handler.get_finished(finished_req_ids)
return None, None
def build_connector_worker_meta(self):
if self.worker_handler is not None:
return self.worker_handler.build_connector_worker_meta()
return None
# --- Scheduler-side methods ---
def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None:
if self.scheduler_manager is not None:
self.scheduler_manager.bind_gpu_block_pool(gpu_block_pool)
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
if self.scheduler_manager is not None:
return self.scheduler_manager.get_num_new_matched_tokens(
request, num_computed_tokens
)
return 0, False
def update_state_after_alloc(
self,
request: "Request",
blocks: "KVCacheBlocks",
num_external_tokens: int,
) -> None:
if self.scheduler_manager is not None:
self.scheduler_manager.update_state_after_alloc(
request, blocks, num_external_tokens
)
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
if self.scheduler_manager is not None:
return self.scheduler_manager.build_connector_meta(scheduler_output)
return SimpleCPUOffloadMetadata()
def update_connector_output(
self,
connector_output: KVConnectorOutput,
) -> None:
if self.scheduler_manager is not None:
self.scheduler_manager.update_connector_output(connector_output)
def request_finished(
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
if self.scheduler_manager is not None:
return self.scheduler_manager.request_finished(request, block_ids)
return False, None
def request_finished_all_groups(
self,
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
if self.scheduler_manager is not None:
return self.scheduler_manager.request_finished_all_groups(
request, block_ids
)
return False, None
def has_pending_push_work(self) -> bool:
if self.scheduler_manager is not None:
return self.scheduler_manager.has_pending_stores()
return False
def get_boundary_store_stats(self) -> BoundaryStoreStats | None:
"""Return cumulative boundary-handoff store diagnostics."""
if self.scheduler_manager is not None:
return self.scheduler_manager.get_boundary_store_stats()
return None
def get_kv_connector_stats(self) -> KVConnectorStats | None:
# Worker-side callers may invoke this hook on connectors constructed
# via __new__ (as in test_worker.py), which lack scheduler_manager.
scheduler_manager: SimpleCPUOffloadScheduler | None = getattr(
self, "scheduler_manager", None
)
if scheduler_manager is not None:
return scheduler_manager.get_stats()
return None
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
SimpleCPUOffloadStats(data=data)
if data is not None
else SimpleCPUOffloadStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return SimpleCPUOffloadPromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
def take_events(self) -> Iterable[KVCacheEvent]:
if self.scheduler_manager is not None:
return self.scheduler_manager.take_events()
return []
# NOTE: Workers are not contacted. In-flight transfers drain naturally,
# and stale completions are ignored by the guarded
# SimpleCPUOffloadScheduler._process_store_event().
def reset_cache(self) -> bool | None:
if self.scheduler_manager is not None:
return self.scheduler_manager.reset()
return None
@@ -1,235 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mamba conv-state sub-projection decomposition for NIXL transfer.
With DS conv state layout (dim, state_len), sub-projections are
contiguous in memory. Each D rank reads its slices via separate
RDMA transfers — no P-side permutation needed.
Supported model types:
- Mamba1: conv = [x], temporal = (intermediate_size, state_size)
- Mamba2: conv = [x, B, C], temporal = (num_heads, head_dim)
- GDN (Gated Delta Net): conv = [Q, K, V] (dim(Q)==dim(K)),
temporal = (num_v_heads, v_dim, k_dim)
"""
import math
from dataclasses import dataclass
import torch
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import MambaSpec
@dataclass(frozen=True)
class MambaConvSplitInfo:
"""Per-rank byte sizes of the conv sub-projections.
Used by both P and D sides for NIXL descriptor registration.
All fields are LOCAL to this engine's TP (already divided by TP size).
DS memory layout within one page (contiguous):
Mamba1: |---- x ----| (single sub-projection, no decomposition)
Mamba2: |-- x --|- B -|- C -| (B == C)
GDN: |- Q -|- K -|-- V --| (dim(Q)==dim(K), V may differ)
"""
conv_rows: int # conv_kernel - 1 (typically 3)
# Per-rank column counts per sub-projection:
# 1 entry for Mamba1, 3 for Mamba2/GDN.
local_proj_dims: tuple[int, ...]
conv_dtype_size: int # bytes per element (e.g. 2 for float16)
ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes)
@property
def local_conv_dim(self) -> int:
"""Total conv columns per rank."""
return sum(self.local_proj_dims)
@property
def proj_bytes(self) -> tuple[int, ...]:
"""Byte sizes of the sub-projections for one rank."""
row_bytes = self.conv_rows * self.conv_dtype_size
return tuple(d * row_bytes for d in self.local_proj_dims)
@property
def local_conv_offsets(self) -> list[tuple[int, int]]:
"""(byte_offset, byte_size) of each sub-projection within this
engine's page.
Used by both P and D for local descriptor registration.
"""
offsets: list[tuple[int, int]] = []
offset = 0
for size in self.proj_bytes:
offsets.append((offset, size))
offset += size
return offsets
def remote_conv_offsets(
self, local_rank_offset: int, tp_ratio: int
) -> list[tuple[int, int]]:
"""(byte_offset, byte_size) of this D rank's sub-projection slices
within one P page.
Used by D side only, during remote descriptor registration.
Args:
local_rank_offset: which slice this D rank reads.
tp_ratio: signed TP ratio.
>= 1: D_TP >= P_TP — P page is larger, D reads its slice.
< 0: P_TP > D_TP — P pages are smaller, D reads entire
P page. Local dims are scaled down by |tp_ratio|
to get P-sized offsets.
"""
offsets: list[tuple[int, int]] = []
if tp_ratio >= 1:
remote_base = 0
for size in self.proj_bytes:
offsets.append((remote_base + local_rank_offset * size, size))
remote_base += size * tp_ratio
else:
# NOTE (ZhanqiuHu): tp_ratio < 0 means P_TP > D_TP, so P pages
# are smaller than D's. Local dims are D-sized, but we need
# P-sized offsets. Scale down by |tp_ratio|.
abs_ratio = -tp_ratio
remote_base = 0
for size in self.proj_bytes:
remote_size = size // abs_ratio
offsets.append((remote_base, remote_size))
remote_base += remote_size
return offsets
def derive_mamba_conv_split(
mamba_spec: MambaSpec,
local_tp: int,
) -> MambaConvSplitInfo:
"""Derive per-rank sub-projection byte sizes from a MambaSpec.
Called once at init on both P and D. Decomposes the conv dimension
into its sub-projection parts based on the model type.
Args:
mamba_spec: MambaSpec whose shapes are:
shapes[0] = conv state: (conv_dim_local, conv_rows) in DS layout.
shapes[1] = temporal state (model-specific shape).
local_tp: this engine's tensor-parallel size.
Returns:
MambaConvSplitInfo with per-rank sub-projection dims, conv_rows,
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
"""
_supported = (
MambaAttentionBackendEnum.MAMBA1,
MambaAttentionBackendEnum.MAMBA2,
MambaAttentionBackendEnum.GDN_ATTN,
)
if mamba_spec.mamba_type not in _supported:
raise NotImplementedError(
f"Conv transfer only supports Mamba1, Mamba2 and GDN models, "
f"got mamba_type={mamba_spec.mamba_type!r}."
)
conv_shape = mamba_spec.shapes[0]
assert len(conv_shape) == 2, f"Expected 2D conv state shape, got {conv_shape}"
# NOTE (ZhanqiuHu): 3-read requires DS layout, which is already asserted
# in nixl worker __init__. Use it directly instead of heuristic detection.
assert is_conv_state_dim_first(), "3-read requires DS conv state layout"
local_conv_dim = conv_shape[0] # DS: (conv_dim_local, conv_rows)
conv_rows = conv_shape[1]
conv_dtype_size = torch.tensor(
[],
dtype=mamba_spec.dtypes[0], # type: ignore[misc]
).element_size()
ssm_dtype_size = torch.tensor(
[],
dtype=mamba_spec.dtypes[1], # type: ignore[misc]
).element_size()
conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size
ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size
local_proj_dims: tuple[int, ...]
if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA1:
# Mamba1 conv state holds only x (no B/C), so it's a single
# contiguous TP shard with no sub-projection decomposition.
temporal_shape = mamba_spec.shapes[1]
assert temporal_shape[0] == local_conv_dim, (
f"Mamba1 temporal state dim ({temporal_shape[0]}) doesn't match "
f"conv dim ({local_conv_dim}); both should be "
f"intermediate_size/TP."
)
local_proj_dims = (local_conv_dim,)
elif mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2:
# NOTE (ZhanqiuHu): intermediate_size (= global x dim) is not stored
# in MambaSpec, so we reconstruct it from the SSM temporal state shape:
# shapes[1] = (local_num_heads, head_dim), already divided by TP.
head_dim = mamba_spec.shapes[1][1]
local_num_heads = mamba_spec.shapes[1][0]
intermediate_size = local_num_heads * local_tp * head_dim
# NOTE (ZhanqiuHu): global conv dim = intermediate_size + 2 * groups_ss,
# where groups_ss is the B (= C) dimension. B and C are always the same
# size, so we recover groups_ss from the remainder after subtracting x.
remainder = local_conv_dim * local_tp - intermediate_size
assert remainder > 0 and remainder % 2 == 0, (
f"Conv dim ({local_conv_dim}*tp={local_tp}) doesn't decompose "
f"into intermediate_size={intermediate_size} + 2*groups_ss. "
f"remainder={remainder}"
)
groups_ss = remainder // 2
# Divide by TP to get per-rank column counts.
x_local = intermediate_size // local_tp
b_local = groups_ss // local_tp
local_proj_dims = (x_local, b_local, b_local)
elif mamba_spec.mamba_type == MambaAttentionBackendEnum.GDN_ATTN:
# GDN: conv = [Q, K, V] where dim(Q) == dim(K) == key_dim.
# conv_dim = key_dim*2 + value_dim (all global, divided by TP).
# Temporal state shape is (num_v_heads/TP, head_v_dim, head_k_dim).
temporal_shape = mamba_spec.shapes[1]
num_v_heads_local = temporal_shape[0]
head_v_dim = temporal_shape[1]
value_dim_local = num_v_heads_local * head_v_dim
remainder = local_conv_dim - value_dim_local
assert remainder > 0 and remainder % 2 == 0, (
f"GDN conv dim ({local_conv_dim}) doesn't decompose into "
f"2*key_dim_local + value_dim_local={value_dim_local}. "
f"remainder={remainder}"
)
key_dim_local = remainder // 2
local_proj_dims = (key_dim_local, key_dim_local, value_dim_local)
else:
raise NotImplementedError(
f"Conv split not supported for mamba_type={mamba_spec.mamba_type!r}"
)
return MambaConvSplitInfo(
conv_rows=conv_rows,
local_proj_dims=local_proj_dims,
conv_dtype_size=conv_dtype_size,
ssm_sizes=(conv_state_bytes, ssm_state_bytes),
)
def compute_physical_blocks_per_logical(
ssm_sizes: tuple[int, ...], block_len: int
) -> int:
"""Derive _physical_blocks_per_logical_kv_block from remote metadata.
The remote engine's ratio is not sent directly in the handshake, so we
reconstruct it: total mamba state per logical block / block_len.
Args:
ssm_sizes: (conv_state_bytes, ssm_state_bytes) from NixlAgentMetadata.
block_len: the engine's block_len in bytes (from block_lens[0]).
"""
return math.ceil((ssm_sizes[0] + ssm_sizes[1]) / block_len)